colddb/colddb.lua

colddb = {}

local function createDir(directory)
	return minetest.mkdir(directory)
end

function colddb.file_Exists(db,name,tag_name)
	local t = ""
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	local f = io.open(string.format("%s%s%s.cold",db.directory,t,name),"r")
	if f ~= nil then 
		io.close(f)
		return true
	else 
		return false 
	end
	return false 
end

function colddb.get_db(directory)
	local directory = string.format("%s/%s",minetest.get_worldpath(),directory)
	if not createDir(directory) then
		error(string.format("%s is not a directory.",directory))
	end
	db = {
		global_tag = "",
		directory = directory,
		tags = {},
		mem_pool = {},
		indexes_pool = {},
		iterate_queue = {},
		indexes = false,
		add_to_mem_pool = true,
		async_pool = extended_api.Async.create_async_pool(),
	}
	extended_api.Async.priority(db.async_pool,150,250)
	-- make tables weak so the garbage-collector will remove unused data
	setmetatable(db.tags, {__mode = "kv"})
	setmetatable(db.mem_pool, {__mode = "kv"})
	setmetatable(db.indexes_pool, {__mode = "kv"})
	return db
end

function colddb.add_global_tag(db,tag)
	local t = ""
	if type(tag) == "table" then
		for index in pairs(tag) do
			t = string.format("%s%s/",t,index)
		end
	else
		t = string.format("%s/",tag)
	end
	db.global_tag = string.format("%s%s",db.global_tag,t)
	db.directory = string.format("%s/%s",db.directory,t)
	if not createDir(db.directory) then
		error(string.format("%s is not a directory.",db.directory))
	end
end

function colddb.add_tag(db,name,tag)
	local t = ""
	if not db.tags[name] then
		db.tags[name] = ""
	end
	if type(tag) == "table" then
		for key,value in pairs(tag) do
			t = string.format("%s%s/",t,value)
		end
	else
		t = string.format("%s/",tag)
	end
	local test_path = string.format("%s%s%s",db.directory,db.tags[name],t)
	if not createDir(test_path) then
		error(string.format("%s is not a directory.",test_path))
	end
	db.tags[name] = string.format("%s%s",db.tags[name],t)
end

function colddb.get_tag(db,name)
	if not name then
		return ""
	end
	local tag = db.tags[name]
	if tag then
		return tag
	end
	return ""
end

function colddb.get_or_add_tag(db,name,tag)
	if not db.tags[name] then
		colddb.add_tag(db,name,tag)
	end
	return name
end

function colddb.remove_tag(db,name)
	if db.tags[name] then
		local delete_path = string.format("%s%s",db.directory,db.tags[name])
		local wc = delete_path:len()
		delete_path = delete_path:sub(0,wc-1)
		db.tags[name] = nil
		os.remove(delete_path)
	end
end

function colddb.delete_file(db,name,tag_name)
	local t = ""
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	local text = string.format("%s%s%s.cold",db.directory,t,name)
	
	local err,msg = os.remove(text) 
	if err == nil then 
		print(string.format("error removing db data %s error message: %s",text,msg))
	end
end

function colddb.load_table(db,name,tag_name)
	local t = ""
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	local f = io.open(string.format("%s%s%s.cold",db.directory,t,name), "r")
	if f then
		local data = minetest.deserialize(f:read("*a"))
		f:close()
		return data
	end
	return nil
end

function colddb.save_table(db,name, _table,tag_name)
	local t = ""
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	return minetest.safe_file_write(string.format("%s%s%s.cold",db.directory,t,name), minetest.serialize(_table))
end

function colddb.save_key(db,name,tag_name)
	local t = ""
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	return minetest.safe_file_write(string.format("%s%s%s.cold",db.directory,t,name), "")
end

function colddb.load_key(db,name,tag_name)
	local t = ""
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	local f = io.open(string.format("%s%s%s.cold",db.directory,t,name), "r")
	if f then
		f:close()
		return true
	end
	return false
end

function colddb.delete_index_table(db,tag_name)
	local t = ""
	local name = "æIndex_table"
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	local p = string.format("%s%sæIndex_table.cold",db.directory,t)
	if colddb.file_Exists(db,name,tag_name) then
		local err,msg = os.remove(p) 
		if err == nil then 
			print(string.format("error removing db data %s error message: %s",p,msg))
		end
		return true
	end
	return false
end

local function delete_lines_func_begin(args)
	local f = io.open(args.copyfile, "w")
	local db = args.db
	local cs = args.cs
	if f then
		args.file = f
		args.removedlist = {}
		return args
	end
	return false
end

local function delete_lines_func_i(line,i,args)
	local f = args.file
	local db = args.db
	local cs = args.cs
	local om = db.indexes_pool[args.cs]
	if om and not om.deleted_items[line] then
		f:write(string.format("\n%s",line))
	else
		args.count = args.count - 1
		args.removedlist[line] = true
	end
end

local function delete_lines_func_end(args)
	local db = args.db
	local cs = args.cs
	if db.indexes_pool[cs] or db.indexes_pool[cs].file then
		db.indexes_pool[cs].file:close()
		db.indexes_pool[cs].file = nil
		args.file:seek("set")
		args.file:write(string.format("%i",args.count))
		args.file:close()
		os.remove(args.oldfile)
		os.rename(args.copyfile, args.oldfile)
		for i,l in pairs(args.removedlist) do
			db.indexes_pool[cs].deleted_items[i] = nil
		end
		db.indexes_pool[cs].deleting = false
	end
	args = nil
end

function colddb.delete_lines(db,_lines,tag_name)
	local t = ""
	local name = "æIndex_table"
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	local cs = string.format("%s%s",t,name)
	local f = db.indexes_pool[cs]
	local k = type(_lines)
	if k == "string" then
		f.deleted_items[_lines] = true
	else
		for i,l in pairs(_lines) do
			f.deleted_items[i] = true
		end
	end
	if not db.indexes_pool[cs].deleting then
		db.indexes_pool[cs].deleting = false
	end
	if f and f.file and not db.indexes_pool[cs].deleting then
		db.indexes_pool[cs].deleting = true
		if db.indexes_pool[cs].needs_flushing == true then
			f.file:flush()
			db.indexes_pool[cs].needs_flushing = false
		end
		local oldfile = string.format("%s%sæIndex_table.cold",db.directory,t)
		local copyfile = string.format("%s%sæIndex_table.cold.replacer",db.directory,t)
		local args = {db=db,cs=cs,oldfile=oldfile,copyfile=copyfile,do_not_skip_removed_items=true}
		db.indexes_pool[cs] = f
		colddb.iterate_index_table(db,delete_lines_func_begin,delete_lines_func_i,delete_lines_func_end,args,tag_name)
	end
end

function colddb.create_index_table(db,tag_name)
	local t = ""
	local name = "æIndex_table"
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	local p = string.format("%s%sæIndex_table.cold",db.directory,t)
	if not colddb.file_Exists(db,name,tag_name) then
		local f = io.open(p, "w")
		if f then
			f:seek("set")
			f:write("0")
			f:close()
		end
	end
	local f = io.open(p, "r+")
	if f then
		f:seek("set")
		f:write("0")
		db.indexes_pool[string.format("%s%s",t,name)] = {file = f,needs_flushing = false,deleted_items = {},iterating = false}
		return true
	end
	return false
end

function colddb.open_index_table(db,tag_name)
	local t = ""
	local name = "æIndex_table"
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	local cs = string.format("%s%s",t,name)
	local fs = db.indexes_pool[cs]
	if not fs then
		local p = string.format("%s%sæIndex_table.cold",db.directory,t)
		if not colddb.file_Exists(db,name,tag_name) then
			local f = io.open(p, "w")
			if f then
				f:seek("set")
				f:write("0")
				f:close()
			end
		end
		local f = io.open(p, "r+")
		if f then
			db.indexes_pool[cs] = {file = f,needs_flushing = false,deleted_items = {},iterating = false}
			return f
		end
		return nil
	else
		return fs.file
	end
	return nil
end

function colddb.append_index_table(db,key,tag_name)
	local t = ""
	local name = "æIndex_table"
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	local cs = string.format("%s%s",t,name)
	local f = db.indexes_pool[cs]
	local k = type(key)
	if f and f.file and k == "string" then
		local fl = f.file
		if f.needs_flushing == true then
			fl:flush()
		end
		db.indexes_pool[cs].needs_flushing = true
		fl:seek("end")
		fl:write(string.format("\n%s",key))
		fl:seek("set")
		local count = tonumber(fl:read("*l"))
		count = count + 1
		fl:seek("set")
		fl:write(string.format("%i",count))
	elseif f and f.file then
		local fl = f.file
		if f.needs_flushing == true then
			fl:flush()
		end
		db.indexes_pool[cs].needs_flushing = true
		local c = 0
		for i in pairs(key) do
			fl:seek("end")
			fl:write(string.format("\n%s",i))
			c = c + 1
		end
		fl:seek("set")
		local count = tonumber(fl:read("*l"))
		count = count + c
		fl:seek("set")
		fl:write(string.format("%i",count))
	else
		return false
	end
end

function colddb.get_count(db,tag_name)
	local t = ""
	local name = "æIndex_table"
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	local cs = string.format("%s%s",t,name)
	local f = db.indexes_pool[cs]
	if f and f.file then
		local fl = f.file
		if f.needs_flushing == true then
			fl:flush()
			f.needs_flushing = true
		end
		fl:seek("set")
		local count = tonumber(fl:read("*l"))
		return count
	end
	return nil
end

local function iterate(db,func_on_iterate,end_func,count,cs,args)
	local f = db.indexes_pool[cs]
	local fl = f.file
	extended_api.Async.iterate(db.async_pool,1,count,function(i)
		local line = fl:read("*l")
		if args.do_not_skip_removed_items or not db.indexes_pool[cs].deleted_items[line] then
			func_on_iterate(line,i,args)
		end
	end,function() 
		if end_func then
			end_func(args)
		end
		if db.iterate_queue[cs] and db.iterate_queue[cs][1] then
			local copy = db.iterate_queue[cs][1]
			f = db.indexes_pool[cs]
			if not f or not f.file then
				colddb.open_index_table(db,copy.tag_name)
				f = db.indexes_pool[cs]
			end
			if copy.begin_func then
				local a = copy.begin_func(copy.args)
				if a and type(a) == "table" then
					copy.args = a
				end
			end
			minetest.after(0,iterate,copy.db,copy.func_on_iterate,copy.end_func,copy.count,copy.cs,copy.args)
			table.remove(db.iterate_queue[cs],1)
			return false
		else
			db.iterate_queue[cs] = nil
		end
			db.indexes_pool[cs].iterating = false
		return false
	end)
end

function colddb.iterate_index_table(db,begin_func,func_on_iterate,end_func,args,tag_name)
	local t = ""
	local name = "æIndex_table"
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	local cs = string.format("%s%s",t,name)
	local f = db.indexes_pool[cs]
	if not f or not f.file then
		colddb.open_index_table(db,tag_name)
		f = db.indexes_pool[cs]
	end
	if f and f.file and db.indexes_pool[cs].iterating == false then
		db.indexes_pool[cs].iterating = true
		local fl = f.file
		if f.needs_flushing == true then
			fl:flush()
			f.needs_flushing = true
		end
		-- Get count
		fl:seek("set")
		local c = tonumber(fl:read("*l"))
		if c < 1 then
			-- If theres nothing to index then return
			return false
		end
		if not args then
			args = {}
		end
		args.count = c
		-- Run the begin function
		if begin_func then
			local a = begin_func(args)
			if a and type(a) == "table" then
				args = a
			end
		end
		-- Start iterating the index table
		iterate(db,func_on_iterate,end_func,c,cs,args)
	elseif f and f.file then
		-- If its iterating some other function then add this one to the queue list
		fl:seek("set")
		local c = tonumber(fl:read("*l"))
		if c < 1 then
			-- If theres nothing to index then return
			return false
		end
		if not db.iterate_queue[cs] then
			db.iterate_queue[cs] = {}
		end
		local _table = {db=db,begin_func=begin_func,func_on_iterate=func_on_iterate,end_func=end_func,count=c,cs=cs,tag_name=tag_name,args=args}
		table.insert(db.iterate_queue[cs],_table)
	end
end

local function load_into_mem(db,name,_table,tag_name)
	if db.add_to_mem_pool then
		local t = ""
		if tag_name then
			t = colddb.get_tag(db,tag_name)
		end
		local cs = string.format("%s%s",t,name)
		if not db.mem_pool[cs] then
			db.mem_pool[cs] = {mem = _table,indexes = db.indexes}
		else
			db.mem_pool[cs].mem = _table
			db.mem_pool[cs].indexes = db.indexes
		end
	end
end

function colddb.set(db,name,_table,tag_name)
	local t = ""
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	if db.indexes and not colddb.file_Exists(db,name,tag_name) then
		extended_api.Async.queue_task(db.async_pool,function() 
			local cs2 = string.format("%s%s",t,"æIndex_table")
			local om = db.indexes_pool[cs2]
			if not colddb.file_Exists(db,"æIndex_table",tag_name) or not (om and om.file) then
				colddb.open_index_table(db,tag_name)
			end
			colddb.append_index_table(db,name,tag_name)
		end)
	end
	extended_api.Async.queue_task(db.async_pool,function() 
		colddb.save_table(db,name, _table,tag_name)
	end)
	if db.add_to_mem_pool then
		load_into_mem(db,name,_table,tag_name)
	end
end

function colddb.set_key(db,name,tag_name)
	local t = ""
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	if db.indexes and not colddb.file_Exists(db,name,tag_name) then
		extended_api.Async.queue_task(db.async_pool,function() 
			local cs2 = string.format("%s%s",t,"æIndex_table")
			local om = db.indexes_pool[cs2]
			if not colddb.file_Exists(db,"æIndex_table",tag_name) or not (om and om.file) then
				colddb.open_index_table(db,tag_name)
			end
			colddb.append_index_table(db,name,tag_name)
		end)
	end
	extended_api.Async.queue_task(db.async_pool,function() 
		colddb.save_key(db,name, tag_name)
	end)
	if db.add_to_mem_pool then
		load_into_mem(db,name,"",tag_name)
	end
end

function colddb.get(db,name,tag_name,callback)
	extended_api.Async.queue_task(db.async_pool,function() 
		local t = ""
		if tag_name then
			t = colddb.get_tag(db,tag_name)
		end
		local cs = string.format("%s%s",t,name)
		local pm = db.mem_pool[cs]
		if pm then
			return pm.mem
		else
			local _table = colddb.load_table(db,name,tag_name)
			if _table then
				load_into_mem(db,name,_table,tag_name)
				return _table
			end
		end
		return nil
	end,callback)
end

function colddb.get_key(db,name,tag_name,callback)
	extended_api.Async.queue_task(db.async_pool,function() 
		local t = ""
		if tag_name then
			t = colddb.get_tag(db,tag_name)
		end
		local cs = string.format("%s%s",t,name)
		local pm = db.mem_pool[cs]
		if pm then
			return true
		else
			local bool = colddb.load_key(db,name,tag_name)
			if bool then
				load_into_mem(db,name,bool,tag_name)
				return bool
			end
		end
		return nil
	end,callback)
end

function colddb.remove(db,name,tag_name)
	local t = ""
	if tag_name then
		t = colddb.get_tag(db,tag_name)
	end
	local cs = string.format("%s%s",t,name)
	if db.mem_pool[cs] then
		db.mem_pool[cs] = nil
	end
	if db.indexes and colddb.file_Exists(db,"æIndex_table",tag_name) then
		extended_api.Async.queue_task(db.async_pool,function()
			local cs2 = string.format("%s%s",t,"æIndex_table")
			if not (db.indexes_pool[cs2] and db.indexes_pool[cs2].file) then
				colddb.open_index_table(db,tag_name)
			end
			colddb.delete_lines(db,name,tag_name)
		end)
	end
	extended_api.Async.queue_task(db.async_pool,function()
		colddb.delete_file(db,name,tag_name)
	end)
end