forked from mtcontrib/colddb
595 lines
15 KiB
Lua
595 lines
15 KiB
Lua
colddb = {}
|
|
|
|
local function createDir(directory)
|
|
return minetest.mkdir(directory)
|
|
end
|
|
|
|
function colddb.file_Exists(db,name,tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local f = io.open(string.format("%s%s%s.cold",db.directory,t,name),"r")
|
|
if f ~= nil then
|
|
io.close(f)
|
|
return true
|
|
else
|
|
return false
|
|
end
|
|
return false
|
|
end
|
|
|
|
function colddb.get_db(directory)
|
|
local directory = string.format("%s/%s",minetest.get_worldpath(),directory)
|
|
if not createDir(directory) then
|
|
error(string.format("%s is not a directory.",directory))
|
|
end
|
|
db = {
|
|
global_tag = "",
|
|
directory = directory,
|
|
tags = {},
|
|
mem_pool = {},
|
|
indexes_pool = {},
|
|
iterate_queue = {},
|
|
indexes = false,
|
|
add_to_mem_pool = true,
|
|
async_pool = extended_api.Async.create_async_pool(),
|
|
}
|
|
extended_api.Async.priority(db.async_pool,150,250)
|
|
-- make tables weak so the garbage-collector will remove unused data
|
|
setmetatable(db.tags, {__mode = "kv"})
|
|
setmetatable(db.mem_pool, {__mode = "kv"})
|
|
setmetatable(db.indexes_pool, {__mode = "kv"})
|
|
return db
|
|
end
|
|
|
|
function colddb.add_global_tag(db,tag)
|
|
local t = ""
|
|
if type(tag) == "table" then
|
|
for index in pairs(tag) do
|
|
t = string.format("%s%s/",t,index)
|
|
end
|
|
else
|
|
t = string.format("%s/",tag)
|
|
end
|
|
db.global_tag = string.format("%s%s",db.global_tag,t)
|
|
db.directory = string.format("%s/%s",db.directory,t)
|
|
if not createDir(db.directory) then
|
|
error(string.format("%s is not a directory.",db.directory))
|
|
end
|
|
end
|
|
|
|
function colddb.add_tag(db,name,tag)
|
|
local t = ""
|
|
if not db.tags[name] then
|
|
db.tags[name] = ""
|
|
end
|
|
if type(tag) == "table" then
|
|
for key,value in pairs(tag) do
|
|
t = string.format("%s%s/",t,value)
|
|
end
|
|
else
|
|
t = string.format("%s/",tag)
|
|
end
|
|
local test_path = string.format("%s%s%s",db.directory,db.tags[name],t)
|
|
if not createDir(test_path) then
|
|
error(string.format("%s is not a directory.",test_path))
|
|
end
|
|
db.tags[name] = string.format("%s%s",db.tags[name],t)
|
|
end
|
|
|
|
function colddb.get_tag(db,name)
|
|
if not name then
|
|
return ""
|
|
end
|
|
local tag = db.tags[name]
|
|
if tag then
|
|
return tag
|
|
end
|
|
return ""
|
|
end
|
|
|
|
function colddb.get_or_add_tag(db,name,tag)
|
|
if not db.tags[name] then
|
|
colddb.add_tag(db,name,tag)
|
|
end
|
|
return name
|
|
end
|
|
|
|
function colddb.remove_tag(db,name)
|
|
if db.tags[name] then
|
|
local delete_path = string.format("%s%s",db.directory,db.tags[name])
|
|
local wc = delete_path:len()
|
|
delete_path = delete_path:sub(0,wc-1)
|
|
db.tags[name] = nil
|
|
os.remove(delete_path)
|
|
end
|
|
end
|
|
|
|
function colddb.delete_file(db,name,tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local text = string.format("%s%s%s.cold",db.directory,t,name)
|
|
|
|
local err,msg = os.remove(text)
|
|
if err == nil then
|
|
print(string.format("error removing db data %s error message: %s",text,msg))
|
|
end
|
|
end
|
|
|
|
function colddb.load_table(db,name,tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local f = io.open(string.format("%s%s%s.cold",db.directory,t,name), "r")
|
|
if f then
|
|
local data = minetest.deserialize(f:read("*a"))
|
|
f:close()
|
|
return data
|
|
end
|
|
return nil
|
|
end
|
|
|
|
function colddb.save_table(db,name, _table,tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
return minetest.safe_file_write(string.format("%s%s%s.cold",db.directory,t,name), minetest.serialize(_table))
|
|
end
|
|
|
|
function colddb.save_key(db,name,tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
return minetest.safe_file_write(string.format("%s%s%s.cold",db.directory,t,name), "")
|
|
end
|
|
|
|
function colddb.load_key(db,name,tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local f = io.open(string.format("%s%s%s.cold",db.directory,t,name), "r")
|
|
if f then
|
|
f:close()
|
|
return true
|
|
end
|
|
return false
|
|
end
|
|
|
|
function colddb.delete_index_table(db,tag_name)
|
|
local t = ""
|
|
local name = "æIndex_table"
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local p = string.format("%s%sæIndex_table.cold",db.directory,t)
|
|
if colddb.file_Exists(db,name,tag_name) then
|
|
local err,msg = os.remove(p)
|
|
if err == nil then
|
|
print(string.format("error removing db data %s error message: %s",p,msg))
|
|
end
|
|
return true
|
|
end
|
|
return false
|
|
end
|
|
|
|
local function delete_lines_func_begin(args)
|
|
local f = io.open(args.copyfile, "w")
|
|
local db = args.db
|
|
local cs = args.cs
|
|
if f then
|
|
args.file = f
|
|
args.removedlist = {}
|
|
return args
|
|
end
|
|
return false
|
|
end
|
|
|
|
local function delete_lines_func_i(line,i,args)
|
|
local f = args.file
|
|
local db = args.db
|
|
local cs = args.cs
|
|
local om = db.indexes_pool[args.cs]
|
|
if om and not om.deleted_items[line] then
|
|
f:write(string.format("\n%s",line))
|
|
else
|
|
args.count = args.count - 1
|
|
args.removedlist[line] = true
|
|
end
|
|
end
|
|
|
|
local function delete_lines_func_end(args)
|
|
local db = args.db
|
|
local cs = args.cs
|
|
if db.indexes_pool[cs] or db.indexes_pool[cs].file then
|
|
db.indexes_pool[cs].file:close()
|
|
db.indexes_pool[cs].file = nil
|
|
args.file:seek("set")
|
|
args.file:write(string.format("%i",args.count))
|
|
args.file:close()
|
|
os.remove(args.oldfile)
|
|
os.rename(args.copyfile, args.oldfile)
|
|
for i,l in pairs(args.removedlist) do
|
|
db.indexes_pool[cs].deleted_items[i] = nil
|
|
end
|
|
db.indexes_pool[cs].deleting = false
|
|
end
|
|
args = nil
|
|
end
|
|
|
|
function colddb.delete_lines(db,_lines,tag_name)
|
|
local t = ""
|
|
local name = "æIndex_table"
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local cs = string.format("%s%s",t,name)
|
|
local f = db.indexes_pool[cs]
|
|
local k = type(_lines)
|
|
if k == "string" then
|
|
f.deleted_items[_lines] = true
|
|
else
|
|
for i,l in pairs(_lines) do
|
|
f.deleted_items[i] = true
|
|
end
|
|
end
|
|
if not db.indexes_pool[cs].deleting then
|
|
db.indexes_pool[cs].deleting = false
|
|
end
|
|
if f and f.file and not db.indexes_pool[cs].deleting then
|
|
db.indexes_pool[cs].deleting = true
|
|
if db.indexes_pool[cs].needs_flushing == true then
|
|
f.file:flush()
|
|
db.indexes_pool[cs].needs_flushing = false
|
|
end
|
|
local oldfile = string.format("%s%sæIndex_table.cold",db.directory,t)
|
|
local copyfile = string.format("%s%sæIndex_table.cold.replacer",db.directory,t)
|
|
local args = {db=db,cs=cs,oldfile=oldfile,copyfile=copyfile,do_not_skip_removed_items=true}
|
|
db.indexes_pool[cs] = f
|
|
colddb.iterate_index_table(db,delete_lines_func_begin,delete_lines_func_i,delete_lines_func_end,args,tag_name)
|
|
end
|
|
end
|
|
|
|
function colddb.create_index_table(db,tag_name)
|
|
local t = ""
|
|
local name = "æIndex_table"
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local p = string.format("%s%sæIndex_table.cold",db.directory,t)
|
|
if not colddb.file_Exists(db,name,tag_name) then
|
|
local f = io.open(p, "w")
|
|
if f then
|
|
f:seek("set")
|
|
f:write("0")
|
|
f:close()
|
|
end
|
|
end
|
|
local f = io.open(p, "r+")
|
|
if f then
|
|
f:seek("set")
|
|
f:write("0")
|
|
db.indexes_pool[string.format("%s%s",t,name)] = {file = f,needs_flushing = false,deleted_items = {},iterating = false}
|
|
return true
|
|
end
|
|
return false
|
|
end
|
|
|
|
function colddb.open_index_table(db,tag_name)
|
|
local t = ""
|
|
local name = "æIndex_table"
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local cs = string.format("%s%s",t,name)
|
|
local fs = db.indexes_pool[cs]
|
|
if not fs then
|
|
local p = string.format("%s%sæIndex_table.cold",db.directory,t)
|
|
if not colddb.file_Exists(db,name,tag_name) then
|
|
local f = io.open(p, "w")
|
|
if f then
|
|
f:seek("set")
|
|
f:write("0")
|
|
f:close()
|
|
end
|
|
end
|
|
local f = io.open(p, "r+")
|
|
if f then
|
|
db.indexes_pool[cs] = {file = f,needs_flushing = false,deleted_items = {},iterating = false}
|
|
return f
|
|
end
|
|
return nil
|
|
else
|
|
return fs.file
|
|
end
|
|
return nil
|
|
end
|
|
|
|
function colddb.append_index_table(db,key,tag_name)
|
|
local t = ""
|
|
local name = "æIndex_table"
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local cs = string.format("%s%s",t,name)
|
|
local f = db.indexes_pool[cs]
|
|
local k = type(key)
|
|
if f and f.file and k == "string" then
|
|
local fl = f.file
|
|
if f.needs_flushing == true then
|
|
fl:flush()
|
|
end
|
|
db.indexes_pool[cs].needs_flushing = true
|
|
fl:seek("end")
|
|
fl:write(string.format("\n%s",key))
|
|
fl:seek("set")
|
|
local count = tonumber(fl:read("*l"))
|
|
count = count + 1
|
|
fl:seek("set")
|
|
fl:write(string.format("%i",count))
|
|
elseif f and f.file then
|
|
local fl = f.file
|
|
if f.needs_flushing == true then
|
|
fl:flush()
|
|
end
|
|
db.indexes_pool[cs].needs_flushing = true
|
|
local c = 0
|
|
for i in pairs(key) do
|
|
fl:seek("end")
|
|
fl:write(string.format("\n%s",i))
|
|
c = c + 1
|
|
end
|
|
fl:seek("set")
|
|
local count = tonumber(fl:read("*l"))
|
|
count = count + c
|
|
fl:seek("set")
|
|
fl:write(string.format("%i",count))
|
|
else
|
|
return false
|
|
end
|
|
end
|
|
|
|
function colddb.get_count(db,tag_name)
|
|
local t = ""
|
|
local name = "æIndex_table"
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local cs = string.format("%s%s",t,name)
|
|
local f = db.indexes_pool[cs]
|
|
if f and f.file then
|
|
local fl = f.file
|
|
if f.needs_flushing == true then
|
|
fl:flush()
|
|
f.needs_flushing = true
|
|
end
|
|
fl:seek("set")
|
|
local count = tonumber(fl:read("*l"))
|
|
return count
|
|
end
|
|
return nil
|
|
end
|
|
|
|
local function iterate(db,func_on_iterate,end_func,count,cs,args)
|
|
local f = db.indexes_pool[cs]
|
|
local fl = f.file
|
|
extended_api.Async.iterate(db.async_pool,1,count,function(i)
|
|
local line = fl:read("*l")
|
|
if args.do_not_skip_removed_items or not db.indexes_pool[cs].deleted_items[line] then
|
|
func_on_iterate(line,i,args)
|
|
end
|
|
end,function()
|
|
if end_func then
|
|
end_func(args)
|
|
end
|
|
if db.iterate_queue[cs] and db.iterate_queue[cs][1] then
|
|
local copy = db.iterate_queue[cs][1]
|
|
f = db.indexes_pool[cs]
|
|
if not f or not f.file then
|
|
colddb.open_index_table(db,copy.tag_name)
|
|
f = db.indexes_pool[cs]
|
|
end
|
|
if copy.begin_func then
|
|
local a = copy.begin_func(copy.args)
|
|
if a and type(a) == "table" then
|
|
copy.args = a
|
|
end
|
|
end
|
|
minetest.after(0,iterate,copy.db,copy.func_on_iterate,copy.end_func,copy.count,copy.cs,copy.args)
|
|
table.remove(db.iterate_queue[cs],1)
|
|
return false
|
|
else
|
|
db.iterate_queue[cs] = nil
|
|
end
|
|
db.indexes_pool[cs].iterating = false
|
|
return false
|
|
end)
|
|
end
|
|
|
|
function colddb.iterate_index_table(db,begin_func,func_on_iterate,end_func,args,tag_name)
|
|
local t = ""
|
|
local name = "æIndex_table"
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local cs = string.format("%s%s",t,name)
|
|
local f = db.indexes_pool[cs]
|
|
if not f or not f.file then
|
|
colddb.open_index_table(db,tag_name)
|
|
f = db.indexes_pool[cs]
|
|
end
|
|
if f and f.file and db.indexes_pool[cs].iterating == false then
|
|
db.indexes_pool[cs].iterating = true
|
|
local fl = f.file
|
|
if f.needs_flushing == true then
|
|
fl:flush()
|
|
f.needs_flushing = true
|
|
end
|
|
-- Get count
|
|
fl:seek("set")
|
|
local c = tonumber(fl:read("*l"))
|
|
if c < 1 then
|
|
-- If theres nothing to index then return
|
|
return false
|
|
end
|
|
if not args then
|
|
args = {}
|
|
end
|
|
args.count = c
|
|
-- Run the begin function
|
|
if begin_func then
|
|
local a = begin_func(args)
|
|
if a and type(a) == "table" then
|
|
args = a
|
|
end
|
|
end
|
|
-- Start iterating the index table
|
|
iterate(db,func_on_iterate,end_func,c,cs,args)
|
|
elseif f and f.file then
|
|
-- If its iterating some other function then add this one to the queue list
|
|
fl:seek("set")
|
|
local c = tonumber(fl:read("*l"))
|
|
if c < 1 then
|
|
-- If theres nothing to index then return
|
|
return false
|
|
end
|
|
if not db.iterate_queue[cs] then
|
|
db.iterate_queue[cs] = {}
|
|
end
|
|
local _table = {db=db,begin_func=begin_func,func_on_iterate=func_on_iterate,end_func=end_func,count=c,cs=cs,tag_name=tag_name,args=args}
|
|
table.insert(db.iterate_queue[cs],_table)
|
|
end
|
|
end
|
|
|
|
local function load_into_mem(db,name,_table,tag_name)
|
|
if db.add_to_mem_pool then
|
|
local t = ""
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local cs = string.format("%s%s",t,name)
|
|
if not db.mem_pool[cs] then
|
|
db.mem_pool[cs] = {mem = _table,indexes = db.indexes}
|
|
else
|
|
db.mem_pool[cs].mem = _table
|
|
db.mem_pool[cs].indexes = db.indexes
|
|
end
|
|
end
|
|
end
|
|
|
|
function colddb.set(db,name,_table,tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
if db.indexes and not colddb.file_Exists(db,name,tag_name) then
|
|
extended_api.Async.queue_task(db.async_pool,function()
|
|
local cs2 = string.format("%s%s",t,"æIndex_table")
|
|
local om = db.indexes_pool[cs2]
|
|
if not colddb.file_Exists(db,"æIndex_table",tag_name) or not (om and om.file) then
|
|
colddb.open_index_table(db,tag_name)
|
|
end
|
|
colddb.append_index_table(db,name,tag_name)
|
|
end)
|
|
end
|
|
extended_api.Async.queue_task(db.async_pool,function()
|
|
colddb.save_table(db,name, _table,tag_name)
|
|
end)
|
|
if db.add_to_mem_pool then
|
|
load_into_mem(db,name,_table,tag_name)
|
|
end
|
|
end
|
|
|
|
function colddb.set_key(db,name,tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
if db.indexes and not colddb.file_Exists(db,name,tag_name) then
|
|
extended_api.Async.queue_task(db.async_pool,function()
|
|
local cs2 = string.format("%s%s",t,"æIndex_table")
|
|
local om = db.indexes_pool[cs2]
|
|
if not colddb.file_Exists(db,"æIndex_table",tag_name) or not (om and om.file) then
|
|
colddb.open_index_table(db,tag_name)
|
|
end
|
|
colddb.append_index_table(db,name,tag_name)
|
|
end)
|
|
end
|
|
extended_api.Async.queue_task(db.async_pool,function()
|
|
colddb.save_key(db,name, tag_name)
|
|
end)
|
|
if db.add_to_mem_pool then
|
|
load_into_mem(db,name,"",tag_name)
|
|
end
|
|
end
|
|
|
|
function colddb.get(db,name,tag_name,callback)
|
|
extended_api.Async.queue_task(db.async_pool,function()
|
|
local t = ""
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local cs = string.format("%s%s",t,name)
|
|
local pm = db.mem_pool[cs]
|
|
if pm then
|
|
return pm.mem
|
|
else
|
|
local _table = colddb.load_table(db,name,tag_name)
|
|
if _table then
|
|
load_into_mem(db,name,_table,tag_name)
|
|
return _table
|
|
end
|
|
end
|
|
return nil
|
|
end,callback)
|
|
end
|
|
|
|
function colddb.get_key(db,name,tag_name,callback)
|
|
extended_api.Async.queue_task(db.async_pool,function()
|
|
local t = ""
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local cs = string.format("%s%s",t,name)
|
|
local pm = db.mem_pool[cs]
|
|
if pm then
|
|
return true
|
|
else
|
|
local bool = colddb.load_key(db,name,tag_name)
|
|
if bool then
|
|
load_into_mem(db,name,bool,tag_name)
|
|
return bool
|
|
end
|
|
end
|
|
return nil
|
|
end,callback)
|
|
end
|
|
|
|
function colddb.remove(db,name,tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = colddb.get_tag(db,tag_name)
|
|
end
|
|
local cs = string.format("%s%s",t,name)
|
|
if db.mem_pool[cs] then
|
|
db.mem_pool[cs] = nil
|
|
end
|
|
if db.indexes and colddb.file_Exists(db,"æIndex_table",tag_name) then
|
|
extended_api.Async.queue_task(db.async_pool,function()
|
|
local cs2 = string.format("%s%s",t,"æIndex_table")
|
|
if not (db.indexes_pool[cs2] and db.indexes_pool[cs2].file) then
|
|
colddb.open_index_table(db,tag_name)
|
|
end
|
|
colddb.delete_lines(db,name,tag_name)
|
|
end)
|
|
end
|
|
extended_api.Async.queue_task(db.async_pool,function()
|
|
colddb.delete_file(db,name,tag_name)
|
|
end)
|
|
end |