forked from mtcontrib/colddb
682 lines
17 KiB
Lua
682 lines
17 KiB
Lua
colddb = {}
|
|
|
|
local function createDir(directory)
|
|
return minetest.mkdir(directory)
|
|
end
|
|
|
|
function colddb.Colddb(directory)
|
|
local directory = string.format("%s/%s/", minetest.get_worldpath(), directory)
|
|
if not createDir(directory) then
|
|
error(string.format("%s is not a directory.", directory))
|
|
end
|
|
|
|
local self = {}
|
|
|
|
self.db = {
|
|
global_tag = "",
|
|
directory = directory,
|
|
tags = {},
|
|
mem_pool = {},
|
|
mem_pool_del = {},
|
|
indexes_pool = {},
|
|
iterate_queue = {},
|
|
indexes = false,
|
|
add_to_mem_pool = true,
|
|
async = extended_api.Async(),
|
|
}
|
|
|
|
self.db.async.priority(150, 250)
|
|
-- make tables weak so the garbage-collector will remove unused data
|
|
setmetatable(self.db.tags, {__mode = "kv"})
|
|
setmetatable(self.db.mem_pool, {__mode = "kv"})
|
|
setmetatable(self.db.mem_pool_del, {__mode = "kv"})
|
|
setmetatable(self.db.indexes_pool, {__mode = "kv"})
|
|
|
|
self.file_Exists = function(name, tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local f = io.open(string.format("%s%s%s.cold", self.db.directory, t, name), "r")
|
|
if f ~= nil then
|
|
io.close(f)
|
|
return true
|
|
else
|
|
return false
|
|
end
|
|
return false
|
|
end
|
|
|
|
local function delete_lines_func_begin(args)
|
|
local f = io.open(args.copyfile, "w")
|
|
if f then
|
|
args.file = f
|
|
args.removedlist = {}
|
|
return args
|
|
end
|
|
return false
|
|
end
|
|
|
|
local function delete_lines_func_i(line, i, args)
|
|
local f = args.file
|
|
local om = self.db.indexes_pool[args.cs]
|
|
if om and not om.deleted_items[line] then
|
|
f:write(string.format("\n%s", line))
|
|
else
|
|
args.count = args.count - 1
|
|
args.removedlist[line] = true
|
|
end
|
|
end
|
|
|
|
local function delete_lines_func_end(args)
|
|
local cs = args.cs
|
|
if self.db.indexes_pool[cs] or self.db.indexes_pool[cs].file then
|
|
self.db.indexes_pool[cs].file:close()
|
|
self.db.indexes_pool[cs].file = nil
|
|
args.file:seek("set")
|
|
args.file:write(string.format("%i", args.count))
|
|
args.file:close()
|
|
if args.count < 1 then
|
|
os.remove(args.oldfile)
|
|
os.remove(args.copyfile)
|
|
else
|
|
os.remove(args.oldfile)
|
|
os.rename(args.copyfile, args.oldfile)
|
|
end
|
|
for i, l in pairs(args.removedlist) do
|
|
self.db.indexes_pool[cs].deleted_items[i] = nil
|
|
end
|
|
self.db.indexes_pool[cs].deleting = false
|
|
end
|
|
args = nil
|
|
end
|
|
|
|
local function iterate(func_on_iterate, end_func, count, cs, args)
|
|
local f = self.db.indexes_pool[cs]
|
|
local fl = f.file
|
|
self.db.async.iterate(1, count, function(i)
|
|
local line = fl:read("*l")
|
|
if args.do_not_skip_removed_items or not self.db.indexes_pool[cs].deleted_items[line] then
|
|
local ar = func_on_iterate(line, i, args)
|
|
if ar ~= nil then
|
|
args = ar
|
|
return args
|
|
end
|
|
end
|
|
end,function()
|
|
if end_func then
|
|
end_func(args)
|
|
end
|
|
if self.db.iterate_queue[cs] and self.db.iterate_queue[cs][1] then
|
|
local copy = self.db.iterate_queue[cs][1]
|
|
f = self.db.indexes_pool[cs]
|
|
if not f or not f.file then
|
|
self.open_index_table(copy.tag_name)
|
|
f = self.db.indexes_pool[cs]
|
|
end
|
|
if copy.begin_func then
|
|
local a = copy.begin_func(copy.args)
|
|
if a and type(a) == "table" then
|
|
copy.args = a
|
|
end
|
|
end
|
|
minetest.after(0, iterate, copy.func_on_iterate, copy.end_func, copy.count, copy.cs, copy.args)
|
|
table.remove(self.db.iterate_queue[cs], 1)
|
|
return false
|
|
else
|
|
fl:close()
|
|
self.db.iterate_queue[cs] = nil
|
|
end
|
|
self.db.indexes_pool[cs].iterating = false
|
|
return false
|
|
end)
|
|
end
|
|
|
|
local function load_into_mem(name, _table, tag_name)
|
|
if self.db.add_to_mem_pool then
|
|
local t = ""
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local cs = string.format("%s%s", t, name)
|
|
if not self.db.mem_pool[cs] then
|
|
self.db.mem_pool[cs] = {mem = _table, indexes = self.db.indexes}
|
|
else
|
|
self.db.mem_pool[cs].mem = _table
|
|
self.db.mem_pool[cs].indexes = self.db.indexes
|
|
end
|
|
end
|
|
end
|
|
|
|
local path_count = {}
|
|
|
|
local function _remove_tag(delete_path, prev_dp)
|
|
if path_count[delete_path] and path_count[delete_path] > 0 then
|
|
minetest.after(1.5, _remove_tag, delete_path)
|
|
return
|
|
elseif path_count[delete_path] and path_count[delete_path] < 1 then
|
|
self.db.mem_pool = {}
|
|
self.db.mem_pool_del = {}
|
|
os.remove(delete_path)
|
|
return
|
|
elseif not path_count[delete_path] then
|
|
path_count[delete_path] = 0
|
|
end
|
|
|
|
local list = minetest.get_dir_list(delete_path)
|
|
self.db.async.foreach(list, function(k, v)
|
|
v = string.format("%s/%s", delete_path, v)
|
|
local err = os.remove(v)
|
|
if err == nil then
|
|
minetest.after(0, _remove_tag, v, delete_path)
|
|
path_count[delete_path] = path_count[delete_path] + 1
|
|
end
|
|
end, function()
|
|
if prev_dp then
|
|
path_count[prev_dp] = path_count[prev_dp] - 1
|
|
end
|
|
if path_count[delete_path] > 0 then
|
|
minetest.after(1.5, _remove_tag, delete_path)
|
|
else
|
|
self.db.mem_pool = {}
|
|
self.db.mem_pool_del = {}
|
|
os.remove(delete_path)
|
|
end
|
|
end)
|
|
end
|
|
|
|
self.add_global_tag = function(tag)
|
|
local t = ""
|
|
if type(tag) == "table" then
|
|
for index in pairs(tag) do
|
|
t = string.format("%s%s/", t, index)
|
|
end
|
|
else
|
|
t = string.format("%s/", tag)
|
|
end
|
|
self.db.global_tag = string.format("%s%s", self.db.global_tag, t)
|
|
self.db.directory = string.format("%s/%s", self.db.directory, t)
|
|
if not createDir(self.db.directory) then
|
|
error(string.format("%s is not a directory.", self.db.directory))
|
|
end
|
|
end
|
|
|
|
self.add_tag = function(name, tag)
|
|
local t = ""
|
|
if not self.db.tags[name] then
|
|
self.db.tags[name] = ""
|
|
end
|
|
if type(tag) == "table" then
|
|
for key, value in pairs(tag) do
|
|
t = string.format("%s%s/", t, value)
|
|
end
|
|
else
|
|
t = string.format("%s/", tag)
|
|
end
|
|
local test_path = string.format("%s%s%s", self.db.directory, self.db.tags[name], t)
|
|
if not createDir(test_path) then
|
|
error(string.format("%s is not a directory.", test_path))
|
|
end
|
|
self.db.tags[name] = string.format("%s%s", self.db.tags[name], t)
|
|
end
|
|
|
|
self.get_tag = function(name)
|
|
if not name then
|
|
return ""
|
|
end
|
|
local tag = self.db.tags[name]
|
|
if tag then
|
|
return tag
|
|
end
|
|
return ""
|
|
end
|
|
|
|
self.get_or_add_tag = function(name, tag)
|
|
if not self.db.tags[name] then
|
|
self.add_tag(name, tag)
|
|
end
|
|
return name
|
|
end
|
|
|
|
self.remove_tag = function(name)
|
|
if self.db.tags[name] then
|
|
local delete_path = string.format("%s%s", self.db.directory, self.db.tags[name])
|
|
local wc = delete_path:len()
|
|
delete_path = delete_path:sub(0, wc-1)
|
|
self.db.tags[name] = nil
|
|
local err = os.remove(delete_path)
|
|
if err == nil then
|
|
minetest.after(0.1, _remove_tag, delete_path)
|
|
end
|
|
end
|
|
end
|
|
|
|
self.delete_file = function(name, tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local text = string.format("%s%s%s.cold", self.db.directory, t, name)
|
|
|
|
local err, msg = os.remove(text)
|
|
if err == nil then
|
|
print(string.format("error removing db data %s error message: %s", text, msg))
|
|
end
|
|
end
|
|
|
|
self.load_table = function(name, tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local f = io.open(string.format("%s%s%s.cold", self.db.directory, t, name), "r")
|
|
if f then
|
|
local data = minetest.deserialize(f:read("*a"))
|
|
f:close()
|
|
return data
|
|
end
|
|
return nil
|
|
end
|
|
|
|
self.save_table = function(name, _table, tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
return minetest.safe_file_write(string.format("%s%s%s.cold", self.db.directory, t, name), minetest.serialize(_table))
|
|
end
|
|
|
|
self.save_key = function(name, tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
return minetest.safe_file_write(string.format("%s%s%s.cold", self.db.directory, t, name), "")
|
|
end
|
|
|
|
self.load_key = function(name, tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local f = io.open(string.format("%s%s%s.cold", self.db.directory, t, name), "r")
|
|
if f then
|
|
f:close()
|
|
return true
|
|
end
|
|
return false
|
|
end
|
|
|
|
self.delete_index_table = function(tag_name)
|
|
local t = ""
|
|
local name = "æIndex_table"
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local p = string.format("%s%sæIndex_table.cold", self.db.directory, t)
|
|
if self.file_Exists(name, tag_name) then
|
|
local err, msg = os.remove(p)
|
|
if err == nil then
|
|
print(string.format("error removing db data %s error message: %s", p, msg))
|
|
end
|
|
return true
|
|
end
|
|
return false
|
|
end
|
|
|
|
self.delete_lines = function(_lines, tag_name)
|
|
local t = ""
|
|
local name = "æIndex_table"
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local cs = string.format("%s%s", t, name)
|
|
local f = self.db.indexes_pool[cs]
|
|
local k = type(_lines)
|
|
if k == "string" then
|
|
f.deleted_items[_lines] = true
|
|
else
|
|
for i, l in pairs(_lines) do
|
|
f.deleted_items[i] = true
|
|
end
|
|
end
|
|
if not self.db.indexes_pool[cs].deleting then
|
|
self.db.indexes_pool[cs].deleting = false
|
|
end
|
|
if f and f.file and not self.db.indexes_pool[cs].deleting then
|
|
self.db.indexes_pool[cs].deleting = true
|
|
if self.db.indexes_pool[cs].needs_flushing == true then
|
|
f.file:flush()
|
|
self.db.indexes_pool[cs].needs_flushing = false
|
|
end
|
|
local oldfile = string.format("%s%sæIndex_table.cold", self.db.directory, t)
|
|
local copyfile = string.format("%s%sæIndex_table.cold.replacer", self.db.directory, t)
|
|
local args = {cs = cs, oldfile = oldfile, copyfile = copyfile, do_not_skip_removed_items = true}
|
|
self.db.indexes_pool[cs] = f
|
|
iterate_index_table(delete_lines_func_begin, delete_lines_func_i, delete_lines_func_end, args, tag_name)
|
|
end
|
|
end
|
|
|
|
self.open_index_table = function(tag_name)
|
|
local t = ""
|
|
local name = "æIndex_table"
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local cs = string.format("%s%s", t, name)
|
|
local fs = self.db.indexes_pool[cs]
|
|
if not fs then
|
|
local p = string.format("%s%sæIndex_table.cold", self.db.directory, t)
|
|
if not self.file_Exists(name,tag_name) then
|
|
local f = io.open(p, "w")
|
|
if f then
|
|
f:seek("set")
|
|
f:write("0")
|
|
f:close()
|
|
end
|
|
end
|
|
local f = io.open(p, "r+")
|
|
if f then
|
|
self.db.indexes_pool[cs] = {file = f, needs_flushing = false, deleted_items = {}, iterating = false}
|
|
return f
|
|
end
|
|
return nil
|
|
else
|
|
return fs.file
|
|
end
|
|
return nil
|
|
end
|
|
|
|
self.append_index_table = function(key, tag_name)
|
|
local t = ""
|
|
local name = "æIndex_table"
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local cs = string.format("%s%s", t, name)
|
|
local f = self.db.indexes_pool[cs]
|
|
local k = type(key)
|
|
if f and f.file and k == "string" then
|
|
local fl = f.file
|
|
if f.needs_flushing == true then
|
|
fl:flush()
|
|
f.needs_flushing = false
|
|
end
|
|
self.db.indexes_pool[cs].needs_flushing = true
|
|
fl:seek("end")
|
|
fl:write(string.format("\n%s", key))
|
|
fl:seek("set")
|
|
local count = tonumber(fl:read("*l"))
|
|
count = count + 1
|
|
fl:seek("set")
|
|
fl:write(string.format("%i", count))
|
|
fl:close()
|
|
elseif f and f.file then
|
|
local fl = f.file
|
|
if f.needs_flushing == true then
|
|
fl:flush()
|
|
f.needs_flushing = false
|
|
end
|
|
self.db.indexes_pool[cs].needs_flushing = true
|
|
local c = 0
|
|
for i in pairs(key) do
|
|
fl:seek("end")
|
|
fl:write(string.format("\n%s", i))
|
|
c = c + 1
|
|
end
|
|
fl:seek("set")
|
|
local count = tonumber(fl:read("*l"))
|
|
count = count + c
|
|
fl:seek("set")
|
|
fl:write(string.format("%i", count))
|
|
fl:close()
|
|
else
|
|
return false
|
|
end
|
|
end
|
|
|
|
self.get_count = function(tag_name)
|
|
local t = ""
|
|
local name = "æIndex_table"
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local cs = string.format("%s%s",t,name)
|
|
local f = self.db.indexes_pool[cs]
|
|
if f and f.file then
|
|
local fl = f.file
|
|
if f.needs_flushing == true then
|
|
fl:flush()
|
|
f.needs_flushing = false
|
|
end
|
|
fl:seek("set")
|
|
local count = tonumber(fl:read("*l"))
|
|
fl:close()
|
|
return count
|
|
end
|
|
return nil
|
|
end
|
|
|
|
self.iterate_index_table = function(begin_func, func_on_iterate, end_func, args, tag_name)
|
|
local t = ""
|
|
local name = "æIndex_table"
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local cs = string.format("%s%s", t, name)
|
|
local f = self.db.indexes_pool[cs]
|
|
if not f or not f.file then
|
|
self.open_index_table(tag_name)
|
|
f = self.db.indexes_pool[cs]
|
|
end
|
|
if f and f.file and self.db.indexes_pool[cs].iterating == false then
|
|
self.db.indexes_pool[cs].iterating = true
|
|
local fl = f.file
|
|
if f.needs_flushing == true then
|
|
fl:flush()
|
|
f.needs_flushing = false
|
|
end
|
|
-- Get count
|
|
fl:seek("set")
|
|
local c = tonumber(fl:read("*l"))
|
|
if not args then
|
|
args = {}
|
|
end
|
|
args.count = c
|
|
-- Run the begin function
|
|
if begin_func then
|
|
local a = begin_func(args)
|
|
if a and type(a) == "table" then
|
|
args = a
|
|
end
|
|
end
|
|
if c < 1 then
|
|
-- If theres nothing to index then return
|
|
end_func(args)
|
|
self.db.indexes_pool[cs].iterating = false
|
|
return false
|
|
end
|
|
-- Start iterating the index table
|
|
iterate(func_on_iterate, end_func, c, cs, args)
|
|
elseif f and f.file then
|
|
local fl = f.file
|
|
-- If its iterating some other function then add this one to the queue list
|
|
fl:seek("set")
|
|
local c = tonumber(fl:read("*l"))
|
|
if c < 1 then
|
|
-- If theres nothing to index then return
|
|
return false
|
|
end
|
|
if not self.db.iterate_queue[cs] then
|
|
self.db.iterate_queue[cs] = {}
|
|
end
|
|
local _table = {begin_func = begin_func, func_on_iterate = func_on_iterate, end_func = end_func, count = c, cs = cs, tag_name = tag_name, args = args}
|
|
table.insert(self.db.iterate_queue[cs], _table)
|
|
end
|
|
end
|
|
|
|
self.set = function(name, _table, tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
if self.db.indexes and not self.file_Exists(name,tag_name) then
|
|
self.db.async.queue_task(function()
|
|
local cs2 = string.format("%s%s", t , "æIndex_table")
|
|
local om = self.db.indexes_pool[cs2]
|
|
if not self.file_Exists("æIndex_table", tag_name) or not (om and om.file) then
|
|
self.open_index_table(tag_name)
|
|
end
|
|
self.append_index_table(name, tag_name)
|
|
end)
|
|
end
|
|
self.db.async.queue_task(function()
|
|
self.save_table(name, _table, tag_name)
|
|
end)
|
|
if self.db.add_to_mem_pool then
|
|
load_into_mem(name, _table, tag_name)
|
|
end
|
|
local cs = string.format("%s%s", t, name)
|
|
self.db.mem_pool_del[cs] = nil
|
|
end
|
|
|
|
self.set_key = function(name, tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
if self.db.indexes and not self.file_Exists(name, tag_name) then
|
|
self.db.async.queue_task(function()
|
|
local cs2 = string.format("%s%s", t, "æIndex_table")
|
|
local om = self.db.indexes_pool[cs2]
|
|
if not self.file_Exists("æIndex_table", tag_name) or not (om and om.file) then
|
|
self.open_index_table(tag_name)
|
|
end
|
|
self.append_index_table(name, tag_name)
|
|
end)
|
|
end
|
|
self.db.async.queue_task(function()
|
|
self.save_key(name, tag_name)
|
|
end)
|
|
if self.db.add_to_mem_pool then
|
|
load_into_mem(name, "", tag_name)
|
|
end
|
|
local cs = string.format("%s%s", t, name)
|
|
self.db.mem_pool_del[cs] = nil
|
|
end
|
|
|
|
self.get = function(name, tag_name, callback)
|
|
local t = ""
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local cs = string.format("%s%s", t, name)
|
|
if self.db.mem_pool_del[cs] then
|
|
if callback then
|
|
callback(nil)
|
|
end
|
|
return nil
|
|
end
|
|
if callback then
|
|
self.db.async.queue_task(function()
|
|
local pm = self.db.mem_pool[cs]
|
|
if pm then
|
|
return pm.mem
|
|
else
|
|
local _table = self.load_table(name, tag_name)
|
|
if _table then
|
|
load_into_mem(name, _table, tag_name)
|
|
return _table
|
|
end
|
|
end
|
|
self.db.mem_pool_del[cs] = true
|
|
return nil
|
|
end,callback)
|
|
else
|
|
local pm = self.db.mem_pool[cs]
|
|
if pm then
|
|
return pm.mem
|
|
else
|
|
local _table = self.load_table(name, tag_name)
|
|
if _table then
|
|
load_into_mem(name, _table, tag_name)
|
|
return _table
|
|
end
|
|
end
|
|
self.db.mem_pool_del[cs] = true
|
|
return nil
|
|
end
|
|
end
|
|
|
|
self.get_key = function(name, tag_name, callback)
|
|
local t = ""
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local cs = string.format("%s%s", t, name)
|
|
if self.db.mem_pool_del[cs] then
|
|
if callback then
|
|
callback(false)
|
|
end
|
|
return false
|
|
end
|
|
if callback then
|
|
self.db.async.queue_task(function()
|
|
local t = ""
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local pm = self.db.mem_pool[cs]
|
|
if pm then
|
|
return true
|
|
else
|
|
local bool = self.load_key(name, tag_name)
|
|
if bool then
|
|
load_into_mem(name, bool, tag_name)
|
|
return bool
|
|
end
|
|
end
|
|
self.db.mem_pool_del[cs] = true
|
|
return nil
|
|
end,callback)
|
|
else
|
|
local pm = self.db.mem_pool[cs]
|
|
if pm then
|
|
return true
|
|
else
|
|
local bool = self.load_key(name, tag_name)
|
|
if bool then
|
|
load_into_mem(name, bool, tag_name)
|
|
return bool
|
|
end
|
|
end
|
|
self.db.mem_pool_del[cs] = true
|
|
return nil
|
|
end
|
|
end
|
|
|
|
self.remove = function(name, tag_name)
|
|
local t = ""
|
|
if tag_name then
|
|
t = self.get_tag(tag_name)
|
|
end
|
|
local cs = string.format("%s%s", t, name)
|
|
self.db.mem_pool[cs] = nil
|
|
self.db.mem_pool_del[cs] = true
|
|
if self.db.indexes and self.file_Exists("æIndex_table", tag_name) then
|
|
self.db.async.queue_task(function()
|
|
local cs2 = string.format("%s%s",t,"æIndex_table")
|
|
if not (self.db.indexes_pool[cs2] and self.db.indexes_pool[cs2].file) then
|
|
self.open_index_table(tag_name)
|
|
end
|
|
self.delete_lines(name, tag_name)
|
|
end)
|
|
end
|
|
self.db.async.queue_task(function()
|
|
self.delete_file(name, tag_name)
|
|
end)
|
|
end
|
|
|
|
return self
|
|
end
|