Code clean up and function rename

This commit is contained in:
Coder12a
2019-01-14 18:17:45 -06:00
parent 69f789e5fd
commit b700bd5c77
2 changed files with 153 additions and 205 deletions

View File

@ -1,42 +1,37 @@
colddb = {}
local function createDir(directory)
return minetest.mkdir(directory)
end
function colddb.Colddb(directory)
local directory = string.format("%s/%s/", minetest.get_worldpath(), directory)
if not createDir(directory) then
error(string.format("%s is not a directory.", directory))
function colddb.Colddb(dir)
local dir = string.format("%s/%s/", minetest.get_worldpath(), dir)
if not minetest.mkdir(dir) then
error(string.format("%s is not a directory.", dir))
end
local self = {}
self.db = {
directory = directory,
tags = {},
mem_pool = {},
mem_pool_del = {},
indexes_pool = {},
iterate_queue = {},
indexes = false,
add_to_mem_pool = true,
async = extended_api.Async(),
}
local directory = dir
local tags = {}
local mem_pool = {}
local mem_pool_del = {}
local indexes_pool = {}
local iterate_queue = {}
local add_to_mem_pool = true
local async = extended_api.Async()
local path_count = {}
async.priority(150, 250)
self.db.async.priority(150, 250)
-- make tables weak so the garbage-collector will remove unused data
setmetatable(self.db.tags, {__mode = "kv"})
setmetatable(self.db.mem_pool, {__mode = "kv"})
setmetatable(self.db.mem_pool_del, {__mode = "kv"})
setmetatable(self.db.indexes_pool, {__mode = "kv"})
setmetatable(tags, {__mode = "kv"})
setmetatable(mem_pool, {__mode = "kv"})
setmetatable(mem_pool_del, {__mode = "kv"})
setmetatable(indexes_pool, {__mode = "kv"})
self.file_Exists = function(name, tag_name)
local function file_Exists(name, tag_name)
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
local f = io.open(string.format("%s%s%s.cold", self.db.directory, t, name), "r")
local f = io.open(string.format("%s%s%s.cold", directory, t, name), "r")
if f ~= nil then
io.close(f)
return true
@ -58,7 +53,7 @@ function colddb.Colddb(directory)
local function delete_lines_func_i(line, i, args)
local f = args.file
local om = self.db.indexes_pool[args.cs]
local om = indexes_pool[args.cs]
if om and not om.deleted_items[line] then
f:write(string.format("\n%s", line))
else
@ -69,7 +64,7 @@ function colddb.Colddb(directory)
local function delete_lines_func_end(args)
local cs = args.cs
local index = self.db.indexes_pool[cs]
local index = indexes_pool[cs]
if index and index.file then
index.file:close()
index.file = nil
@ -83,20 +78,20 @@ function colddb.Colddb(directory)
os.remove(args.oldfile)
os.rename(args.copyfile, args.oldfile)
end
local pool = self.db.indexes_pool[cs]
local pool = indexes_pool[cs]
for i, l in pairs(args.removedlist) do
pool.deleted_items[i] = nil
end
pool.deleting = false
self.db.indexes_pool[cs] = pool
indexes_pool[cs] = pool
end
args = nil
end
local function iterate(func_on_iterate, end_func, count, cs, args)
local f = self.db.indexes_pool[cs]
local f = indexes_pool[cs]
local fl = f.file
self.db.async.iterate(1, count, function(i)
async.iterate(1, count, function(i)
local line = fl:read("*l")
if args.do_not_skip_removed_items or not f.deleted_items[line] then
local ar = func_on_iterate(line, i, args)
@ -105,17 +100,17 @@ function colddb.Colddb(directory)
return args
end
end
end,function()
end, function()
if end_func then
end_func(args)
end
local iterate_queue = self.db.iterate_queue[cs]
local iterate_queue = iterate_queue[cs]
if iterate_queue and iterate_queue[1] then
local copy = iterate_queue[1]
f = self.db.indexes_pool[cs]
f = indexes_pool[cs]
if not f or not f.file then
self.open_index_table(copy.tag_name)
f = self.db.indexes_pool[cs]
f = indexes_pool[cs]
end
if copy.begin_func then
local a = copy.begin_func(copy.args)
@ -124,52 +119,46 @@ function colddb.Colddb(directory)
end
end
minetest.after(0, iterate, copy.func_on_iterate, copy.end_func, copy.count, copy.cs, copy.args)
table.remove(self.db.iterate_queue[cs], 1)
table.remove(iterate_queue[cs], 1)
return false
else
fl:close()
self.db.iterate_queue[cs] = nil
iterate_queue[cs] = nil
end
self.db.indexes_pool[cs].iterating = false
indexes_pool[cs].iterating = false
return false
end)
end
local function load_into_mem(name, _table, tag_name)
if self.db.add_to_mem_pool then
if add_to_mem_pool then
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
local mem_pool = self.db.mem_pool[cs]
if not mem_pool then
mem_pool = {mem = _table, indexes = self.db.indexes}
else
mem_pool.mem = _table
mem_pool.indexes = self.db.indexes
end
self.db.mem_pool[cs] = mem_pool
mem_pool[cs] = {mem = _table}
end
end
local path_count = {}
local function _remove_tag(delete_path, prev_dp)
if path_count[delete_path] and path_count[delete_path] > 0 then
local pc = path_count[delete_path]
if pc and pc > 0 then
minetest.after(1.5, _remove_tag, delete_path)
return
elseif path_count[delete_path] and path_count[delete_path] < 1 then
self.db.mem_pool = {}
self.db.mem_pool_del = {}
elseif pc and pc < 1 then
mem_pool = {}
mem_pool_del = {}
os.remove(delete_path)
path_count[delete_path] = nil
return
elseif not path_count[delete_path] then
elseif not pc then
path_count[delete_path] = 0
end
local list = minetest.get_dir_list(delete_path)
self.db.async.foreach(list, function(k, v)
async.foreach(list, function(k, v)
v = string.format("%s/%s", delete_path, v)
local err = os.remove(v)
if err == nil then
@ -180,20 +169,21 @@ function colddb.Colddb(directory)
if prev_dp then
path_count[prev_dp] = path_count[prev_dp] - 1
end
if path_count[delete_path] > 0 then
if pc > 0 then
minetest.after(1.5, _remove_tag, delete_path)
else
self.db.mem_pool = {}
self.db.mem_pool_del = {}
mem_pool = {}
mem_pool_del = {}
os.remove(delete_path)
path_count[delete_path] = nil
end
end)
end
self.add_tag = function(name, tag)
self.add_tag = function(name, tag, no_new_path)
local t = ""
if not self.db.tags[name] then
self.db.tags[name] = ""
if not tags[name] then
tags[name] = ""
end
if type(tag) == "table" then
for key, value in pairs(tag) do
@ -202,37 +192,46 @@ function colddb.Colddb(directory)
else
t = string.format("%s/", tag)
end
local test_path = string.format("%s%s%s", self.db.directory, self.db.tags[name], t)
if not createDir(test_path) then
tags[name] = string.format("%s%s", tags[name], t)
if no_new_path then
return
end
local test_path = string.format("%s%s%s", directory, tags[name], t)
if not minetest.mkdir(test_path) then
error(string.format("%s is not a directory.", test_path))
end
self.db.tags[name] = string.format("%s%s", self.db.tags[name], t)
end
self.get_tag = function(name)
if not name then
return ""
end
local tag = self.db.tags[name]
local tag = tags[name]
if tag then
return tag
end
return ""
end
self.get_or_add_tag = function(name, tag)
if not self.db.tags[name] then
self.add_tag(name, tag)
self.get_or_add_tag = function(name, tag, no_new_path)
if not tags[name] then
self.add_tag(name, tag, no_new_path)
end
return name
end
self.remove_tag = function(name)
if self.db.tags[name] then
local delete_path = string.format("%s%s", self.db.directory, self.db.tags[name])
if not name then
local delete_path = directory
local wc = delete_path:len()
delete_path = delete_path:sub(0, wc-1)
self.db.tags[name] = nil
return
end
if tags[name] then
local delete_path = string.format("%s%s", directory, tags[name])
local wc = delete_path:len()
delete_path = delete_path:sub(0, wc-1)
tags[name] = nil
local err = os.remove(delete_path)
if err == nil then
minetest.after(0.1, _remove_tag, delete_path)
@ -240,12 +239,12 @@ function colddb.Colddb(directory)
end
end
self.delete_file = function(name, tag_name)
local function delete_file(name, tag_name)
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
local text = string.format("%s%s%s.cold", self.db.directory, t, name)
local text = string.format("%s%s%s.cold", directory, t, name)
local err, msg = os.remove(text)
if err == nil then
@ -253,12 +252,12 @@ function colddb.Colddb(directory)
end
end
self.load_table = function(name, tag_name)
local function load_table(name, tag_name)
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
local f = io.open(string.format("%s%s%s.cold", self.db.directory, t, name), "r")
local f = io.open(string.format("%s%s%s.cold", directory, t, name), "r")
if f then
local data = minetest.deserialize(f:read("*a"))
f:close()
@ -267,28 +266,28 @@ function colddb.Colddb(directory)
return nil
end
self.save_table = function(name, _table, tag_name)
local function save_table(name, _table, tag_name)
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
return minetest.safe_file_write(string.format("%s%s%s.cold", self.db.directory, t, name), minetest.serialize(_table))
return minetest.safe_file_write(string.format("%s%s%s.cold", directory, t, name), minetest.serialize(_table))
end
self.save_key = function(name, tag_name)
local function save_key(name, tag_name)
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
return minetest.safe_file_write(string.format("%s%s%s.cold", self.db.directory, t, name), "")
return minetest.safe_file_write(string.format("%s%s%s.cold", directory, t, name), "")
end
self.load_key = function(name, tag_name)
local function load_key(name, tag_name)
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
local f = io.open(string.format("%s%s%s.cold", self.db.directory, t, name), "r")
local f = io.open(string.format("%s%s%s.cold", directory, t, name), "r")
if f then
f:close()
return true
@ -296,31 +295,13 @@ function colddb.Colddb(directory)
return false
end
self.delete_index_table = function(tag_name)
self.delete_entries = function(name ,_lines, tag_name)
local t = ""
local name = "æIndex_table"
if tag_name then
t = self.get_tag(tag_name)
end
local p = string.format("%s%sæIndex_table.cold", self.db.directory, t)
if self.file_Exists(name, tag_name) then
local err, msg = os.remove(p)
if err == nil then
print(string.format("error removing db data %s error message: %s", p, msg))
end
return true
end
return false
end
self.delete_lines = function(_lines, tag_name)
local t = ""
local name = "æIndex_table"
if tag_name then
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
local f = self.db.indexes_pool[cs]
local f = indexes_pool[cs]
local k = type(_lines)
if k == "string" then
f.deleted_items[_lines] = true
@ -330,33 +311,32 @@ function colddb.Colddb(directory)
end
end
if not f.deleting then
self.db.indexes_pool[cs].deleting = false
indexes_pool[cs].deleting = false
end
if f and f.file and not f.deleting then
self.db.indexes_pool[cs].deleting = true
indexes_pool[cs].deleting = true
if f.needs_flushing == true then
f.file:flush()
self.db.indexes_pool[cs].needs_flushing = false
indexes_pool[cs].needs_flushing = false
end
local oldfile = string.format("%s%sæIndex_table.cold", self.db.directory, t)
local copyfile = string.format("%s%sæIndex_table.cold.replacer", self.db.directory, t)
local oldfile = string.format("%s%s%s.cold", directory, t, name)
local copyfile = string.format("%s%s%s.cold.replacer", directory, t, name)
local args = {cs = cs, oldfile = oldfile, copyfile = copyfile, do_not_skip_removed_items = true}
self.db.indexes_pool[cs] = f
iterate_index_table(delete_lines_func_begin, delete_lines_func_i, delete_lines_func_end, args, tag_name)
indexes_pool[cs] = f
iterate_index_table(delete_entries_func_begin, delete_entries_func_i, delete_entries_func_end, args, tag_name)
end
end
self.open_index_table = function(tag_name)
self.open_entry_file = function(name, tag_name)
local t = ""
local name = "æIndex_table"
if tag_name then
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
local fs = self.db.indexes_pool[cs]
local fs = indexes_pool[cs]
if not fs then
local p = string.format("%s%sæIndex_table.cold", self.db.directory, t)
if not self.file_Exists(name,tag_name) then
local p = string.format("%s%s%s.cold", directory, t, name)
if not file_Exists(name,tag_name) then
local f = io.open(p, "w")
if f then
f:seek("set")
@ -366,7 +346,7 @@ function colddb.Colddb(directory)
end
local f = io.open(p, "r+")
if f then
self.db.indexes_pool[cs] = {file = f, needs_flushing = false, deleted_items = {}, iterating = false}
indexes_pool[cs] = {file = f, needs_flushing = false, deleted_items = {}, iterating = false}
return f
end
return nil
@ -376,14 +356,13 @@ function colddb.Colddb(directory)
return nil
end
self.append_index_table = function(key, tag_name)
self.append_entry_file = function(name, key, tag_name)
local t = ""
local name = "æIndex_table"
if tag_name then
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
local f = self.db.indexes_pool[cs]
local f = indexes_pool[cs]
local k = type(key)
if f and f.file and k == "string" then
local fl = f.file
@ -391,7 +370,7 @@ function colddb.Colddb(directory)
fl:flush()
f.needs_flushing = false
end
self.db.indexes_pool[cs].needs_flushing = true
indexes_pool[cs].needs_flushing = true
fl:seek("end")
fl:write(string.format("\n%s", key))
fl:seek("set")
@ -406,7 +385,7 @@ function colddb.Colddb(directory)
fl:flush()
f.needs_flushing = false
end
self.db.indexes_pool[cs].needs_flushing = true
indexes_pool[cs].needs_flushing = true
local c = 0
for i in pairs(key) do
fl:seek("end")
@ -424,14 +403,17 @@ function colddb.Colddb(directory)
end
end
self.get_count = function(tag_name)
self.get_entry_file_count = function(name, tag_name)
local t = ""
local name = "æIndex_table"
if tag_name then
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s",t,name)
local f = self.db.indexes_pool[cs]
local cs = string.format("%s%s", t, name)
local f = indexes_pool[cs]
if not f or not f.file then
self.open_entry_file(name, tag_name)
f = indexes_pool[cs]
end
if f and f.file then
local fl = f.file
if f.needs_flushing == true then
@ -446,20 +428,19 @@ function colddb.Colddb(directory)
return nil
end
self.iterate_index_table = function(begin_func, func_on_iterate, end_func, args, tag_name)
self.iterate_entry_file = function(name, begin_func, func_on_iterate, end_func, args, tag_name)
local t = ""
local name = "æIndex_table"
if tag_name then
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
local f = self.db.indexes_pool[cs]
local f = indexes_pool[cs]
if not f or not f.file then
self.open_index_table(tag_name)
f = self.db.indexes_pool[cs]
self.open_entry_file(tag_name)
f = indexes_pool[cs]
end
if f and f.file and self.db.indexes_pool[cs].iterating == false then
self.db.indexes_pool[cs].iterating = true
if f and f.file and indexes_pool[cs].iterating == false then
indexes_pool[cs].iterating = true
local fl = f.file
if f.needs_flushing == true then
fl:flush()
@ -482,7 +463,7 @@ function colddb.Colddb(directory)
if c < 1 then
-- If theres nothing to index then return
end_func(args)
self.db.indexes_pool[cs].iterating = false
indexes_pool[cs].iterating = false
return false
end
-- Start iterating the index table
@ -496,11 +477,11 @@ function colddb.Colddb(directory)
-- If theres nothing to index then return
return false
end
if not self.db.iterate_queue[cs] then
self.db.iterate_queue[cs] = {}
if not iterate_queue[cs] then
iterate_queue[cs] = {}
end
local _table = {begin_func = begin_func, func_on_iterate = func_on_iterate, end_func = end_func, count = c, cs = cs, tag_name = tag_name, args = args}
table.insert(self.db.iterate_queue[cs], _table)
table.insert(iterate_queue[cs], _table)
end
end
@ -509,24 +490,14 @@ function colddb.Colddb(directory)
if tag_name then
t = self.get_tag(tag_name)
end
if self.db.indexes and not self.file_Exists(name,tag_name) then
self.db.async.queue_task(function()
local cs2 = string.format("%s%s", t , "æIndex_table")
local om = self.db.indexes_pool[cs2]
if not self.file_Exists("æIndex_table", tag_name) or not (om and om.file) then
self.open_index_table(tag_name)
end
self.append_index_table(name, tag_name)
end)
end
self.db.async.queue_task(function()
self.save_table(name, _table, tag_name)
async.queue_task(function()
save_table(name, _table, tag_name)
end)
if self.db.add_to_mem_pool then
if add_to_mem_pool then
load_into_mem(name, _table, tag_name)
end
local cs = string.format("%s%s", t, name)
self.db.mem_pool_del[cs] = nil
mem_pool_del[cs] = nil
end
self.set_key = function(name, tag_name)
@ -534,24 +505,14 @@ function colddb.Colddb(directory)
if tag_name then
t = self.get_tag(tag_name)
end
if self.db.indexes and not self.file_Exists(name, tag_name) then
self.db.async.queue_task(function()
local cs2 = string.format("%s%s", t, "æIndex_table")
local om = self.db.indexes_pool[cs2]
if not self.file_Exists("æIndex_table", tag_name) or not (om and om.file) then
self.open_index_table(tag_name)
end
self.append_index_table(name, tag_name)
end)
end
self.db.async.queue_task(function()
self.save_key(name, tag_name)
async.queue_task(function()
save_key(name, tag_name)
end)
if self.db.add_to_mem_pool then
if add_to_mem_pool then
load_into_mem(name, "", tag_name)
end
local cs = string.format("%s%s", t, name)
self.db.mem_pool_del[cs] = nil
mem_pool_del[cs] = nil
end
self.get = function(name, tag_name, callback)
@ -560,39 +521,39 @@ function colddb.Colddb(directory)
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
if self.db.mem_pool_del[cs] then
if mem_pool_del[cs] then
if callback then
callback(nil)
end
return nil
end
if callback then
self.db.async.queue_task(function()
local pm = self.db.mem_pool[cs]
async.queue_task(function()
local pm = mem_pool[cs]
if pm then
return pm.mem
else
local _table = self.load_table(name, tag_name)
local _table = load_table(name, tag_name)
if _table then
load_into_mem(name, _table, tag_name)
return _table
end
end
self.db.mem_pool_del[cs] = true
mem_pool_del[cs] = true
return nil
end,callback)
else
local pm = self.db.mem_pool[cs]
local pm = mem_pool[cs]
if pm then
return pm.mem
else
local _table = self.load_table(name, tag_name)
local _table = load_table(name, tag_name)
if _table then
load_into_mem(name, _table, tag_name)
return _table
end
end
self.db.mem_pool_del[cs] = true
mem_pool_del[cs] = true
return nil
end
end
@ -603,43 +564,43 @@ function colddb.Colddb(directory)
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
if self.db.mem_pool_del[cs] then
if mem_pool_del[cs] then
if callback then
callback(false)
end
return false
end
if callback then
self.db.async.queue_task(function()
async.queue_task(function()
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
local pm = self.db.mem_pool[cs]
local pm = mem_pool[cs]
if pm then
return true
else
local bool = self.load_key(name, tag_name)
local bool = load_key(name, tag_name)
if bool then
load_into_mem(name, bool, tag_name)
return bool
end
end
self.db.mem_pool_del[cs] = true
mem_pool_del[cs] = true
return nil
end,callback)
else
local pm = self.db.mem_pool[cs]
local pm = mem_pool[cs]
if pm then
return true
else
local bool = self.load_key(name, tag_name)
local bool = load_key(name, tag_name)
if bool then
load_into_mem(name, bool, tag_name)
return bool
end
end
self.db.mem_pool_del[cs] = true
mem_pool_del[cs] = true
return nil
end
end
@ -650,19 +611,10 @@ function colddb.Colddb(directory)
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
self.db.mem_pool[cs] = nil
self.db.mem_pool_del[cs] = true
if self.db.indexes and self.file_Exists("æIndex_table", tag_name) then
self.db.async.queue_task(function()
local cs2 = string.format("%s%s", t, "æIndex_table")
if not (self.db.indexes_pool[cs2] and self.db.indexes_pool[cs2].file) then
self.open_index_table(tag_name)
end
self.delete_lines(name, tag_name)
end)
end
self.db.async.queue_task(function()
self.delete_file(name, tag_name)
mem_pool[cs] = nil
mem_pool_del[cs] = true
async.queue_task(function()
delete_file(name, tag_name)
end)
end