Code clean up and function rename

This commit is contained in:
Coder12a 2019-01-14 18:17:45 -06:00
parent 69f789e5fd
commit b700bd5c77
2 changed files with 153 additions and 205 deletions

View File

@ -2,7 +2,7 @@ ColdDB
===========
ColdDB is a minetest mod that implements a serverless, asynchronous, NoSQL database engine.<br>
It provides a key or key-value storage system using plain Lua tables, also it can iterate through the keys.<br>
It provides a key or key-value based storage system using plain Lua tables.<br>
Usage
===========
@ -44,27 +44,23 @@ coldbase.remove("MyKeyAndValue")
```lua
coldbase.add_to_mem_pool = true
```
7. if indexes is true(false by default). When a key file is created an indexing file stores the key for look-ups. this makes it possible to iterate through keys.
7. returns the amount of entries from the entry file.
```lua
coldbase.indexes = true
coldbase.get_entry_file_count(name, tag_name)
```
8. only if coldbase.indexes is true. returns the amount of keys that are in the indexing file.
8. iterates through the entry file(breaks and ends if it reaches the end of the file).
```lua
coldbase.get_count()
coldbase.iterate_entry_file(name, begin_func, func_on_iterate, end_func, args, tag_name)
```
9. only if coldbase.indexes is true. iterates through the indexing file(breaks and ends if it reaches the end of the file).
```lua
coldbase.iterate_index_table(nil, func_list_keys, nil)
```
10. adds a folder which can be used in other functions that have tag_name arg.
9. adds a folder which can be used in other functions that have tag_name arg.
```lua
coldbase.add_tag("Extra_Folder", {"Extra", "Folder"})
```
11. returns the tag name if the tag does not exists it creates one.
10. returns the tag name if the tag does not exists it creates one.
```lua
coldbase.get_or_add_tag("Extra_Folder", {"Extra", "Folder"})
coldbase.get_or_add_tag("Extra_Folder", {"Extra", "Folder"}, no_new_path)
```
12. remove tag by name.
11. remove tag by name.
```lua
coldbase.remove_tag("Extra_Folder")
```

View File

@ -1,42 +1,37 @@
colddb = {}
local function createDir(directory)
return minetest.mkdir(directory)
end
function colddb.Colddb(directory)
local directory = string.format("%s/%s/", minetest.get_worldpath(), directory)
if not createDir(directory) then
error(string.format("%s is not a directory.", directory))
function colddb.Colddb(dir)
local dir = string.format("%s/%s/", minetest.get_worldpath(), dir)
if not minetest.mkdir(dir) then
error(string.format("%s is not a directory.", dir))
end
local self = {}
self.db = {
directory = directory,
tags = {},
mem_pool = {},
mem_pool_del = {},
indexes_pool = {},
iterate_queue = {},
indexes = false,
add_to_mem_pool = true,
async = extended_api.Async(),
}
local directory = dir
local tags = {}
local mem_pool = {}
local mem_pool_del = {}
local indexes_pool = {}
local iterate_queue = {}
local add_to_mem_pool = true
local async = extended_api.Async()
local path_count = {}
async.priority(150, 250)
self.db.async.priority(150, 250)
-- make tables weak so the garbage-collector will remove unused data
setmetatable(self.db.tags, {__mode = "kv"})
setmetatable(self.db.mem_pool, {__mode = "kv"})
setmetatable(self.db.mem_pool_del, {__mode = "kv"})
setmetatable(self.db.indexes_pool, {__mode = "kv"})
setmetatable(tags, {__mode = "kv"})
setmetatable(mem_pool, {__mode = "kv"})
setmetatable(mem_pool_del, {__mode = "kv"})
setmetatable(indexes_pool, {__mode = "kv"})
self.file_Exists = function(name, tag_name)
local function file_Exists(name, tag_name)
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
local f = io.open(string.format("%s%s%s.cold", self.db.directory, t, name), "r")
local f = io.open(string.format("%s%s%s.cold", directory, t, name), "r")
if f ~= nil then
io.close(f)
return true
@ -58,7 +53,7 @@ function colddb.Colddb(directory)
local function delete_lines_func_i(line, i, args)
local f = args.file
local om = self.db.indexes_pool[args.cs]
local om = indexes_pool[args.cs]
if om and not om.deleted_items[line] then
f:write(string.format("\n%s", line))
else
@ -69,7 +64,7 @@ function colddb.Colddb(directory)
local function delete_lines_func_end(args)
local cs = args.cs
local index = self.db.indexes_pool[cs]
local index = indexes_pool[cs]
if index and index.file then
index.file:close()
index.file = nil
@ -83,20 +78,20 @@ function colddb.Colddb(directory)
os.remove(args.oldfile)
os.rename(args.copyfile, args.oldfile)
end
local pool = self.db.indexes_pool[cs]
local pool = indexes_pool[cs]
for i, l in pairs(args.removedlist) do
pool.deleted_items[i] = nil
end
pool.deleting = false
self.db.indexes_pool[cs] = pool
indexes_pool[cs] = pool
end
args = nil
end
local function iterate(func_on_iterate, end_func, count, cs, args)
local f = self.db.indexes_pool[cs]
local f = indexes_pool[cs]
local fl = f.file
self.db.async.iterate(1, count, function(i)
async.iterate(1, count, function(i)
local line = fl:read("*l")
if args.do_not_skip_removed_items or not f.deleted_items[line] then
local ar = func_on_iterate(line, i, args)
@ -109,13 +104,13 @@ function colddb.Colddb(directory)
if end_func then
end_func(args)
end
local iterate_queue = self.db.iterate_queue[cs]
local iterate_queue = iterate_queue[cs]
if iterate_queue and iterate_queue[1] then
local copy = iterate_queue[1]
f = self.db.indexes_pool[cs]
f = indexes_pool[cs]
if not f or not f.file then
self.open_index_table(copy.tag_name)
f = self.db.indexes_pool[cs]
f = indexes_pool[cs]
end
if copy.begin_func then
local a = copy.begin_func(copy.args)
@ -124,52 +119,46 @@ function colddb.Colddb(directory)
end
end
minetest.after(0, iterate, copy.func_on_iterate, copy.end_func, copy.count, copy.cs, copy.args)
table.remove(self.db.iterate_queue[cs], 1)
table.remove(iterate_queue[cs], 1)
return false
else
fl:close()
self.db.iterate_queue[cs] = nil
iterate_queue[cs] = nil
end
self.db.indexes_pool[cs].iterating = false
indexes_pool[cs].iterating = false
return false
end)
end
local function load_into_mem(name, _table, tag_name)
if self.db.add_to_mem_pool then
if add_to_mem_pool then
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
local mem_pool = self.db.mem_pool[cs]
if not mem_pool then
mem_pool = {mem = _table, indexes = self.db.indexes}
else
mem_pool.mem = _table
mem_pool.indexes = self.db.indexes
end
self.db.mem_pool[cs] = mem_pool
mem_pool[cs] = {mem = _table}
end
end
local path_count = {}
local function _remove_tag(delete_path, prev_dp)
if path_count[delete_path] and path_count[delete_path] > 0 then
local pc = path_count[delete_path]
if pc and pc > 0 then
minetest.after(1.5, _remove_tag, delete_path)
return
elseif path_count[delete_path] and path_count[delete_path] < 1 then
self.db.mem_pool = {}
self.db.mem_pool_del = {}
elseif pc and pc < 1 then
mem_pool = {}
mem_pool_del = {}
os.remove(delete_path)
path_count[delete_path] = nil
return
elseif not path_count[delete_path] then
elseif not pc then
path_count[delete_path] = 0
end
local list = minetest.get_dir_list(delete_path)
self.db.async.foreach(list, function(k, v)
async.foreach(list, function(k, v)
v = string.format("%s/%s", delete_path, v)
local err = os.remove(v)
if err == nil then
@ -180,20 +169,21 @@ function colddb.Colddb(directory)
if prev_dp then
path_count[prev_dp] = path_count[prev_dp] - 1
end
if path_count[delete_path] > 0 then
if pc > 0 then
minetest.after(1.5, _remove_tag, delete_path)
else
self.db.mem_pool = {}
self.db.mem_pool_del = {}
mem_pool = {}
mem_pool_del = {}
os.remove(delete_path)
path_count[delete_path] = nil
end
end)
end
self.add_tag = function(name, tag)
self.add_tag = function(name, tag, no_new_path)
local t = ""
if not self.db.tags[name] then
self.db.tags[name] = ""
if not tags[name] then
tags[name] = ""
end
if type(tag) == "table" then
for key, value in pairs(tag) do
@ -202,37 +192,46 @@ function colddb.Colddb(directory)
else
t = string.format("%s/", tag)
end
local test_path = string.format("%s%s%s", self.db.directory, self.db.tags[name], t)
if not createDir(test_path) then
tags[name] = string.format("%s%s", tags[name], t)
if no_new_path then
return
end
local test_path = string.format("%s%s%s", directory, tags[name], t)
if not minetest.mkdir(test_path) then
error(string.format("%s is not a directory.", test_path))
end
self.db.tags[name] = string.format("%s%s", self.db.tags[name], t)
end
self.get_tag = function(name)
if not name then
return ""
end
local tag = self.db.tags[name]
local tag = tags[name]
if tag then
return tag
end
return ""
end
self.get_or_add_tag = function(name, tag)
if not self.db.tags[name] then
self.add_tag(name, tag)
self.get_or_add_tag = function(name, tag, no_new_path)
if not tags[name] then
self.add_tag(name, tag, no_new_path)
end
return name
end
self.remove_tag = function(name)
if self.db.tags[name] then
local delete_path = string.format("%s%s", self.db.directory, self.db.tags[name])
if not name then
local delete_path = directory
local wc = delete_path:len()
delete_path = delete_path:sub(0, wc-1)
self.db.tags[name] = nil
return
end
if tags[name] then
local delete_path = string.format("%s%s", directory, tags[name])
local wc = delete_path:len()
delete_path = delete_path:sub(0, wc-1)
tags[name] = nil
local err = os.remove(delete_path)
if err == nil then
minetest.after(0.1, _remove_tag, delete_path)
@ -240,12 +239,12 @@ function colddb.Colddb(directory)
end
end
self.delete_file = function(name, tag_name)
local function delete_file(name, tag_name)
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
local text = string.format("%s%s%s.cold", self.db.directory, t, name)
local text = string.format("%s%s%s.cold", directory, t, name)
local err, msg = os.remove(text)
if err == nil then
@ -253,12 +252,12 @@ function colddb.Colddb(directory)
end
end
self.load_table = function(name, tag_name)
local function load_table(name, tag_name)
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
local f = io.open(string.format("%s%s%s.cold", self.db.directory, t, name), "r")
local f = io.open(string.format("%s%s%s.cold", directory, t, name), "r")
if f then
local data = minetest.deserialize(f:read("*a"))
f:close()
@ -267,28 +266,28 @@ function colddb.Colddb(directory)
return nil
end
self.save_table = function(name, _table, tag_name)
local function save_table(name, _table, tag_name)
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
return minetest.safe_file_write(string.format("%s%s%s.cold", self.db.directory, t, name), minetest.serialize(_table))
return minetest.safe_file_write(string.format("%s%s%s.cold", directory, t, name), minetest.serialize(_table))
end
self.save_key = function(name, tag_name)
local function save_key(name, tag_name)
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
return minetest.safe_file_write(string.format("%s%s%s.cold", self.db.directory, t, name), "")
return minetest.safe_file_write(string.format("%s%s%s.cold", directory, t, name), "")
end
self.load_key = function(name, tag_name)
local function load_key(name, tag_name)
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
local f = io.open(string.format("%s%s%s.cold", self.db.directory, t, name), "r")
local f = io.open(string.format("%s%s%s.cold", directory, t, name), "r")
if f then
f:close()
return true
@ -296,31 +295,13 @@ function colddb.Colddb(directory)
return false
end
self.delete_index_table = function(tag_name)
self.delete_entries = function(name ,_lines, tag_name)
local t = ""
local name = "æIndex_table"
if tag_name then
t = self.get_tag(tag_name)
end
local p = string.format("%s%sæIndex_table.cold", self.db.directory, t)
if self.file_Exists(name, tag_name) then
local err, msg = os.remove(p)
if err == nil then
print(string.format("error removing db data %s error message: %s", p, msg))
end
return true
end
return false
end
self.delete_lines = function(_lines, tag_name)
local t = ""
local name = "æIndex_table"
if tag_name then
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
local f = self.db.indexes_pool[cs]
local f = indexes_pool[cs]
local k = type(_lines)
if k == "string" then
f.deleted_items[_lines] = true
@ -330,33 +311,32 @@ function colddb.Colddb(directory)
end
end
if not f.deleting then
self.db.indexes_pool[cs].deleting = false
indexes_pool[cs].deleting = false
end
if f and f.file and not f.deleting then
self.db.indexes_pool[cs].deleting = true
indexes_pool[cs].deleting = true
if f.needs_flushing == true then
f.file:flush()
self.db.indexes_pool[cs].needs_flushing = false
indexes_pool[cs].needs_flushing = false
end
local oldfile = string.format("%s%sæIndex_table.cold", self.db.directory, t)
local copyfile = string.format("%s%sæIndex_table.cold.replacer", self.db.directory, t)
local oldfile = string.format("%s%s%s.cold", directory, t, name)
local copyfile = string.format("%s%s%s.cold.replacer", directory, t, name)
local args = {cs = cs, oldfile = oldfile, copyfile = copyfile, do_not_skip_removed_items = true}
self.db.indexes_pool[cs] = f
iterate_index_table(delete_lines_func_begin, delete_lines_func_i, delete_lines_func_end, args, tag_name)
indexes_pool[cs] = f
iterate_index_table(delete_entries_func_begin, delete_entries_func_i, delete_entries_func_end, args, tag_name)
end
end
self.open_index_table = function(tag_name)
self.open_entry_file = function(name, tag_name)
local t = ""
local name = "æIndex_table"
if tag_name then
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
local fs = self.db.indexes_pool[cs]
local fs = indexes_pool[cs]
if not fs then
local p = string.format("%s%sæIndex_table.cold", self.db.directory, t)
if not self.file_Exists(name,tag_name) then
local p = string.format("%s%s%s.cold", directory, t, name)
if not file_Exists(name,tag_name) then
local f = io.open(p, "w")
if f then
f:seek("set")
@ -366,7 +346,7 @@ function colddb.Colddb(directory)
end
local f = io.open(p, "r+")
if f then
self.db.indexes_pool[cs] = {file = f, needs_flushing = false, deleted_items = {}, iterating = false}
indexes_pool[cs] = {file = f, needs_flushing = false, deleted_items = {}, iterating = false}
return f
end
return nil
@ -376,14 +356,13 @@ function colddb.Colddb(directory)
return nil
end
self.append_index_table = function(key, tag_name)
self.append_entry_file = function(name, key, tag_name)
local t = ""
local name = "æIndex_table"
if tag_name then
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
local f = self.db.indexes_pool[cs]
local f = indexes_pool[cs]
local k = type(key)
if f and f.file and k == "string" then
local fl = f.file
@ -391,7 +370,7 @@ function colddb.Colddb(directory)
fl:flush()
f.needs_flushing = false
end
self.db.indexes_pool[cs].needs_flushing = true
indexes_pool[cs].needs_flushing = true
fl:seek("end")
fl:write(string.format("\n%s", key))
fl:seek("set")
@ -406,7 +385,7 @@ function colddb.Colddb(directory)
fl:flush()
f.needs_flushing = false
end
self.db.indexes_pool[cs].needs_flushing = true
indexes_pool[cs].needs_flushing = true
local c = 0
for i in pairs(key) do
fl:seek("end")
@ -424,14 +403,17 @@ function colddb.Colddb(directory)
end
end
self.get_count = function(tag_name)
self.get_entry_file_count = function(name, tag_name)
local t = ""
local name = "æIndex_table"
if tag_name then
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
local f = self.db.indexes_pool[cs]
local f = indexes_pool[cs]
if not f or not f.file then
self.open_entry_file(name, tag_name)
f = indexes_pool[cs]
end
if f and f.file then
local fl = f.file
if f.needs_flushing == true then
@ -446,20 +428,19 @@ function colddb.Colddb(directory)
return nil
end
self.iterate_index_table = function(begin_func, func_on_iterate, end_func, args, tag_name)
self.iterate_entry_file = function(name, begin_func, func_on_iterate, end_func, args, tag_name)
local t = ""
local name = "æIndex_table"
if tag_name then
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
local f = self.db.indexes_pool[cs]
local f = indexes_pool[cs]
if not f or not f.file then
self.open_index_table(tag_name)
f = self.db.indexes_pool[cs]
self.open_entry_file(tag_name)
f = indexes_pool[cs]
end
if f and f.file and self.db.indexes_pool[cs].iterating == false then
self.db.indexes_pool[cs].iterating = true
if f and f.file and indexes_pool[cs].iterating == false then
indexes_pool[cs].iterating = true
local fl = f.file
if f.needs_flushing == true then
fl:flush()
@ -482,7 +463,7 @@ function colddb.Colddb(directory)
if c < 1 then
-- If theres nothing to index then return
end_func(args)
self.db.indexes_pool[cs].iterating = false
indexes_pool[cs].iterating = false
return false
end
-- Start iterating the index table
@ -496,11 +477,11 @@ function colddb.Colddb(directory)
-- If theres nothing to index then return
return false
end
if not self.db.iterate_queue[cs] then
self.db.iterate_queue[cs] = {}
if not iterate_queue[cs] then
iterate_queue[cs] = {}
end
local _table = {begin_func = begin_func, func_on_iterate = func_on_iterate, end_func = end_func, count = c, cs = cs, tag_name = tag_name, args = args}
table.insert(self.db.iterate_queue[cs], _table)
table.insert(iterate_queue[cs], _table)
end
end
@ -509,24 +490,14 @@ function colddb.Colddb(directory)
if tag_name then
t = self.get_tag(tag_name)
end
if self.db.indexes and not self.file_Exists(name,tag_name) then
self.db.async.queue_task(function()
local cs2 = string.format("%s%s", t , "æIndex_table")
local om = self.db.indexes_pool[cs2]
if not self.file_Exists("æIndex_table", tag_name) or not (om and om.file) then
self.open_index_table(tag_name)
end
self.append_index_table(name, tag_name)
async.queue_task(function()
save_table(name, _table, tag_name)
end)
end
self.db.async.queue_task(function()
self.save_table(name, _table, tag_name)
end)
if self.db.add_to_mem_pool then
if add_to_mem_pool then
load_into_mem(name, _table, tag_name)
end
local cs = string.format("%s%s", t, name)
self.db.mem_pool_del[cs] = nil
mem_pool_del[cs] = nil
end
self.set_key = function(name, tag_name)
@ -534,24 +505,14 @@ function colddb.Colddb(directory)
if tag_name then
t = self.get_tag(tag_name)
end
if self.db.indexes and not self.file_Exists(name, tag_name) then
self.db.async.queue_task(function()
local cs2 = string.format("%s%s", t, "æIndex_table")
local om = self.db.indexes_pool[cs2]
if not self.file_Exists("æIndex_table", tag_name) or not (om and om.file) then
self.open_index_table(tag_name)
end
self.append_index_table(name, tag_name)
async.queue_task(function()
save_key(name, tag_name)
end)
end
self.db.async.queue_task(function()
self.save_key(name, tag_name)
end)
if self.db.add_to_mem_pool then
if add_to_mem_pool then
load_into_mem(name, "", tag_name)
end
local cs = string.format("%s%s", t, name)
self.db.mem_pool_del[cs] = nil
mem_pool_del[cs] = nil
end
self.get = function(name, tag_name, callback)
@ -560,39 +521,39 @@ function colddb.Colddb(directory)
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
if self.db.mem_pool_del[cs] then
if mem_pool_del[cs] then
if callback then
callback(nil)
end
return nil
end
if callback then
self.db.async.queue_task(function()
local pm = self.db.mem_pool[cs]
async.queue_task(function()
local pm = mem_pool[cs]
if pm then
return pm.mem
else
local _table = self.load_table(name, tag_name)
local _table = load_table(name, tag_name)
if _table then
load_into_mem(name, _table, tag_name)
return _table
end
end
self.db.mem_pool_del[cs] = true
mem_pool_del[cs] = true
return nil
end,callback)
else
local pm = self.db.mem_pool[cs]
local pm = mem_pool[cs]
if pm then
return pm.mem
else
local _table = self.load_table(name, tag_name)
local _table = load_table(name, tag_name)
if _table then
load_into_mem(name, _table, tag_name)
return _table
end
end
self.db.mem_pool_del[cs] = true
mem_pool_del[cs] = true
return nil
end
end
@ -603,43 +564,43 @@ function colddb.Colddb(directory)
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
if self.db.mem_pool_del[cs] then
if mem_pool_del[cs] then
if callback then
callback(false)
end
return false
end
if callback then
self.db.async.queue_task(function()
async.queue_task(function()
local t = ""
if tag_name then
t = self.get_tag(tag_name)
end
local pm = self.db.mem_pool[cs]
local pm = mem_pool[cs]
if pm then
return true
else
local bool = self.load_key(name, tag_name)
local bool = load_key(name, tag_name)
if bool then
load_into_mem(name, bool, tag_name)
return bool
end
end
self.db.mem_pool_del[cs] = true
mem_pool_del[cs] = true
return nil
end,callback)
else
local pm = self.db.mem_pool[cs]
local pm = mem_pool[cs]
if pm then
return true
else
local bool = self.load_key(name, tag_name)
local bool = load_key(name, tag_name)
if bool then
load_into_mem(name, bool, tag_name)
return bool
end
end
self.db.mem_pool_del[cs] = true
mem_pool_del[cs] = true
return nil
end
end
@ -650,19 +611,10 @@ function colddb.Colddb(directory)
t = self.get_tag(tag_name)
end
local cs = string.format("%s%s", t, name)
self.db.mem_pool[cs] = nil
self.db.mem_pool_del[cs] = true
if self.db.indexes and self.file_Exists("æIndex_table", tag_name) then
self.db.async.queue_task(function()
local cs2 = string.format("%s%s", t, "æIndex_table")
if not (self.db.indexes_pool[cs2] and self.db.indexes_pool[cs2].file) then
self.open_index_table(tag_name)
end
self.delete_lines(name, tag_name)
end)
end
self.db.async.queue_task(function()
self.delete_file(name, tag_name)
mem_pool[cs] = nil
mem_pool_del[cs] = true
async.queue_task(function()
delete_file(name, tag_name)
end)
end