diff --git a/README.md b/README.md index 8fe69f0..c949a2e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ColdDB =========== ColdDB is a minetest mod that implements a serverless, asynchronous, NoSQL database engine.
-It provides a key or key-value storage system using plain Lua tables, also it can iterate through the keys.
+It provides a key or key-value based storage system using plain Lua tables.
Usage =========== @@ -44,27 +44,23 @@ coldbase.remove("MyKeyAndValue") ```lua coldbase.add_to_mem_pool = true ``` -7. if indexes is true(false by default). When a key file is created an indexing file stores the key for look-ups. this makes it possible to iterate through keys. +7. returns the amount of entries from the entry file. ```lua -coldbase.indexes = true +coldbase.get_entry_file_count(name, tag_name) ``` -8. only if coldbase.indexes is true. returns the amount of keys that are in the indexing file. +8. iterates through the entry file(breaks and ends if it reaches the end of the file). ```lua -coldbase.get_count() +coldbase.iterate_entry_file(name, begin_func, func_on_iterate, end_func, args, tag_name) ``` -9. only if coldbase.indexes is true. iterates through the indexing file(breaks and ends if it reaches the end of the file). -```lua -coldbase.iterate_index_table(nil, func_list_keys, nil) -``` -10. adds a folder which can be used in other functions that have tag_name arg. +9. adds a folder which can be used in other functions that have tag_name arg. ```lua coldbase.add_tag("Extra_Folder", {"Extra", "Folder"}) ``` -11. returns the tag name if the tag does not exists it creates one. +10. returns the tag name if the tag does not exists it creates one. ```lua -coldbase.get_or_add_tag("Extra_Folder", {"Extra", "Folder"}) +coldbase.get_or_add_tag("Extra_Folder", {"Extra", "Folder"}, no_new_path) ``` -12. remove tag by name. +11. remove tag by name. ```lua coldbase.remove_tag("Extra_Folder") ``` diff --git a/colddb.lua b/colddb.lua index 31936f2..b3de00c 100644 --- a/colddb.lua +++ b/colddb.lua @@ -1,42 +1,37 @@ colddb = {} -local function createDir(directory) - return minetest.mkdir(directory) -end - -function colddb.Colddb(directory) - local directory = string.format("%s/%s/", minetest.get_worldpath(), directory) - if not createDir(directory) then - error(string.format("%s is not a directory.", directory)) +function colddb.Colddb(dir) + local dir = string.format("%s/%s/", minetest.get_worldpath(), dir) + if not minetest.mkdir(dir) then + error(string.format("%s is not a directory.", dir)) end local self = {} - self.db = { - directory = directory, - tags = {}, - mem_pool = {}, - mem_pool_del = {}, - indexes_pool = {}, - iterate_queue = {}, - indexes = false, - add_to_mem_pool = true, - async = extended_api.Async(), - } + local directory = dir + local tags = {} + local mem_pool = {} + local mem_pool_del = {} + local indexes_pool = {} + local iterate_queue = {} + local add_to_mem_pool = true + local async = extended_api.Async() + local path_count = {} + + async.priority(150, 250) - self.db.async.priority(150, 250) -- make tables weak so the garbage-collector will remove unused data - setmetatable(self.db.tags, {__mode = "kv"}) - setmetatable(self.db.mem_pool, {__mode = "kv"}) - setmetatable(self.db.mem_pool_del, {__mode = "kv"}) - setmetatable(self.db.indexes_pool, {__mode = "kv"}) + setmetatable(tags, {__mode = "kv"}) + setmetatable(mem_pool, {__mode = "kv"}) + setmetatable(mem_pool_del, {__mode = "kv"}) + setmetatable(indexes_pool, {__mode = "kv"}) - self.file_Exists = function(name, tag_name) + local function file_Exists(name, tag_name) local t = "" if tag_name then t = self.get_tag(tag_name) end - local f = io.open(string.format("%s%s%s.cold", self.db.directory, t, name), "r") + local f = io.open(string.format("%s%s%s.cold", directory, t, name), "r") if f ~= nil then io.close(f) return true @@ -58,7 +53,7 @@ function colddb.Colddb(directory) local function delete_lines_func_i(line, i, args) local f = args.file - local om = self.db.indexes_pool[args.cs] + local om = indexes_pool[args.cs] if om and not om.deleted_items[line] then f:write(string.format("\n%s", line)) else @@ -69,7 +64,7 @@ function colddb.Colddb(directory) local function delete_lines_func_end(args) local cs = args.cs - local index = self.db.indexes_pool[cs] + local index = indexes_pool[cs] if index and index.file then index.file:close() index.file = nil @@ -83,20 +78,20 @@ function colddb.Colddb(directory) os.remove(args.oldfile) os.rename(args.copyfile, args.oldfile) end - local pool = self.db.indexes_pool[cs] + local pool = indexes_pool[cs] for i, l in pairs(args.removedlist) do pool.deleted_items[i] = nil end pool.deleting = false - self.db.indexes_pool[cs] = pool + indexes_pool[cs] = pool end args = nil end local function iterate(func_on_iterate, end_func, count, cs, args) - local f = self.db.indexes_pool[cs] + local f = indexes_pool[cs] local fl = f.file - self.db.async.iterate(1, count, function(i) + async.iterate(1, count, function(i) local line = fl:read("*l") if args.do_not_skip_removed_items or not f.deleted_items[line] then local ar = func_on_iterate(line, i, args) @@ -105,17 +100,17 @@ function colddb.Colddb(directory) return args end end - end,function() + end, function() if end_func then end_func(args) end - local iterate_queue = self.db.iterate_queue[cs] + local iterate_queue = iterate_queue[cs] if iterate_queue and iterate_queue[1] then local copy = iterate_queue[1] - f = self.db.indexes_pool[cs] + f = indexes_pool[cs] if not f or not f.file then self.open_index_table(copy.tag_name) - f = self.db.indexes_pool[cs] + f = indexes_pool[cs] end if copy.begin_func then local a = copy.begin_func(copy.args) @@ -124,52 +119,46 @@ function colddb.Colddb(directory) end end minetest.after(0, iterate, copy.func_on_iterate, copy.end_func, copy.count, copy.cs, copy.args) - table.remove(self.db.iterate_queue[cs], 1) + table.remove(iterate_queue[cs], 1) return false else fl:close() - self.db.iterate_queue[cs] = nil + iterate_queue[cs] = nil end - self.db.indexes_pool[cs].iterating = false + indexes_pool[cs].iterating = false return false end) end local function load_into_mem(name, _table, tag_name) - if self.db.add_to_mem_pool then + if add_to_mem_pool then local t = "" if tag_name then t = self.get_tag(tag_name) end local cs = string.format("%s%s", t, name) - local mem_pool = self.db.mem_pool[cs] - if not mem_pool then - mem_pool = {mem = _table, indexes = self.db.indexes} - else - mem_pool.mem = _table - mem_pool.indexes = self.db.indexes - end - self.db.mem_pool[cs] = mem_pool + mem_pool[cs] = {mem = _table} end end - local path_count = {} - local function _remove_tag(delete_path, prev_dp) - if path_count[delete_path] and path_count[delete_path] > 0 then + local pc = path_count[delete_path] + if pc and pc > 0 then minetest.after(1.5, _remove_tag, delete_path) return - elseif path_count[delete_path] and path_count[delete_path] < 1 then - self.db.mem_pool = {} - self.db.mem_pool_del = {} + elseif pc and pc < 1 then + mem_pool = {} + mem_pool_del = {} os.remove(delete_path) + path_count[delete_path] = nil + return - elseif not path_count[delete_path] then + elseif not pc then path_count[delete_path] = 0 end local list = minetest.get_dir_list(delete_path) - self.db.async.foreach(list, function(k, v) + async.foreach(list, function(k, v) v = string.format("%s/%s", delete_path, v) local err = os.remove(v) if err == nil then @@ -180,20 +169,21 @@ function colddb.Colddb(directory) if prev_dp then path_count[prev_dp] = path_count[prev_dp] - 1 end - if path_count[delete_path] > 0 then + if pc > 0 then minetest.after(1.5, _remove_tag, delete_path) else - self.db.mem_pool = {} - self.db.mem_pool_del = {} + mem_pool = {} + mem_pool_del = {} os.remove(delete_path) + path_count[delete_path] = nil end end) end - self.add_tag = function(name, tag) + self.add_tag = function(name, tag, no_new_path) local t = "" - if not self.db.tags[name] then - self.db.tags[name] = "" + if not tags[name] then + tags[name] = "" end if type(tag) == "table" then for key, value in pairs(tag) do @@ -202,37 +192,46 @@ function colddb.Colddb(directory) else t = string.format("%s/", tag) end - local test_path = string.format("%s%s%s", self.db.directory, self.db.tags[name], t) - if not createDir(test_path) then + tags[name] = string.format("%s%s", tags[name], t) + if no_new_path then + return + end + local test_path = string.format("%s%s%s", directory, tags[name], t) + if not minetest.mkdir(test_path) then error(string.format("%s is not a directory.", test_path)) end - self.db.tags[name] = string.format("%s%s", self.db.tags[name], t) end self.get_tag = function(name) if not name then return "" end - local tag = self.db.tags[name] + local tag = tags[name] if tag then return tag end return "" end - self.get_or_add_tag = function(name, tag) - if not self.db.tags[name] then - self.add_tag(name, tag) + self.get_or_add_tag = function(name, tag, no_new_path) + if not tags[name] then + self.add_tag(name, tag, no_new_path) end return name end self.remove_tag = function(name) - if self.db.tags[name] then - local delete_path = string.format("%s%s", self.db.directory, self.db.tags[name]) + if not name then + local delete_path = directory local wc = delete_path:len() delete_path = delete_path:sub(0, wc-1) - self.db.tags[name] = nil + return + end + if tags[name] then + local delete_path = string.format("%s%s", directory, tags[name]) + local wc = delete_path:len() + delete_path = delete_path:sub(0, wc-1) + tags[name] = nil local err = os.remove(delete_path) if err == nil then minetest.after(0.1, _remove_tag, delete_path) @@ -240,12 +239,12 @@ function colddb.Colddb(directory) end end - self.delete_file = function(name, tag_name) + local function delete_file(name, tag_name) local t = "" if tag_name then t = self.get_tag(tag_name) end - local text = string.format("%s%s%s.cold", self.db.directory, t, name) + local text = string.format("%s%s%s.cold", directory, t, name) local err, msg = os.remove(text) if err == nil then @@ -253,12 +252,12 @@ function colddb.Colddb(directory) end end - self.load_table = function(name, tag_name) + local function load_table(name, tag_name) local t = "" if tag_name then t = self.get_tag(tag_name) end - local f = io.open(string.format("%s%s%s.cold", self.db.directory, t, name), "r") + local f = io.open(string.format("%s%s%s.cold", directory, t, name), "r") if f then local data = minetest.deserialize(f:read("*a")) f:close() @@ -267,28 +266,28 @@ function colddb.Colddb(directory) return nil end - self.save_table = function(name, _table, tag_name) + local function save_table(name, _table, tag_name) local t = "" if tag_name then t = self.get_tag(tag_name) end - return minetest.safe_file_write(string.format("%s%s%s.cold", self.db.directory, t, name), minetest.serialize(_table)) + return minetest.safe_file_write(string.format("%s%s%s.cold", directory, t, name), minetest.serialize(_table)) end - self.save_key = function(name, tag_name) + local function save_key(name, tag_name) local t = "" if tag_name then t = self.get_tag(tag_name) end - return minetest.safe_file_write(string.format("%s%s%s.cold", self.db.directory, t, name), "") + return minetest.safe_file_write(string.format("%s%s%s.cold", directory, t, name), "") end - self.load_key = function(name, tag_name) + local function load_key(name, tag_name) local t = "" if tag_name then t = self.get_tag(tag_name) end - local f = io.open(string.format("%s%s%s.cold", self.db.directory, t, name), "r") + local f = io.open(string.format("%s%s%s.cold", directory, t, name), "r") if f then f:close() return true @@ -296,31 +295,13 @@ function colddb.Colddb(directory) return false end - self.delete_index_table = function(tag_name) + self.delete_entries = function(name ,_lines, tag_name) local t = "" - local name = "æIndex_table" - if tag_name then - t = self.get_tag(tag_name) - end - local p = string.format("%s%sæIndex_table.cold", self.db.directory, t) - if self.file_Exists(name, tag_name) then - local err, msg = os.remove(p) - if err == nil then - print(string.format("error removing db data %s error message: %s", p, msg)) - end - return true - end - return false - end - - self.delete_lines = function(_lines, tag_name) - local t = "" - local name = "æIndex_table" if tag_name then t = self.get_tag(tag_name) end local cs = string.format("%s%s", t, name) - local f = self.db.indexes_pool[cs] + local f = indexes_pool[cs] local k = type(_lines) if k == "string" then f.deleted_items[_lines] = true @@ -330,33 +311,32 @@ function colddb.Colddb(directory) end end if not f.deleting then - self.db.indexes_pool[cs].deleting = false + indexes_pool[cs].deleting = false end if f and f.file and not f.deleting then - self.db.indexes_pool[cs].deleting = true + indexes_pool[cs].deleting = true if f.needs_flushing == true then f.file:flush() - self.db.indexes_pool[cs].needs_flushing = false + indexes_pool[cs].needs_flushing = false end - local oldfile = string.format("%s%sæIndex_table.cold", self.db.directory, t) - local copyfile = string.format("%s%sæIndex_table.cold.replacer", self.db.directory, t) + local oldfile = string.format("%s%s%s.cold", directory, t, name) + local copyfile = string.format("%s%s%s.cold.replacer", directory, t, name) local args = {cs = cs, oldfile = oldfile, copyfile = copyfile, do_not_skip_removed_items = true} - self.db.indexes_pool[cs] = f - iterate_index_table(delete_lines_func_begin, delete_lines_func_i, delete_lines_func_end, args, tag_name) + indexes_pool[cs] = f + iterate_index_table(delete_entries_func_begin, delete_entries_func_i, delete_entries_func_end, args, tag_name) end end - self.open_index_table = function(tag_name) + self.open_entry_file = function(name, tag_name) local t = "" - local name = "æIndex_table" if tag_name then t = self.get_tag(tag_name) end local cs = string.format("%s%s", t, name) - local fs = self.db.indexes_pool[cs] + local fs = indexes_pool[cs] if not fs then - local p = string.format("%s%sæIndex_table.cold", self.db.directory, t) - if not self.file_Exists(name,tag_name) then + local p = string.format("%s%s%s.cold", directory, t, name) + if not file_Exists(name,tag_name) then local f = io.open(p, "w") if f then f:seek("set") @@ -366,7 +346,7 @@ function colddb.Colddb(directory) end local f = io.open(p, "r+") if f then - self.db.indexes_pool[cs] = {file = f, needs_flushing = false, deleted_items = {}, iterating = false} + indexes_pool[cs] = {file = f, needs_flushing = false, deleted_items = {}, iterating = false} return f end return nil @@ -376,14 +356,13 @@ function colddb.Colddb(directory) return nil end - self.append_index_table = function(key, tag_name) + self.append_entry_file = function(name, key, tag_name) local t = "" - local name = "æIndex_table" if tag_name then t = self.get_tag(tag_name) end local cs = string.format("%s%s", t, name) - local f = self.db.indexes_pool[cs] + local f = indexes_pool[cs] local k = type(key) if f and f.file and k == "string" then local fl = f.file @@ -391,7 +370,7 @@ function colddb.Colddb(directory) fl:flush() f.needs_flushing = false end - self.db.indexes_pool[cs].needs_flushing = true + indexes_pool[cs].needs_flushing = true fl:seek("end") fl:write(string.format("\n%s", key)) fl:seek("set") @@ -406,7 +385,7 @@ function colddb.Colddb(directory) fl:flush() f.needs_flushing = false end - self.db.indexes_pool[cs].needs_flushing = true + indexes_pool[cs].needs_flushing = true local c = 0 for i in pairs(key) do fl:seek("end") @@ -424,14 +403,17 @@ function colddb.Colddb(directory) end end - self.get_count = function(tag_name) + self.get_entry_file_count = function(name, tag_name) local t = "" - local name = "æIndex_table" if tag_name then t = self.get_tag(tag_name) end - local cs = string.format("%s%s",t,name) - local f = self.db.indexes_pool[cs] + local cs = string.format("%s%s", t, name) + local f = indexes_pool[cs] + if not f or not f.file then + self.open_entry_file(name, tag_name) + f = indexes_pool[cs] + end if f and f.file then local fl = f.file if f.needs_flushing == true then @@ -446,20 +428,19 @@ function colddb.Colddb(directory) return nil end - self.iterate_index_table = function(begin_func, func_on_iterate, end_func, args, tag_name) + self.iterate_entry_file = function(name, begin_func, func_on_iterate, end_func, args, tag_name) local t = "" - local name = "æIndex_table" if tag_name then t = self.get_tag(tag_name) end local cs = string.format("%s%s", t, name) - local f = self.db.indexes_pool[cs] + local f = indexes_pool[cs] if not f or not f.file then - self.open_index_table(tag_name) - f = self.db.indexes_pool[cs] + self.open_entry_file(tag_name) + f = indexes_pool[cs] end - if f and f.file and self.db.indexes_pool[cs].iterating == false then - self.db.indexes_pool[cs].iterating = true + if f and f.file and indexes_pool[cs].iterating == false then + indexes_pool[cs].iterating = true local fl = f.file if f.needs_flushing == true then fl:flush() @@ -482,7 +463,7 @@ function colddb.Colddb(directory) if c < 1 then -- If theres nothing to index then return end_func(args) - self.db.indexes_pool[cs].iterating = false + indexes_pool[cs].iterating = false return false end -- Start iterating the index table @@ -496,11 +477,11 @@ function colddb.Colddb(directory) -- If theres nothing to index then return return false end - if not self.db.iterate_queue[cs] then - self.db.iterate_queue[cs] = {} + if not iterate_queue[cs] then + iterate_queue[cs] = {} end local _table = {begin_func = begin_func, func_on_iterate = func_on_iterate, end_func = end_func, count = c, cs = cs, tag_name = tag_name, args = args} - table.insert(self.db.iterate_queue[cs], _table) + table.insert(iterate_queue[cs], _table) end end @@ -509,24 +490,14 @@ function colddb.Colddb(directory) if tag_name then t = self.get_tag(tag_name) end - if self.db.indexes and not self.file_Exists(name,tag_name) then - self.db.async.queue_task(function() - local cs2 = string.format("%s%s", t , "æIndex_table") - local om = self.db.indexes_pool[cs2] - if not self.file_Exists("æIndex_table", tag_name) or not (om and om.file) then - self.open_index_table(tag_name) - end - self.append_index_table(name, tag_name) - end) - end - self.db.async.queue_task(function() - self.save_table(name, _table, tag_name) + async.queue_task(function() + save_table(name, _table, tag_name) end) - if self.db.add_to_mem_pool then + if add_to_mem_pool then load_into_mem(name, _table, tag_name) end local cs = string.format("%s%s", t, name) - self.db.mem_pool_del[cs] = nil + mem_pool_del[cs] = nil end self.set_key = function(name, tag_name) @@ -534,24 +505,14 @@ function colddb.Colddb(directory) if tag_name then t = self.get_tag(tag_name) end - if self.db.indexes and not self.file_Exists(name, tag_name) then - self.db.async.queue_task(function() - local cs2 = string.format("%s%s", t, "æIndex_table") - local om = self.db.indexes_pool[cs2] - if not self.file_Exists("æIndex_table", tag_name) or not (om and om.file) then - self.open_index_table(tag_name) - end - self.append_index_table(name, tag_name) - end) - end - self.db.async.queue_task(function() - self.save_key(name, tag_name) + async.queue_task(function() + save_key(name, tag_name) end) - if self.db.add_to_mem_pool then + if add_to_mem_pool then load_into_mem(name, "", tag_name) end local cs = string.format("%s%s", t, name) - self.db.mem_pool_del[cs] = nil + mem_pool_del[cs] = nil end self.get = function(name, tag_name, callback) @@ -560,39 +521,39 @@ function colddb.Colddb(directory) t = self.get_tag(tag_name) end local cs = string.format("%s%s", t, name) - if self.db.mem_pool_del[cs] then + if mem_pool_del[cs] then if callback then callback(nil) end return nil end if callback then - self.db.async.queue_task(function() - local pm = self.db.mem_pool[cs] + async.queue_task(function() + local pm = mem_pool[cs] if pm then return pm.mem else - local _table = self.load_table(name, tag_name) + local _table = load_table(name, tag_name) if _table then load_into_mem(name, _table, tag_name) return _table end end - self.db.mem_pool_del[cs] = true + mem_pool_del[cs] = true return nil end,callback) else - local pm = self.db.mem_pool[cs] + local pm = mem_pool[cs] if pm then return pm.mem else - local _table = self.load_table(name, tag_name) + local _table = load_table(name, tag_name) if _table then load_into_mem(name, _table, tag_name) return _table end end - self.db.mem_pool_del[cs] = true + mem_pool_del[cs] = true return nil end end @@ -603,43 +564,43 @@ function colddb.Colddb(directory) t = self.get_tag(tag_name) end local cs = string.format("%s%s", t, name) - if self.db.mem_pool_del[cs] then + if mem_pool_del[cs] then if callback then callback(false) end return false end if callback then - self.db.async.queue_task(function() + async.queue_task(function() local t = "" if tag_name then t = self.get_tag(tag_name) end - local pm = self.db.mem_pool[cs] + local pm = mem_pool[cs] if pm then return true else - local bool = self.load_key(name, tag_name) + local bool = load_key(name, tag_name) if bool then load_into_mem(name, bool, tag_name) return bool end end - self.db.mem_pool_del[cs] = true + mem_pool_del[cs] = true return nil end,callback) else - local pm = self.db.mem_pool[cs] + local pm = mem_pool[cs] if pm then return true else - local bool = self.load_key(name, tag_name) + local bool = load_key(name, tag_name) if bool then load_into_mem(name, bool, tag_name) return bool end end - self.db.mem_pool_del[cs] = true + mem_pool_del[cs] = true return nil end end @@ -650,19 +611,10 @@ function colddb.Colddb(directory) t = self.get_tag(tag_name) end local cs = string.format("%s%s", t, name) - self.db.mem_pool[cs] = nil - self.db.mem_pool_del[cs] = true - if self.db.indexes and self.file_Exists("æIndex_table", tag_name) then - self.db.async.queue_task(function() - local cs2 = string.format("%s%s", t, "æIndex_table") - if not (self.db.indexes_pool[cs2] and self.db.indexes_pool[cs2].file) then - self.open_index_table(tag_name) - end - self.delete_lines(name, tag_name) - end) - end - self.db.async.queue_task(function() - self.delete_file(name, tag_name) + mem_pool[cs] = nil + mem_pool_del[cs] = true + async.queue_task(function() + delete_file(name, tag_name) end) end