From 4c1334e26136b9938d61ad5c47fb8891ca64394a Mon Sep 17 00:00:00 2001 From: y5nw Date: Tue, 30 Apr 2024 19:08:20 +0200 Subject: [PATCH 01/11] Implement API to replace async jobs --- builtin/game/async.lua | 28 +++++++++++---- src/script/cpp_api/s_async.cpp | 66 +++++++++++++++++++++++++--------- src/script/cpp_api/s_async.h | 37 +++++++++++++++++++ 3 files changed, 109 insertions(+), 22 deletions(-) diff --git a/builtin/game/async.lua b/builtin/game/async.lua index 469f179d7..04bcd3913 100644 --- a/builtin/game/async.lua +++ b/builtin/game/async.lua @@ -8,15 +8,31 @@ function core.async_event_handler(jobid, retval) core.async_jobs[jobid] = nil end -function core.handle_async(func, callback, ...) +local function prepare_async_args(func, callback, ...) assert(type(func) == "function" and type(callback) == "function", - "Invalid minetest.handle_async invocation") + "Invalid invocation of minetest.handle_async or minetest.replace_async") local args = {n = select("#", ...), ...} local mod_origin = core.get_last_run_mod() - local jobid = core.do_async_callback(func, args, mod_origin) - core.async_jobs[jobid] = callback - - return true + return func, args, mod_origin end +function core.handle_async(func, callback, ...) + local jobid = core.do_async_callback(prepare_async_args(func, callback, ...)) + core.async_jobs[jobid] = callback + + return jobid +end + +function core.replace_async(jobid, func, callback, ...) + assert(type(jobid) == "number", + "Invalid jobid for minetest.replace_async") + + local newid = core.replace_async_callback(jobid, prepare_async_args(func, callback, ...)) + return newid +end + +local dummy = function() end +function core.cancel_asyunc(jobid) + core.replace_async(jobid, dummy, dummy) +end diff --git a/src/script/cpp_api/s_async.cpp b/src/script/cpp_api/s_async.cpp index 75b1a8205..7790598fc 100644 --- a/src/script/cpp_api/s_async.cpp +++ b/src/script/cpp_api/s_async.cpp @@ -35,6 +35,16 @@ extern "C" { #include "common/c_packer.h" #include "lua_api/l_base.h" +LuaJobInfo::LuaJobInfo(std::string &&func, std::string &¶ms, const std::string &mod_origin): + function(func), params(params), mod_origin(mod_origin) +{} + +LuaJobInfo::LuaJobInfo(std::string &&func, PackedValue *params, const std::string &mod_origin): + function(func), mod_origin(mod_origin) +{ + params_ext.reset(params); +} + /******************************************************************************/ AsyncEngine::~AsyncEngine() { @@ -101,40 +111,64 @@ void AsyncEngine::addWorkerThread() } /******************************************************************************/ -u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &¶ms, - const std::string &mod_origin) + +u32 AsyncEngine::queueAsyncJob(LuaJobInfo &&job) { MutexAutoLock autolock(jobQueueMutex); u32 jobId = jobIdCounter++; - jobQueue.emplace_back(); - auto &to_add = jobQueue.back(); - to_add.id = jobId; - to_add.function = std::move(func); - to_add.params = std::move(params); - to_add.mod_origin = mod_origin; + job.id = jobId; + jobQueue.push_back(std::move(job)); jobQueueCounter.post(); return jobId; } +u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &¶ms, + const std::string &mod_origin) +{ + LuaJobInfo to_add(std::move(func), std::move(params), mod_origin); + return queueAsyncJob(std::move(to_add)); +} + u32 AsyncEngine::queueAsyncJob(std::string &&func, PackedValue *params, const std::string &mod_origin) { - MutexAutoLock autolock(jobQueueMutex); - u32 jobId = jobIdCounter++; + LuaJobInfo to_add(std::move(func), params, mod_origin); + return queueAsyncJob(std::move(to_add)); +} - jobQueue.emplace_back(); - auto &to_add = jobQueue.back(); - to_add.id = jobId; - to_add.function = std::move(func); - to_add.params_ext.reset(params); - to_add.mod_origin = mod_origin; +u32 AsyncEngine::replaceAsyncJob(const u32 &oldId, LuaJobInfo &&job) +{ + MutexAutoLock autolock(jobQueueMutex); + int pos = oldId - (jobIdCounter - jobQueue.size()); + u32 jobId = oldId; + if (pos < 0 || pos >= jobQueue.size()) { + job.id = jobId = jobIdCounter++; + jobQueue.push_back(std::move(job)); + } else { + job.id = jobId; + jobQueue[pos] = std::move(job); + } jobQueueCounter.post(); return jobId; } +u32 AsyncEngine::replaceAsyncJob(const u32 &oldId, std::string &&func, std::string &¶ms, + const std::string &mod_origin) +{ + LuaJobInfo to_add(std::move(func), std::move(params), mod_origin); + return replaceAsyncJob(oldId, std::move(to_add)); +} + +u32 AsyncEngine::replaceAsyncJob(const u32 &oldId, std::string &&func, PackedValue *params, + const std::string &mod_origin) +{ + LuaJobInfo to_add(std::move(func), params, mod_origin); + return replaceAsyncJob(oldId, std::move(to_add)); +} + /******************************************************************************/ bool AsyncEngine::getJob(LuaJobInfo *job) { diff --git a/src/script/cpp_api/s_async.h b/src/script/cpp_api/s_async.h index 1e34e40ea..e7dbd77a7 100644 --- a/src/script/cpp_api/s_async.h +++ b/src/script/cpp_api/s_async.h @@ -41,6 +41,8 @@ class AsyncEngine; struct LuaJobInfo { LuaJobInfo() = default; + LuaJobInfo(std::string &&func, std::string &¶ms, const std::string &mod_origin = ""); + LuaJobInfo(std::string &&func, PackedValue *params, const std::string &mod_origin = ""); // Function to be called in async environment (from string.dump) std::string function; @@ -114,6 +116,26 @@ public: u32 queueAsyncJob(std::string &&func, PackedValue *params, const std::string &mod_origin = ""); + /** + * Replace an async job if possible or queue the job otherwise + * @param oldId The ID of the job to replace + * @param func Serialized lua function + * @param params Serialized parameters + * @return jobid The job is replaced or queued + */ + u32 replaceAsyncJob(const u32 &oldId, std::string &&func, std::string &¶ms, + const std::string &mod_origin = ""); + + /** + * Replace an async job if possible or queue the job otherwise + * @param oldId The ID of the job to replace + * @param func Serialized lua function + * @param params Serialized parameters (takes ownership!) + * @return ID of queued job + */ + u32 replaceAsyncJob(const u32 &oldId, std::string &&func, PackedValue *params, + const std::string &mod_origin = ""); + /** * Engine step to process finished jobs * @param L The Lua stack @@ -129,6 +151,21 @@ protected: */ bool getJob(LuaJobInfo *job); + /** + * Queue an async job + * @param job The job to queue (takes ownership!) + * @return Id of the queued job + */ + u32 queueAsyncJob(LuaJobInfo &&job); + + /** + * Replace an async job if possible or queue the job otherwise + * @param jobId The Id of the job to replace + * @param job The new job to use in-place (takes ownership!) + * @return Id of the new job + */ + u32 replaceAsyncJob(const u32 &jobId, LuaJobInfo &&job); + /** * Put a Job result back to result queue * @param result result of completed job From c7e204315592d57d9815ab26618d45a7c2f336c9 Mon Sep 17 00:00:00 2001 From: y5nw Date: Tue, 30 Apr 2024 19:31:31 +0200 Subject: [PATCH 02/11] add test + fixup --- builtin/game/async.lua | 1 + games/devtest/mods/unittests/async_env.lua | 19 ++++++++++ src/script/lua_api/l_server.cpp | 43 ++++++++++++++++++---- src/script/lua_api/l_server.h | 3 ++ src/script/scripting_server.cpp | 7 ++++ src/script/scripting_server.h | 3 ++ 6 files changed, 69 insertions(+), 7 deletions(-) diff --git a/builtin/game/async.lua b/builtin/game/async.lua index 04bcd3913..b9972a3fa 100644 --- a/builtin/game/async.lua +++ b/builtin/game/async.lua @@ -29,6 +29,7 @@ function core.replace_async(jobid, func, callback, ...) "Invalid jobid for minetest.replace_async") local newid = core.replace_async_callback(jobid, prepare_async_args(func, callback, ...)) + core.async_jobs[jobid] = callback return newid end diff --git a/games/devtest/mods/unittests/async_env.lua b/games/devtest/mods/unittests/async_env.lua index d7a714941..f70210c4f 100644 --- a/games/devtest/mods/unittests/async_env.lua +++ b/games/devtest/mods/unittests/async_env.lua @@ -207,3 +207,22 @@ local function test_vector_preserve(cb) end, {vec}) end unittests.register("test_async_vector", test_vector_preserve, {async=true}) + +local function test_async_job_replacement(cb) + local id = core.handle_async(function(x) + return x + end, function(ret) + print(ret) + cb("Replaced async callback still run") + end, 1) + local new_id = core.replace_async(id, function(x) + return -x + end, function(ret) + if ret ~= -2 then + return cb("Wrong async value passed") + end + cb() + end, 2) + assert(id == new_id, "core.replace_async sanity check failed") +end +unittests.register("test_async_job_replacement", test_async_job_replacement, {async=true}) diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 185287dd7..47e2bc5f4 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -644,26 +644,54 @@ int ModApiServer::l_notify_authentication_modified(lua_State *L) return 0; } +static std::string get_serialized_function(lua_State *L, int index) +{ + luaL_checktype(L, index, LUA_TFUNCTION); + call_string_dump(L, index); + size_t func_length; + const char *serialized_func_raw = lua_tolstring(L, -1, &func_length); + return std::string(serialized_func_raw, func_length); +} + // do_async_callback(func, params, mod_origin) int ModApiServer::l_do_async_callback(lua_State *L) { NO_MAP_LOCK_REQUIRED; ServerScripting *script = getScriptApi(L); - luaL_checktype(L, 1, LUA_TFUNCTION); luaL_checktype(L, 2, LUA_TTABLE); luaL_checktype(L, 3, LUA_TSTRING); - call_string_dump(L, 1); - size_t func_length; - const char *serialized_func_raw = lua_tolstring(L, -1, &func_length); - + auto serialized_func = get_serialized_function(L, 1); PackedValue *param = script_pack(L, 2); - std::string mod_origin = readParam(L, 3); u32 jobId = script->queueAsync( - std::string(serialized_func_raw, func_length), + std::move(serialized_func), + param, mod_origin); + + lua_settop(L, 0); + lua_pushinteger(L, jobId); + return 1; +} + +// replace_async_callback(id, func, params, mod_origin) +int ModApiServer::l_replace_async_callback(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + ServerScripting *script = getScriptApi(L); + + luaL_checktype(L, 1, LUA_TNUMBER); + luaL_checktype(L, 3, LUA_TTABLE); + luaL_checktype(L, 4, LUA_TSTRING); + + u32 id = lua_tointeger(L, 1); + auto serialized_func = get_serialized_function(L, 2); + PackedValue *param = script_pack(L, 3); + std::string mod_origin = readParam(L, 4); + + u32 jobId = script->replaceAsync(id, + std::move(serialized_func), param, mod_origin); lua_settop(L, 0); @@ -767,6 +795,7 @@ void ModApiServer::Initialize(lua_State *L, int top) API_FCT(notify_authentication_modified); API_FCT(do_async_callback); + API_FCT(replace_async_callback); API_FCT(register_async_dofile); API_FCT(serialize_roundtrip); diff --git a/src/script/lua_api/l_server.h b/src/script/lua_api/l_server.h index 505dce735..45de81216 100644 --- a/src/script/lua_api/l_server.h +++ b/src/script/lua_api/l_server.h @@ -118,6 +118,9 @@ private: // do_async_callback(func, params, mod_origin) static int l_do_async_callback(lua_State *L); + // replace_async_callback(id, func, params, mod_origin) + static int l_replace_async_callback(lua_State *L); + // register_async_dofile(path) static int l_register_async_dofile(lua_State *L); diff --git a/src/script/scripting_server.cpp b/src/script/scripting_server.cpp index 324850011..b63bdeaee 100644 --- a/src/script/scripting_server.cpp +++ b/src/script/scripting_server.cpp @@ -140,6 +140,13 @@ u32 ServerScripting::queueAsync(std::string &&serialized_func, param, mod_origin); } +u32 ServerScripting::replaceAsync(const u32 &id, std::string &&serialized_func, + PackedValue *param, const std::string &mod_origin) +{ + return asyncEngine.replaceAsyncJob(id, std::move(serialized_func), + param, mod_origin); +} + void ServerScripting::InitializeModApi(lua_State *L, int top) { // Register reference classes (userdata) diff --git a/src/script/scripting_server.h b/src/script/scripting_server.h index 20d99313a..efd544399 100644 --- a/src/script/scripting_server.h +++ b/src/script/scripting_server.h @@ -65,6 +65,9 @@ public: u32 queueAsync(std::string &&serialized_func, PackedValue *param, const std::string &mod_origin); + u32 replaceAsync(const u32 &id, std::string &&serialized_func, + PackedValue *param, const std::string &mod_origin); + private: void InitializeModApi(lua_State *L, int top); From d08962e407cc5910ac7f1d33c6ea45bbc262dd36 Mon Sep 17 00:00:00 2001 From: y5nw Date: Tue, 30 Apr 2024 19:32:55 +0200 Subject: [PATCH 03/11] fix typo --- builtin/game/async.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/game/async.lua b/builtin/game/async.lua index b9972a3fa..d1e262509 100644 --- a/builtin/game/async.lua +++ b/builtin/game/async.lua @@ -34,6 +34,6 @@ function core.replace_async(jobid, func, callback, ...) end local dummy = function() end -function core.cancel_asyunc(jobid) +function core.cancel_async(jobid) core.replace_async(jobid, dummy, dummy) end From 83f485d50be0c96115c71e62644cf1858d023396 Mon Sep 17 00:00:00 2001 From: y5nw Date: Tue, 30 Apr 2024 19:52:44 +0200 Subject: [PATCH 04/11] test + fixup --- builtin/game/async.lua | 4 ++-- games/devtest/mods/unittests/async_env.lua | 27 +++++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/builtin/game/async.lua b/builtin/game/async.lua index d1e262509..5558126e9 100644 --- a/builtin/game/async.lua +++ b/builtin/game/async.lua @@ -29,11 +29,11 @@ function core.replace_async(jobid, func, callback, ...) "Invalid jobid for minetest.replace_async") local newid = core.replace_async_callback(jobid, prepare_async_args(func, callback, ...)) - core.async_jobs[jobid] = callback + core.async_jobs[newid] = callback return newid end local dummy = function() end function core.cancel_async(jobid) - core.replace_async(jobid, dummy, dummy) + return jobid == core.replace_async(jobid, dummy, dummy) end diff --git a/games/devtest/mods/unittests/async_env.lua b/games/devtest/mods/unittests/async_env.lua index f70210c4f..908771eeb 100644 --- a/games/devtest/mods/unittests/async_env.lua +++ b/games/devtest/mods/unittests/async_env.lua @@ -221,8 +221,33 @@ local function test_async_job_replacement(cb) if ret ~= -2 then return cb("Wrong async value passed") end - cb() end, 2) assert(id == new_id, "core.replace_async sanity check failed") + + id = core.handle_async(function(x) + return x + end, function() + return cb("Canceled async job run") + end) + assert(core.cancel_async(id), "core.cancel_async sanity check failed") + + -- Try to replace a job that is already run. Do this by delaying the main thread by some time. + -- Canceling the job does not need to be tested again since it is a thing wrapper that is already tested earlier. + id = core.handle_async(function(x) + return x + end, function(ret) + if ret ~= 1 then + return cb("Wrong async value passed to old handler") + end + + core.replace_async(id, function(x) + return -x + end, function(new_ret) + if new_ret ~= -2 then + return cb("wrong async value passed to new handler") + end + cb() + end, 2) + end, 1) end unittests.register("test_async_job_replacement", test_async_job_replacement, {async=true}) From 679bdbf3996d0be549a757f7d367a6fc17d3c0fa Mon Sep 17 00:00:00 2001 From: y5nw Date: Tue, 30 Apr 2024 19:56:38 +0200 Subject: [PATCH 05/11] test --- games/devtest/mods/unittests/async_env.lua | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/games/devtest/mods/unittests/async_env.lua b/games/devtest/mods/unittests/async_env.lua index 908771eeb..af62d5312 100644 --- a/games/devtest/mods/unittests/async_env.lua +++ b/games/devtest/mods/unittests/async_env.lua @@ -222,17 +222,20 @@ local function test_async_job_replacement(cb) return cb("Wrong async value passed") end end, 2) - assert(id == new_id, "core.replace_async sanity check failed") + if id ~= new_id then + cb("core.replace_async sanity check failed") + end id = core.handle_async(function(x) return x end, function() return cb("Canceled async job run") end) - assert(core.cancel_async(id), "core.cancel_async sanity check failed") + if not core.cancel_async(id) then + cb("core.cancel_async sanity check failed") + end -- Try to replace a job that is already run. Do this by delaying the main thread by some time. - -- Canceling the job does not need to be tested again since it is a thing wrapper that is already tested earlier. id = core.handle_async(function(x) return x end, function(ret) @@ -240,14 +243,20 @@ local function test_async_job_replacement(cb) return cb("Wrong async value passed to old handler") end - core.replace_async(id, function(x) + new_id = core.replace_async(id, function(x) return -x end, function(new_ret) if new_ret ~= -2 then - return cb("wrong async value passed to new handler") + return cb("Wrong async value passed to new handler") end cb() end, 2) + if id == new_id then + cb("core.replace_async replaced a completed job") + end + if core.cancel_async(id) then + cb("core.relpace_async canceled a completed job") + end end, 1) end unittests.register("test_async_job_replacement", test_async_job_replacement, {async=true}) From d7343b58962e3087f88ce6eb79b98cc1cf639f4b Mon Sep 17 00:00:00 2001 From: y5nw Date: Tue, 30 Apr 2024 20:03:37 +0200 Subject: [PATCH 06/11] add documentation --- doc/lua_api.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/lua_api.md b/doc/lua_api.md index b4c1613c2..decbcb5d3 100644 --- a/doc/lua_api.md +++ b/doc/lua_api.md @@ -6678,6 +6678,16 @@ This allows you easy interoperability for delegating work to jobs. * When `func` returns the callback is called (in the normal environment) with all of the return values as arguments. * Optional: Variable number of arguments that are passed to `func` + * Returns the ID of the async job. +* `minetest.replace_async(id, func, callback, ...)`: + * Try to replace the job with the given ID. The new job is otherwise queued + as if by `minetest.handle_async`. + * Note that the `func`, `callback` and variable arguments are all replaced. + * Returns the ID of the new async job. This is the same as the previous id + if the previous job was canceled. +* `minetest.cancel_async(id)`: + * Try to cancel the job with the given ID. + * Returns whether the job was canceled. * `minetest.register_async_dofile(path)`: * Register a path to a Lua file to be imported when an async environment is initialized. You can use this to preload code which you can then call From 21a7ee52ce21dcac71f5d00e89b3833866895acd Mon Sep 17 00:00:00 2001 From: y5nw Date: Fri, 17 May 2024 20:24:14 +0200 Subject: [PATCH 07/11] Minor refactor --- src/script/cpp_api/s_async.cpp | 18 ++++++++++ src/script/cpp_api/s_async.h | 18 ++++++++++ src/script/lua_api/CMakeLists.txt | 1 + src/script/lua_api/l_server.cpp | 57 ------------------------------- src/script/lua_api/l_server.h | 6 ---- src/script/scripting_server.cpp | 23 ++----------- src/script/scripting_server.h | 13 +------ 7 files changed, 41 insertions(+), 95 deletions(-) diff --git a/src/script/cpp_api/s_async.cpp b/src/script/cpp_api/s_async.cpp index 7790598fc..fc19dbb8b 100644 --- a/src/script/cpp_api/s_async.cpp +++ b/src/script/cpp_api/s_async.cpp @@ -423,3 +423,21 @@ void* AsyncWorkerThread::run() return 0; } +u32 ScriptApiAsync::queueAsync(std::string &&serialized_func, + PackedValue *param, const std::string &mod_origin) +{ + return asyncEngine.queueAsyncJob(std::move(serialized_func), + param, mod_origin); +} + +u32 ScriptApiAsync::replaceAsync(const u32 &id, std::string &&serialized_func, + PackedValue *param, const std::string &mod_origin) +{ + return asyncEngine.replaceAsyncJob(id, std::move(serialized_func), + param, mod_origin); +} + +void ScriptApiAsync::stepAsync() +{ + asyncEngine.step(getStack()); +} diff --git a/src/script/cpp_api/s_async.h b/src/script/cpp_api/s_async.h index e7dbd77a7..691242dbc 100644 --- a/src/script/cpp_api/s_async.h +++ b/src/script/cpp_api/s_async.h @@ -232,3 +232,21 @@ private: // Counter semaphore for job dispatching Semaphore jobQueueCounter; }; + +class ScriptApiAsync: + virtual public ScriptApiBase +{ +public: + ScriptApiAsync(Server *server): asyncEngine(server) {} + + virtual void initAsync() = 0; + void stepAsync(); + + u32 queueAsync(std::string &&serialized_func, + PackedValue *param, const std::string &mod_origin); + u32 replaceAsync(const u32 &id, std::string &&serialized_func, + PackedValue *param, const std::string &mod_origin); + +protected: + AsyncEngine asyncEngine; +}; diff --git a/src/script/lua_api/CMakeLists.txt b/src/script/lua_api/CMakeLists.txt index d9405e4fe..dac9f40c8 100644 --- a/src/script/lua_api/CMakeLists.txt +++ b/src/script/lua_api/CMakeLists.txt @@ -1,5 +1,6 @@ set(common_SCRIPT_LUA_API_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/l_areastore.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/l_async.cpp ${CMAKE_CURRENT_SOURCE_DIR}/l_auth.cpp ${CMAKE_CURRENT_SOURCE_DIR}/l_base.cpp ${CMAKE_CURRENT_SOURCE_DIR}/l_craft.cpp diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 47e2bc5f4..092a44049 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -644,61 +644,6 @@ int ModApiServer::l_notify_authentication_modified(lua_State *L) return 0; } -static std::string get_serialized_function(lua_State *L, int index) -{ - luaL_checktype(L, index, LUA_TFUNCTION); - call_string_dump(L, index); - size_t func_length; - const char *serialized_func_raw = lua_tolstring(L, -1, &func_length); - return std::string(serialized_func_raw, func_length); -} - -// do_async_callback(func, params, mod_origin) -int ModApiServer::l_do_async_callback(lua_State *L) -{ - NO_MAP_LOCK_REQUIRED; - ServerScripting *script = getScriptApi(L); - - luaL_checktype(L, 2, LUA_TTABLE); - luaL_checktype(L, 3, LUA_TSTRING); - - auto serialized_func = get_serialized_function(L, 1); - PackedValue *param = script_pack(L, 2); - std::string mod_origin = readParam(L, 3); - - u32 jobId = script->queueAsync( - std::move(serialized_func), - param, mod_origin); - - lua_settop(L, 0); - lua_pushinteger(L, jobId); - return 1; -} - -// replace_async_callback(id, func, params, mod_origin) -int ModApiServer::l_replace_async_callback(lua_State *L) -{ - NO_MAP_LOCK_REQUIRED; - ServerScripting *script = getScriptApi(L); - - luaL_checktype(L, 1, LUA_TNUMBER); - luaL_checktype(L, 3, LUA_TTABLE); - luaL_checktype(L, 4, LUA_TSTRING); - - u32 id = lua_tointeger(L, 1); - auto serialized_func = get_serialized_function(L, 2); - PackedValue *param = script_pack(L, 3); - std::string mod_origin = readParam(L, 4); - - u32 jobId = script->replaceAsync(id, - std::move(serialized_func), - param, mod_origin); - - lua_settop(L, 0); - lua_pushinteger(L, jobId); - return 1; -} - // register_async_dofile(path) int ModApiServer::l_register_async_dofile(lua_State *L) { @@ -794,8 +739,6 @@ void ModApiServer::Initialize(lua_State *L, int top) API_FCT(unban_player_or_ip); API_FCT(notify_authentication_modified); - API_FCT(do_async_callback); - API_FCT(replace_async_callback); API_FCT(register_async_dofile); API_FCT(serialize_roundtrip); diff --git a/src/script/lua_api/l_server.h b/src/script/lua_api/l_server.h index 45de81216..880a22953 100644 --- a/src/script/lua_api/l_server.h +++ b/src/script/lua_api/l_server.h @@ -115,12 +115,6 @@ private: // notify_authentication_modified(name) static int l_notify_authentication_modified(lua_State *L); - // do_async_callback(func, params, mod_origin) - static int l_do_async_callback(lua_State *L); - - // replace_async_callback(id, func, params, mod_origin) - static int l_replace_async_callback(lua_State *L); - // register_async_dofile(path) static int l_register_async_dofile(lua_State *L); diff --git a/src/script/scripting_server.cpp b/src/script/scripting_server.cpp index b63bdeaee..e93a5a942 100644 --- a/src/script/scripting_server.cpp +++ b/src/script/scripting_server.cpp @@ -24,6 +24,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "filesys.h" #include "cpp_api/s_internal.h" #include "lua_api/l_areastore.h" +#include "lua_api/l_async.h" #include "lua_api/l_auth.h" #include "lua_api/l_base.h" #include "lua_api/l_craft.h" @@ -53,7 +54,7 @@ extern "C" { ServerScripting::ServerScripting(Server* server): ScriptApiBase(ScriptingType::Server), - asyncEngine(server) + ScriptApiAsync(server) { setGameDef(server); @@ -128,25 +129,6 @@ void ServerScripting::initAsync() asyncEngine.initialize(0); } -void ServerScripting::stepAsync() -{ - asyncEngine.step(getStack()); -} - -u32 ServerScripting::queueAsync(std::string &&serialized_func, - PackedValue *param, const std::string &mod_origin) -{ - return asyncEngine.queueAsyncJob(std::move(serialized_func), - param, mod_origin); -} - -u32 ServerScripting::replaceAsync(const u32 &id, std::string &&serialized_func, - PackedValue *param, const std::string &mod_origin) -{ - return asyncEngine.replaceAsyncJob(id, std::move(serialized_func), - param, mod_origin); -} - void ServerScripting::InitializeModApi(lua_State *L, int top) { // Register reference classes (userdata) @@ -170,6 +152,7 @@ void ServerScripting::InitializeModApi(lua_State *L, int top) ModChannelRef::Register(L); // Initialize mod api modules + ModApiAsync::Initialize(L, top); ModApiAuth::Initialize(L, top); ModApiCraft::Initialize(L, top); ModApiEnv::Initialize(L, top); diff --git a/src/script/scripting_server.h b/src/script/scripting_server.h index efd544399..23f008b0c 100644 --- a/src/script/scripting_server.h +++ b/src/script/scripting_server.h @@ -37,6 +37,7 @@ struct PackedValue; class ServerScripting: virtual public ScriptApiBase, + public ScriptApiAsync, public ScriptApiDetached, public ScriptApiEntity, public ScriptApiEnv, @@ -58,20 +59,8 @@ public: // Initialize async engine, call this AFTER loading all mods void initAsync(); - // Global step handler to collect async results - void stepAsync(); - - // Pass job to async threads - u32 queueAsync(std::string &&serialized_func, - PackedValue *param, const std::string &mod_origin); - - u32 replaceAsync(const u32 &id, std::string &&serialized_func, - PackedValue *param, const std::string &mod_origin); - private: void InitializeModApi(lua_State *L, int top); static void InitializeAsync(lua_State *L, int top); - - AsyncEngine asyncEngine; }; From e834d2a11564d06ce6f675ca5b50fc7f2d5b4a31 Mon Sep 17 00:00:00 2001 From: y5nw Date: Fri, 17 May 2024 20:58:05 +0200 Subject: [PATCH 08/11] add missing files --- src/script/lua_api/l_async.cpp | 64 ++++++++++++++++++++++++++++++++++ src/script/lua_api/l_async.h | 12 +++++++ 2 files changed, 76 insertions(+) create mode 100644 src/script/lua_api/l_async.cpp create mode 100644 src/script/lua_api/l_async.h diff --git a/src/script/lua_api/l_async.cpp b/src/script/lua_api/l_async.cpp new file mode 100644 index 000000000..373b645fe --- /dev/null +++ b/src/script/lua_api/l_async.cpp @@ -0,0 +1,64 @@ +#include "lua_api/l_internal.h" +#include "lua_api/l_async.h" +#include "cpp_api/s_async.h" + +static std::string get_serialized_function(lua_State *L, int index) +{ + luaL_checktype(L, index, LUA_TFUNCTION); + call_string_dump(L, index); + size_t func_length; + const char *serialized_func_raw = lua_tolstring(L, -1, &func_length); + return std::string(serialized_func_raw, func_length); +} + +// do_async_callback(func, params, mod_origin) +int ModApiAsync::l_do_async_callback(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + ScriptApiAsync *script = getScriptApi(L); + + luaL_checktype(L, 2, LUA_TTABLE); + luaL_checktype(L, 3, LUA_TSTRING); + + auto serialized_func = get_serialized_function(L, 1); + PackedValue *param = script_pack(L, 2); + std::string mod_origin = readParam(L, 3); + + u32 jobId = script->queueAsync( + std::move(serialized_func), + param, mod_origin); + + lua_settop(L, 0); + lua_pushinteger(L, jobId); + return 1; +} + +// replace_async_callback(id, func, params, mod_origin) +int ModApiAsync::l_replace_async_callback(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + ScriptApiAsync *script = getScriptApi(L); + + luaL_checktype(L, 1, LUA_TNUMBER); + luaL_checktype(L, 3, LUA_TTABLE); + luaL_checktype(L, 4, LUA_TSTRING); + + u32 id = lua_tointeger(L, 1); + auto serialized_func = get_serialized_function(L, 2); + PackedValue *param = script_pack(L, 3); + std::string mod_origin = readParam(L, 4); + + u32 jobId = script->replaceAsync(id, + std::move(serialized_func), + param, mod_origin); + + lua_settop(L, 0); + lua_pushinteger(L, jobId); + return 1; +} + +void ModApiAsync::Initialize(lua_State *L, int top) +{ + API_FCT(do_async_callback); + API_FCT(replace_async_callback); +} diff --git a/src/script/lua_api/l_async.h b/src/script/lua_api/l_async.h new file mode 100644 index 000000000..a793893e6 --- /dev/null +++ b/src/script/lua_api/l_async.h @@ -0,0 +1,12 @@ +#pragma once + +#include "lua_api/l_base.h" + +class ModApiAsync : public ModApiBase +{ +public: + static void Initialize(lua_State *L, int top); +private: + static int l_do_async_callback(lua_State *L); + static int l_replace_async_callback(lua_State *L); +}; From cca26c0eb295d5c7daa0378811f4664391e47dcb Mon Sep 17 00:00:00 2001 From: y5nw Date: Fri, 17 May 2024 21:28:09 +0200 Subject: [PATCH 09/11] Introduce AsyncJob object --- builtin/game/async.lua | 30 +++--- games/devtest/mods/unittests/async_env.lua | 18 ++-- src/script/lua_api/l_async.cpp | 104 +++++++++++++++------ src/script/lua_api/l_async.h | 34 ++++++- src/script/scripting_server.cpp | 1 + 5 files changed, 137 insertions(+), 50 deletions(-) diff --git a/builtin/game/async.lua b/builtin/game/async.lua index 5558126e9..d184440c9 100644 --- a/builtin/game/async.lua +++ b/builtin/game/async.lua @@ -10,7 +10,7 @@ end local function prepare_async_args(func, callback, ...) assert(type(func) == "function" and type(callback) == "function", - "Invalid invocation of minetest.handle_async or minetest.replace_async") + "Invalid invocation of minetest.handle_async or AsyncJob:replace") local args = {n = select("#", ...), ...} local mod_origin = core.get_last_run_mod() @@ -18,22 +18,24 @@ local function prepare_async_args(func, callback, ...) end function core.handle_async(func, callback, ...) - local jobid = core.do_async_callback(prepare_async_args(func, callback, ...)) - core.async_jobs[jobid] = callback + local job = core.do_async_callback(prepare_async_args(func, callback, ...)) + core.async_jobs[job:get_id()] = callback - return jobid + return job end -function core.replace_async(jobid, func, callback, ...) - assert(type(jobid) == "number", - "Invalid jobid for minetest.replace_async") +if core.async_job_methods then + local replace_job = core.async_job_methods.replace + function core.async_job_methods:replace(func, callback, ...) + local newjob = replace_job(self, prepare_async_args(func, callback, ...)) + core.async_jobs[newjob:get_id()] = callback + return newjob + end - local newid = core.replace_async_callback(jobid, prepare_async_args(func, callback, ...)) - core.async_jobs[newid] = callback - return newid -end + local dummy = function() end + function core.async_job_methods:cancel() + return self:get_id() == self:replace(dummy, dummy) + end -local dummy = function() end -function core.cancel_async(jobid) - return jobid == core.replace_async(jobid, dummy, dummy) + core.async_job_methods = nil end diff --git a/games/devtest/mods/unittests/async_env.lua b/games/devtest/mods/unittests/async_env.lua index af62d5312..fb081614c 100644 --- a/games/devtest/mods/unittests/async_env.lua +++ b/games/devtest/mods/unittests/async_env.lua @@ -209,41 +209,41 @@ end unittests.register("test_async_vector", test_vector_preserve, {async=true}) local function test_async_job_replacement(cb) - local id = core.handle_async(function(x) + local job = core.handle_async(function(x) return x end, function(ret) print(ret) cb("Replaced async callback still run") end, 1) - local new_id = core.replace_async(id, function(x) + local newjob = job:replace(function(x) return -x end, function(ret) if ret ~= -2 then return cb("Wrong async value passed") end end, 2) - if id ~= new_id then + if job:get_id() ~= newjob:get_id() then cb("core.replace_async sanity check failed") end - id = core.handle_async(function(x) + job = core.handle_async(function(x) return x end, function() return cb("Canceled async job run") end) - if not core.cancel_async(id) then + if not job:cancel() then cb("core.cancel_async sanity check failed") end -- Try to replace a job that is already run. Do this by delaying the main thread by some time. - id = core.handle_async(function(x) + job = core.handle_async(function(x) return x end, function(ret) if ret ~= 1 then return cb("Wrong async value passed to old handler") end - new_id = core.replace_async(id, function(x) + newjob = job:replace(id, function(x) return -x end, function(new_ret) if new_ret ~= -2 then @@ -251,10 +251,10 @@ local function test_async_job_replacement(cb) end cb() end, 2) - if id == new_id then + if job:get_id() == newjob:get_id() then cb("core.replace_async replaced a completed job") end - if core.cancel_async(id) then + if id:cancel() then cb("core.relpace_async canceled a completed job") end end, 1) diff --git a/src/script/lua_api/l_async.cpp b/src/script/lua_api/l_async.cpp index 373b645fe..8c0e52f8f 100644 --- a/src/script/lua_api/l_async.cpp +++ b/src/script/lua_api/l_async.cpp @@ -2,6 +2,23 @@ #include "lua_api/l_async.h" #include "cpp_api/s_async.h" +// garbage collector +int LuaAsyncJob::gc_object(lua_State *L) +{ + LuaAsyncJob *o = *(LuaAsyncJob **)(lua_touserdata(L, 1)); + delete o; + return 0; +} + +// get_id() -> id +int LuaAsyncJob::l_get_id(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + LuaAsyncJob *o = checkObject(L, 1); + lua_pushinteger(L, o->get_id()); + return 1; +} + static std::string get_serialized_function(lua_State *L, int index) { luaL_checktype(L, index, LUA_TFUNCTION); @@ -11,6 +28,66 @@ static std::string get_serialized_function(lua_State *L, int index) return std::string(serialized_func_raw, func_length); } +// replace(self, func, params, mod_origin) -> new_job +// This implements the part that is strictly needed for replacing the job. +// The actual LuaAsyncJob:replace used by mods is implemented in Lua based on this. +int LuaAsyncJob::l_replace(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + ScriptApiAsync *script = getScriptApi(L); + + luaL_checktype(L, 3, LUA_TTABLE); + luaL_checktype(L, 4, LUA_TSTRING); + + LuaAsyncJob *o = checkObject(L, 1); + u32 id = o->get_id(); + auto serialized_func = get_serialized_function(L, 2); + PackedValue *param = script_pack(L, 3); + std::string mod_origin = readParam(L, 4); + + u32 jobId = script->replaceAsync(id, + std::move(serialized_func), + param, mod_origin); + + lua_settop(L, 0); + create(L, jobId); + return 1; +} + +int LuaAsyncJob::create(lua_State *L, const int id) +{ + NO_MAP_LOCK_REQUIRED; + LuaAsyncJob *o = new LuaAsyncJob(id); + *(void **)(lua_newuserdata(L, sizeof(void *))) = o; + luaL_getmetatable(L, className); + lua_setmetatable(L, -2); + return 1; +} + +void LuaAsyncJob::Register(lua_State *L) +{ + static const luaL_Reg metamethods[] = { + {"__gc", gc_object}, + {0, 0} + }; + + registerClass(L, className, methods, metamethods); + + // Expose __index to be complemented by Lua. + lua_getglobal(L, "core"); + luaL_getmetatable(L, className); + lua_getfield(L, -1, "__index"); + lua_setfield(L, -2, "async_job_methods"); +} + +const char LuaAsyncJob::className[] = "AsyncJob"; +const luaL_Reg LuaAsyncJob::methods[] = { + luamethod(LuaAsyncJob, get_id), + luamethod(LuaAsyncJob, replace), + {0, 0} +}; + + // do_async_callback(func, params, mod_origin) int ModApiAsync::l_do_async_callback(lua_State *L) { @@ -29,36 +106,11 @@ int ModApiAsync::l_do_async_callback(lua_State *L) param, mod_origin); lua_settop(L, 0); - lua_pushinteger(L, jobId); - return 1; -} - -// replace_async_callback(id, func, params, mod_origin) -int ModApiAsync::l_replace_async_callback(lua_State *L) -{ - NO_MAP_LOCK_REQUIRED; - ScriptApiAsync *script = getScriptApi(L); - - luaL_checktype(L, 1, LUA_TNUMBER); - luaL_checktype(L, 3, LUA_TTABLE); - luaL_checktype(L, 4, LUA_TSTRING); - - u32 id = lua_tointeger(L, 1); - auto serialized_func = get_serialized_function(L, 2); - PackedValue *param = script_pack(L, 3); - std::string mod_origin = readParam(L, 4); - - u32 jobId = script->replaceAsync(id, - std::move(serialized_func), - param, mod_origin); - - lua_settop(L, 0); - lua_pushinteger(L, jobId); + LuaAsyncJob::create(L, jobId); return 1; } void ModApiAsync::Initialize(lua_State *L, int top) { API_FCT(do_async_callback); - API_FCT(replace_async_callback); } diff --git a/src/script/lua_api/l_async.h b/src/script/lua_api/l_async.h index a793893e6..719a3cc90 100644 --- a/src/script/lua_api/l_async.h +++ b/src/script/lua_api/l_async.h @@ -1,12 +1,44 @@ #pragma once #include "lua_api/l_base.h" +#include "util/basic_macros.h" + +class LuaAsyncJob : public ModApiBase +{ +private: + const int id; + + LuaAsyncJob(const int id): id(id) {} + + static const luaL_Reg methods[]; + + // garbage collector + static int gc_object(lua_State *L); + + // get_id(self) -> id + static int l_get_id(lua_State *L); + + // replace(self, func, params, mod_origin) -> id; partially implemented in Lua + static int l_replace(lua_State *L); + + // cancel(self) -> boolean: implemented in Lua + +public: + DISABLE_CLASS_COPY(LuaAsyncJob) + + int get_id() const { return id; } + + static int create(lua_State *L, const int id); + + static void Register(lua_State *L); + static const char className[]; +}; class ModApiAsync : public ModApiBase { public: static void Initialize(lua_State *L, int top); private: + // do_async_callback(func, params, mod_origin) static int l_do_async_callback(lua_State *L); - static int l_replace_async_callback(lua_State *L); }; diff --git a/src/script/scripting_server.cpp b/src/script/scripting_server.cpp index e93a5a942..eaea5a57a 100644 --- a/src/script/scripting_server.cpp +++ b/src/script/scripting_server.cpp @@ -135,6 +135,7 @@ void ServerScripting::InitializeModApi(lua_State *L, int top) InvRef::Register(L); ItemStackMetaRef::Register(L); LuaAreaStore::Register(L); + LuaAsyncJob::Register(L); LuaItemStack::Register(L); LuaPerlinNoise::Register(L); LuaPerlinNoiseMap::Register(L); From 96dfd5cc0f5e8a4e885cb35a8c54852437251ccc Mon Sep 17 00:00:00 2001 From: y5nw Date: Fri, 17 May 2024 21:55:14 +0200 Subject: [PATCH 10/11] minor --- builtin/game/async.lua | 2 +- games/devtest/mods/unittests/async_env.lua | 11 +++++------ src/script/lua_api/l_async.cpp | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/builtin/game/async.lua b/builtin/game/async.lua index d184440c9..f85864649 100644 --- a/builtin/game/async.lua +++ b/builtin/game/async.lua @@ -34,7 +34,7 @@ if core.async_job_methods then local dummy = function() end function core.async_job_methods:cancel() - return self:get_id() == self:replace(dummy, dummy) + return self:get_id() == self:replace(dummy, dummy):get_id() end core.async_job_methods = nil diff --git a/games/devtest/mods/unittests/async_env.lua b/games/devtest/mods/unittests/async_env.lua index fb081614c..7d222c148 100644 --- a/games/devtest/mods/unittests/async_env.lua +++ b/games/devtest/mods/unittests/async_env.lua @@ -212,7 +212,6 @@ local function test_async_job_replacement(cb) local job = core.handle_async(function(x) return x end, function(ret) - print(ret) cb("Replaced async callback still run") end, 1) local newjob = job:replace(function(x) @@ -223,7 +222,7 @@ local function test_async_job_replacement(cb) end end, 2) if job:get_id() ~= newjob:get_id() then - cb("core.replace_async sanity check failed") + cb("AsyncJob:replace sanity check failed") end job = core.handle_async(function(x) @@ -243,7 +242,7 @@ local function test_async_job_replacement(cb) return cb("Wrong async value passed to old handler") end - newjob = job:replace(id, function(x) + newjob = job:replace(function(x) return -x end, function(new_ret) if new_ret ~= -2 then @@ -252,10 +251,10 @@ local function test_async_job_replacement(cb) cb() end, 2) if job:get_id() == newjob:get_id() then - cb("core.replace_async replaced a completed job") + cb("AsyncJob:replace replaced a completed job") end - if id:cancel() then - cb("core.relpace_async canceled a completed job") + if job:cancel() then + cb("AsyncJob:replace canceled a completed job") end end, 1) end diff --git a/src/script/lua_api/l_async.cpp b/src/script/lua_api/l_async.cpp index 8c0e52f8f..e2c8ec7be 100644 --- a/src/script/lua_api/l_async.cpp +++ b/src/script/lua_api/l_async.cpp @@ -77,7 +77,7 @@ void LuaAsyncJob::Register(lua_State *L) lua_getglobal(L, "core"); luaL_getmetatable(L, className); lua_getfield(L, -1, "__index"); - lua_setfield(L, -2, "async_job_methods"); + lua_setfield(L, -3, "async_job_methods"); } const char LuaAsyncJob::className[] = "AsyncJob"; From 0ae56118de417fa61c9d08abf90a77daa5475c11 Mon Sep 17 00:00:00 2001 From: y5nw Date: Fri, 17 May 2024 22:50:46 +0200 Subject: [PATCH 11/11] minor --- games/devtest/mods/unittests/async_env.lua | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/games/devtest/mods/unittests/async_env.lua b/games/devtest/mods/unittests/async_env.lua index 7d222c148..5e16a4339 100644 --- a/games/devtest/mods/unittests/async_env.lua +++ b/games/devtest/mods/unittests/async_env.lua @@ -212,7 +212,7 @@ local function test_async_job_replacement(cb) local job = core.handle_async(function(x) return x end, function(ret) - cb("Replaced async callback still run") + return cb("Replaced async callback still run") end, 1) local newjob = job:replace(function(x) return -x @@ -222,7 +222,7 @@ local function test_async_job_replacement(cb) end end, 2) if job:get_id() ~= newjob:get_id() then - cb("AsyncJob:replace sanity check failed") + return cb("AsyncJob:replace sanity check failed") end job = core.handle_async(function(x) @@ -231,7 +231,7 @@ local function test_async_job_replacement(cb) return cb("Canceled async job run") end) if not job:cancel() then - cb("core.cancel_async sanity check failed") + return cb("core.cancel_async sanity check failed") end -- Try to replace a job that is already run. Do this by delaying the main thread by some time. @@ -251,10 +251,10 @@ local function test_async_job_replacement(cb) cb() end, 2) if job:get_id() == newjob:get_id() then - cb("AsyncJob:replace replaced a completed job") + return cb("AsyncJob:replace replaced a completed job") end if job:cancel() then - cb("AsyncJob:replace canceled a completed job") + return cb("AsyncJob:replace canceled a completed job") end end, 1) end