Merge aa9057218b into c4703a7f19

doc/lua_api.md

@@ -6731,6 +6731,7 @@ Functions:
 * Standalone helpers such as logging, filesystem, encoding,
   hashing or compression APIs
 * `minetest.register_async_metatable` (see above)
+* IPC
 
 Variables:
 * `minetest.settings`
@@ -6805,6 +6806,7 @@ Functions:
 * `minetest.get_node`, `set_node`, `find_node_near`, `find_nodes_in_area`,
   `spawn_tree` and similar
     * these only operate on the current chunk (if inside a callback)
+* IPC
 
 Variables:
 * `minetest.settings`
@@ -6881,6 +6883,33 @@ Server
       this can make transfer of bigger files painless (if set up). Nevertheless
       it is advised not to use dynamic media for big media files.
 
+IPC
+---
+
+The engine provides a generalized mechanism to enable sharing data between the
+different Lua environments (main, mapgen and async).
+
+* `minetest.ipc_get(key)`:
+  * Read a value from the shared data area.
+  * `key`: string, should use the `"modname:thing"` convention to avoid conflicts.
+  * return an arbitrary Lua value, or `nil` if this key does not exist
+* `minetest.ipc_set(key, value)`:
+  * Write a value to the shared data area.
+  * `key`: as above
+  * `value`: an arbitrary Lua value, cannot be or contain userdata.
+* `minetest.ipc_swap(key, value)`:
+  * Like `minetest.ipc_set` but returns the value that was at the same key
+    before writing the new one. This is atomic.
+
+Interacting with the shared data will perform an operation comparable to
+(de)serialization.
+Modifying references will not have any effect, as in this example:
+```lua
+minetest.ipc_set("test:foo", {})
+minetest.ipc_get("test:foo").subkey = "value"
+minetest.ipc_get("test:foo") -- returns an empty table
+```
+
 Bans
 ----
 

games/devtest/mods/unittests/inside_mapgen_env.lua

@@ -22,9 +22,11 @@ local function do_tests()
 	assert(core.registered_items["unittests:description_test"].on_place == true)
 end
 
--- there's no (usable) communcation path between mapgen and the regular env
--- so we just run the test unconditionally
-do_tests()
+-- first thread to get here runs the tests
+if core.ipc_swap("unittests:mg_once", 1) == nil then
+	-- this is checked from the main env
+	core.ipc_set("unittests:mg", { pcall(do_tests) })
+end
 
 core.register_on_generated(function(vm, pos1, pos2, blockseed)
 	local n = tonumber(core.get_mapgen_setting("chunksize")) * 16 - 1

games/devtest/mods/unittests/misc.lua

@@ -254,3 +254,28 @@ local function test_gennotify_api()
 	assert(#custom == 0, "custom ids not empty")
 end
 unittests.register("test_gennotify_api", test_gennotify_api)
+
+-- <=> inside_mapgen_env.lua
+local function test_mapgen_env(cb)
+	-- emerge threads start delayed so this can take a second
+	local res = core.ipc_get("unittests:mg")
+	if res == nil then
+		return core.after(0, test_mapgen_env, cb)
+	end
+	-- handle error status
+	if res[1] then
+		cb()
+	else
+		cb(res[2])
+	end
+end
+unittests.register("test_mapgen_env", test_mapgen_env, {async=true})
+
+local function test_ipc_vector_preserve(cb)
+	-- the IPC uses the same mechanism as register_async_metatable to preserve metatables
+	core.ipc_set("unittests:v", vector.new(4, 0, 4))
+	local v = core.ipc_get("unittests:v")
+	assert(type(v) == "table")
+	assert(vector.check(v))
+end
+unittests.register("test_ipc_vector_preserve", test_ipc_vector_preserve)

src/gamedef.h

@@ -34,19 +34,19 @@ class Camera;
 class ModChannel;
 class ModStorage;
 class ModStorageDatabase;
+struct SubgameSpec;
+struct ModSpec;
+struct ModIPCStore;
 
 namespace irr::scene {
 	class IAnimatedMesh;
 	class ISceneManager;
 }
 
-struct SubgameSpec;
-struct ModSpec;
 /*
 	An interface for fetching game-global definitions like tool and
 	mapnode properties
 */
-
 class IGameDef
 {
 public:
@@ -63,6 +63,9 @@ public:
 	// environment thread.
 	virtual IRollbackManager* getRollbackManager() { return NULL; }
 
+	// Only usable on server.
+	virtual ModIPCStore *getModIPCStore() { return nullptr; }
+
 	// Shorthands
 	// TODO: these should be made const-safe so that a const IGameDef* is
 	//       actually usable

src/log.cpp

@@ -34,6 +34,10 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include <android/log.h>
 #endif
 
+#if !defined(_WIN32)
+#include <unistd.h> // isatty
+#endif
+
 #include <sstream>
 #include <iostream>
 #include <algorithm>
@@ -104,21 +108,19 @@ thread_local LogStream dout_con(trace_target);
 
 static unsigned int g_level_to_android[] = {
 	ANDROID_LOG_INFO,     // LL_NONE
-	//ANDROID_LOG_FATAL,
 	ANDROID_LOG_ERROR,    // LL_ERROR
 	ANDROID_LOG_WARN,     // LL_WARNING
-	ANDROID_LOG_WARN,     // LL_ACTION
-	//ANDROID_LOG_INFO,
+	ANDROID_LOG_INFO,     // LL_ACTION
 	ANDROID_LOG_DEBUG,    // LL_INFO
 	ANDROID_LOG_VERBOSE,  // LL_VERBOSE
 	ANDROID_LOG_VERBOSE,  // LL_TRACE
 };
 
-void AndroidLogOutput::logRaw(LogLevel lev, const std::string &line) {
+void AndroidLogOutput::logRaw(LogLevel lev, const std::string &line)
+{
 	static_assert(ARRLEN(g_level_to_android) == LL_MAX,
 		"mismatch between android and internal loglevels");
-	__android_log_print(g_level_to_android[lev],
-		PROJECT_NAME_C, "%s", line.c_str());
+	__android_log_write(g_level_to_android[lev], PROJECT_NAME_C, line.c_str());
 }
 #endif
 
@@ -156,9 +158,7 @@ void Logger::addOutput(ILogOutput *out)
 
 void Logger::addOutput(ILogOutput *out, LogLevel lev)
 {
-	MutexAutoLock lock(m_mutex);
-	m_outputs[lev].push_back(out);
-	m_has_outputs[lev] = true;
+	addOutputMasked(out, LOGLEVEL_TO_MASKLEVEL(lev));
 }
 
 void Logger::addOutputMasked(ILogOutput *out, LogLevelMask mask)
@@ -187,9 +187,7 @@ LogLevelMask Logger::removeOutput(ILogOutput *out)
 	MutexAutoLock lock(m_mutex);
 	LogLevelMask ret_mask = 0;
 	for (size_t i = 0; i < LL_MAX; i++) {
-		std::vector<ILogOutput *>::iterator it;
-
-		it = std::find(m_outputs[i].begin(), m_outputs[i].end(), out);
+		auto it = std::find(m_outputs[i].begin(), m_outputs[i].end(), out);
 		if (it != m_outputs[i].end()) {
 			ret_mask |= LOGLEVEL_TO_MASKLEVEL(i);
 			m_outputs[i].erase(it);
@@ -218,9 +216,9 @@ void Logger::deregisterThread()
 	m_thread_names.erase(id);
 }
 
-const std::string Logger::getLevelLabel(LogLevel lev)
+const char *Logger::getLevelLabel(LogLevel lev)
 {
-	static const std::string names[] = {
+	static const char *names[] = {
 		"",
 		"ERROR",
 		"WARNING",
@@ -229,45 +227,50 @@ const std::string Logger::getLevelLabel(LogLevel lev)
 		"VERBOSE",
 		"TRACE",
 	};
-	assert(lev < LL_MAX && lev >= 0);
 	static_assert(ARRLEN(names) == LL_MAX,
 		"mismatch between loglevel names and enum");
+	assert(lev < LL_MAX && lev >= 0);
 	return names[lev];
 }
 
 LogColor Logger::color_mode = LOG_COLOR_AUTO;
 
-const std::string Logger::getThreadName()
+const std::string &Logger::getThreadName()
 {
-	std::map<std::thread::id, std::string>::const_iterator it;
-
 	std::thread::id id = std::this_thread::get_id();
-	it = m_thread_names.find(id);
+
+	auto it = m_thread_names.find(id);
 	if (it != m_thread_names.end())
 		return it->second;
 
-	std::ostringstream os;
-	os << "#0x" << std::hex << id;
-	return os.str();
+	thread_local std::string fallback_name;
+	if (!fallback_name.empty()) {
+		std::ostringstream os;
+		os << "#0x" << std::hex << id;
+		fallback_name = os.str();
+	}
+	return fallback_name;
 }
 
 void Logger::log(LogLevel lev, const std::string &text)
 {
-	if (m_silenced_levels[lev])
+	if (isLevelSilenced(lev))
 		return;
 
-	const std::string thread_name = getThreadName();
-	const std::string label = getLevelLabel(lev);
+	const std::string &thread_name = getThreadName();
+	const char *label = getLevelLabel(lev);
 	const std::string timestamp = getTimestamp();
-	std::ostringstream os(std::ios_base::binary);
-	os << timestamp << ": " << label << "[" << thread_name << "]: " << text;
 
-	logToOutputs(lev, os.str(), timestamp, thread_name, text);
+	std::string line = timestamp;
+	line.append(": ").append(label).append("[").append(thread_name)
+		.append("]: ").append(text);
+
+	logToOutputs(lev, line, timestamp, thread_name, text);
 }
 
 void Logger::logRaw(LogLevel lev, const std::string &text)
 {
-	if (m_silenced_levels[lev])
+	if (isLevelSilenced(lev))
 		return;
 
 	logToOutputsRaw(lev, text);
@@ -320,6 +323,17 @@ void FileLogOutput::setFile(const std::string &filename, s64 file_size_max)
 		"-------------\n" << std::endl;
 }
 
+StreamLogOutput::StreamLogOutput(std::ostream &stream) :
+	m_stream(stream)
+{
+#if !defined(_WIN32)
+	if (&stream == &std::cout)
+		is_tty = isatty(STDOUT_FILENO);
+	else if (&stream == &std::cerr)
+		is_tty = isatty(STDERR_FILENO);
+#endif
+}
+
 void StreamLogOutput::logRaw(LogLevel lev, const std::string &line)
 {
 	bool colored_message = (Logger::color_mode == LOG_COLOR_ALWAYS) ||

src/log.h

@@ -26,9 +26,6 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include <fstream>
 #include <thread>
 #include <mutex>
-#if !defined(_WIN32)  // POSIX
-	#include <unistd.h>
-#endif
 #include "threading/mutex_auto_lock.h"
 #include "util/basic_macros.h"
 #include "util/stream.h"
@@ -73,12 +70,16 @@ public:
 	void logRaw(LogLevel lev, const std::string &text);
 
 	static LogLevel stringToLevel(const std::string &name);
-	static const std::string getLevelLabel(LogLevel lev);
+	static const char *getLevelLabel(LogLevel lev);
 
 	bool hasOutput(LogLevel level) {
 		return m_has_outputs[level].load(std::memory_order_relaxed);
 	}
 
+	bool isLevelSilenced(LogLevel level) {
+		return m_silenced_levels[level].load(std::memory_order_relaxed);
+	}
+
 	static LogColor color_mode;
 
 private:
@@ -87,15 +88,11 @@ private:
 		const std::string &time, const std::string &thread_name,
 		const std::string &payload_text);
 
-	const std::string getThreadName();
+	const std::string &getThreadName();
 
 	std::vector<ILogOutput *> m_outputs[LL_MAX];
 	std::atomic<bool> m_has_outputs[LL_MAX];
-
-	// Should implement atomic loads and stores (even though it's only
-	// written to when one thread has access currently).
-	// Works on all known architectures (x86, ARM, MIPS).
-	volatile bool m_silenced_levels[LL_MAX];
+	std::atomic<bool> m_silenced_levels[LL_MAX];
 	std::map<std::thread::id, std::string> m_thread_names;
 	mutable std::mutex m_mutex;
 };
@@ -120,16 +117,7 @@ public:
 
 class StreamLogOutput : public ICombinedLogOutput {
 public:
-	StreamLogOutput(std::ostream &stream) :
-		m_stream(stream)
-	{
-#if !defined(_WIN32)
-		if (&stream == &std::cout)
-			is_tty = isatty(STDOUT_FILENO);
-		else if (&stream == &std::cerr)
-			is_tty = isatty(STDERR_FILENO);
-#endif
-	}
+	StreamLogOutput(std::ostream &stream);
 
 	void logRaw(LogLevel lev, const std::string &line);
 

src/network/connection.cpp

@@ -1078,7 +1078,7 @@ bool UDPPeer::processReliableSendCommand(
 	bool have_sequence_number = false;
 	bool have_initial_sequence_number = false;
 	std::queue<BufferedPacketPtr> toadd;
-	volatile u16 initial_sequence_number = 0;
+	u16 initial_sequence_number = 0;
 
 	for (SharedBuffer<u8> &original : originals) {
 		u16 seqnum = chan.getOutgoingSequenceNumber(have_sequence_number);
@@ -1118,7 +1118,7 @@ bool UDPPeer::processReliableSendCommand(
 		return true;
 	}
 
-	volatile u16 packets_available = toadd.size();
+	u16 packets_available = toadd.size();
 	/* we didn't get a single sequence number no need to fill queue */
 	if (!have_initial_sequence_number) {
 		LOG(derr_con << m_connection->getDesc() << "Ran out of sequence numbers!" << std::endl);

src/script/lua_api/CMakeLists.txt

@@ -6,6 +6,7 @@ set(common_SCRIPT_LUA_API_SRCS
 	${CMAKE_CURRENT_SOURCE_DIR}/l_env.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_http.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_inventory.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/l_ipc.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_item.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_itemstackmeta.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_mapgen.cpp

src/script/lua_api/l_ipc.cpp

@@ -0,0 +1,81 @@
+// Minetest
+// SPDX-License-Identifier: LGPL-2.1-or-later
+
+#include "lua_api/l_ipc.h"
+#include "lua_api/l_internal.h"
+#include "common/c_packer.h"
+#include "server.h"
+#include "debug.h"
+
+typedef std::shared_lock<std::shared_mutex> SharedReadLock;
+typedef std::unique_lock<std::shared_mutex> SharedWriteLock;
+
+int ModApiIPC::l_ipc_get(lua_State *L)
+{
+	auto *store = getGameDef(L)->getModIPCStore();
+
+	auto key = readParam<std::string>(L, 1);
+
+	{
+		SharedReadLock autolock(store->mutex);
+		auto it = store->map.find(key);
+		if (it == store->map.end())
+			lua_pushnil(L);
+		else
+			script_unpack(L, it->second.get());
+	}
+	return 1;
+}
+
+int ModApiIPC::ipc_put(lua_State *L, bool push_old)
+{
+	auto *store = getGameDef(L)->getModIPCStore();
+
+	auto key = readParam<std::string>(L, 1);
+
+	std::unique_ptr<PackedValue> vnew, vold;
+	luaL_checkany(L, 2);
+	if (!lua_isnil(L, 2)) {
+		vnew.reset(script_pack(L, 2));
+		if (vnew->contains_userdata)
+			throw LuaError("Userdata not allowed");
+	}
+
+	{
+		SharedWriteLock autolock(store->mutex);
+		auto it = store->map.find(key);
+		if (it == store->map.end()) {
+			[[maybe_unused]] auto r = store->map.emplace(std::move(key), std::move(vnew));
+			assert(r.second);
+		} else {
+			vold = std::move(it->second);
+			it->second = std::move(vnew);
+		}
+	}
+
+	if (push_old) {
+		if (vold)
+			script_unpack(L, vold.get());
+		else
+			lua_pushnil(L);
+		return 1;
+	}
+	return 0;
+}
+
+/*
+ * Implementation note:
+ * Iterating over the IPC table is intentionally not supported.
+ * Mods should know what they have set.
+ * This has the nice side effect that mods are able to use a randomly generated key
+ * if they really *really* want to avoid other code touching their data. 
+ */
+
+void ModApiIPC::Initialize(lua_State *L, int top)
+{
+	sanity_check(getGameDef(L)->getModIPCStore() != nullptr);
+
+	API_FCT(ipc_get);
+	API_FCT(ipc_set);
+	API_FCT(ipc_swap);
+}

src/script/lua_api/l_ipc.h

@@ -0,0 +1,17 @@
+// Minetest
+// SPDX-License-Identifier: LGPL-2.1-or-later
+
+#pragma once
+
+#include "lua_api/l_base.h"
+
+class ModApiIPC : public ModApiBase {
+private:
+	static int l_ipc_get(lua_State *L);
+	static int ipc_put(lua_State *L, bool push_old);
+	static int l_ipc_set(lua_State *L) { return ipc_put(L, false); }
+	static int l_ipc_swap(lua_State *L) { return ipc_put(L, true); }
+
+public:
+	static void Initialize(lua_State *L, int top);
+};

src/script/scripting_emerge.cpp

@@ -35,6 +35,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_util.h"
 #include "lua_api/l_vmanip.h"
 #include "lua_api/l_settings.h"
+#include "lua_api/l_ipc.h"
 
 extern "C" {
 #include <lualib.h>
@@ -89,5 +90,6 @@ void EmergeScripting::InitializeModApi(lua_State *L, int top)
 	ModApiMapgen::InitializeEmerge(L, top);
 	ModApiServer::InitializeAsync(L, top);
 	ModApiUtil::InitializeAsync(L, top);
+	ModApiIPC::Initialize(L, top);
 	// TODO ^ these should also be renamed to InitializeRO or such
 }

src/script/scripting_server.cpp

@@ -46,6 +46,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_settings.h"
 #include "lua_api/l_http.h"
 #include "lua_api/l_storage.h"
+#include "lua_api/l_ipc.h"
 
 extern "C" {
 #include <lualib.h>
@@ -121,6 +122,7 @@ void ServerScripting::initAsync()
 	asyncEngine.registerStateInitializer(ModApiCraft::InitializeAsync);
 	asyncEngine.registerStateInitializer(ModApiItem::InitializeAsync);
 	asyncEngine.registerStateInitializer(ModApiServer::InitializeAsync);
+	asyncEngine.registerStateInitializer(ModApiIPC::Initialize);
 	// not added: ModApiMapgen is a minefield for thread safety
 	// not added: ModApiHttp async api can't really work together with our jobs
 	// not added: ModApiStorage is probably not thread safe(?)
@@ -176,6 +178,7 @@ void ServerScripting::InitializeModApi(lua_State *L, int top)
 	ModApiHttp::Initialize(L, top);
 	ModApiStorage::Initialize(L, top);
 	ModApiChannels::Initialize(L, top);
+	ModApiIPC::Initialize(L, top);
 }
 
 void ServerScripting::InitializeAsync(lua_State *L, int top)

src/server.h

@@ -46,6 +46,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include <unordered_set>
 #include <optional>
 #include <string_view>
+#include <shared_mutex>
 
 class ChatEvent;
 struct ChatEventChat;
@@ -141,6 +142,11 @@ struct ClientInfo {
 	std::string vers_string, lang_code;
 };
 
+struct ModIPCStore {
+	std::shared_mutex mutex;
+	std::unordered_map<std::string, std::unique_ptr<PackedValue>> map;
+};
+
 class Server : public con::PeerHandler, public MapEventReceiver,
 		public IGameDef
 {
@@ -301,12 +307,14 @@ public:
 	NodeDefManager* getWritableNodeDefManager();
 	IWritableCraftDefManager* getWritableCraftDefManager();
 
+	// Not under envlock
 	virtual const std::vector<ModSpec> &getMods() const;
 	virtual const ModSpec* getModSpec(const std::string &modname) const;
 	virtual const SubgameSpec* getGameSpec() const { return &m_gamespec; }
 	static std::string getBuiltinLuaPath();
 	virtual std::string getWorldPath() const { return m_path_world; }
 	virtual std::string getModDataPath() const { return m_path_mod_data; }
+	virtual ModIPCStore *getModIPCStore() { return &m_ipcstore; }
 
 	inline bool isSingleplayer() const
 			{ return m_simple_singleplayer_mode; }
@@ -662,6 +670,8 @@ private:
 
 	std::unordered_map<std::string, Translations> server_translations;
 
+	ModIPCStore m_ipcstore;
+
 	/*
 		Threads
 	*/

src/terminal_chat_console.cpp

@@ -426,7 +426,7 @@ void TerminalChatConsole::step(int ch)
 		printw("[ESC] Toggle ESC mode |"
 			" [CTRL+C] Shut down |"
 			" (L) in-, (l) decrease loglevel %s",
-			Logger::getLevelLabel((LogLevel) m_log_level).c_str());
+			Logger::getLevelLabel((LogLevel) m_log_level));
 	}
 
 	refresh();

src/unittest/test_threading.cpp

@@ -161,7 +161,7 @@ void TestThreading::testAtomicSemaphoreThread()
 
 
 
-static volatile bool g_tls_broken;
+static std::atomic<bool> g_tls_broken;
 
 class TLSTestThread : public Thread {
 public:
@@ -226,7 +226,7 @@ private:
 */
 void TestThreading::testTLS()
 {
-	static const int num_threads = 10;
+	constexpr int num_threads = 10;
 
 	for (int j = 0; j < num_threads; j++) {
 		g_tls_broken = false;