Automatically choose multiple emerge threads for singlenode (#16634)

builtin/settingtypes.txt

@@ -2367,17 +2367,12 @@ emergequeue_limit_diskonly (Per-player limit of queued blocks load from disk) in
 #    This limit is enforced per player.
 emergequeue_limit_generate (Per-player limit of queued blocks to generate) int 128 1 1000000
 
-#    Number of emerge threads to use.
-#    Value 0:
-#    -    Automatic selection. The number of emerge threads will be
-#    -    'number of processors - 2', with a lower limit of 1.
-#    Any other value:
-#    -    Specifies the number of emerge threads, with a lower limit of 1.
-#    WARNING: Increasing the number of emerge threads increases engine mapgen
-#    speed, but this may harm game performance by interfering with other
-#    processes, especially in singleplayer and/or when running Lua code in
-#    'on_generated'. For many users the optimum setting may be '1'.
-num_emerge_threads (Number of emerge threads) int 1 0 32767
+#    Number of emerge threads (responsible for map generation and loading) to use.
+#    If 0 then the engine will automatically choose a suitable value depending
+#    on the hardware and type of map generator.
+#    WARNING: There are known bugs in the default map generators (v6, v7, ...)
+#    when using more than 1 thread. The automatic choice will avoid this.
+num_emerge_threads (Number of emerge threads) int 0 0 32767
 
 [**cURL] [common]
 

src/defaultsettings.cpp

@@ -490,7 +490,7 @@ void set_default_settings()
 	settings->setDefault("emergequeue_limit_total", "1024");
 	settings->setDefault("emergequeue_limit_diskonly", "128");
 	settings->setDefault("emergequeue_limit_generate", "128");
-	settings->setDefault("num_emerge_threads", "1");
+	settings->setDefault("num_emerge_threads", "0");
 	settings->setDefault("secure.enable_security", "true");
 	settings->setDefault("secure.trusted_mods", "");
 	settings->setDefault("secure.http_mods", "");

src/emerge.cpp

@@ -6,8 +6,8 @@
 
 #include "emerge_internal.h"
 
+#include <cmath>
 #include <iostream>
-
 #include "config.h"
 #include "constants.h"
 #include "irrlicht_changes/printing.h"
@@ -30,7 +30,6 @@
 
 EmergeParams::~EmergeParams()
 {
-	infostream << "EmergeParams: destroying " << this << std::endl;
 	// Delete everything that was cloned on creation of EmergeParams
 	delete biomegen;
 	delete biomemgr;
@@ -60,7 +59,9 @@ EmergeParams::EmergeParams(EmergeManager *parent, const BiomeGen *biomegen,
 
 EmergeManager::EmergeManager(Server *server, MetricsBackend *mb)
 {
-	this->ndef      = server->getNodeDefManager();
+	assert(server);
+	this->m_server  = server;
+	this->ndef      = server->ndef();
 	this->biomemgr  = new BiomeManager(server);
 	this->oremgr    = new OreManager(server);
 	this->decomgr   = new DecorationManager(server);
@@ -87,31 +88,14 @@ EmergeManager::EmergeManager(Server *server, MetricsBackend *mb)
 		);
 	}
 
-	s16 nthreads = 1;
-	g_settings->getS16NoEx("num_emerge_threads", nthreads);
-	// If automatic, leave a proc for the main thread and one for
-	// some other misc thread
-	if (nthreads <= 0)
-		nthreads = Thread::getNumberOfProcessors() - 2;
-	if (nthreads < 1)
-		nthreads = 1;
-
 	m_qlimit_total = g_settings->getU32("emergequeue_limit_total");
-	// FIXME: these fallback values are probably not good
-	if (!g_settings->getU32NoEx("emergequeue_limit_diskonly", m_qlimit_diskonly))
-		m_qlimit_diskonly = nthreads * 5 + 1;
-	if (!g_settings->getU32NoEx("emergequeue_limit_generate", m_qlimit_generate))
-		m_qlimit_generate = nthreads + 1;
+	m_qlimit_diskonly = g_settings->getU32("emergequeue_limit_diskonly");
+	m_qlimit_generate = g_settings->getU32("emergequeue_limit_generate");
 
 	// don't trust user input for something very important like this
 	m_qlimit_diskonly = rangelim(m_qlimit_diskonly, 2, 1000000);
 	m_qlimit_generate = rangelim(m_qlimit_generate, 1, 1000000);
 	m_qlimit_total = std::max(m_qlimit_total, std::max(m_qlimit_diskonly, m_qlimit_generate));
-
-	for (s16 i = 0; i < nthreads; i++)
-		m_threads.push_back(new EmergeThread(server, i));
-
-	infostream << "EmergeManager: using " << nthreads << " threads" << std::endl;
 }
 
 
@@ -188,18 +172,58 @@ void EmergeManager::initMapgens(MapgenParams *params)
 
 	mgparams = params;
 
+	infostream << "EmergeManager: initializing for mapgen="
+		<< Mapgen::getMapgenName(params->mgtype)
+		<< " and chunksize=" << params->chunksize << std::endl;
+
+	/*
+	 * Singlenode is currently the only mapgen not affected by the
+	 * unfinished slice bug, so allow multiple threads by default.
+	 * We do this for the Lua mapgens who benefit from this (since singlenode
+	 * itself isn't very useful).
+	 * see <https://github.com/luanti-org/luanti/issues/9357>
+	 */
+	bool multithread = params->mgtype == MAPGEN_SINGLENODE;
+	initThreads(multithread);
+
 	v3s16 csize = params->chunksize * MAP_BLOCKSIZE;
 	biomegen = biomemgr->createBiomeGen(BIOMEGEN_ORIGINAL, params->bparams, csize);
 
 	for (u32 i = 0; i != m_threads.size(); i++) {
 		EmergeParams *p = new EmergeParams(this, biomegen,
 			biomemgr, oremgr, decomgr, schemmgr);
-		infostream << "EmergeManager: Created params " << p
-			<< " for thread " << i << std::endl;
 		m_mapgens.push_back(Mapgen::createMapgen(params->mgtype, params, p));
 	}
 }
 
+void EmergeManager::initThreads(bool should_multithread)
+{
+	s16 nthreads = g_settings->getS16("num_emerge_threads");
+	if (nthreads <= 0 && should_multithread) {
+		u32 concurrency = Thread::getNumberOfProcessors();
+		u32 memoryMB = porting::getMemorySizeMB();
+		if (memoryMB) {
+			// Cap threads according to total RAM with a conservative 1 GB per thread.
+			// This is for the sake of Android phones, where many cores & low RAM
+			// is not uncommon (e.g. 8C + 3GB).
+			concurrency = std::min<u32>(concurrency, std::roundf(memoryMB / 1024.0f));
+		}
+		// Leave 2 cores for main thread and whatever else.
+		nthreads = (concurrency > 2) ? (concurrency - 2) : 1;
+		// Testing has shown that more than 4 threads don't become any faster:
+		// <https://github.com/luanti-org/luanti/pull/16634>
+		// May have to be revisited after emerge code is refactored to be less
+		// lock heavy.
+		nthreads = std::min<s16>(4, nthreads);
+	}
+	nthreads = std::max<s16>(1, nthreads);
+
+	FATAL_ERROR_IF(!m_threads.empty(), "Threads already initialized.");
+	for (s16 i = 0; i < nthreads; i++)
+		m_threads.push_back(new EmergeThread(m_server, i));
+
+	infostream << "EmergeManager: using " << nthreads << " thread(s)" << std::endl;
+}
 
 Mapgen *EmergeManager::getCurrentMapgen()
 {
@@ -247,12 +271,6 @@ void EmergeManager::stopThreads()
 }
 
 
-bool EmergeManager::isRunning()
-{
-	return m_threads_active;
-}
-
-
 bool EmergeManager::enqueueBlockEmerge(
 	session_t peer_id,
 	v3s16 blockpos,

src/emerge.h

@@ -165,7 +165,6 @@ public:
 
 	void startThreads();
 	void stopThreads();
-	bool isRunning();
 
 	bool enqueueBlockEmerge(
 		session_t peer_id,
@@ -193,10 +192,14 @@ public:
 	static v3s16 getContainingChunk(v3s16 blockpos, v3s16 chunksize);
 
 private:
+	void initThreads(bool should_multithread);
+
 	std::vector<Mapgen *> m_mapgens;
 	std::vector<EmergeThread *> m_threads;
 	bool m_threads_active = false;
 
+	// Server reference
+	Server *m_server = nullptr;
 	// The map database
 	MapDatabaseAccessor *m_db = nullptr;
 

src/mapgen/mapgen.cpp

@@ -144,7 +144,9 @@ const char *Mapgen::getMapgenName(MapgenType mgtype)
 	if (index == MAPGEN_INVALID || index >= ARRLEN(g_reg_mapgens))
 		return "invalid";
 
-	return g_reg_mapgens[index].name;
+	auto &it = g_reg_mapgens[index];
+	assert(it.name);
+	return it.name;
 }
 
 

src/porting.cpp

@@ -40,6 +40,8 @@
 #if defined(__APPLE__)
 	#include <mach-o/dyld.h>
 	#include <CoreFoundation/CoreFoundation.h>
+	#include <sys/types.h>
+	#include <sys/sysctl.h>
 	// For _NSGetEnviron()
 	// Related: https://gitlab.haskell.org/ghc/ghc/issues/2458
 	#include <crt_externs.h>
@@ -67,7 +69,7 @@
 #include <atomic>
 
 #if CHECK_CLIENT_BUILD() && defined(_WIN32)
-// On Windows export some driver-specific variables to encourage Minetest to be
+// On Windows export some driver-specific variables to encourage Luanti to be
 // executed on the discrete GPU in case of systems with two. Portability is fun.
 extern "C" {
 	__declspec(dllexport) DWORD NvOptimusEnablement = 1;
@@ -267,6 +269,26 @@ const std::string &get_sysinfo()
 	return ret;
 }
 
+u32 getMemorySizeMB()
+{
+#ifdef _WIN32
+	MEMORYSTATUSEX status;
+	status.dwLength = sizeof(status);
+	if (GlobalMemoryStatusEx(&status))
+		return status.ullTotalPhys >> 20;
+#elif defined(__unix__) && defined(_SC_PHYS_PAGES) && defined(_SC_PAGE_SIZE)
+	long pages = sysconf(_SC_PHYS_PAGES);
+	long page_size = sysconf(_SC_PAGE_SIZE);
+	if (pages != -1 && page_size != -1)
+		return (pages * page_size) >> 20;
+#elif defined(__APPLE__)
+	int64_t memsize;
+	size_t len = sizeof(memsize);
+	if (sysctlbyname("hw.memsize", &memsize, &len, nullptr, 0) == 0)
+		return memsize >> 20;
+#endif
+	return 0;
+}
 
 [[maybe_unused]] static bool getCurrentWorkingDir(char *buf, size_t len)
 {

src/porting.h

@@ -127,6 +127,12 @@ void initializePaths();
 const std::string &get_sysinfo();
 
 
+/*
+	Return size of system RAM in MB
+	(or 0 if unavailable/error)
+*/
+u32 getMemorySizeMB();
+
 // Monotonic timer
 
 #ifdef _WIN32 // Windows

src/unittest/test_utilities.cpp

@@ -50,6 +50,7 @@ public:
 	void testSanitizeUntrusted();
 	void testReadSeed();
 	void testMyDoubleStringConversions();
+	void testGetMemorySize();
 };
 
 static TestUtilities g_test_instance;
@@ -87,6 +88,7 @@ void TestUtilities::runTests(IGameDef *gamedef)
 	TEST(testSanitizeUntrusted);
 	TEST(testReadSeed);
 	TEST(testMyDoubleStringConversions);
+	TEST(testGetMemorySize);
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -805,3 +807,23 @@ void TestUtilities::testMyDoubleStringConversions()
 	test_round_trip(0.3);
 	test_round_trip(0.1 + 0.2);
 }
+
+void TestUtilities::testGetMemorySize()
+{
+#if defined(_WIN32) || defined(__linux__) || defined(__APPLE__)
+	const bool fail_ok = false;
+#else
+	const bool fail_ok = true;
+#endif
+
+	u32 total = porting::getMemorySizeMB();
+	UASSERT(total != 0 || fail_ok);
+	if (total != 0) {
+		infostream << "memory size in MB = " << total << std::endl;
+		// should be a sane value
+		UASSERTCMP(u32, >=, total, 130);
+		UASSERTCMP(u32, <, total, 8 * 1024 * 1024);
+	} else {
+		warningstream << "testGetMemorySize: retrieving failed" << std::endl;
+	}
+}