From c625fa71e108ecbf99a2ac5a99b7f59e20008978 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Thu, 13 Nov 2025 20:17:24 +0100 Subject: [PATCH] Automatically choose multiple emerge threads for singlenode (#16634) --- builtin/settingtypes.txt | 17 +++---- src/defaultsettings.cpp | 2 +- src/emerge.cpp | 78 ++++++++++++++++++++------------- src/emerge.h | 5 ++- src/mapgen/mapgen.cpp | 4 +- src/porting.cpp | 24 +++++++++- src/porting.h | 6 +++ src/unittest/test_utilities.cpp | 22 ++++++++++ 8 files changed, 113 insertions(+), 45 deletions(-) diff --git a/builtin/settingtypes.txt b/builtin/settingtypes.txt index 0600488c1..2d6f8bc0c 100644 --- a/builtin/settingtypes.txt +++ b/builtin/settingtypes.txt @@ -2367,17 +2367,12 @@ emergequeue_limit_diskonly (Per-player limit of queued blocks load from disk) in # This limit is enforced per player. emergequeue_limit_generate (Per-player limit of queued blocks to generate) int 128 1 1000000 -# Number of emerge threads to use. -# Value 0: -# - Automatic selection. The number of emerge threads will be -# - 'number of processors - 2', with a lower limit of 1. -# Any other value: -# - Specifies the number of emerge threads, with a lower limit of 1. -# WARNING: Increasing the number of emerge threads increases engine mapgen -# speed, but this may harm game performance by interfering with other -# processes, especially in singleplayer and/or when running Lua code in -# 'on_generated'. For many users the optimum setting may be '1'. -num_emerge_threads (Number of emerge threads) int 1 0 32767 +# Number of emerge threads (responsible for map generation and loading) to use. +# If 0 then the engine will automatically choose a suitable value depending +# on the hardware and type of map generator. +# WARNING: There are known bugs in the default map generators (v6, v7, ...) +# when using more than 1 thread. The automatic choice will avoid this. +num_emerge_threads (Number of emerge threads) int 0 0 32767 [**cURL] [common] diff --git a/src/defaultsettings.cpp b/src/defaultsettings.cpp index e074f8137..4e7de6ebb 100644 --- a/src/defaultsettings.cpp +++ b/src/defaultsettings.cpp @@ -490,7 +490,7 @@ void set_default_settings() settings->setDefault("emergequeue_limit_total", "1024"); settings->setDefault("emergequeue_limit_diskonly", "128"); settings->setDefault("emergequeue_limit_generate", "128"); - settings->setDefault("num_emerge_threads", "1"); + settings->setDefault("num_emerge_threads", "0"); settings->setDefault("secure.enable_security", "true"); settings->setDefault("secure.trusted_mods", ""); settings->setDefault("secure.http_mods", ""); diff --git a/src/emerge.cpp b/src/emerge.cpp index f4af7a265..3b7e205fe 100644 --- a/src/emerge.cpp +++ b/src/emerge.cpp @@ -6,8 +6,8 @@ #include "emerge_internal.h" +#include #include - #include "config.h" #include "constants.h" #include "irrlicht_changes/printing.h" @@ -30,7 +30,6 @@ EmergeParams::~EmergeParams() { - infostream << "EmergeParams: destroying " << this << std::endl; // Delete everything that was cloned on creation of EmergeParams delete biomegen; delete biomemgr; @@ -60,7 +59,9 @@ EmergeParams::EmergeParams(EmergeManager *parent, const BiomeGen *biomegen, EmergeManager::EmergeManager(Server *server, MetricsBackend *mb) { - this->ndef = server->getNodeDefManager(); + assert(server); + this->m_server = server; + this->ndef = server->ndef(); this->biomemgr = new BiomeManager(server); this->oremgr = new OreManager(server); this->decomgr = new DecorationManager(server); @@ -87,31 +88,14 @@ EmergeManager::EmergeManager(Server *server, MetricsBackend *mb) ); } - s16 nthreads = 1; - g_settings->getS16NoEx("num_emerge_threads", nthreads); - // If automatic, leave a proc for the main thread and one for - // some other misc thread - if (nthreads <= 0) - nthreads = Thread::getNumberOfProcessors() - 2; - if (nthreads < 1) - nthreads = 1; - m_qlimit_total = g_settings->getU32("emergequeue_limit_total"); - // FIXME: these fallback values are probably not good - if (!g_settings->getU32NoEx("emergequeue_limit_diskonly", m_qlimit_diskonly)) - m_qlimit_diskonly = nthreads * 5 + 1; - if (!g_settings->getU32NoEx("emergequeue_limit_generate", m_qlimit_generate)) - m_qlimit_generate = nthreads + 1; + m_qlimit_diskonly = g_settings->getU32("emergequeue_limit_diskonly"); + m_qlimit_generate = g_settings->getU32("emergequeue_limit_generate"); // don't trust user input for something very important like this m_qlimit_diskonly = rangelim(m_qlimit_diskonly, 2, 1000000); m_qlimit_generate = rangelim(m_qlimit_generate, 1, 1000000); m_qlimit_total = std::max(m_qlimit_total, std::max(m_qlimit_diskonly, m_qlimit_generate)); - - for (s16 i = 0; i < nthreads; i++) - m_threads.push_back(new EmergeThread(server, i)); - - infostream << "EmergeManager: using " << nthreads << " threads" << std::endl; } @@ -188,18 +172,58 @@ void EmergeManager::initMapgens(MapgenParams *params) mgparams = params; + infostream << "EmergeManager: initializing for mapgen=" + << Mapgen::getMapgenName(params->mgtype) + << " and chunksize=" << params->chunksize << std::endl; + + /* + * Singlenode is currently the only mapgen not affected by the + * unfinished slice bug, so allow multiple threads by default. + * We do this for the Lua mapgens who benefit from this (since singlenode + * itself isn't very useful). + * see + */ + bool multithread = params->mgtype == MAPGEN_SINGLENODE; + initThreads(multithread); + v3s16 csize = params->chunksize * MAP_BLOCKSIZE; biomegen = biomemgr->createBiomeGen(BIOMEGEN_ORIGINAL, params->bparams, csize); for (u32 i = 0; i != m_threads.size(); i++) { EmergeParams *p = new EmergeParams(this, biomegen, biomemgr, oremgr, decomgr, schemmgr); - infostream << "EmergeManager: Created params " << p - << " for thread " << i << std::endl; m_mapgens.push_back(Mapgen::createMapgen(params->mgtype, params, p)); } } +void EmergeManager::initThreads(bool should_multithread) +{ + s16 nthreads = g_settings->getS16("num_emerge_threads"); + if (nthreads <= 0 && should_multithread) { + u32 concurrency = Thread::getNumberOfProcessors(); + u32 memoryMB = porting::getMemorySizeMB(); + if (memoryMB) { + // Cap threads according to total RAM with a conservative 1 GB per thread. + // This is for the sake of Android phones, where many cores & low RAM + // is not uncommon (e.g. 8C + 3GB). + concurrency = std::min(concurrency, std::roundf(memoryMB / 1024.0f)); + } + // Leave 2 cores for main thread and whatever else. + nthreads = (concurrency > 2) ? (concurrency - 2) : 1; + // Testing has shown that more than 4 threads don't become any faster: + // + // May have to be revisited after emerge code is refactored to be less + // lock heavy. + nthreads = std::min(4, nthreads); + } + nthreads = std::max(1, nthreads); + + FATAL_ERROR_IF(!m_threads.empty(), "Threads already initialized."); + for (s16 i = 0; i < nthreads; i++) + m_threads.push_back(new EmergeThread(m_server, i)); + + infostream << "EmergeManager: using " << nthreads << " thread(s)" << std::endl; +} Mapgen *EmergeManager::getCurrentMapgen() { @@ -247,12 +271,6 @@ void EmergeManager::stopThreads() } -bool EmergeManager::isRunning() -{ - return m_threads_active; -} - - bool EmergeManager::enqueueBlockEmerge( session_t peer_id, v3s16 blockpos, diff --git a/src/emerge.h b/src/emerge.h index 982296d50..2fb2c055f 100644 --- a/src/emerge.h +++ b/src/emerge.h @@ -165,7 +165,6 @@ public: void startThreads(); void stopThreads(); - bool isRunning(); bool enqueueBlockEmerge( session_t peer_id, @@ -193,10 +192,14 @@ public: static v3s16 getContainingChunk(v3s16 blockpos, v3s16 chunksize); private: + void initThreads(bool should_multithread); + std::vector m_mapgens; std::vector m_threads; bool m_threads_active = false; + // Server reference + Server *m_server = nullptr; // The map database MapDatabaseAccessor *m_db = nullptr; diff --git a/src/mapgen/mapgen.cpp b/src/mapgen/mapgen.cpp index d5866460c..9d2d2b006 100644 --- a/src/mapgen/mapgen.cpp +++ b/src/mapgen/mapgen.cpp @@ -144,7 +144,9 @@ const char *Mapgen::getMapgenName(MapgenType mgtype) if (index == MAPGEN_INVALID || index >= ARRLEN(g_reg_mapgens)) return "invalid"; - return g_reg_mapgens[index].name; + auto &it = g_reg_mapgens[index]; + assert(it.name); + return it.name; } diff --git a/src/porting.cpp b/src/porting.cpp index 711b65db6..1df84b5c6 100644 --- a/src/porting.cpp +++ b/src/porting.cpp @@ -40,6 +40,8 @@ #if defined(__APPLE__) #include #include + #include + #include // For _NSGetEnviron() // Related: https://gitlab.haskell.org/ghc/ghc/issues/2458 #include @@ -67,7 +69,7 @@ #include #if CHECK_CLIENT_BUILD() && defined(_WIN32) -// On Windows export some driver-specific variables to encourage Minetest to be +// On Windows export some driver-specific variables to encourage Luanti to be // executed on the discrete GPU in case of systems with two. Portability is fun. extern "C" { __declspec(dllexport) DWORD NvOptimusEnablement = 1; @@ -267,6 +269,26 @@ const std::string &get_sysinfo() return ret; } +u32 getMemorySizeMB() +{ +#ifdef _WIN32 + MEMORYSTATUSEX status; + status.dwLength = sizeof(status); + if (GlobalMemoryStatusEx(&status)) + return status.ullTotalPhys >> 20; +#elif defined(__unix__) && defined(_SC_PHYS_PAGES) && defined(_SC_PAGE_SIZE) + long pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGE_SIZE); + if (pages != -1 && page_size != -1) + return (pages * page_size) >> 20; +#elif defined(__APPLE__) + int64_t memsize; + size_t len = sizeof(memsize); + if (sysctlbyname("hw.memsize", &memsize, &len, nullptr, 0) == 0) + return memsize >> 20; +#endif + return 0; +} [[maybe_unused]] static bool getCurrentWorkingDir(char *buf, size_t len) { diff --git a/src/porting.h b/src/porting.h index 1a4bb9e7b..3f60a51bd 100644 --- a/src/porting.h +++ b/src/porting.h @@ -127,6 +127,12 @@ void initializePaths(); const std::string &get_sysinfo(); +/* + Return size of system RAM in MB + (or 0 if unavailable/error) +*/ +u32 getMemorySizeMB(); + // Monotonic timer #ifdef _WIN32 // Windows diff --git a/src/unittest/test_utilities.cpp b/src/unittest/test_utilities.cpp index fedacb311..4b034fda0 100644 --- a/src/unittest/test_utilities.cpp +++ b/src/unittest/test_utilities.cpp @@ -50,6 +50,7 @@ public: void testSanitizeUntrusted(); void testReadSeed(); void testMyDoubleStringConversions(); + void testGetMemorySize(); }; static TestUtilities g_test_instance; @@ -87,6 +88,7 @@ void TestUtilities::runTests(IGameDef *gamedef) TEST(testSanitizeUntrusted); TEST(testReadSeed); TEST(testMyDoubleStringConversions); + TEST(testGetMemorySize); } //////////////////////////////////////////////////////////////////////////////// @@ -805,3 +807,23 @@ void TestUtilities::testMyDoubleStringConversions() test_round_trip(0.3); test_round_trip(0.1 + 0.2); } + +void TestUtilities::testGetMemorySize() +{ +#if defined(_WIN32) || defined(__linux__) || defined(__APPLE__) + const bool fail_ok = false; +#else + const bool fail_ok = true; +#endif + + u32 total = porting::getMemorySizeMB(); + UASSERT(total != 0 || fail_ok); + if (total != 0) { + infostream << "memory size in MB = " << total << std::endl; + // should be a sane value + UASSERTCMP(u32, >=, total, 130); + UASSERTCMP(u32, <, total, 8 * 1024 * 1024); + } else { + warningstream << "testGetMemorySize: retrieving failed" << std::endl; + } +}