night rendering

also properly drop support for version < 22, which hasn't worked in years

.travis.yml

@@ -3,7 +3,15 @@ compiler:
   - gcc
   - clang
 dist: bionic
-before_install: sudo apt-get install -y cmake libgd-dev libsqlite3-dev libleveldb-dev
+addons:
+  apt:
+    packages:
+      - cmake
+      - libgd-dev
+      - libsqlite3-dev
+      - libleveldb-dev
+      - libpq-dev
+      - postgresql-server-dev-all
 script: ./util/travis/script.sh
 notifications:
   email: false

BlockDecoder.cpp

@@ -11,20 +11,18 @@ static inline uint16_t readU16(const unsigned char *data)
 	return data[0] << 8 | data[1];
 }
 
-static int readBlockContent(const unsigned char *mapData, u8 version, unsigned int datapos)
+static int readBlockContent(const unsigned char *mapData, u8 contentWidth, unsigned int datapos)
 {
-	if (version >= 24) {
+	if (contentWidth == 2) {
 		size_t index = datapos << 1;
 		return (mapData[index] << 8) | mapData[index + 1];
-	} else if (version >= 20) {
+	} else {
-		if (mapData[datapos] <= 0x80)
+		u8 param = mapData[datapos];
-			return mapData[datapos];
+		if (param <= 0x7f)
+			return param;
 		else
-			return (int(mapData[datapos]) << 4) | (int(mapData[datapos + 0x2000]) >> 4);
+			return (int(param) << 4) | (int(mapData[datapos + 0x2000]) >> 4);
 	}
-	std::ostringstream oss;
-	oss << "Unsupported map version " << version;
-	throw std::runtime_error(oss.str());
 }
 
 BlockDecoder::BlockDecoder()
@@ -39,6 +37,7 @@ void BlockDecoder::reset()
 	m_nameMap.clear();
 
 	m_version = 0;
+	m_contentWidth = 0;
 	m_mapData = ustring();
 }
 
@@ -50,16 +49,30 @@ void BlockDecoder::decode(const ustring &datastr)
 
 	uint8_t version = data[0];
 	//uint8_t flags = data[1];
+	if (version < 22) {
+		std::ostringstream oss;
+		oss << "Unsupported map version " << (int)version;
+		throw std::runtime_error(oss.str());
+	}
 	m_version = version;
 
 	size_t dataOffset = 0;
 	if (version >= 27)
-		dataOffset = 6;
-	else if (version >= 22)
 		dataOffset = 4;
 	else
 		dataOffset = 2;
 
+	uint8_t contentWidth = data[dataOffset];
+	dataOffset++;
+	uint8_t paramsWidth = data[dataOffset];
+	dataOffset++;
+	if (contentWidth != 1 && contentWidth != 2)
+		throw std::runtime_error("unsupported map version (contentWidth)");
+	if (paramsWidth != 2)
+		throw std::runtime_error("unsupported map version (paramsWidth)");
+	m_contentWidth = contentWidth;
+
+
 	ZlibDecompressor decompressor(data, length);
 	decompressor.setSeekPos(dataOffset);
 	m_mapData = decompressor.decompress();
@@ -67,8 +80,6 @@ void BlockDecoder::decode(const ustring &datastr)
 	dataOffset = decompressor.seekPos();
 
 	// Skip unused data
-	if (version <= 21)
-		dataOffset += 2;
 	if (version == 23)
 		dataOffset += 1;
 	if (version == 24) {
@@ -92,7 +103,7 @@ void BlockDecoder::decode(const ustring &datastr)
 	dataOffset += 4; // Skip timestamp
 
 	// Read mapping
-	if (version >= 22) {
+	{
 		dataOffset++; // mapping version
 		uint16_t numMappings = readU16(data + dataOffset);
 		dataOffset += 2;
@@ -130,7 +141,7 @@ bool BlockDecoder::isEmpty() const
 std::string BlockDecoder::getNode(u8 x, u8 y, u8 z) const
 {
 	unsigned int position = x + (y << 4) + (z << 8);
-	int content = readBlockContent(m_mapData.c_str(), m_version, position);
+	int content = readBlockContent(m_mapData.c_str(), m_contentWidth, position);
 	if (content == m_blockAirId || content == m_blockIgnoreId)
 		return "";
 	NameMap::const_iterator it = m_nameMap.find(content);
@@ -140,3 +151,10 @@ std::string BlockDecoder::getNode(u8 x, u8 y, u8 z) const
 	}
 	return it->second;
 }
+
+u8 BlockDecoder::getParam1(u8 x, u8 y, u8 z) const
+{
+	unsigned int position = x + (y << 4) + (z << 8);
+	unsigned int offset = (m_contentWidth == 2) ? 0x2000 : 0x1000;
+	return m_mapData.c_str()[offset + position];
+}

CMakeLists.txt

@@ -1,7 +1,6 @@
 project(minetestmapper CXX)
 cmake_minimum_required(VERSION 2.6)
 cmake_policy(SET CMP0003 NEW)
-set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
 
 set(VERSION_MAJOR 1)
 set(VERSION_MINOR 0)
@@ -13,6 +12,7 @@ if(NOT CMAKE_BUILD_TYPE)
 	set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build." FORCE)
 endif()
 
+set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_FLAGS_RELEASE "-O3 -Wall -DNDEBUG")
 set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g2 -Wall")
 
@@ -45,6 +45,9 @@ if(NOT CUSTOM_DOCDIR STREQUAL "")
         message(STATUS "Using DOCDIR=${DOCDIR}")
 endif()
 
+#set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
+find_package(PkgConfig)
+include(FindPackageHandleStandardArgs)
 
 # Libraries: gd
 
@@ -66,9 +69,6 @@ if(NOT ZLIB_LIBRARY OR NOT ZLIB_INCLUDE_DIR)
 	message(FATAL_ERROR "zlib not found!")
 endif(NOT ZLIB_LIBRARY OR NOT ZLIB_INCLUDE_DIR)
 
-find_package(PkgConfig)
-include(FindPackageHandleStandardArgs)
-
 # Libraries: sqlite3
 
 find_library(SQLITE3_LIBRARY sqlite3)
@@ -82,77 +82,61 @@ endif(NOT SQLITE3_LIBRARY OR NOT SQLITE3_INCLUDE_DIR)
 # Libraries: postgresql
 
 option(ENABLE_POSTGRESQL "Enable PostgreSQL backend" TRUE)
-set(USE_POSTGRESQL 0)
+set(USE_POSTGRESQL FALSE)
 
 if(ENABLE_POSTGRESQL)
-	find_program(POSTGRESQL_CONFIG_EXECUTABLE pg_config DOC "pg_config")
-	find_library(POSTGRESQL_LIBRARY pq)
-	if(POSTGRESQL_CONFIG_EXECUTABLE)
-		execute_process(COMMAND ${POSTGRESQL_CONFIG_EXECUTABLE} --includedir-server
-			OUTPUT_VARIABLE POSTGRESQL_SERVER_INCLUDE_DIRS
-			OUTPUT_STRIP_TRAILING_WHITESPACE)
-		execute_process(COMMAND ${POSTGRESQL_CONFIG_EXECUTABLE}
-			OUTPUT_VARIABLE POSTGRESQL_CLIENT_INCLUDE_DIRS
-			OUTPUT_STRIP_TRAILING_WHITESPACE)
-		# This variable is case sensitive for the cmake PostgreSQL module
-		set(PostgreSQL_ADDITIONAL_SEARCH_PATHS ${POSTGRESQL_SERVER_INCLUDE_DIRS} ${POSTGRESQL_CLIENT_INCLUDE_DIRS})
-	endif()
-
 	find_package("PostgreSQL")
 
-	if(POSTGRESQL_FOUND)
+	if(PostgreSQL_FOUND)
-		set(USE_POSTGRESQL 1)
+		set(USE_POSTGRESQL TRUE)
 		message(STATUS "PostgreSQL backend enabled")
 		# This variable is case sensitive, don't try to change it to POSTGRESQL_INCLUDE_DIR
-		message(STATUS "PostgreSQL includes: ${PostgreSQL_INCLUDE_DIR}")
+		message(STATUS "PostgreSQL includes: ${PostgreSQL_INCLUDE_DIRS}")
-		include_directories(${PostgreSQL_INCLUDE_DIR})
+		include_directories(${PostgreSQL_INCLUDE_DIRS})
-		set(POSTGRESQL_LIBRARY ${PostgreSQL_LIBRARIES})
 	else()
-		message(STATUS "PostgreSQL not found.")
+		message(STATUS "PostgreSQL not found!")
-		set(POSTGRESQL_LIBRARY "")
+		set(PostgreSQL_LIBRARIES "")
 	endif()
 endif(ENABLE_POSTGRESQL)
 
 # Libraries: leveldb
 
-set(USE_LEVELDB 0)
+OPTION(ENABLE_LEVELDB "Enable LevelDB backend" TRUE)
-
+set(USE_LEVELDB FALSE)
-OPTION(ENABLE_LEVELDB "Enable LevelDB backend")
 
 if(ENABLE_LEVELDB)
 	find_library(LEVELDB_LIBRARY leveldb)
-	find_path(LEVELDB_INCLUDE_DIR db.h PATH_SUFFIXES leveldb)
+	find_path(LEVELDB_INCLUDE_DIR leveldb/db.h)
 	message (STATUS "LevelDB library: ${LEVELDB_LIBRARY}")
 	message (STATUS "LevelDB headers: ${LEVELDB_INCLUDE_DIR}")
 	if(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR)
-		set(USE_LEVELDB 1)
+		set(USE_LEVELDB TRUE)
 		message(STATUS "LevelDB backend enabled")
 		include_directories(${LEVELDB_INCLUDE_DIR})
-	else(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR)
+	else()
-		set(USE_LEVELDB 0)
 		message(STATUS "LevelDB not found!")
-	endif(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR)
+		set(LEVELDB_LIBRARY "")
+	endif()
 endif(ENABLE_LEVELDB)
 
 # Libraries: redis
 
-set(USE_REDIS 0)
+OPTION(ENABLE_REDIS "Enable redis backend" TRUE)
-
+set(USE_REDIS FALSE)
-OPTION(ENABLE_REDIS "Enable redis backend")
 
 if(ENABLE_REDIS)
 	find_library(REDIS_LIBRARY hiredis)
-	find_path(REDIS_INCLUDE_DIR hiredis.h PATH_SUFFIXES hiredis)
+	find_path(REDIS_INCLUDE_DIR hiredis/hiredis.h)
 	message (STATUS "redis library: ${REDIS_LIBRARY}")
 	message (STATUS "redis headers: ${REDIS_INCLUDE_DIR}")
 	if(REDIS_LIBRARY AND REDIS_INCLUDE_DIR)
-		set(USE_REDIS 1)
+		set(USE_REDIS TRUE)
 		message(STATUS "redis backend enabled")
 		include_directories(${REDIS_INCLUDE_DIR})
-	else(REDIS_LIBRARY AND REDIS_INCLUDE_DIR)
+	else()
-		set(USE_REDIS 0)
 		message(STATUS "redis not found!")
-	endif(REDIS_LIBRARY AND REDIS_INCLUDE_DIR)
+		set(REDIS_LIBRARY "")
+	endif()
 endif(ENABLE_REDIS)
 
 # Compiling & Linking
@@ -203,7 +187,7 @@ add_executable(minetestmapper
 target_link_libraries(
 	minetestmapper
 	${SQLITE3_LIBRARY}
-	${POSTGRESQL_LIBRARY}
+	${PostgreSQL_LIBRARIES}
 	${LEVELDB_LIBRARY}
 	${REDIS_LIBRARY}
 	${LIBGD_LIBRARY}

PixelAttributes.cpp

@@ -7,17 +7,14 @@
  * =====================================================================
  */
 
-#include <cstdlib>
-#include <cstring>
 #include "PixelAttributes.h"
-
+#include <cstring>
-using namespace std;
 
 PixelAttributes::PixelAttributes():
 	m_width(0)
 {
 	for (size_t i = 0; i < LineCount; ++i) {
-		m_pixelAttributes[i] = 0;
+		m_pixelAttributes[i] = nullptr;
 	}
 }
 
@@ -47,9 +44,9 @@ void PixelAttributes::scroll()
 void PixelAttributes::freeAttributes()
 {
 	for (size_t i = 0; i < LineCount; ++i) {
-		if (m_pixelAttributes[i] != 0) {
+		if (m_pixelAttributes[i] != nullptr) {
 			delete[] m_pixelAttributes[i];
-			m_pixelAttributes[i] = 0;
+			m_pixelAttributes[i] = nullptr;
 		}
 	}
 }

README.rst

@@ -106,3 +106,8 @@ colors:
 
 scales:
     Draw scales on specified image edges (letters *t b l r* meaning top, bottom, left and right), e.g. ``--scales tbr``
+
+exhaustive:
+    | Select if database should be traversed exhaustively or using range queries, available: *never*, *y*, *full*, *auto*
+    | Defaults to *auto*. You shouldn't need to change this, but doing so can improve rendering times on large maps.
+    | For these optimizations to work it is important that you set ``min-y`` and ``max-y`` when you don't care about the world below e.g. -60 and above 1000 nodes.

TileGenerator.cpp

@@ -1,12 +1,14 @@
 #include <cstdio>
 #include <cstdlib>
 #include <climits>
+#include <cassert>
 #include <fstream>
 #include <iostream>
 #include <sstream>
 #include <stdexcept>
 #include <cstring>
 #include <vector>
+#include <cmath>
 
 #include "TileGenerator.h"
 #include "config.h"
@@ -89,12 +91,13 @@ TileGenerator::TileGenerator():
 	m_xMax(INT_MIN),
 	m_zMin(INT_MAX),
 	m_zMax(INT_MIN),
-	m_yMin(-30000),
+	m_yMin(INT16_MIN),
-	m_yMax(30000),
+	m_yMax(INT16_MAX),
 	m_geomX(-2048),
 	m_geomY(-2048),
 	m_geomX2(2048),
 	m_geomY2(2048),
+	m_exhaustiveSearch(EXH_AUTO),
 	m_zoom(1),
 	m_scales(SCALE_LEFT | SCALE_TOP)
 {
@@ -184,6 +187,7 @@ void TileGenerator::setBackend(std::string backend)
 
 void TileGenerator::setGeometry(int x, int y, int w, int h)
 {
+	assert(w > 0 && h > 0);
 	m_geomX  = round_multiple_nosign(x, 16) / 16;
 	m_geomY  = round_multiple_nosign(y, 16) / 16;
 	m_geomX2 = round_multiple_nosign(x + w, 16) / 16;
@@ -193,11 +197,20 @@ void TileGenerator::setGeometry(int x, int y, int w, int h)
 void TileGenerator::setMinY(int y)
 {
 	m_yMin = y;
+	if (m_yMin > m_yMax)
+		std::swap(m_yMin, m_yMax);
 }
 
 void TileGenerator::setMaxY(int y)
 {
 	m_yMax = y;
+	if (m_yMin > m_yMax)
+		std::swap(m_yMin, m_yMax);
+}
+
+void TileGenerator::setExhaustiveSearch(int mode)
+{
+	m_exhaustiveSearch = mode;
 }
 
 void TileGenerator::parseColorsFile(const std::string &fileName)
@@ -216,6 +229,7 @@ void TileGenerator::printGeometry(const std::string &input)
 		input_path += PATH_SEPARATOR;
 	}
 
+	setExhaustiveSearch(EXH_NEVER);
 	openDb(input_path);
 	loadBlocks();
 
@@ -241,10 +255,12 @@ void TileGenerator::generate(const std::string &input, const std::string &output
 		input_path += PATH_SEPARATOR;
 	}
 
+	if (m_dontWriteEmpty) // FIXME: possible too, just needs to be done differently
+		setExhaustiveSearch(EXH_NEVER);
 	openDb(input_path);
 	loadBlocks();
 
-	if (m_dontWriteEmpty  && ! m_positions.size())
+	if (m_dontWriteEmpty && m_positions.empty())
 	{
 		closeDatabase();
 		return;
@@ -268,9 +284,9 @@ void TileGenerator::generate(const std::string &input, const std::string &output
 
 void TileGenerator::parseColorsStream(std::istream &in)
 {
-	char line[128];
+	char line[512];
 	while (in.good()) {
-		in.getline(line, 128);
+		in.getline(line, sizeof(line));
 
 		for(char *p = line; *p; p++) {
 			if(*p != '#')
@@ -281,29 +297,45 @@ void TileGenerator::parseColorsStream(std::istream &in)
 		if(strlen(line) == 0)
 			continue;
 
-		char name[64 + 1];
+		char name[128 + 1] = {0};
 		unsigned int r, g, b, a, t;
 		a = 255;
 		t = 0;
-		int items = sscanf(line, "%64s %u %u %u %u %u", name, &r, &g, &b, &a, &t);
+		int items = sscanf(line, "%128s %u %u %u %u %u", name, &r, &g, &b, &a, &t);
 		if(items < 4) {
 			std::cerr << "Failed to parse color entry '" << line << "'" << std::endl;
 			continue;
 		}
 	
-		ColorEntry color = ColorEntry(r, g, b, a, t);
+		ColorEntry color(r, g, b, a, t);
 		m_colorMap[name] = color;
 	}
 }
 
+std::set<std::string> TileGenerator::getSupportedBackends()
+{
+	std::set<std::string> r;
+	r.insert("sqlite3");
+#if USE_POSTGRESQL
+	r.insert("postgresql");
+#endif
+#if USE_LEVELDB
+	r.insert("leveldb");
+#endif
+#if USE_REDIS
+	r.insert("redis");
+#endif
+	return r;
+}
+
 void TileGenerator::openDb(const std::string &input)
 {
 	std::string backend = m_backend;
-	if(backend == "") {
+	if (backend == "") {
-		std::ifstream ifs((input + "/world.mt").c_str());
+		std::ifstream ifs(input + "/world.mt");
 		if(!ifs.good())
-			throw std::runtime_error("Failed to read world.mt");
+			throw std::runtime_error("Failed to open world.mt");
-		backend = read_setting("backend", ifs);
+		backend = read_setting_default("backend", ifs, "sqlite3");
 		ifs.close();
 	}
 
@@ -323,6 +355,33 @@ void TileGenerator::openDb(const std::string &input)
 #endif
 	else
 		throw std::runtime_error(((std::string) "Unknown map backend: ") + backend);
+
+	// Determine how we're going to traverse the database (heuristic)
+	if (m_exhaustiveSearch == EXH_AUTO) {
+		using u64 = uint64_t;
+		u64 y_range = (m_yMax / 16 + 1) - (m_yMin / 16);
+		u64 blocks = (u64)(m_geomX2 - m_geomX) * y_range * (u64)(m_geomY2 - m_geomY);
+#ifndef NDEBUG
+		std::cout << "Heuristic parameters:"
+			<< " preferRangeQueries()=" << m_db->preferRangeQueries()
+			<< " y_range=" << y_range << " blocks=" << blocks << std::endl;
+#endif
+		if (m_db->preferRangeQueries())
+			m_exhaustiveSearch = EXH_NEVER;
+		else if (blocks < 200000)
+			m_exhaustiveSearch = EXH_FULL;
+		else if (y_range < 42)
+			m_exhaustiveSearch = EXH_Y;
+		else
+			m_exhaustiveSearch = EXH_NEVER;
+	} else if (m_exhaustiveSearch == EXH_FULL || m_exhaustiveSearch == EXH_Y) {
+		if (m_db->preferRangeQueries()) {
+			std::cerr << "Note: The current database backend supports efficient "
+				"range queries, forcing exhaustive search should always result "
+				" in worse performance." << std::endl;
+		}
+	}
+	assert(m_exhaustiveSearch != EXH_AUTO);
 }
 
 void TileGenerator::closeDatabase()
@@ -333,15 +392,19 @@ void TileGenerator::closeDatabase()
 
 void TileGenerator::loadBlocks()
 {
-	std::vector<BlockPos> vec = m_db->getBlockPos();
+	const int16_t yMax = m_yMax / 16 + 1;
-	for (std::vector<BlockPos>::iterator it = vec.begin(); it != vec.end(); ++it) {
+
-		BlockPos pos = *it;
+	if (m_exhaustiveSearch == EXH_NEVER || m_exhaustiveSearch == EXH_Y) {
-		// Check that it's in geometry (from --geometry option)
+		std::vector<BlockPos> vec = m_db->getBlockPos(
-		if (pos.x < m_geomX || pos.x >= m_geomX2 || pos.z < m_geomY || pos.z >= m_geomY2)
+			BlockPos(m_geomX, m_yMin / 16, m_geomY),
-			continue;
+			BlockPos(m_geomX2, yMax, m_geomY2)
-		// Check that it's between --min-y and --max-y
+		);
-		if (pos.y * 16 < m_yMin || pos.y * 16 > m_yMax)
+
-			continue;
+		for (auto pos : vec) {
+			assert(pos.x >= m_geomX && pos.x < m_geomX2);
+			assert(pos.y >= m_yMin / 16 && pos.y < yMax);
+			assert(pos.z >= m_geomY && pos.z < m_geomY2);
+
 			// Adjust minimum and maximum positions to the nearest block
 			if (pos.x < m_xMin)
 				m_xMin = pos.x;
@@ -352,10 +415,18 @@ void TileGenerator::loadBlocks()
 				m_zMin = pos.z;
 			if (pos.z > m_zMax)
 				m_zMax = pos.z;
-		m_positions.push_back(std::pair<int, int>(pos.x, pos.z));
+
+			m_positions[pos.z].emplace(pos.x);
+		}
+
+#ifndef NDEBUG
+		int count = 0;
+		for (const auto &it : m_positions)
+			count += it.second.size();
+		std::cout << "Loaded " << count
+			<< " positions (across Z: " << m_positions.size() << ") for rendering" << std::endl;
+#endif
 	}
-	m_positions.sort();
-	m_positions.unique();
 }
 
 void TileGenerator::createImage()
@@ -393,10 +464,11 @@ void TileGenerator::createImage()
 	image_height = (m_mapHeight * m_zoom) + m_yBorder;
 	image_height += (m_scales & SCALE_BOTTOM) ? scale_d : 0;
 
-	if(image_width > 4096 || image_height > 4096)
+	if(image_width > 4096 || image_height > 4096) {
 		std::cerr << "Warning: The width or height of the image to be created exceeds 4096 pixels!"
 			<< " (Dimensions: " << image_width << "x" << image_height << ")"
 			<< std::endl;
+	}
 	m_image = new Image(image_width, image_height);
 	m_image->drawFilledRect(0, 0, image_width, image_height, m_bgColor); // Background
 }
@@ -404,15 +476,9 @@ void TileGenerator::createImage()
 void TileGenerator::renderMap()
 {
 	BlockDecoder blk;
-	std::list<int> zlist = getZValueList();
+	const int16_t yMax = m_yMax / 16 + 1;
-	for (std::list<int>::iterator zPosition = zlist.begin(); zPosition != zlist.end(); ++zPosition) {
-		int zPos = *zPosition;
-		std::map<int16_t, BlockList> blocks;
-		m_db->getBlocksOnZ(blocks, zPos);
-		for (std::list<std::pair<int, int> >::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
-			if (position->second != zPos)
-				continue;
 
+	auto renderSingle = [&] (int16_t xPos, int16_t zPos, BlockList &blockStack) {
 		m_readPixels.reset();
 		m_readInfo.reset();
 		for (int i = 0; i < 16; i++) {
@@ -423,16 +489,13 @@ void TileGenerator::renderMap()
 			}
 		}
 
-			int xPos = position->first;
+		for (const auto &it : blockStack) {
-			blocks[xPos].sort();
+			const BlockPos pos = it.first;
-			const BlockList &blockStack = blocks[xPos];
+			assert(pos.x == xPos && pos.z == zPos);
-			for (BlockList::const_iterator it = blockStack.begin(); it != blockStack.end(); ++it) {
+			assert(pos.y >= m_yMin / 16 && pos.y < yMax);
-				const BlockPos &pos = it->first;
 
 			blk.reset();
-				blk.decode(it->second);
+			blk.decode(it.second);
-				if (blk.isEmpty())
-					continue;
 			renderMapBlock(blk, pos);
 
 			// Exit out if all pixels for this MapBlock are covered
@@ -441,18 +504,103 @@ void TileGenerator::renderMap()
 		}
 		if (!m_readPixels.full())
 			renderMapBlockBottom(blockStack.begin()->first);
-		}
+	};
+	auto postRenderRow = [&] (int16_t zPos) {
 		if (m_shading)
 			renderShading(zPos);
+	};
+
+	if (m_exhaustiveSearch == EXH_NEVER) {
+		for (auto it = m_positions.rbegin(); it != m_positions.rend(); ++it) {
+			int16_t zPos = it->first;
+			for (auto it2 = it->second.rbegin(); it2 != it->second.rend(); ++it2) {
+				int16_t xPos = *it2;
+
+				BlockList blockStack;
+				m_db->getBlocksOnXZ(blockStack, xPos, zPos, m_yMin / 16, yMax);
+				blockStack.sort();
+
+				renderSingle(xPos, zPos, blockStack);
+			}
+			postRenderRow(zPos);
+		}
+	} else if (m_exhaustiveSearch == EXH_Y) {
+#ifndef NDEBUG
+		std::cerr << "Exhaustively searching height of "
+			<< (yMax - (m_yMin / 16)) << " blocks" << std::endl;
+#endif
+		std::vector<BlockPos> positions;
+		positions.reserve(yMax - (m_yMin / 16));
+		for (auto it = m_positions.rbegin(); it != m_positions.rend(); ++it) {
+			int16_t zPos = it->first;
+			for (auto it2 = it->second.rbegin(); it2 != it->second.rend(); ++it2) {
+				int16_t xPos = *it2;
+
+				positions.clear();
+				for (int16_t yPos = m_yMin / 16; yPos < yMax; yPos++)
+					positions.emplace_back(xPos, yPos, zPos);
+
+				BlockList blockStack;
+				m_db->getBlocksByPos(blockStack, positions);
+				blockStack.sort();
+
+				renderSingle(xPos, zPos, blockStack);
+			}
+			postRenderRow(zPos);
+		}
+	} else if (m_exhaustiveSearch == EXH_FULL) {
+#ifndef NDEBUG
+		std::cerr << "Exhaustively searching "
+			<< (m_geomX2 - m_geomX) << "x" << (yMax - (m_yMin / 16)) << "x"
+			<< (m_geomY2 - m_geomY) << " blocks" << std::endl;
+#endif
+		std::vector<BlockPos> positions;
+		positions.reserve(yMax - (m_yMin / 16));
+		for (int16_t zPos = m_geomY2 - 1; zPos >= m_geomY; zPos--) {
+			for (int16_t xPos = m_geomX2 - 1; xPos >= m_geomX; xPos--) {
+				positions.clear();
+				for (int16_t yPos = m_yMin / 16; yPos < yMax; yPos++)
+					positions.emplace_back(xPos, yPos, zPos);
+
+				BlockList blockStack;
+				m_db->getBlocksByPos(blockStack, positions);
+				blockStack.sort();
+
+				renderSingle(xPos, zPos, blockStack);
+			}
+			postRenderRow(zPos);
+		}
 	}
 }
 
 void TileGenerator::renderMapBlock(const BlockDecoder &blk, const BlockPos &pos)
 {
+	/***/
+	static bool light_curve_init = false;
+	static float light_curve[16];
+	if (!light_curve_init) {
+		for (u8 i = 0; i < 16; i++)
+			light_curve[i] = expf((i - 15) / 12.0f);
+		light_curve_init = true;
+	}
+	/***/
+	auto light_at = [blk] (u8 x, u8 y, u8 z) -> u8 {
+		return blk.getParam1(x, y, z) >> 4; // night bank
+	};
+	static u8 m_light[16][16];
+	if (blk.isEmpty()) {
+		for (int z = 0; z < 16; ++z) {
+			for (int x = 0; x < 16; ++x) {
+				m_light[z][x] = light_at(x, 0, z);
+			}
+		}
+		return;
+	}
+
 	int xBegin = (pos.x - m_xMin) * 16;
 	int zBegin = (m_zMax - pos.z) * 16;
 	int minY = (pos.y * 16 > m_yMin) ? 0 : m_yMin - pos.y * 16;
-	int maxY = (pos.y * 16 < m_yMax) ? 15 : m_yMax - pos.y * 16;
+	int maxY = (pos.y * 16 + 15 < m_yMax) ? 15 : m_yMax - pos.y * 16;
 	for (int z = 0; z < 16; ++z) {
 		int imageY = zBegin + 15 - z;
 		for (int x = 0; x < 16; ++x) {
@@ -462,14 +610,25 @@ void TileGenerator::renderMapBlock(const BlockDecoder &blk, const BlockPos &pos)
 
 			for (int y = maxY; y >= minY; --y) {
 				string name = blk.getNode(x, y, z);
-				if (name == "")
+				if (name == "") {
+					if (y == 0) m_light[z][x] = light_at(x, 0, z);
 					continue;
+				}
 				ColorMap::const_iterator it = m_colorMap.find(name);
 				if (it == m_colorMap.end()) {
 					m_unknownNodes.insert(name);
 					continue;
 				}
-				const Color c = it->second.to_color();
+				Color c = it->second.to_color();
+				u8 light = (y == 15) ? m_light[z][x] : light_at(x, y+1, z);
+				if (light < 15) light = mymax(light, light_at(x, y, z));
+				if (1) {
+					float l2 = light_curve[light];
+					c.r = colorSafeBounds(c.r * l2);
+					c.g = colorSafeBounds(c.g * l2);
+					c.b = colorSafeBounds(c.b * l2);
+				} else
+					c = Color(light * 17, light * 17, light * 17);
 				if (m_drawAlpha) {
 					if (m_color[z][x].a == 0)
 						m_color[z][x] = c; // first visible time, no color mixing
@@ -636,32 +795,21 @@ void TileGenerator::renderOrigin()
 void TileGenerator::renderPlayers(const std::string &inputPath)
 {
 	PlayerAttributes players(inputPath);
-	for (PlayerAttributes::Players::iterator player = players.begin(); player != players.end(); ++player) {
+	for (auto &player : players) {
-		if (player->x < m_xMin * 16 || player->x > m_xMax * 16 ||
+		if (player.x < m_xMin * 16 || player.x > m_xMax * 16 ||
-			player->z < m_zMin * 16 || player->z > m_zMax * 16)
+			player.z < m_zMin * 16 || player.z > m_zMax * 16)
 			continue;
-		if (player->y < m_yMin || player->y > m_yMax)
+		if (player.y < m_yMin || player.y > m_yMax)
 			continue;
-		int imageX = getImageX(player->x, true),
+		int imageX = getImageX(player.x, true),
-			imageY = getImageY(player->z, true);
+			imageY = getImageY(player.z, true);
 
 		m_image->drawFilledRect(imageX - 1, imageY, 3, 1, m_playerColor);
 		m_image->drawFilledRect(imageX, imageY - 1, 1, 3, m_playerColor);
-		m_image->drawText(imageX + 2, imageY, player->name, m_playerColor);
+		m_image->drawText(imageX + 2, imageY, player.name, m_playerColor);
 	}
 }
 
-inline std::list<int> TileGenerator::getZValueList() const
-{
-	std::list<int> zlist;
-	for (std::list<std::pair<int, int> >::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position)
-		zlist.push_back(position->second);
-	zlist.sort();
-	zlist.unique();
-	zlist.reverse();
-	return zlist;
-}
-
 void TileGenerator::writeImage(const std::string &output)
 {
 	m_image->save(output);
@@ -674,8 +822,8 @@ void TileGenerator::printUnknown()
 	if (m_unknownNodes.size() == 0)
 		return;
 	std::cerr << "Unknown nodes:" << std::endl;
-	for (NameSet::iterator node = m_unknownNodes.begin(); node != m_unknownNodes.end(); ++node)
+	for (const auto &node : m_unknownNodes)
-		std::cerr << "\t" << *node << std::endl;
+		std::cerr << "\t" << node << std::endl;
 }
 
 inline int TileGenerator::getImageX(int val, bool absolute) const

db-leveldb.cpp

@@ -11,7 +11,6 @@ static inline int64_t stoi64(const std::string &s)
 	return t;
 }
 
-
 static inline std::string i64tos(int64_t i)
 {
 	std::ostringstream os;
@@ -19,6 +18,7 @@ static inline std::string i64tos(int64_t i)
 	return os.str();
 }
 
+
 DBLevelDB::DBLevelDB(const std::string &mapdir)
 {
 	leveldb::Options options;
@@ -28,6 +28,9 @@ DBLevelDB::DBLevelDB(const std::string &mapdir)
 		throw std::runtime_error(std::string("Failed to open Database: ") + status.ToString());
 	}
 
+	/* LevelDB is a dumb key-value store, so the only optimization we can do
+	 * is to cache the block positions that exist in the db.
+	 */
 	loadPosCache();
 }
 
@@ -38,9 +41,21 @@ DBLevelDB::~DBLevelDB()
 }
 
 
-std::vector<BlockPos> DBLevelDB::getBlockPos()
+std::vector<BlockPos> DBLevelDB::getBlockPos(BlockPos min, BlockPos max)
 {
-	return posCache;
+	std::vector<BlockPos> res;
+	for (const auto &it : posCache) {
+		if (it.first < min.z || it.first >= max.z)
+			continue;
+		for (auto pos2 : it.second) {
+			if (pos2.first < min.x || pos2.first >= max.x)
+				continue;
+			if (pos2.second < min.y || pos2.second >= max.y)
+				continue;
+			res.emplace_back(pos2.first, pos2.second, it.first);
+		}
+	}
+	return res;
 }
 
 
@@ -49,26 +64,51 @@ void DBLevelDB::loadPosCache()
 	leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions());
 	for (it->SeekToFirst(); it->Valid(); it->Next()) {
 		int64_t posHash = stoi64(it->key().ToString());
-		posCache.push_back(decodeBlockPos(posHash));
+		BlockPos pos = decodeBlockPos(posHash);
+
+		posCache[pos.z].emplace_back(pos.x, pos.y);
 	}
 	delete it;
 }
 
 
-void DBLevelDB::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
+void DBLevelDB::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+		int16_t min_y, int16_t max_y)
 {
 	std::string datastr;
 	leveldb::Status status;
 
-	for (std::vector<BlockPos>::iterator it = posCache.begin(); it != posCache.end(); ++it) {
+	auto it = posCache.find(z);
-		if (it->z != zPos) {
+	if (it == posCache.cend())
+		return;
+	for (auto pos2 : it->second) {
+		if (pos2.first != x)
 			continue;
-		}
+		if (pos2.second < min_y || pos2.second >= max_y)
-		status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(*it)), &datastr);
+			continue;
+
+		BlockPos pos(x, pos2.second, z);
+		status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(pos)), &datastr);
 		if (status.ok()) {
-			Block b(*it, ustring((const unsigned char *) datastr.data(), datastr.size()));
+			blocks.emplace_back(
-			blocks[b.first.x].push_back(b);
+				pos, ustring((unsigned char *) datastr.data(), datastr.size())
+			);
 		}
 	}
 }
 
+void DBLevelDB::getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions)
+{
+	std::string datastr;
+	leveldb::Status status;
+
+	for (auto pos : positions) {
+		status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(pos)), &datastr);
+		if (status.ok()) {
+			blocks.emplace_back(
+				pos, ustring((unsigned char *) datastr.data(), datastr.size())
+			);
+		}
+	}
+}

db-postgresql.cpp

@@ -27,11 +27,21 @@ DBPostgreSQL::DBPostgreSQL(const std::string &mapdir)
 
 	prepareStatement(
 		"get_block_pos",
-		"SELECT posX, posY, posZ FROM blocks"
+		"SELECT posX::int4, posY::int4, posZ::int4 FROM blocks WHERE"
+		" (posX BETWEEN $1::int4 AND $2::int4) AND"
+		" (posY BETWEEN $3::int4 AND $4::int4) AND"
+		" (posZ BETWEEN $5::int4 AND $6::int4)"
 	);
 	prepareStatement(
-		"get_blocks_z",
+		"get_blocks",
-		"SELECT posX, posY, data FROM blocks WHERE posZ = $1::int4"
+		"SELECT posY::int4, data FROM blocks WHERE"
+		" posX = $1::int4 AND posZ = $2::int4"
+		" AND (posY BETWEEN $3::int4 AND $4::int4)"
+	);
+	prepareStatement(
+		"get_block_exact",
+		"SELECT data FROM blocks WHERE"
+		" posX = $1::int4 AND posY = $2::int4 AND posZ = $3::int4"
 	);
 
 	checkResults(PQexec(db, "START TRANSACTION;"));
@@ -43,25 +53,38 @@ DBPostgreSQL::~DBPostgreSQL()
 {
 	try {
 		checkResults(PQexec(db, "COMMIT;"));
-	} catch (std::exception& caught) {
+	} catch (const std::exception& caught) {
 		std::cerr << "could not finalize: " << caught.what() << std::endl;
 	}
 	PQfinish(db);
 }
 
-std::vector<BlockPos> DBPostgreSQL::getBlockPos()
+
+std::vector<BlockPos> DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max)
 {
-	std::vector<BlockPos> positions;
+	int32_t const x1 = htonl(min.x);
+	int32_t const x2 = htonl(max.x - 1);
+	int32_t const y1 = htonl(min.y);
+	int32_t const y2 = htonl(max.y - 1);
+	int32_t const z1 = htonl(min.z);
+	int32_t const z2 = htonl(max.z - 1);
+
+	const void *args[] = { &x1, &x2, &y1, &y2, &z1, &z2 };
+	const int argLen[] = { 4, 4, 4, 4, 4, 4 };
+	const int argFmt[] = { 1, 1, 1, 1, 1, 1 };
 
 	PGresult *results = execPrepared(
-		"get_block_pos", 0,
+		"get_block_pos", ARRLEN(args), args,
-		NULL, NULL, NULL, false, false
+		argLen, argFmt, false
 	);
 
 	int numrows = PQntuples(results);
 
+	std::vector<BlockPos> positions;
+	positions.reserve(numrows);
+
 	for (int row = 0; row < numrows; ++row)
-		positions.push_back(pg_to_blockpos(results, row, 0));
+		positions.emplace_back(pg_to_blockpos(results, row, 0));
 
 	PQclear(results);
 
@@ -69,16 +92,20 @@ std::vector<BlockPos> DBPostgreSQL::getBlockPos()
 }
 
 
-void DBPostgreSQL::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
+void DBPostgreSQL::getBlocksOnXZ(BlockList &blocks, int16_t xPos, int16_t zPos,
+		int16_t min_y, int16_t max_y)
 {
+	int32_t const x = htonl(xPos);
 	int32_t const z = htonl(zPos);
+	int32_t const y1 = htonl(min_y);
+	int32_t const y2 = htonl(max_y - 1);
 
-	const void *args[] = { &z };
+	const void *args[] = { &x, &z, &y1, &y2 };
-	const int argLen[] = { sizeof(z) };
+	const int argLen[] = { 4, 4, 4, 4 };
-	const int argFmt[] = { 1 };
+	const int argFmt[] = { 1, 1, 1, 1 };
 
 	PGresult *results = execPrepared(
-		"get_blocks_z", ARRLEN(args), args,
+		"get_blocks", ARRLEN(args), args,
 		argLen, argFmt, false
 	);
 
@@ -86,24 +113,60 @@ void DBPostgreSQL::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zP
 
 	for (int row = 0; row < numrows; ++row) {
 		BlockPos position;
-		position.x = pg_binary_to_int(results, row, 0);
+		position.x = xPos;
-		position.y = pg_binary_to_int(results, row, 1);
+		position.y = pg_binary_to_int(results, row, 0);
 		position.z = zPos;
-		Block const b(
+		blocks.emplace_back(
 			position,
 			ustring(
 				reinterpret_cast<unsigned char*>(
-					PQgetvalue(results, row, 2)
+					PQgetvalue(results, row, 1)
 				),
-				PQgetlength(results, row, 2)
+				PQgetlength(results, row, 1)
 			)
 		);
-		blocks[position.x].push_back(b);
 	}
 
 	PQclear(results);
 }
 
+
+void DBPostgreSQL::getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions)
+{
+	int32_t x, y, z;
+
+	const void *args[] = { &x, &y, &z };
+	const int argLen[] = { 4, 4, 4 };
+	const int argFmt[] = { 1, 1, 1 };
+
+	for (auto pos : positions) {
+		x = htonl(pos.x);
+		y = htonl(pos.y);
+		z = htonl(pos.z);
+
+		PGresult *results = execPrepared(
+			"get_block_exact", ARRLEN(args), args,
+			argLen, argFmt, false
+		);
+
+		if (PQntuples(results) > 0) {
+			blocks.emplace_back(
+				pos,
+				ustring(
+					reinterpret_cast<unsigned char*>(
+						PQgetvalue(results, 0, 0)
+					),
+					PQgetlength(results, 0, 0)
+				)
+			);
+		}
+
+		PQclear(results);
+	}
+}
+
+
 PGresult *DBPostgreSQL::checkResults(PGresult *res, bool clear)
 {
 	ExecStatusType statusType = PQresultStatus(res);
@@ -138,20 +201,15 @@ PGresult *DBPostgreSQL::execPrepared(
 	const char *stmtName, const int paramsNumber,
 	const void **params,
 	const int *paramsLengths, const int *paramsFormats,
-	bool clear, bool nobinary
+	bool clear
 )
 {
 	return checkResults(PQexecPrepared(db, stmtName, paramsNumber,
 		(const char* const*) params, paramsLengths, paramsFormats,
-		nobinary ? 1 : 0), clear
+		1 /* binary output */), clear
 	);
 }
 
-int DBPostgreSQL::pg_to_int(PGresult *res, int row, int col)
-{
-	return atoi(PQgetvalue(res, row, col));
-}
-
 int DBPostgreSQL::pg_binary_to_int(PGresult *res, int row, int col)
 {
 	int32_t* raw = reinterpret_cast<int32_t*>(PQgetvalue(res, row, col));
@@ -161,8 +219,8 @@ int DBPostgreSQL::pg_binary_to_int(PGresult *res, int row, int col)
 BlockPos DBPostgreSQL::pg_to_blockpos(PGresult *res, int row, int col)
 {
 	BlockPos result;
-	result.x = pg_to_int(res, row, col);
+	result.x = pg_binary_to_int(res, row, col);
-	result.y = pg_to_int(res, row, col + 1);
+	result.y = pg_binary_to_int(res, row, col + 1);
-	result.z = pg_to_int(res, row, col + 2);
+	result.z = pg_binary_to_int(res, row, col + 2);
 	return result;
 }

db-redis.cpp

@@ -51,6 +51,9 @@ DBRedis::DBRedis(const std::string &mapdir)
 		throw std::runtime_error(err);
 	}
 
+	/* Redis is just a key-value store, so the only optimization we can do
+	 * is to cache the block positions that exist in the db.
+	 */
 	loadPosCache();
 }
 
@@ -61,13 +64,25 @@ DBRedis::~DBRedis()
 }
 
 
-std::vector<BlockPos> DBRedis::getBlockPos()
+std::vector<BlockPos> DBRedis::getBlockPos(BlockPos min, BlockPos max)
 {
-	return posCache;
+	std::vector<BlockPos> res;
+	for (const auto &it : posCache) {
+		if (it.first < min.z || it.first >= max.z)
+			continue;
+		for (auto pos2 : it.second) {
+			if (pos2.first < min.x || pos2.first >= max.x)
+				continue;
+			if (pos2.second < min.y || pos2.second >= max.y)
+				continue;
+			res.emplace_back(pos2.first, pos2.second, it.first);
+		}
+	}
+	return res;
 }
 
 
-std::string DBRedis::replyTypeStr(int type) {
+const char *DBRedis::replyTypeStr(int type) {
 	switch(type) {
 		case REDIS_REPLY_STATUS:
 			return "REDIS_REPLY_STATUS";
@@ -98,14 +113,16 @@ void DBRedis::loadPosCache()
 	for(size_t i = 0; i < reply->elements; i++) {
 		if(reply->element[i]->type != REDIS_REPLY_STRING)
 			REPLY_TYPE_ERR(reply->element[i], "HKEYS subreply");
-		posCache.push_back(decodeBlockPos(stoi64(reply->element[i]->str)));
+		BlockPos pos = decodeBlockPos(stoi64(reply->element[i]->str));
+		posCache[pos.z].emplace_back(pos.x, pos.y);
 	}
 
 	freeReplyObject(reply);
 }
 
 
-void DBRedis::HMGET(const std::vector<BlockPos> &positions, std::vector<ustring> *result)
+void DBRedis::HMGET(const std::vector<BlockPos> &positions,
+	std::function<void(std::size_t, ustring)> result)
 {
 	const char *argv[DB_REDIS_HMGET_NUMFIELDS + 2];
 	argv[0] = "HMGET";
@@ -113,6 +130,7 @@ void DBRedis::HMGET(const std::vector<BlockPos> &positions, std::vector<ustring>
 
 	std::vector<BlockPos>::const_iterator position = positions.begin();
 	std::size_t remaining = positions.size();
+	std::size_t abs_i = 0;
 	while (remaining > 0) {
 		const std::size_t batch_size =
 			(remaining > DB_REDIS_HMGET_NUMFIELDS) ? DB_REDIS_HMGET_NUMFIELDS : remaining;
@@ -130,50 +148,51 @@ void DBRedis::HMGET(const std::vector<BlockPos> &positions, std::vector<ustring>
 
 		if(!reply)
 			throw std::runtime_error("Redis command HMGET failed");
-		if (reply->type != REDIS_REPLY_ARRAY) {
+		if (reply->type != REDIS_REPLY_ARRAY)
-			freeReplyObject(reply);
+			REPLY_TYPE_ERR(reply, "HMGET reply");
-			REPLY_TYPE_ERR(reply, "HKEYS subreply");
-		}
 		if (reply->elements != batch_size) {
 			freeReplyObject(reply);
 			throw std::runtime_error("HMGET wrong number of elements");
 		}
-		for (std::size_t i = 0; i < batch_size; ++i) {
+		for (std::size_t i = 0; i < reply->elements; ++i) {
 			redisReply *subreply = reply->element[i];
-			if(!subreply)
+			if (subreply->type == REDIS_REPLY_NIL)
-				throw std::runtime_error("Redis command HMGET failed");
+				continue;
-			if (subreply->type != REDIS_REPLY_STRING) {
+			else if (subreply->type != REDIS_REPLY_STRING)
-				freeReplyObject(reply);
+				REPLY_TYPE_ERR(subreply, "HMGET subreply");
-				REPLY_TYPE_ERR(reply, "HKEYS subreply");
+			if (subreply->len == 0)
-			}
-			if (subreply->len == 0) {
-				freeReplyObject(reply);
 				throw std::runtime_error("HMGET empty string");
-			}
+			result(abs_i + i, ustring((const unsigned char *) subreply->str, subreply->len));
-			result->push_back(ustring((const unsigned char *) subreply->str, subreply->len));
 		}
 		freeReplyObject(reply);
+		abs_i += reply->elements;
 		remaining -= batch_size;
 	}
 }
 
 
-void DBRedis::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
+void DBRedis::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+		int16_t min_y, int16_t max_y)
 {
-	std::vector<BlockPos> z_positions;
+	auto it = posCache.find(z);
-	for (std::vector<BlockPos>::const_iterator it = posCache.begin(); it != posCache.end(); ++it) {
+	if (it == posCache.cend())
-		if (it->z != zPos) {
+		return;
-			continue;
-		}
-		z_positions.push_back(*it);
-	}
-	std::vector<ustring> z_blocks;
-	HMGET(z_positions, &z_blocks);
 
-	std::vector<ustring>::const_iterator z_block = z_blocks.begin();
+	std::vector<BlockPos> positions;
-	for (std::vector<BlockPos>::const_iterator pos = z_positions.begin();
+	for (auto pos2 : it->second) {
-			pos != z_positions.end();
+		if (pos2.first == x && pos2.second >= min_y && pos2.second < max_y)
-			++pos, ++z_block) {
+			positions.emplace_back(x, pos2.second, z);
-		blocks[pos->x].push_back(Block(*pos, *z_block));
 	}
+
+	getBlocksByPos(blocks, positions);
+}
+
+
+void DBRedis::getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions)
+{
+	auto result = [&] (std::size_t i, ustring data) {
+		blocks.emplace_back(positions[i], std::move(data));
+	};
+	HMGET(positions, result);
 }

db-sqlite3.cpp

@@ -1,6 +1,8 @@
 #include <stdexcept>
 #include <unistd.h> // for usleep
 #include <iostream>
+#include <algorithm>
+#include <time.h>
 #include "db-sqlite3.h"
 #include "types.h"
 
@@ -11,7 +13,6 @@
 	}
 #define SQLOK(f) SQLRES(f, SQLITE_OK)
 
-
 DBSQLite3::DBSQLite3(const std::string &mapdir)
 {
 	int result;
@@ -24,9 +25,17 @@ DBSQLite3::DBSQLite3(const std::string &mapdir)
 			"SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ?",
 		-1, &stmt_get_blocks_z, NULL))
 
+	SQLOK(prepare_v2(db,
+			"SELECT data FROM blocks WHERE pos = ?",
+		-1, &stmt_get_block_exact, NULL))
+
 	SQLOK(prepare_v2(db,
 			"SELECT pos FROM blocks",
 		-1, &stmt_get_block_pos, NULL))
+
+	SQLOK(prepare_v2(db,
+			"SELECT pos FROM blocks WHERE pos BETWEEN ? AND ?",
+		-1, &stmt_get_block_pos_z, NULL))
 }
 
 
@@ -34,59 +43,154 @@ DBSQLite3::~DBSQLite3()
 {
 	sqlite3_finalize(stmt_get_blocks_z);
 	sqlite3_finalize(stmt_get_block_pos);
+	sqlite3_finalize(stmt_get_block_pos_z);
+	sqlite3_finalize(stmt_get_block_exact);
 
 	if (sqlite3_close(db) != SQLITE_OK) {
 		std::cerr << "Error closing SQLite database." << std::endl;
 	};
 }
 
-std::vector<BlockPos> DBSQLite3::getBlockPos()
+
+inline void DBSQLite3::getPosRange(int64_t &min, int64_t &max, int16_t zPos,
+		int16_t zPos2) const
+{
+	/* The range of block positions is [-2048, 2047], which turns into [0, 4095]
+	 * when casted to unsigned. This didn't actually help me understand the
+	 * numbers below, but I wanted to write it down.
+	 */
+
+	// Magic numbers!
+	min = encodeBlockPos(BlockPos(0, -2048, zPos));
+	max = encodeBlockPos(BlockPos(0, 2048, zPos2)) - 1;
+}
+
+
+std::vector<BlockPos> DBSQLite3::getBlockPos(BlockPos min, BlockPos max)
 {
 	int result;
-	std::vector<BlockPos> positions;
+	sqlite3_stmt *stmt;
-	while ((result = sqlite3_step(stmt_get_block_pos)) != SQLITE_DONE) {
+
-		if (result == SQLITE_ROW) {
+	if(min.z <= -2048 && max.z >= 2048) {
-			int64_t posHash = sqlite3_column_int64(stmt_get_block_pos, 0);
+		stmt = stmt_get_block_pos;
-			positions.push_back(decodeBlockPos(posHash));
-		} else if (result == SQLITE_BUSY) { // Wait some time and try again
-			usleep(10000);
 	} else {
+		stmt = stmt_get_block_pos_z;
+		int64_t minPos, maxPos;
+		if (min.z < -2048)
+			min.z = -2048;
+		if (max.z > 2048)
+			max.z = 2048;
+		getPosRange(minPos, maxPos, min.z, max.z - 1);
+		SQLOK(bind_int64(stmt, 1, minPos))
+		SQLOK(bind_int64(stmt, 2, maxPos))
+	}
+
+	std::vector<BlockPos> positions;
+	while ((result = sqlite3_step(stmt)) != SQLITE_DONE) {
+		if (result == SQLITE_BUSY) { // Wait some time and try again
+			usleep(10000);
+		} else if (result != SQLITE_ROW) {
 			throw std::runtime_error(sqlite3_errmsg(db));
 		}
+
+		int64_t posHash = sqlite3_column_int64(stmt, 0);
+		BlockPos pos = decodeBlockPos(posHash);
+		if(pos.x >= min.x && pos.x < max.x && pos.y >= min.y && pos.y < max.y)
+			positions.emplace_back(pos);
 	}
-	SQLOK(reset(stmt_get_block_pos));
+	SQLOK(reset(stmt));
 	return positions;
 }
 
 
-void DBSQLite3::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
+void DBSQLite3::loadBlockCache(int16_t zPos)
 {
 	int result;
+	blockCache.clear();
 
-	// Magic numbers!
+	int64_t minPos, maxPos;
-	int64_t minPos = encodeBlockPos(BlockPos(0, -2048, zPos));
+	getPosRange(minPos, maxPos, zPos, zPos);
-	int64_t maxPos = encodeBlockPos(BlockPos(0, 2048, zPos)) - 1;
 
 	SQLOK(bind_int64(stmt_get_blocks_z, 1, minPos));
 	SQLOK(bind_int64(stmt_get_blocks_z, 2, maxPos));
 
 	while ((result = sqlite3_step(stmt_get_blocks_z)) != SQLITE_DONE) {
-		if (result == SQLITE_ROW) {
+		if (result == SQLITE_BUSY) { // Wait some time and try again
+			usleep(10000);
+		} else if (result != SQLITE_ROW) {
+			throw std::runtime_error(sqlite3_errmsg(db));
+		}
+
 		int64_t posHash = sqlite3_column_int64(stmt_get_blocks_z, 0);
+		BlockPos pos = decodeBlockPos(posHash);
 		const unsigned char *data = reinterpret_cast<const unsigned char *>(
 				sqlite3_column_blob(stmt_get_blocks_z, 1));
 		size_t size = sqlite3_column_bytes(stmt_get_blocks_z, 1);
-			Block b(decodeBlockPos(posHash), ustring(data, size));
+		blockCache[pos.x].emplace_back(pos, ustring(data, size));
-			blocks[b.first.x].push_back(b);
-		} else if (result == SQLITE_BUSY) { // Wait some time and try again
-			usleep(10000);
-		} else {
-			throw std::runtime_error(sqlite3_errmsg(db));
 	}
-	}
+	SQLOK(reset(stmt_get_blocks_z))
-	SQLOK(reset(stmt_get_blocks_z));
 }
 
-#undef SQLRES
-#undef SQLOK
 
+void DBSQLite3::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+		int16_t min_y, int16_t max_y)
+{
+	/* Cache the blocks on the given Z coordinate between calls, this only
+	 * works due to order in which the TileGenerator asks for blocks. */
+	if (z != blockCachedZ) {
+		loadBlockCache(z);
+		blockCachedZ = z;
+	}
+
+	auto it = blockCache.find(x);
+	if (it == blockCache.end())
+		return;
+
+	if (it->second.empty()) {
+		/* We have swapped this list before, this is not supposed to happen
+		 * because it's bad for performance. But rather than silently breaking
+		 * do the right thing and load the blocks again. */
+#ifndef NDEBUG
+		std::cout << "Warning: suboptimal access pattern for sqlite3 backend" << std::endl;
+#endif
+		loadBlockCache(z);
+	}
+	// Swap lists to avoid copying contents
+	blocks.clear();
+	std::swap(blocks, it->second);
+
+	for (auto it = blocks.begin(); it != blocks.end(); ) {
+		if (it->first.y < min_y || it->first.y >= max_y)
+			it = blocks.erase(it);
+		else
+			it++;
+	}
+}
+
+
+void DBSQLite3::getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions)
+{
+	int result;
+
+	for (auto pos : positions) {
+		int64_t dbPos = encodeBlockPos(pos);
+		SQLOK(bind_int64(stmt_get_block_exact, 1, dbPos));
+
+		while ((result = sqlite3_step(stmt_get_block_exact)) == SQLITE_BUSY) {
+			usleep(10000); // Wait some time and try again
+		}
+		if (result == SQLITE_DONE) {
+			// no data
+		} else if (result != SQLITE_ROW) {
+			throw std::runtime_error(sqlite3_errmsg(db));
+		} else {
+			const unsigned char *data = reinterpret_cast<const unsigned char *>(
+					sqlite3_column_blob(stmt_get_block_exact, 0));
+			size_t size = sqlite3_column_bytes(stmt_get_block_exact, 0);
+			blocks.emplace_back(pos, ustring(data, size));
+		}
+
+		SQLOK(reset(stmt_get_block_exact))
+	}
+}

include/BlockDecoder.h

@@ -1,11 +1,7 @@
 #ifndef BLOCKDECODER_H
 #define BLOCKDECODER_H
 
-#if __cplusplus >= 201103L
 #include <unordered_map>
-#else
-#include <map>
-#endif
 
 #include "types.h"
 
@@ -17,18 +13,15 @@ public:
 	void decode(const ustring &data);
 	bool isEmpty() const;
 	std::string getNode(u8 x, u8 y, u8 z) const; // returns "" for air, ignore and invalid nodes
+	u8 getParam1(u8 x, u8 y, u8 z) const;
 
 private:
-#if __cplusplus >= 201103L
 	typedef std::unordered_map<int, std::string> NameMap;
-#else
-	typedef std::map<int, std::string> NameMap;
-#endif
 	NameMap m_nameMap;
 	int m_blockAirId;
 	int m_blockIgnoreId;
 
-	u8 m_version;
+	u8 m_version, m_contentWidth;
 	ustring m_mapData;
 };
 

include/TileGenerator.h

@@ -2,15 +2,11 @@
 #define TILEGENERATOR_HEADER
 
 #include <iosfwd>
-#include <list>
-#include <config.h>
-#if __cplusplus >= 201103L
-#include <unordered_map>
-#include <unordered_set>
-#else
 #include <map>
 #include <set>
-#endif
+#include <config.h>
+#include <unordered_map>
+#include <unordered_set>
 #include <stdint.h>
 #include <string>
 
@@ -27,6 +23,13 @@ enum {
 	SCALE_RIGHT = (1 << 3),
 };
 
+enum {
+	EXH_NEVER, // Always use range queries
+	EXH_Y,     // Exhaustively search Y space, range queries for X/Z
+	EXH_FULL,  // Exhaustively search entire requested geometry
+	EXH_AUTO,  // Automatically pick one of the previous modes
+};
+
 struct ColorEntry {
 	ColorEntry(): r(0), g(0), b(0), a(0), t(0) {};
 	ColorEntry(uint8_t r, uint8_t g, uint8_t b, uint8_t a, uint8_t t): r(r), g(g), b(b), a(a), t(t) {};
@@ -60,13 +63,8 @@ struct BitmapThing { // 16x16 bitmap
 class TileGenerator
 {
 private:
-#if __cplusplus >= 201103L
 	typedef std::unordered_map<std::string, ColorEntry> ColorMap;
 	typedef std::unordered_set<std::string> NameSet;
-#else
-	typedef std::map<std::string, ColorEntry> ColorMap;
-	typedef std::set<std::string> NameSet;
-#endif
 
 public:
 	TileGenerator();
@@ -74,7 +72,8 @@ public:
 	void setBgColor(const std::string &bgColor);
 	void setScaleColor(const std::string &scaleColor);
 	void setOriginColor(const std::string &originColor);
-	void setPlayerColor(const std::string &playerColor); Color parseColor(const std::string &color);
+	void setPlayerColor(const std::string &playerColor);
+	Color parseColor(const std::string &color);
 	void setDrawOrigin(bool drawOrigin);
 	void setDrawPlayers(bool drawPlayers);
 	void setDrawScale(bool drawScale);
@@ -83,14 +82,17 @@ public:
 	void setGeometry(int x, int y, int w, int h);
 	void setMinY(int y);
 	void setMaxY(int y);
+	void setExhaustiveSearch(int mode);
 	void parseColorsFile(const std::string &fileName);
 	void setBackend(std::string backend);
-	void generate(const std::string &input, const std::string &output);
-	void printGeometry(const std::string &input);
 	void setZoom(int zoom);
 	void setScales(uint flags);
 	void setDontWriteEmpty(bool f);
 
+	void generate(const std::string &input, const std::string &output);
+	void printGeometry(const std::string &input);
+	static std::set<std::string> getSupportedBackends();
+
 private:
 	void parseColorsStream(std::istream &in);
 	void openDb(const std::string &input);
@@ -98,7 +100,6 @@ private:
 	void loadBlocks();
 	void createImage();
 	void renderMap();
-	std::list<int> getZValueList() const;
 	void renderMapBlock(const BlockDecoder &blk, const BlockPos &pos);
 	void renderMapBlockBottom(const BlockPos &pos);
 	void renderShading(int zPos);
@@ -128,19 +129,24 @@ private:
 	DB *m_db;
 	Image *m_image;
 	PixelAttributes m_blockPixelAttributes;
+	/* smallest/largest seen X or Z block coordinate */
 	int m_xMin;
 	int m_xMax;
 	int m_zMin;
 	int m_zMax;
+	/* Y limits for rendered area (node units) */
 	int m_yMin;
 	int m_yMax;
-	int m_geomX;
+	/* limits for rendered area (block units) */
-	int m_geomY;
+	int16_t m_geomX;
-	int m_geomX2;
+	int16_t m_geomY; /* Y in terms of rendered image, Z in the world */
-	int m_geomY2;
+	int16_t m_geomX2;
+	int16_t m_geomY2;
+	/* */
 	int m_mapWidth;
 	int m_mapHeight;
-	std::list<std::pair<int, int> > m_positions;
+	int m_exhaustiveSearch;
+	std::map<int16_t, std::set<int16_t>> m_positions; /* indexed by Z, contains X coords */
 	ColorMap m_colorMap;
 	BitmapThing m_readPixels;
 	BitmapThing m_readInfo;

include/cmake_config.h.in

@@ -3,9 +3,9 @@
 #ifndef CMAKE_CONFIG_H
 #define CMAKE_CONFIG_H
 
-#define USE_POSTGRESQL @USE_POSTGRESQL@
+#cmakedefine01 USE_POSTGRESQL
-#define USE_LEVELDB @USE_LEVELDB@
+#cmakedefine01 USE_LEVELDB
-#define USE_REDIS @USE_REDIS@
+#cmakedefine01 USE_REDIS
 
 #define SHAREDIR "@SHAREDIR@"
 

include/db-leveldb.h

@@ -2,19 +2,29 @@
 #define DB_LEVELDB_HEADER
 
 #include "db.h"
+#include <unordered_map>
+#include <utility>
 #include <leveldb/db.h>
 
 class DBLevelDB : public DB {
 public:
 	DBLevelDB(const std::string &mapdir);
-	virtual std::vector<BlockPos> getBlockPos();
+	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
-	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
+	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
-	virtual ~DBLevelDB();
+			int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
+	~DBLevelDB() override;
+
+	bool preferRangeQueries() const override { return false; }
+
 private:
+	using pos2d = std::pair<int16_t, int16_t>;
+
 	void loadPosCache();
 
-	std::vector<BlockPos> posCache;
+	// indexed by Z, contains all (x,y) position pairs
-
+	std::unordered_map<int16_t, std::vector<pos2d>> posCache;
 	leveldb::DB *db;
 };
 

include/db-postgresql.h

@@ -7,9 +7,15 @@
 class DBPostgreSQL : public DB {
 public:
 	DBPostgreSQL(const std::string &mapdir);
-	virtual std::vector<BlockPos> getBlockPos();
+	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
-	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
+	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
-	virtual ~DBPostgreSQL();
+		int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
+	~DBPostgreSQL() override;
+
+	bool preferRangeQueries() const override { return true; }
+
 protected:
 	PGresult *checkResults(PGresult *res, bool clear = true);
 	void prepareStatement(const std::string &name, const std::string &sql);
@@ -17,11 +23,11 @@ protected:
 		const char *stmtName, const int paramsNumber,
 		const void **params,
 		const int *paramsLengths = NULL, const int *paramsFormats = NULL,
-		bool clear = true, bool nobinary = true
+		bool clear = true
 	);
-	int pg_to_int(PGresult *res, int row, int col);
 	int pg_binary_to_int(PGresult *res, int row, int col);
 	BlockPos pg_to_blockpos(PGresult *res, int row, int col);
+
 private:
 	PGconn *db;
 };

include/db-redis.h

@@ -2,21 +2,33 @@
 #define DB_REDIS_HEADER
 
 #include "db.h"
-#include <hiredis.h>
+#include <unordered_map>
+#include <utility>
+#include <functional>
+#include <hiredis/hiredis.h>
 
 class DBRedis : public DB {
 public:
 	DBRedis(const std::string &mapdir);
-	virtual std::vector<BlockPos> getBlockPos();
+	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
-	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
+	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
-	virtual ~DBRedis();
+		int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
+	~DBRedis() override;
+
+	bool preferRangeQueries() const override { return false; }
+
 private:
-	static std::string replyTypeStr(int type);
+	using pos2d = std::pair<int16_t, int16_t>;
+	static const char *replyTypeStr(int type);
 
 	void loadPosCache();
-	void HMGET(const std::vector<BlockPos> &positions, std::vector<ustring> *result);
+	void HMGET(const std::vector<BlockPos> &positions,
+			std::function<void(std::size_t, ustring)> result);
 
-	std::vector<BlockPos> posCache;
+	// indexed by Z, contains all (x,y) position pairs
+	std::unordered_map<int16_t, std::vector<pos2d>> posCache;
 
 	redisContext *ctx;
 	std::string hash;

include/db-sqlite3.h

@@ -2,19 +2,35 @@
 #define _DB_SQLITE3_H
 
 #include "db.h"
+#include <unordered_map>
 #include <sqlite3.h>
 
 class DBSQLite3 : public DB {
 public:
 	DBSQLite3(const std::string &mapdir);
-	virtual std::vector<BlockPos> getBlockPos();
+	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
-	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
+	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
-	virtual ~DBSQLite3();
+			int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
+	~DBSQLite3() override;
+
+	bool preferRangeQueries() const override { return false; }
+
 private:
+	inline void getPosRange(int64_t &min, int64_t &max, int16_t zPos,
+			int16_t zPos2) const;
+	void loadBlockCache(int16_t zPos);
+
 	sqlite3 *db;
 
 	sqlite3_stmt *stmt_get_block_pos;
+	sqlite3_stmt *stmt_get_block_pos_z;
 	sqlite3_stmt *stmt_get_blocks_z;
+	sqlite3_stmt *stmt_get_block_exact;
+
+	int16_t blockCachedZ = -10000;
+	std::unordered_map<int16_t, BlockList> blockCache; // indexed by X
 };
 
 #endif // _DB_SQLITE3_H

include/db.h

@@ -5,19 +5,19 @@
 #include <map>
 #include <list>
 #include <vector>
-#include <string>
 #include <utility>
 #include "types.h"
 
 
-class BlockPos {
+struct BlockPos {
-public:
 	int16_t x;
 	int16_t y;
 	int16_t z;
 
 	BlockPos() : x(0), y(0), z(0) {}
 	BlockPos(int16_t x, int16_t y, int16_t z) : x(x), y(y), z(z) {}
+
+	// Implements the inverse ordering so that (2,2,2) < (1,1,1)
 	bool operator < (const BlockPos &p) const
 	{
 		if (z > p.z)
@@ -43,13 +43,31 @@ typedef std::list<Block> BlockList;
 
 class DB {
 protected:
+	// Helpers that implement the hashed positions used by most backends
 	inline int64_t  encodeBlockPos(const BlockPos pos) const;
 	inline BlockPos decodeBlockPos(int64_t hash) const;
 
 public:
-	virtual std::vector<BlockPos> getBlockPos() = 0;
+	/* Return all block positions inside the range given by min and max,
-	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos) = 0;
+	 * so that min.x <= x < max.x, ...
-	virtual ~DB() {};
+	 */
+	virtual std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) = 0;
+	/* Read all blocks in column given by x and z
+	 * and inside the given Y range (min_y <= y < max_y) into list
+	 */
+	virtual void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+			int16_t min_y, int16_t max_y) = 0;
+	/* Read blocks at given positions into list
+	 */
+	virtual void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) = 0;
+	/* Can this database efficiently do range queries?
+	 * (for large data sets, more efficient that brute force)
+	 */
+	virtual bool preferRangeQueries() const = 0;
+
+
+	virtual ~DB() {}
 };
 
 

include/util.h

@@ -10,7 +10,7 @@ inline std::string read_setting_default(const std::string &name, std::istream &i
 {
 	try {
 		return read_setting(name, is);
-	} catch(std::runtime_error &e) {
+	} catch(const std::runtime_error &e) {
 		return def;
 	}
 }

mapper.cpp

@@ -1,49 +1,68 @@
+#include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <getopt.h>
 #include <fstream>
 #include <iostream>
-#include <map>
+#include <utility>
 #include <string>
 #include <sstream>
 #include <stdexcept>
 #include "cmake_config.h"
 #include "TileGenerator.h"
 
-void usage()
+static void usage()
 {
-	const char *usage_text = "minetestmapper [options]\n"
+	const std::pair<const char*, const char*> options[] = {
-			"  -i/--input <world_path>\n"
+		{"-i/--input", "<world_path>"},
-			"  -o/--output <output_image.png>\n"
+		{"-o/--output", "<output_image.png>"},
-			"  --bgcolor <color>\n"
+		{"--bgcolor", "<color>"},
-			"  --scalecolor <color>\n"
+		{"--scalecolor", "<color>"},
-			"  --playercolor <color>\n"
+		{"--playercolor", "<color>"},
-			"  --origincolor <color>\n"
+		{"--origincolor", "<color>"},
-			"  --drawscale\n"
+		{"--drawscale", ""},
-			"  --drawplayers\n"
+		{"--drawplayers", ""},
-			"  --draworigin\n"
+		{"--draworigin", ""},
-			"  --drawalpha\n"
+		{"--drawalpha", ""},
-			"  --noshading\n"
+		{"--noshading", ""},
-			"  --noemptyimage\n"
+		{"--noemptyimage", ""},
-			"  --min-y <y>\n"
+		{"--min-y", "<y>"},
-			"  --max-y <y>\n"
+		{"--max-y", "<y>"},
-			"  --backend <backend>\n"
+		{"--backend", "<backend>"},
-			"  --geometry x:y+w+h\n"
+		{"--geometry", "x:y+w+h"},
-			"  --extent\n"
+		{"--extent", ""},
-			"  --zoom <zoomlevel>\n"
+		{"--zoom", "<zoomlevel>"},
-			"  --colors <colors.txt>\n"
+		{"--colors", "<colors.txt>"},
-			"  --scales [t][b][l][r]\n"
+		{"--scales", "[t][b][l][r]"},
-			"Color format: '#000000'\n";
+		{"--exhaustive", "never|y|full|auto"},
-	std::cout << usage_text;
+	};
+	const char *top_text =
+		"minetestmapper -i <world_path> -o <output_image.png> [options]\n"
+		"Generate an overview image of a Minetest map.\n"
+		"\n"
+		"Options:\n";
+	const char *bottom_text =
+		"\n"
+		"Color format: hexadecimal '#RRGGBB', e.g. '#FF0000' = red\n";
+
+	printf("%s", top_text);
+	for (const auto &p : options)
+		printf("  %-18s%s\n", p.first, p.second);
+	printf("%s", bottom_text);
+	auto backends = TileGenerator::getSupportedBackends();
+	printf("Supported backends: ");
+	for (auto s : backends)
+		printf("%s ", s.c_str());
+	printf("\n");
 }
 
-bool file_exists(const std::string &path)
+static bool file_exists(const std::string &path)
 {
 	std::ifstream ifs(path.c_str());
 	return ifs.is_open();
 }
 
-std::string search_colors(const std::string &worldpath)
+static std::string search_colors(const std::string &worldpath)
 {
 	if(file_exists(worldpath + "/colors.txt"))
 		return worldpath + "/colors.txt";
@@ -57,7 +76,8 @@ std::string search_colors(const std::string &worldpath)
 	}
 #endif
 
-	if(!(SHAREDIR[0] == '.' || SHAREDIR[0] == '\0') && file_exists(SHAREDIR "/colors.txt"))
+	constexpr bool sharedir_valid = !(SHAREDIR[0] == '.' || SHAREDIR[0] == '\0');
+	if(sharedir_valid && file_exists(SHAREDIR "/colors.txt"))
 		return SHAREDIR "/colors.txt";
 
 	std::cerr << "Warning: Falling back to using colors.txt from current directory." << std::endl;
@@ -66,7 +86,7 @@ std::string search_colors(const std::string &worldpath)
 
 int main(int argc, char *argv[])
 {
-	static struct option long_options[] =
+	const static struct option long_options[] =
 	{
 		{"help", no_argument, 0, 'h'},
 		{"input", required_argument, 0, 'i'},
@@ -89,6 +109,7 @@ int main(int argc, char *argv[])
 		{"colors", required_argument, 0, 'C'},
 		{"scales", required_argument, 0, 'f'},
 		{"noemptyimage", no_argument, 0, 'n'},
+		{"exhaustive", required_argument, 0, 'j'},
 		{0, 0, 0, 0}
 	};
 
@@ -200,6 +221,19 @@ int main(int argc, char *argv[])
 			case 'n':
 				generator.setDontWriteEmpty(true);
 				break;
+			case 'j': {
+					int mode;
+					if (!strcmp(optarg, "never"))
+						mode = EXH_NEVER;
+					else if (!strcmp(optarg, "y"))
+						mode = EXH_Y;
+					else if (!strcmp(optarg, "full"))
+						mode = EXH_FULL;
+					else
+						mode = EXH_AUTO;
+					generator.setExhaustiveSearch(mode);
+				}
+				break;
 			default:
 				exit(1);
 		}

minetestmapper.6

@@ -70,15 +70,15 @@ Don't draw nodes above this y value, e.g. "--max-y 75"
 
 .TP
 .BR \-\-backend " " \fIbackend\fR
-Use specific map backend; supported: *sqlite3*, *leveldb*, *redis*, *postgresql*, e.g. "--backend leveldb"
+Use specific map backend; supported: \fIsqlite3\fP, \fIleveldb\fP, \fIredis\fP, \fIpostgresql\fP, e.g. "--backend leveldb"
 
 .TP
 .BR \-\-geometry " " \fIgeometry\fR
-Limit area to specific geometry (*x:y+w+h* where x and y specify the lower left corner), e.g. "--geometry -800:-800+1600+1600"
+Limit area to specific geometry (\fIx:y+w+h\fP where x and y specify the lower left corner), e.g. "--geometry -800:-800+1600+1600"
 
 .TP
 .BR \-\-extent " " \fIextent\fR
-Dont render the image, just print the extent of the map that would be generated, in the same format as the geometry above.
+Don't render the image, just print the extent of the map that would be generated, in the same format as the geometry above.
 
 .TP
 .BR \-\-zoom " " \fIfactor\fR
@@ -90,7 +90,18 @@ Forcefully set path to colors.txt file (it's autodetected otherwise), e.g. "--co
 
 .TP
 .BR \-\-scales " " \fIedges\fR
-Draw scales on specified image edges (letters *t b l r* meaning top, bottom, left and right), e.g. "--scales tbr"
+Draw scales on specified image edges (letters \fIt b l r\fP meaning top, bottom, left and right), e.g. "--scales tbr"
+
+.TP
+.BR \-\-exhaustive " \fImode\fR
+Select if database should be traversed exhaustively or using range queries, available: \fInever\fP, \fIy\fP, \fIfull\fP, \fIauto\fP
+
+Defaults to \fIauto\fP. You shouldn't need to change this, but doing so can improve rendering times on large maps.
+For these optimizations to work it is important that you set
+.B min-y
+and
+.B max-y
+when you don't care about the world below e.g. -60 and above 1000 nodes.
 
 .SH MORE INFORMATION
 Website: https://github.com/minetest/minetestmapper

util.cpp

@@ -3,61 +3,43 @@
 
 #include "util.h"
 
-inline std::string trim(const std::string &s)
+static inline std::string trim(const std::string &s)
 {
-  size_t front = 0;
+	auto isspace = [] (char c) -> bool { return c == ' ' || c == '\t' || c == '\r' || c == '\n'; };
-  while(s[front] == ' '    ||
-        s[front] == '\t'   ||
-        s[front] == '\r'   ||
-        s[front] == '\n'
-       )
-    ++front;
 
-  size_t back = s.size();
+	size_t front = 0;
-  while(back > front &&
+	while(isspace(s[front]))
-        (s[back-1] == ' '  ||
+		++front;
-         s[back-1] == '\t' ||
+	size_t back = s.size() - 1;
-         s[back-1] == '\r' ||
+	while(back > front && isspace(s[back]))
-         s[back-1] == '\n'
-        )
-       )
 		--back;
 
-  return s.substr(front, back - front);
+	return s.substr(front, back - front + 1);
 }
 
-#define EOFCHECK() do { \
-    if (is.eof()) { \
-      std::ostringstream oss; \
-      oss << "Setting '" << name << "' not found."; \
-      throw std::runtime_error(oss.str()); \
-    } \
-  } while(0)
-
 std::string read_setting(const std::string &name, std::istream &is)
 {
-  char c;
+	char linebuf[512];
-  char s[256];
+	while (is.good()) {
-  std::string nm, value;
+		is.getline(linebuf, sizeof(linebuf));
 
-  next:
+		for(char *p = linebuf; *p; p++) {
-  while((c = is.get()) == ' ' || c == '\t' || c == '\r' || c == '\n')
+			if(*p != '#')
-    ;
+				continue;
-  EOFCHECK();
+			*p = '\0'; // Cut off at the first #
-  if(c == '#') // Ignore comments
+			break;
-    is.ignore(0xffff, '\n');
+		}
-  EOFCHECK();
+		std::string line(linebuf);
-  s[0] = c; // The current char belongs to the name too
+
-  is.get(&s[1], 255, '=');
+		auto pos = line.find('=');
-  is.ignore(1); // Jump over the =
+		if (pos == std::string::npos)
-  EOFCHECK();
+			continue;
-  nm = trim(std::string(s));
+		auto key = trim(line.substr(0, pos));
-  is.get(s, 256, '\n');
+		if (key != name)
-  value = trim(std::string(s));
+			continue;
-  if(name == nm)
+		return trim(line.substr(pos+1));
-    return value;
+	}
-  else
+	std::ostringstream oss;
-    goto next;
+	oss << "Setting '" << name << "' not found";
+	throw std::runtime_error(oss.str());
 }
-
-#undef EOFCHECK