Cover different db fetch strategies in tests

40a5e16e21ba9a0663fc93919291c1bf0e5c2e3a added the broken reserve() call
and only by chance did 0a56b18cfbb583649bdbc7f0a379df5d63ea2880 not break
it further because I forgot to remove the unconditional resize().

I should read my own code changes more often.

.github/workflows/build.yml

@@ -1,6 +1,6 @@
 name: build
 
-# build on c/cpp changes or workflow changes
+# build on source or workflow changes
 on:
   push:
     paths:
@@ -37,8 +37,11 @@ jobs:
 
       - name: Test
         run: |
-          source util/ci/script.sh
+          ./util/ci/test.sh
-          do_functional_test
+
+      - name: Test Install
+        run: |
+          make DESTDIR=/tmp/install install
 
   clang:
     runs-on: ubuntu-22.04
@@ -58,6 +61,25 @@ jobs:
           CXX: clang++
 
       - name: Test
+        run: |
+          ./util/ci/test.sh
+
+  gcc_fedora:
+    runs-on: ubuntu-latest
+    container:
+      image: fedora:latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install deps
         run: |
           source util/ci/script.sh
-          do_functional_test
+          install_linux_deps
+
+      - name: Build
+        run: |
+          source util/ci/script.sh
+          run_build
+
+      - name: Test
+        run: |
+          ./util/ci/test.sh

.gitignore

@@ -1,8 +1,8 @@
 *~
 
-minetestmapper
+/minetestmapper
-minetestmapper.exe
+/minetestmapper.exe
-colors.txt
+/colors.txt
 
 CMakeCache.txt
 CMakeFiles/
@@ -12,3 +12,5 @@ install_manifest.txt
 Makefile
 cmake_install.cmake
 cmake_config.h
+compile_commands.json
+.vscode/

CMakeLists.txt

@@ -43,7 +43,7 @@ if(NOT CUSTOM_DOCDIR STREQUAL "")
         message(STATUS "Using DOCDIR=${DOCDIR}")
 endif()
 
-set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
+list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
 
 # Libraries: gd
 
@@ -57,7 +57,17 @@ endif(NOT LIBGD_LIBRARY OR NOT LIBGD_INCLUDE_DIR)
 
 # Libraries: zlib
 
-find_package(ZLIB REQUIRED)
+find_package(zlib-ng QUIET)
+if(zlib-ng_FOUND)
+	set(ZLIB_INCLUDE_DIR zlib-ng::zlib)
+	set(ZLIB_LIBRARY zlib-ng::zlib)
+	set(USE_ZLIB_NG TRUE)
+	message(STATUS "Found zlib-ng, using it instead of zlib.")
+else()
+	message(STATUS "zlib-ng not found, falling back to zlib.")
+	find_package(ZLIB REQUIRED)
+	set(USE_ZLIB_NG FALSE)
+endif()
 
 # Libraries: zstd
 
@@ -146,9 +156,48 @@ endif(ENABLE_REDIS)
 
 # Compiling & Linking
 
-include_directories(
+configure_file(
-	"${PROJECT_BINARY_DIR}"
+	"${CMAKE_CURRENT_SOURCE_DIR}/src/cmake_config.h.in"
-	"${CMAKE_CURRENT_SOURCE_DIR}/include"
+	"${CMAKE_CURRENT_BINARY_DIR}/cmake_config.h"
+)
+
+if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang)$")
+	set(CMAKE_CXX_FLAGS_RELEASE "-O2")
+	set(CMAKE_CXX_FLAGS_DEBUG "-Og -g2")
+	add_compile_options(-Wall -pipe)
+elseif(MSVC)
+	add_compile_options(/GR- /Zl)
+endif()
+if(CMAKE_BUILD_TYPE STREQUAL "Release")
+	add_definitions(-DNDEBUG)
+endif()
+
+add_executable(minetestmapper)
+
+target_include_directories(minetestmapper PRIVATE
+	"${CMAKE_CURRENT_SOURCE_DIR}"
+	"${CMAKE_CURRENT_BINARY_DIR}"
+)
+
+target_sources(minetestmapper PRIVATE
+	src/BlockDecoder.cpp
+	src/PixelAttributes.cpp
+	src/PlayerAttributes.cpp
+	src/TileGenerator.cpp
+	src/ZlibDecompressor.cpp
+	src/ZstdDecompressor.cpp
+	src/Image.cpp
+	src/mapper.cpp
+	src/util.cpp
+	src/log.cpp
+	src/db-sqlite3.cpp
+	$<$<BOOL:${USE_POSTGRESQL}>:src/db-postgresql.cpp>
+	$<$<BOOL:${USE_LEVELDB}>:src/db-leveldb.cpp>
+	$<$<BOOL:${USE_REDIS}>:src/db-redis.cpp>
+)
+
+target_include_directories(minetestmapper PRIVATE
+	"${CMAKE_CURRENT_SOURCE_DIR}/src"
 	"${CMAKE_CURRENT_BINARY_DIR}"
 	${SQLITE3_INCLUDE_DIR}
 	${LIBGD_INCLUDE_DIR}
@@ -156,39 +205,7 @@ include_directories(
 	${ZSTD_INCLUDE_DIR}
 )
 
-configure_file(
+target_link_libraries(minetestmapper
-	"${PROJECT_SOURCE_DIR}/include/cmake_config.h.in"
-	"${PROJECT_BINARY_DIR}/cmake_config.h"
-)
-add_definitions(-DUSE_CMAKE_CONFIG_H)
-
-if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang)$")
-	set(CMAKE_CXX_FLAGS_RELEASE "-O2")
-	set(CMAKE_CXX_FLAGS_DEBUG "-Og -g2")
-	add_compile_options(-Wall -pipe)
-endif()
-if(CMAKE_BUILD_TYPE STREQUAL "Release")
-	add_definitions(-DNDEBUG)
-endif()
-
-add_executable(minetestmapper
-	BlockDecoder.cpp
-	PixelAttributes.cpp
-	PlayerAttributes.cpp
-	TileGenerator.cpp
-	ZlibDecompressor.cpp
-	ZstdDecompressor.cpp
-	Image.cpp
-	mapper.cpp
-	util.cpp
-	db-sqlite3.cpp
-	$<$<BOOL:${USE_POSTGRESQL}>:db-postgresql.cpp>
-	$<$<BOOL:${USE_LEVELDB}>:db-leveldb.cpp>
-	$<$<BOOL:${USE_REDIS}>:db-redis.cpp>
-)
-
-target_link_libraries(
-	minetestmapper
 	${SQLITE3_LIBRARY}
 	${PostgreSQL_LIBRARIES}
 	${LEVELDB_LIBRARY}

PlayerAttributes.cpp

@@ -1,131 +0,0 @@
-#include <fstream>
-#include <sstream>
-#include <stdexcept>
-#include <dirent.h>
-#include <unistd.h> // for usleep
-#include <sqlite3.h>
-
-#include "config.h"
-#include "PlayerAttributes.h"
-#include "util.h"
-
-PlayerAttributes::PlayerAttributes(const std::string &worldDir)
-{
-	std::ifstream ifs(worldDir + "world.mt");
-	if (!ifs.good())
-		throw std::runtime_error("Failed to read world.mt");
-	std::string backend = read_setting_default("player_backend", ifs, "files");
-	ifs.close();
-
-	if (backend == "files")
-		readFiles(worldDir + "players");
-	else if (backend == "sqlite3")
-		readSqlite(worldDir + "players.sqlite");
-	else
-		throw std::runtime_error(std::string("Unknown player backend: ") + backend);
-}
-
-void PlayerAttributes::readFiles(const std::string &playersPath)
-{
-	DIR *dir;
-	dir = opendir (playersPath.c_str());
-	if (!dir)
-		return;
-
-	struct dirent *ent;
-	while ((ent = readdir (dir)) != NULL) {
-		if (ent->d_name[0] == '.')
-			continue;
-
-		std::ifstream in(playersPath + PATH_SEPARATOR + ent->d_name);
-		if (!in.good())
-			continue;
-
-		std::string name, position;
-		name = read_setting("name", in);
-		in.seekg(0);
-		position = read_setting("position", in);
-
-		Player player;
-		std::istringstream iss(position);
-		char tmp;
-		iss >> tmp; // '('
-		iss >> player.x;
-		iss >> tmp; // ','
-		iss >> player.y;
-		iss >> tmp; // ','
-		iss >> player.z;
-		iss >> tmp; // ')'
-		if (tmp != ')')
-			continue;
-		player.name = name;
-
-		player.x /= 10.0f;
-		player.y /= 10.0f;
-		player.z /= 10.0f;
-
-		m_players.push_back(player);
-	}
-
-	closedir(dir);
-}
-
-/**********/
-
-#define SQLRES(f, good) \
-	result = (sqlite3_##f); \
-	if (result != good) { \
-		throw std::runtime_error(sqlite3_errmsg(db));\
-	}
-#define SQLOK(f) SQLRES(f, SQLITE_OK)
-
-void PlayerAttributes::readSqlite(const std::string &db_name)
-{
-	int result;
-	sqlite3 *db;
-	sqlite3_stmt *stmt_get_player_pos;
-
-	SQLOK(open_v2(db_name.c_str(), &db, SQLITE_OPEN_READONLY |
-			SQLITE_OPEN_PRIVATECACHE, 0))
-
-	SQLOK(prepare_v2(db,
-			"SELECT name, posX, posY, posZ FROM player",
-		-1, &stmt_get_player_pos, NULL))
-
-	while ((result = sqlite3_step(stmt_get_player_pos)) != SQLITE_DONE) {
-		if (result == SQLITE_BUSY) { // Wait some time and try again
-			usleep(10000);
-		} else if (result != SQLITE_ROW) {
-			throw std::runtime_error(sqlite3_errmsg(db));
-		}
-
-		Player player;
-		const unsigned char *name_ = sqlite3_column_text(stmt_get_player_pos, 0);
-		player.name = reinterpret_cast<const char*>(name_);
-		player.x = sqlite3_column_double(stmt_get_player_pos, 1);
-		player.y = sqlite3_column_double(stmt_get_player_pos, 2);
-		player.z = sqlite3_column_double(stmt_get_player_pos, 3);
-
-		player.x /= 10.0f;
-		player.y /= 10.0f;
-		player.z /= 10.0f;
-
-		m_players.push_back(player);
-	}
-
-	sqlite3_finalize(stmt_get_player_pos);
-	sqlite3_close(db);
-}
-
-/**********/
-
-PlayerAttributes::Players::const_iterator PlayerAttributes::begin() const
-{
-	return m_players.cbegin();
-}
-
-PlayerAttributes::Players::const_iterator PlayerAttributes::end() const
-{
-	return m_players.cend();
-}
-

README.rst

@@ -4,16 +4,18 @@ Minetest Mapper C++
 .. image:: https://github.com/minetest/minetestmapper/workflows/build/badge.svg
     :target: https://github.com/minetest/minetestmapper/actions/workflows/build.yml
 
-Minetestmapper generates an overview image from a Luanti map.
+Minetestmapper generates a top-down overview image from a Luanti map.
 
 A port of minetestmapper.py to C++ from `the obsolete Python script
 <https://github.com/minetest/minetest/tree/0.4.17/util>`_.
 This version is both faster and provides more features.
 
-Minetestmapper ships with a colors.txt file for Minetest Game, if you use a different game or have
+Minetestmapper ships with a colors.txt file suitable for Minetest Game,
-many mods installed you should generate a matching colors.txt for better results.
+if you use a different game or have mods installed you should generate a
+matching colors.txt for better results (colors will be missing otherwise).
 The `generate_colorstxt.py script 
-<./util/generate_colorstxt.py>`_ in the util folder exists for this purpose, detailed instructions can be found within.
+<./util/generate_colorstxt.py>`_ in the util folder exists for this purpose,
+detailed instructions can be found within.
 
 Requirements
 ------------
@@ -41,7 +43,8 @@ Minetestmapper for Windows can be downloaded `from the Releases section
 <https://github.com/minetest/minetestmapper/releases>`_.
 
 After extracting the archive, it can be invoked from cmd.exe or PowerShell:
-::
+
+.. code-block:: dos
 
 	cd C:\Users\yourname\Desktop\example\path
 	minetestmapper.exe --help
@@ -49,7 +52,7 @@ After extracting the archive, it can be invoked from cmd.exe or PowerShell:
 Compilation
 -----------
 
-::
+.. code-block:: bash
 
     cmake . -DENABLE_LEVELDB=1
     make -j$(nproc)
@@ -57,8 +60,8 @@ Compilation
 Usage
 -----
 
-`minetestmapper` has two mandatory paremeters, `-i` (input world path)
+``minetestmapper`` has two mandatory paremeters, ``-i`` (input world path)
-and `-o` (output image path).
+and ``-o`` (output image path).
 
 ::
 
@@ -90,7 +93,7 @@ draworigin:
     Draw origin indicator, ``--draworigin``
 
 drawalpha:
-    Allow nodes to be drawn with transparency (e.g. water), ``--drawalpha``
+    Allow nodes to be drawn with transparency (such as water), ``--drawalpha``
 
 extent:
     Don't output any imagery, just print the extent of the full map, ``--extent``
@@ -101,11 +104,14 @@ noshading:
 noemptyimage:
     Don't output anything when the image would be empty, ``--noemptyimage``
 
+verbose:
+    Enable verbose log putput, ``--verbose``
+
 min-y:
-    Don't draw nodes below this y value, e.g. ``--min-y -25``
+    Don't draw nodes below this Y value, e.g. ``--min-y -25``
 
 max-y:
-    Don't draw nodes above this y value, e.g. ``--max-y 75``
+    Don't draw nodes above this Y value, e.g. ``--max-y 75``
 
 backend:
     Override auto-detected map backend; supported: *sqlite3*, *leveldb*, *redis*, *postgresql*, e.g. ``--backend leveldb``
@@ -113,8 +119,10 @@ backend:
 geometry:
     Limit area to specific geometry (*x:z+w+h* where x and z specify the lower left corner), e.g. ``--geometry -800:-800+1600+1600``
 
+    The coordinates are specified with the same axes as in-game. The Z axis becomes Y when projected on the image.
+
 zoom:
-    Apply zoom to drawn nodes by enlarging them to n*n squares, e.g. ``--zoom 4``
+    Zoom the image by using more than one pixel per node, e.g. ``--zoom 4``
 
 colors:
     Override auto-detected path to colors.txt, e.g. ``--colors ../world/mycolors.txt``
@@ -123,6 +131,9 @@ scales:
     Draw scales on specified image edges (letters *t b l r* meaning top, bottom, left and right), e.g. ``--scales tbr``
 
 exhaustive:
-    | Select if database should be traversed exhaustively or using range queries, available: *never*, *y*, *full*, *auto*
+    Select if database should be traversed exhaustively or using range queries, available: *never*, *y*, *full*, *auto*
-    | Defaults to *auto*. You shouldn't need to change this, but doing so can improve rendering times on large maps.
+
-    | For these optimizations to work it is important that you set ``min-y`` and ``max-y`` when you don't care about the world below e.g. -60 and above 1000 nodes.
+    Defaults to *auto*. You shouldn't need to change this, as minetestmapper tries to automatically picks the best option.
+
+dumpblock:
+    Instead of rendering anything try to load the block at the given position (*x,y,z*) and print its raw data as hexadecimal.

db-sqlite3.cpp

@@ -1,196 +0,0 @@
-#include <stdexcept>
-#include <unistd.h> // for usleep
-#include <iostream>
-#include <algorithm>
-#include <time.h>
-#include "db-sqlite3.h"
-#include "types.h"
-
-#define SQLRES(f, good) \
-	result = (sqlite3_##f);\
-	if (result != good) {\
-		throw std::runtime_error(sqlite3_errmsg(db));\
-	}
-#define SQLOK(f) SQLRES(f, SQLITE_OK)
-
-DBSQLite3::DBSQLite3(const std::string &mapdir)
-{
-	int result;
-	std::string db_name = mapdir + "map.sqlite";
-
-	SQLOK(open_v2(db_name.c_str(), &db, SQLITE_OPEN_READONLY |
-			SQLITE_OPEN_PRIVATECACHE, 0))
-
-	SQLOK(prepare_v2(db,
-			"SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ?",
-		-1, &stmt_get_blocks_z, NULL))
-
-	SQLOK(prepare_v2(db,
-			"SELECT data FROM blocks WHERE pos = ?",
-		-1, &stmt_get_block_exact, NULL))
-
-	SQLOK(prepare_v2(db,
-			"SELECT pos FROM blocks",
-		-1, &stmt_get_block_pos, NULL))
-
-	SQLOK(prepare_v2(db,
-			"SELECT pos FROM blocks WHERE pos BETWEEN ? AND ?",
-		-1, &stmt_get_block_pos_z, NULL))
-}
-
-
-DBSQLite3::~DBSQLite3()
-{
-	sqlite3_finalize(stmt_get_blocks_z);
-	sqlite3_finalize(stmt_get_block_pos);
-	sqlite3_finalize(stmt_get_block_pos_z);
-	sqlite3_finalize(stmt_get_block_exact);
-
-	if (sqlite3_close(db) != SQLITE_OK) {
-		std::cerr << "Error closing SQLite database." << std::endl;
-	};
-}
-
-
-inline void DBSQLite3::getPosRange(int64_t &min, int64_t &max, int16_t zPos,
-		int16_t zPos2) const
-{
-	/* The range of block positions is [-2048, 2047], which turns into [0, 4095]
-	 * when casted to unsigned. This didn't actually help me understand the
-	 * numbers below, but I wanted to write it down.
-	 */
-
-	// Magic numbers!
-	min = encodeBlockPos(BlockPos(0, -2048, zPos));
-	max = encodeBlockPos(BlockPos(0, 2048, zPos2)) - 1;
-}
-
-
-std::vector<BlockPos> DBSQLite3::getBlockPos(BlockPos min, BlockPos max)
-{
-	int result;
-	sqlite3_stmt *stmt;
-
-	if(min.z <= -2048 && max.z >= 2048) {
-		stmt = stmt_get_block_pos;
-	} else {
-		stmt = stmt_get_block_pos_z;
-		int64_t minPos, maxPos;
-		if (min.z < -2048)
-			min.z = -2048;
-		if (max.z > 2048)
-			max.z = 2048;
-		getPosRange(minPos, maxPos, min.z, max.z - 1);
-		SQLOK(bind_int64(stmt, 1, minPos))
-		SQLOK(bind_int64(stmt, 2, maxPos))
-	}
-
-	std::vector<BlockPos> positions;
-	while ((result = sqlite3_step(stmt)) != SQLITE_DONE) {
-		if (result == SQLITE_BUSY) { // Wait some time and try again
-			usleep(10000);
-		} else if (result != SQLITE_ROW) {
-			throw std::runtime_error(sqlite3_errmsg(db));
-		}
-
-		int64_t posHash = sqlite3_column_int64(stmt, 0);
-		BlockPos pos = decodeBlockPos(posHash);
-		if(pos.x >= min.x && pos.x < max.x && pos.y >= min.y && pos.y < max.y)
-			positions.emplace_back(pos);
-	}
-	SQLOK(reset(stmt));
-	return positions;
-}
-
-
-void DBSQLite3::loadBlockCache(int16_t zPos)
-{
-	int result;
-	blockCache.clear();
-
-	int64_t minPos, maxPos;
-	getPosRange(minPos, maxPos, zPos, zPos);
-
-	SQLOK(bind_int64(stmt_get_blocks_z, 1, minPos));
-	SQLOK(bind_int64(stmt_get_blocks_z, 2, maxPos));
-
-	while ((result = sqlite3_step(stmt_get_blocks_z)) != SQLITE_DONE) {
-		if (result == SQLITE_BUSY) { // Wait some time and try again
-			usleep(10000);
-		} else if (result != SQLITE_ROW) {
-			throw std::runtime_error(sqlite3_errmsg(db));
-		}
-
-		int64_t posHash = sqlite3_column_int64(stmt_get_blocks_z, 0);
-		BlockPos pos = decodeBlockPos(posHash);
-		const unsigned char *data = reinterpret_cast<const unsigned char *>(
-				sqlite3_column_blob(stmt_get_blocks_z, 1));
-		size_t size = sqlite3_column_bytes(stmt_get_blocks_z, 1);
-		blockCache[pos.x].emplace_back(pos, ustring(data, size));
-	}
-	SQLOK(reset(stmt_get_blocks_z))
-}
-
-
-void DBSQLite3::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
-		int16_t min_y, int16_t max_y)
-{
-	/* Cache the blocks on the given Z coordinate between calls, this only
-	 * works due to order in which the TileGenerator asks for blocks. */
-	if (z != blockCachedZ) {
-		loadBlockCache(z);
-		blockCachedZ = z;
-	}
-
-	auto it = blockCache.find(x);
-	if (it == blockCache.end())
-		return;
-
-	if (it->second.empty()) {
-		/* We have swapped this list before, this is not supposed to happen
-		 * because it's bad for performance. But rather than silently breaking
-		 * do the right thing and load the blocks again. */
-#ifndef NDEBUG
-		std::cerr << "Warning: suboptimal access pattern for sqlite3 backend" << std::endl;
-#endif
-		loadBlockCache(z);
-	}
-	// Swap lists to avoid copying contents
-	blocks.clear();
-	std::swap(blocks, it->second);
-
-	for (auto it = blocks.begin(); it != blocks.end(); ) {
-		if (it->first.y < min_y || it->first.y >= max_y)
-			it = blocks.erase(it);
-		else
-			it++;
-	}
-}
-
-
-void DBSQLite3::getBlocksByPos(BlockList &blocks,
-			const std::vector<BlockPos> &positions)
-{
-	int result;
-
-	for (auto pos : positions) {
-		int64_t dbPos = encodeBlockPos(pos);
-		SQLOK(bind_int64(stmt_get_block_exact, 1, dbPos));
-
-		while ((result = sqlite3_step(stmt_get_block_exact)) == SQLITE_BUSY) {
-			usleep(10000); // Wait some time and try again
-		}
-		if (result == SQLITE_DONE) {
-			// no data
-		} else if (result != SQLITE_ROW) {
-			throw std::runtime_error(sqlite3_errmsg(db));
-		} else {
-			const unsigned char *data = reinterpret_cast<const unsigned char *>(
-					sqlite3_column_blob(stmt_get_block_exact, 0));
-			size_t size = sqlite3_column_bytes(stmt_get_block_exact, 0);
-			blocks.emplace_back(pos, ustring(data, size));
-		}
-
-		SQLOK(reset(stmt_get_block_exact))
-	}
-}

include/config.h

@@ -1,11 +0,0 @@
-#if MSDOS || __OS2__ || __NT__ || _WIN32
-#define PATH_SEPARATOR '\\'
-#else
-#define PATH_SEPARATOR '/'
-#endif
-
-#ifdef USE_CMAKE_CONFIG_H
-#include "cmake_config.h"
-#else
-#error missing config
-#endif

include/db-sqlite3.h

@@ -1,33 +0,0 @@
-#pragma once
-
-#include "db.h"
-#include <unordered_map>
-#include <sqlite3.h>
-
-class DBSQLite3 : public DB {
-public:
-	DBSQLite3(const std::string &mapdir);
-	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
-	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
-			int16_t min_y, int16_t max_y) override;
-	void getBlocksByPos(BlockList &blocks,
-			const std::vector<BlockPos> &positions) override;
-	~DBSQLite3() override;
-
-	bool preferRangeQueries() const override { return false; }
-
-private:
-	inline void getPosRange(int64_t &min, int64_t &max, int16_t zPos,
-			int16_t zPos2) const;
-	void loadBlockCache(int16_t zPos);
-
-	sqlite3 *db;
-
-	sqlite3_stmt *stmt_get_block_pos;
-	sqlite3_stmt *stmt_get_block_pos_z;
-	sqlite3_stmt *stmt_get_blocks_z;
-	sqlite3_stmt *stmt_get_block_exact;
-
-	int16_t blockCachedZ = -10000;
-	std::unordered_map<int16_t, BlockList> blockCache; // indexed by X
-};

include/types.h

@@ -1,5 +0,0 @@
-#include <string>
-
-typedef std::basic_string<unsigned char> ustring;
-typedef unsigned int uint;
-typedef unsigned char u8;

minetestmapper.6

@@ -9,16 +9,22 @@ minetestmapper \- generate an overview image of a Luanti map
 See additional optional parameters below.
 .SH DESCRIPTION
 .B minetestmapper
-generates an overview image of a Luanti map. This is a port of
+generates a top-down overview image of a Luanti map.
-the original minetestmapper.py to C++, that is both faster and
+This is a port of the obsolete minetestmapper.py script to C++,
-provides more functionality than the obsolete Python script.
+that is both faster and provides more features.
+
+Minetestmapper ships with a colors.txt file suitable for Minetest Game,
+if you use a different game or have mods installed you should generate a
+matching colors.txt for better results (colors will be missing otherwise).
+
 .SH MANDATORY PARAMETERS
 .TP
 .BR \-i " " \fIworld_path\fR
-Input world path.
+Input world path
 .TP
 .BR \-o " " \fIoutput_image\fR
-Path to output image. (only PNG supported currently)
+Path to output image
+
 .SH OPTIONAL PARAMETERS
 .TP
 .BR \-\-bgcolor " " \fIcolor\fR
@@ -26,7 +32,7 @@ Background color of image, e.g. "--bgcolor #ffffff"
 
 .TP
 .BR \-\-scalecolor " " \fIcolor\fR
-Color of scale, e.g. "--scalecolor #000000"
+Color of scale marks and text, e.g. "--scalecolor #000000"
 
 .TP
 .BR \-\-playercolor " " \fIcolor\fR
@@ -38,11 +44,11 @@ Color of origin indicator, e.g. "--origincolor #ff0000"
 
 .TP
 .BR \-\-drawscale
-Draw tick marks
+Draw scale(s) with tick marks and numbers
 
 .TP
 .BR \-\-drawplayers
-Draw player indicators
+Draw player indicators with name
 
 .TP
 .BR \-\-draworigin
@@ -50,7 +56,7 @@ Draw origin indicator
 
 .TP
 .BR \-\-drawalpha
-Allow nodes to be drawn with transparency
+Allow nodes to be drawn with transparency (such as water)
 
 .TP
 .BR \-\-noshading
@@ -58,23 +64,29 @@ Don't draw shading on nodes
 
 .TP
 .BR \-\-noemptyimage
-Don't output anything when the image would be empty.
+Don't output anything when the image would be empty
+
+.TP
+.BR \-\-verbose
+Enable verbose log output.
 
 .TP
 .BR \-\-min-y " " \fInumber\fR
-Don't draw nodes below this y value, e.g. "--min-y -25"
+Don't draw nodes below this Y value, e.g. "--min-y -25"
 
 .TP
 .BR \-\-max-y " " \fInumber\fR
-Don't draw nodes above this y value, e.g. "--max-y 75"
+Don't draw nodes above this Y value, e.g. "--max-y 75"
 
 .TP
 .BR \-\-backend " " \fIbackend\fR
-Use specific map backend; supported: \fIsqlite3\fP, \fIleveldb\fP, \fIredis\fP, \fIpostgresql\fP, e.g. "--backend leveldb"
+Override auto-detected map backend; supported: \fIsqlite3\fP, \fIleveldb\fP, \fIredis\fP, \fIpostgresql\fP, e.g. "--backend leveldb"
 
 .TP
 .BR \-\-geometry " " \fIgeometry\fR
-Limit area to specific geometry (\fIx:y+w+h\fP where x and y specify the lower left corner), e.g. "--geometry -800:-800+1600+1600"
+Limit area to specific geometry (\fIx:z+w+h\fP where x and z specify the lower left corner), e.g. "--geometry -800:-800+1600+1600"
+
+The coordinates are specified with the same axes as in-game. The Z axis becomes Y when projected on the image.
 
 .TP
 .BR \-\-extent
@@ -86,7 +98,7 @@ Zoom the image by using more than one pixel per node, e.g. "--zoom 4"
 
 .TP
 .BR \-\-colors " " \fIpath\fR
-Forcefully set path to colors.txt file (autodetected otherwise), e.g. "--colors ../world/mycolors.txt"
+Override auto-detected path to colors.txt, e.g. "--colors ../world/mycolors.txt"
 
 .TP
 .BR \-\-scales " " \fIedges\fR
@@ -96,19 +108,14 @@ Draw scales on specified image edges (letters \fIt b l r\fP meaning top, bottom,
 .BR \-\-exhaustive " " \fImode\fR
 Select if database should be traversed exhaustively or using range queries, available: \fInever\fP, \fIy\fP, \fIfull\fP, \fIauto\fP
 
-Defaults to \fIauto\fP. You shouldn't need to change this, but doing so can improve rendering times on large maps.
+Defaults to \fIauto\fP. You shouldn't need to change this, as minetestmapper tries to automatically picks the best option.
-For these optimizations to work it is important that you set
-.B min-y
-and
-.B max-y
-when you don't care about the world below e.g. -60 and above 1000 nodes.
 
 .TP
 .BR \-\-dumpblock " " \fIpos\fR
 Instead of rendering anything try to load the block at the given position (\fIx,y,z\fR) and print its raw data as hexadecimal.
 
 .SH MORE INFORMATION
-Website: https://github.com/minetest/minetestmapper
+Website: https://github.com/luanti-org/minetestmapper
 
 .SH MAN PAGE AUTHOR
 Daniel Moerner

src/BlockDecoder.cpp

@@ -4,6 +4,7 @@
 
 #include "BlockDecoder.h"
 #include "ZlibDecompressor.h"
+#include "log.h"
 
 static inline uint16_t readU16(const unsigned char *data)
 {
@@ -55,13 +56,12 @@ void BlockDecoder::decode(const ustring &datastr)
 	}
 	m_version = version;
 
-	ustring datastr2;
 	if (version >= 29) {
 		// decompress whole block at once
 		m_zstd_decompressor.setData(data, length, 1);
-		datastr2 = m_zstd_decompressor.decompress();
+		m_zstd_decompressor.decompress(m_scratch);
-		data = datastr2.c_str();
+		data = m_scratch.c_str();
-		length = datastr2.size();
+		length = m_scratch.size();
 	}
 
 	size_t dataOffset = 0;
@@ -106,16 +106,16 @@ void BlockDecoder::decode(const ustring &datastr)
 	m_contentWidth = contentWidth;
 
 	if (version >= 29) {
-		m_mapData.resize((contentWidth + paramsWidth) * 4096);
+		size_t mapDataSize = (contentWidth + paramsWidth) * 4096;
-		m_mapData.assign(data + dataOffset, m_mapData.size());
+		m_mapData.assign(data + dataOffset, mapDataSize);
 		return; // we have read everything we need and can return early
 	}
 
 	// version < 29
 	ZlibDecompressor decompressor(data, length);
 	decompressor.setSeekPos(dataOffset);
-	m_mapData = decompressor.decompress();
+	decompressor.decompress(m_mapData);
-	decompressor.decompress(); // unused metadata
+	decompressor.decompress(m_scratch); // unused metadata
 	dataOffset = decompressor.seekPos();
 
 	// Skip unused node timers
@@ -161,7 +161,7 @@ const std::string &BlockDecoder::getNode(u8 x, u8 y, u8 z) const
 		return empty;
 	NameMap::const_iterator it = m_nameMap.find(content);
 	if (it == m_nameMap.end()) {
-		std::cerr << "Skipping node with invalid ID." << std::endl;
+		errorstream << "Skipping node with invalid ID." << std::endl;
 		return empty;
 	}
 	return it->second;

src/BlockDecoder.h

@@ -23,6 +23,7 @@ private:
 	u8 m_version, m_contentWidth;
 	ustring m_mapData;
 
-	// one instance for performance
+	// cached allocations/instances for performance
 	ZstdDecompressor m_zstd_decompressor;
+	ustring m_scratch;
 };

src/PlayerAttributes.cpp

@@ -0,0 +1,159 @@
+#include <fstream>
+#include <sstream>
+#include <stdexcept>
+#include <dirent.h>
+#include <unistd.h> // usleep
+
+#include "config.h"
+#include "PlayerAttributes.h"
+#include "util.h"
+#include "log.h"
+#include "db-sqlite3.h" // SQLite3Base
+
+namespace {
+	bool parse_pos(std::string position, Player &dst)
+	{
+		if (position.empty())
+			return false;
+		if (position.front() == '(' && position.back() == ')')
+			position = position.substr(1, position.size() - 2);
+		std::istringstream iss(position);
+		if (!(iss >> dst.x))
+			return false;
+		if (iss.get() != ',')
+			return false;
+		if (!(iss >> dst.y))
+			return false;
+		if (iss.get() != ',')
+			return false;
+		if (!(iss >> dst.z))
+			return false;
+		return iss.eof();
+	}
+
+	// Helper classes per backend
+
+	class FilesReader {
+		std::string path;
+		DIR *dir = nullptr;
+	public:
+		FilesReader(const std::string &path) : path(path) {
+			dir = opendir(path.c_str());
+		}
+		~FilesReader() {
+			if (dir)
+				closedir(dir);
+		}
+
+		void read(PlayerAttributes::Players &dest);
+	};
+
+	class SQLiteReader : SQLite3Base {
+		sqlite3_stmt *stmt_get_player_pos = NULL;
+	public:
+		SQLiteReader(const std::string &database) {
+			openDatabase(database.c_str());
+		}
+		~SQLiteReader() {
+			sqlite3_finalize(stmt_get_player_pos);
+		}
+
+		void read(PlayerAttributes::Players &dest);
+	};
+}
+
+void FilesReader::read(PlayerAttributes::Players &dest)
+{
+	if (!dir)
+		return;
+
+	struct dirent *ent;
+	std::string name, position;
+	while ((ent = readdir(dir)) != NULL) {
+		if (ent->d_name[0] == '.')
+			continue;
+
+		std::ifstream in(path + PATH_SEPARATOR + ent->d_name);
+		if (!in.good())
+			continue;
+
+		name = read_setting("name", in);
+		position = read_setting("position", in);
+
+		Player player;
+		player.name = name;
+		if (!parse_pos(position, player)) {
+			errorstream << "Failed to parse position '" << position << "' in "
+				<< ent->d_name << std::endl;
+			continue;
+		}
+
+		player.x /= 10.0f;
+		player.y /= 10.0f;
+		player.z /= 10.0f;
+
+		dest.push_back(std::move(player));
+	}
+}
+
+#define SQLRES(r, good) check_result(r, good)
+#define SQLOK(r) SQLRES(r, SQLITE_OK)
+
+void SQLiteReader::read(PlayerAttributes::Players &dest)
+{
+	SQLOK(prepare(stmt_get_player_pos,
+			"SELECT name, posX, posY, posZ FROM player"));
+
+	int result;
+	while ((result = sqlite3_step(stmt_get_player_pos)) != SQLITE_DONE) {
+		if (result == SQLITE_BUSY) { // Wait some time and try again
+			usleep(10000);
+		} else if (result != SQLITE_ROW) {
+			throw std::runtime_error(sqlite3_errmsg(db));
+		}
+
+		Player player;
+		player.name = read_str(stmt_get_player_pos, 0);
+		player.x = sqlite3_column_double(stmt_get_player_pos, 1);
+		player.y = sqlite3_column_double(stmt_get_player_pos, 2);
+		player.z = sqlite3_column_double(stmt_get_player_pos, 3);
+
+		player.x /= 10.0f;
+		player.y /= 10.0f;
+		player.z /= 10.0f;
+
+		dest.push_back(std::move(player));
+	}
+}
+
+/**********/
+
+PlayerAttributes::PlayerAttributes(const std::string &worldDir)
+{
+	std::ifstream ifs(worldDir + "world.mt");
+	if (!ifs.good())
+		throw std::runtime_error("Failed to read world.mt");
+	std::string backend = read_setting_default("player_backend", ifs, "files");
+	ifs.close();
+
+	verbosestream << "Player backend: " << backend << std::endl;
+	if (backend == "files")
+		FilesReader(worldDir + "players").read(m_players);
+	else if (backend == "sqlite3")
+		SQLiteReader(worldDir + "players.sqlite").read(m_players);
+	else
+		throw std::runtime_error(std::string("Unknown player backend: ") + backend);
+
+	verbosestream << "Loaded " << m_players.size() << " players" << std::endl;
+}
+
+PlayerAttributes::Players::const_iterator PlayerAttributes::begin() const
+{
+	return m_players.cbegin();
+}
+
+PlayerAttributes::Players::const_iterator PlayerAttributes::end() const
+{
+	return m_players.cend();
+}
+

src/PlayerAttributes.h

@@ -19,8 +19,5 @@ public:
 	Players::const_iterator end() const;
 
 private:
-	void readFiles(const std::string &playersPath);
-	void readSqlite(const std::string &db_name);
-
 	Players m_players;
 };

src/TileGenerator.cpp

@@ -17,6 +17,7 @@
 #include "BlockDecoder.h"
 #include "Image.h"
 #include "util.h"
+#include "log.h"
 
 #include "db-sqlite3.h"
 #if USE_POSTGRESQL
@@ -104,8 +105,8 @@ static Color parseColor(const std::string &color)
 static Color mixColors(Color a, Color b)
 {
 	Color result;
-	double a1 = a.a / 255.0;
+	float a1 = a.a / 255.0f;
-	double a2 = b.a / 255.0;
+	float a2 = b.a / 255.0f;
 
 	result.r = (int) (a1 * a.r + a2 * (1 - a1) * b.r);
 	result.g = (int) (a1 * a.g + a2 * (1 - a1) * b.g);
@@ -153,6 +154,8 @@ TileGenerator::TileGenerator():
 TileGenerator::~TileGenerator()
 {
 	closeDatabase();
+	delete m_image;
+	m_image = nullptr;
 }
 
 void TileGenerator::setBgColor(const std::string &bgColor)
@@ -255,6 +258,7 @@ void TileGenerator::parseColorsFile(const std::string &fileName)
 	std::ifstream in(fileName);
 	if (!in.good())
 		throw std::runtime_error("Specified colors file could not be found");
+	verbosestream << "Parsing colors.txt: " << fileName << std::endl;
 	parseColorsStream(in);
 }
 
@@ -293,19 +297,26 @@ void TileGenerator::dumpBlock(const std::string &input_path, BlockPos pos)
 
 void TileGenerator::generate(const std::string &input_path, const std::string &output)
 {
-	if (m_dontWriteEmpty) // FIXME: possible too, just needs to be done differently
-		setExhaustiveSearch(EXH_NEVER);
 	openDb(input_path);
 	loadBlocks();
 
-	if (m_dontWriteEmpty && m_positions.empty())
+	// If we needed to load positions and there are none, that means the
-	{
+	// result will be empty.
-		closeDatabase();
+	if (m_dontWriteEmpty && (m_exhaustiveSearch == EXH_NEVER ||
+		m_exhaustiveSearch == EXH_Y) && m_positions.empty()) {
+		verbosestream << "Result is empty (no positions)" << std::endl;
 		return;
 	}
 
 	createImage();
 	renderMap();
+
+	if (m_dontWriteEmpty && !m_renderedAny) {
+		verbosestream << "Result is empty (no pixels)" << std::endl;
+		printUnknown();
+		return;
+	}
+
 	closeDatabase();
 	if (m_drawScale) {
 		renderScale();
@@ -339,7 +350,7 @@ void TileGenerator::parseColorsStream(std::istream &in)
 		unsigned int r, g, b, a = 255, t = 0;
 		int items = sscanf(line, "%200s %u %u %u %u %u", name, &r, &g, &b, &a, &t);
 		if (items < 4) {
-			std::cerr << "Failed to parse color entry '" << line << "'" << std::endl;
+			errorstream << "Failed to parse color entry '" << line << "'" << std::endl;
 			continue;
 		}
 
@@ -365,45 +376,59 @@ std::set<std::string> TileGenerator::getSupportedBackends()
 
 void TileGenerator::openDb(const std::string &input_path)
 {
+	if (dir_exists(input_path.c_str())) {
+		// ok
+	} else if (file_exists(input_path.c_str())) {
+		throw std::runtime_error("Input path is a file, it should point to the world folder instead");
+	} else {
+		throw std::runtime_error("Input path does not exist");
+	}
+
 	std::string input = input_path;
 	if (input.back() != PATH_SEPARATOR)
 		input += PATH_SEPARATOR;
 
-	std::string backend = m_backend;
-	if (backend.empty()) {
 	std::ifstream ifs(input + "world.mt");
-		if(!ifs.good())
+
+	std::string backend = m_backend;
+	if (backend.empty() && !ifs.good()) {
 		throw std::runtime_error("Failed to open world.mt");
+	} else if (backend.empty()) {
 		backend = read_setting_default("backend", ifs, "sqlite3");
-		ifs.close();
 	}
 
-	if (backend == "sqlite3")
+	if (backend == "dummy") {
+		throw std::runtime_error("This map uses the dummy backend and contains no data");
+	} else if (backend == "sqlite3") {
 		m_db = new DBSQLite3(input);
 #if USE_POSTGRESQL
-	else if (backend == "postgresql")
+	} else if (backend == "postgresql") {
 		m_db = new DBPostgreSQL(input);
 #endif
 #if USE_LEVELDB
-	else if (backend == "leveldb")
+	} else if (backend == "leveldb") {
 		m_db = new DBLevelDB(input);
 #endif
 #if USE_REDIS
-	else if (backend == "redis")
+	} else if (backend == "redis") {
 		m_db = new DBRedis(input);
 #endif
-	else
+	} else {
 		throw std::runtime_error(std::string("Unknown map backend: ") + backend);
+	}
+
+	if (!read_setting_default("readonly_backend", ifs, "").empty()) {
+		errorstream << "Warning: Map with readonly_backend is not supported. "
+			"The result may be incomplete." << std::endl;
+	}
 
 	// Determine how we're going to traverse the database (heuristic)
 	if (m_exhaustiveSearch == EXH_AUTO) {
 		size_t y_range = (m_yMax / 16 + 1) - (m_yMin / 16);
 		size_t blocks = sat_mul<size_t>(m_geomX2 - m_geomX, y_range, m_geomY2 - m_geomY);
-#ifndef NDEBUG
+		verbosestream << "Heuristic parameters:"
-		std::cerr << "Heuristic parameters:"
 			<< " preferRangeQueries()=" << m_db->preferRangeQueries()
 			<< " y_range=" << y_range << " blocks=" << blocks << std::endl;
-#endif
 		if (m_db->preferRangeQueries())
 			m_exhaustiveSearch = EXH_NEVER;
 		else if (blocks < 200000)
@@ -414,9 +439,9 @@ void TileGenerator::openDb(const std::string &input_path)
 			m_exhaustiveSearch = EXH_NEVER;
 	} else if (m_exhaustiveSearch == EXH_FULL || m_exhaustiveSearch == EXH_Y) {
 		if (m_db->preferRangeQueries()) {
-			std::cerr << "Note: The current database backend supports efficient "
+			errorstream << "Note: The current database backend supports efficient "
-				"range queries, forcing exhaustive search should always result "
+				"range queries, forcing exhaustive search will generally result "
-				" in worse performance." << std::endl;
+				"in worse performance." << std::endl;
 		}
 	}
 	assert(m_exhaustiveSearch != EXH_AUTO);
@@ -441,26 +466,20 @@ void TileGenerator::loadBlocks()
 	const int16_t yMin = mod16(m_yMin);
 
 	if (m_exhaustiveSearch == EXH_NEVER || m_exhaustiveSearch == EXH_Y) {
-		std::vector<BlockPos> vec = m_db->getBlockPos(
+		std::vector<BlockPos> vec = m_db->getBlockPosXZ(
 			BlockPos(m_geomX, yMin, m_geomY),
 			BlockPos(m_geomX2, yMax, m_geomY2)
 		);
 
 		for (auto pos : vec) {
 			assert(pos.x >= m_geomX && pos.x < m_geomX2);
-			assert(pos.y >= yMin && pos.y < yMax);
 			assert(pos.z >= m_geomY && pos.z < m_geomY2);
 
 			// Adjust minimum and maximum positions to the nearest block
-			if (pos.x < m_xMin)
+			m_xMin = mymin<int>(m_xMin, pos.x);
-				m_xMin = pos.x;
+			m_xMax = mymax<int>(m_xMax, pos.x);
-			if (pos.x > m_xMax)
+			m_zMin = mymin<int>(m_zMin, pos.z);
-				m_xMax = pos.x;
+			m_zMax = mymax<int>(m_zMax, pos.z);
-
-			if (pos.z < m_zMin)
-				m_zMin = pos.z;
-			if (pos.z > m_zMax)
-				m_zMax = pos.z;
 
 			m_positions[pos.z].emplace(pos.x);
 		}
@@ -469,10 +488,8 @@ void TileGenerator::loadBlocks()
 		for (const auto &it : m_positions)
 			count += it.second.size();
 		m_progressMax = count;
-#ifndef NDEBUG
+		verbosestream << "Loaded " << count
-		std::cerr << "Loaded " << count
 			<< " positions (across Z: " << m_positions.size() << ") for rendering" << std::endl;
-#endif
 	}
 }
 
@@ -511,8 +528,11 @@ void TileGenerator::createImage()
 	image_height += (m_scales & SCALE_BOTTOM) ? scale_d : 0;
 
 	if(image_width > 4096 || image_height > 4096) {
-		std::cerr << "Warning: The width or height of the image to be created exceeds 4096 pixels!"
+		errorstream << "Warning: The side length of the image to be created exceeds 4096 pixels!"
-			<< " (Dimensions: " << image_width << "x" << image_height << ")"
+			<< " (dimensions: " << image_width << "x" << image_height << ")"
+			<< std::endl;
+	} else {
+		verbosestream << "Creating image with size " << image_width << "x" << image_height
 			<< std::endl;
 	}
 	m_image = new Image(image_width, image_height);
@@ -577,10 +597,8 @@ void TileGenerator::renderMap()
 			postRenderRow(zPos);
 		}
 	} else if (m_exhaustiveSearch == EXH_Y) {
-#ifndef NDEBUG
+		verbosestream << "Exhaustively searching height of "
-		std::cerr << "Exhaustively searching height of "
 			<< (yMax - yMin) << " blocks" << std::endl;
-#endif
 		std::vector<BlockPos> positions;
 		positions.reserve(yMax - yMin);
 		for (auto it = m_positions.rbegin(); it != m_positions.rend(); ++it) {
@@ -604,11 +622,9 @@ void TileGenerator::renderMap()
 	} else if (m_exhaustiveSearch == EXH_FULL) {
 		const size_t span_y = yMax - yMin;
 		m_progressMax = (m_geomX2 - m_geomX) * span_y * (m_geomY2 - m_geomY);
-#ifndef NDEBUG
+		verbosestream << "Exhaustively searching "
-		std::cerr << "Exhaustively searching "
 			<< (m_geomX2 - m_geomX) << "x" << span_y << "x"
 			<< (m_geomY2 - m_geomY) << " blocks" << std::endl;
-#endif
 
 		std::vector<BlockPos> positions;
 		positions.reserve(span_y);
@@ -836,8 +852,8 @@ void TileGenerator::renderPlayers(const std::string &input_path)
 
 	PlayerAttributes players(input);
 	for (auto &player : players) {
-		if (player.x < m_xMin * 16 || player.x > m_xMax * 16 ||
+		if (player.x < m_xMin * 16 || player.x >= (m_xMax+1) * 16 ||
-			player.z < m_zMin * 16 || player.z > m_zMax * 16)
+			player.z < m_zMin * 16 || player.z >= (m_zMax+1) * 16)
 			continue;
 		if (player.y < m_yMin || player.y > m_yMax)
 			continue;
@@ -846,6 +862,7 @@ void TileGenerator::renderPlayers(const std::string &input_path)
 
 		m_image->drawFilledRect(imageX - 1, imageY, 3, 1, m_playerColor);
 		m_image->drawFilledRect(imageX, imageY - 1, 1, 3, m_playerColor);
+		assert(!player.name.empty());
 		m_image->drawText(imageX + 2, imageY, player.name, m_playerColor);
 	}
 }
@@ -861,14 +878,15 @@ void TileGenerator::printUnknown()
 {
 	if (m_unknownNodes.empty())
 		return;
-	std::cerr << "Unknown nodes:" << std::endl;
+	errorstream << "Unknown nodes:\n";
 	for (const auto &node : m_unknownNodes)
-		std::cerr << "\t" << node << std::endl;
+		errorstream << "\t" << node << '\n';
 	if (!m_renderedAny) {
-		std::cerr << "The map was read successfully and not empty, but none of the "
+		errorstream << "The map was read successfully and not empty, but none of the "
 			"encountered nodes had a color associated.\nCheck that you're using "
-			"the right colors.txt. It should match the game you have installed." << std::endl;
+			"the right colors.txt. It should match the game you have installed.\n";
 	}
+	errorstream << std::flush;
 }
 
 void TileGenerator::reportProgress(size_t count)

src/ZlibDecompressor.cpp

@@ -1,6 +1,16 @@
-#include <zlib.h>
+#include <cstdint>
-#include <stdint.h>
 #include "ZlibDecompressor.h"
+#include "config.h"
+
+// for convenient usage of both
+#if USE_ZLIB_NG
+#include <zlib-ng.h>
+#define z_stream zng_stream
+#define Z(x) zng_ ## x
+#else
+#include <zlib.h>
+#define Z(x) x
+#endif
 
 ZlibDecompressor::ZlibDecompressor(const u8 *data, size_t size):
 	m_data(data),
@@ -18,18 +28,13 @@ void ZlibDecompressor::setSeekPos(size_t seekPos)
 	m_seekPos = seekPos;
 }
 
-size_t ZlibDecompressor::seekPos() const
+void ZlibDecompressor::decompress(ustring &buffer)
-{
-	return m_seekPos;
-}
-
-ustring ZlibDecompressor::decompress()
 {
 	const unsigned char *data = m_data + m_seekPos;
 	const size_t size = m_size - m_seekPos;
 
-	ustring buffer;
+	// output space is extended in chunks of this size
-	constexpr size_t BUFSIZE = 32 * 1024;
+	constexpr size_t BUFSIZE = 8 * 1024;
 
 	z_stream strm;
 	strm.zalloc = Z_NULL;
@@ -38,21 +43,22 @@ ustring ZlibDecompressor::decompress()
 	strm.next_in = Z_NULL;
 	strm.avail_in = 0;
 
-	if (inflateInit(&strm) != Z_OK)
+	if (Z(inflateInit)(&strm) != Z_OK)
 		throw DecompressError();
 
 	strm.next_in = const_cast<unsigned char *>(data);
 	strm.avail_in = size;
+	if (buffer.empty())
 		buffer.resize(BUFSIZE);
 	strm.next_out = &buffer[0];
-	strm.avail_out = BUFSIZE;
+	strm.avail_out = buffer.size();
 
 	int ret = 0;
 	do {
-		ret = inflate(&strm, Z_NO_FLUSH);
+		ret = Z(inflate)(&strm, Z_NO_FLUSH);
 		if (strm.avail_out == 0) {
 			const auto off = buffer.size();
-			buffer.reserve(off + BUFSIZE);
+			buffer.resize(off + BUFSIZE);
 			strm.next_out = &buffer[off];
 			strm.avail_out = BUFSIZE;
 		}
@@ -62,8 +68,6 @@ ustring ZlibDecompressor::decompress()
 
 	m_seekPos += strm.next_in - data;
 	buffer.resize(buffer.size() - strm.avail_out);
-	(void) inflateEnd(&strm);
+	(void) Z(inflateEnd)(&strm);
-
-	return buffer;
 }
 

src/ZlibDecompressor.h

@@ -11,8 +11,10 @@ public:
 	ZlibDecompressor(const u8 *data, size_t size);
 	~ZlibDecompressor();
 	void setSeekPos(size_t seekPos);
-	size_t seekPos() const;
+	size_t seekPos() const { return m_seekPos; }
-	ustring decompress();
+	// Decompress and return one zlib stream from the buffer
+	// Advances seekPos as appropriate.
+	void decompress(ustring &dst);
 
 private:
 	const u8 *m_data;

src/ZstdDecompressor.cpp

@@ -21,19 +21,15 @@ void ZstdDecompressor::setData(const u8 *data, size_t size, size_t seekPos)
 	m_size = size;
 }
 
-std::size_t ZstdDecompressor::seekPos() const
+void ZstdDecompressor::decompress(ustring &buffer)
-{
-	return m_seekPos;
-}
-
-ustring ZstdDecompressor::decompress()
 {
 	ZSTD_DStream *stream = reinterpret_cast<ZSTD_DStream*>(m_stream);
 	ZSTD_inBuffer inbuf = { m_data, m_size, m_seekPos };
 
-	ustring buffer;
+	// output space is extended in chunks of this size
-	constexpr size_t BUFSIZE = 32 * 1024;
+	constexpr size_t BUFSIZE = 8 * 1024;
 
+	if (buffer.empty())
 		buffer.resize(BUFSIZE);
 	ZSTD_outBuffer outbuf = { &buffer[0], buffer.size(), 0 };
 
@@ -42,17 +38,15 @@ ustring ZstdDecompressor::decompress()
 	size_t ret;
 	do {
 		ret = ZSTD_decompressStream(stream, &outbuf, &inbuf);
+		if (ret && ZSTD_isError(ret))
+			throw DecompressError();
 		if (outbuf.size == outbuf.pos) {
 			outbuf.size += BUFSIZE;
 			buffer.resize(outbuf.size);
 			outbuf.dst = &buffer[0];
 		}
-		if (ret && ZSTD_isError(ret))
-			throw DecompressError();
 	} while (ret != 0);
 
 	m_seekPos = inbuf.pos;
 	buffer.resize(outbuf.pos);
-
-	return buffer;
 }

src/ZstdDecompressor.h

@@ -11,8 +11,10 @@ public:
 	ZstdDecompressor();
 	~ZstdDecompressor();
 	void setData(const u8 *data, size_t size, size_t seekPos);
-	size_t seekPos() const;
+	size_t seekPos() const { return m_seekPos; }
-	ustring decompress();
+	// Decompress and return one zstd stream from the buffer
+	// Advances seekPos as appropriate.
+	void decompress(ustring &dst);
 
 private:
 	void *m_stream; // ZSTD_DStream

src/cmake_config.h.in

@@ -6,6 +6,7 @@
 #cmakedefine01 USE_POSTGRESQL
 #cmakedefine01 USE_LEVELDB
 #cmakedefine01 USE_REDIS
+#cmakedefine01 USE_ZLIB_NG
 
 #define SHAREDIR "@SHAREDIR@"
 

src/config.h

@@ -0,0 +1,7 @@
+#if defined(MSDOS) || defined(__OS2__) || defined(__NT__) || defined(_WIN32)
+#define PATH_SEPARATOR '\\'
+#else
+#define PATH_SEPARATOR '/'
+#endif
+
+#include "cmake_config.h"

src/db-leveldb.cpp

@@ -1,5 +1,6 @@
 #include <stdexcept>
 #include <sstream>
+#include <algorithm>
 #include "db-leveldb.h"
 #include "types.h"
 
@@ -18,6 +19,12 @@ static inline std::string i64tos(int64_t i)
 	return os.str();
 }
 
+// finds the first position in the list where it.x >= x
+#define lower_bound_x(container, find_x) \
+	std::lower_bound((container).begin(), (container).end(), (find_x), \
+		[] (const vec2 &left, int16_t right) { \
+			return left.x < right; \
+	})
 
 DBLevelDB::DBLevelDB(const std::string &mapdir)
 {
@@ -25,7 +32,7 @@ DBLevelDB::DBLevelDB(const std::string &mapdir)
 	options.create_if_missing = false;
 	leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &db);
 	if (!status.ok()) {
-		throw std::runtime_error(std::string("Failed to open Database: ") + status.ToString());
+		throw std::runtime_error(std::string("Failed to open database: ") + status.ToString());
 	}
 
 	/* LevelDB is a dumb key-value store, so the only optimization we can do
@@ -41,18 +48,24 @@ DBLevelDB::~DBLevelDB()
 }
 
 
-std::vector<BlockPos> DBLevelDB::getBlockPos(BlockPos min, BlockPos max)
+std::vector<BlockPos> DBLevelDB::getBlockPosXZ(BlockPos min, BlockPos max)
 {
 	std::vector<BlockPos> res;
 	for (const auto &it : posCache) {
-		if (it.first < min.z || it.first >= max.z)
+		const int16_t zpos = it.first;
+		if (zpos < min.z || zpos >= max.z)
 			continue;
-		for (auto pos2 : it.second) {
+		auto it2 = lower_bound_x(it.second, min.x);
-			if (pos2.first < min.x || pos2.first >= max.x)
+		for (; it2 != it.second.end(); it2++) {
+			const auto &pos2 = *it2;
+			if (pos2.x >= max.x)
+				break; // went past
+			if (pos2.y < min.y || pos2.y >= max.y)
 				continue;
-			if (pos2.second < min.y || pos2.second >= max.y)
+			// skip duplicates
+			if (!res.empty() && res.back().x == pos2.x && res.back().z == zpos)
 				continue;
-			res.emplace_back(pos2.first, pos2.second, it.first);
+			res.emplace_back(pos2.x, pos2.y, zpos);
 		}
 	}
 	return res;
@@ -61,7 +74,7 @@ std::vector<BlockPos> DBLevelDB::getBlockPos(BlockPos min, BlockPos max)
 
 void DBLevelDB::loadPosCache()
 {
-	leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions());
+	leveldb::Iterator *it = db->NewIterator(leveldb::ReadOptions());
 	for (it->SeekToFirst(); it->Valid(); it->Next()) {
 		int64_t posHash = stoi64(it->key().ToString());
 		BlockPos pos = decodeBlockPos(posHash);
@@ -69,6 +82,9 @@ void DBLevelDB::loadPosCache()
 		posCache[pos.z].emplace_back(pos.x, pos.y);
 	}
 	delete it;
+
+	for (auto &it : posCache)
+		std::sort(it.second.begin(), it.second.end());
 }
 
 
@@ -81,13 +97,18 @@ void DBLevelDB::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 	auto it = posCache.find(z);
 	if (it == posCache.cend())
 		return;
-	for (auto pos2 : it->second) {
+	auto it2 = lower_bound_x(it->second, x);
-		if (pos2.first != x)
+	if (it2 == it->second.end() || it2->x != x)
-			continue;
+		return;
-		if (pos2.second < min_y || pos2.second >= max_y)
+	// it2 is now pointing to a contigous part where it2->x == x
+	for (; it2 != it->second.end(); it2++) {
+		const auto &pos2 = *it2;
+		if (pos2.x != x)
+			break; // went past
+		if (pos2.y < min_y || pos2.y >= max_y)
 			continue;
 
-		BlockPos pos(x, pos2.second, z);
+		BlockPos pos(x, pos2.y, z);
 		status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(pos)), &datastr);
 		if (status.ok()) {
 			blocks.emplace_back(

src/db-leveldb.h

@@ -8,7 +8,7 @@
 class DBLevelDB : public DB {
 public:
 	DBLevelDB(const std::string &mapdir);
-	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
+	std::vector<BlockPos> getBlockPosXZ(BlockPos min, BlockPos max) override;
 	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 			int16_t min_y, int16_t max_y) override;
 	void getBlocksByPos(BlockList &blocks,
@@ -18,11 +18,24 @@ public:
 	bool preferRangeQueries() const override { return false; }
 
 private:
-	using pos2d = std::pair<int16_t, int16_t>;
+	struct vec2 {
+		int16_t x, y;
+		constexpr vec2() : x(0), y(0) {}
+		constexpr vec2(int16_t x, int16_t y) : x(x), y(y) {}
+
+		inline bool operator<(const vec2 &p) const
+		{
+			if (x < p.x)
+				return true;
+			if (x > p.x)
+				return false;
+			return y < p.y;
+		}
+	};
 
 	void loadPosCache();
 
 	// indexed by Z, contains all (x,y) position pairs
-	std::unordered_map<int16_t, std::vector<pos2d>> posCache;
+	std::unordered_map<int16_t, std::vector<vec2>> posCache;
-	leveldb::DB *db;
+	leveldb::DB *db = NULL;
 };

src/db-postgresql.cpp

@@ -3,11 +3,71 @@
 #include <fstream>
 #include <cstdlib>
 #include <arpa/inet.h>
+
 #include "db-postgresql.h"
 #include "util.h"
+#include "log.h"
 #include "types.h"
 
-#define ARRLEN(x) (sizeof(x) / sizeof((x)[0]))
+/* PostgreSQLBase */
+
+PostgreSQLBase::~PostgreSQLBase()
+{
+	if (db)
+		PQfinish(db);
+}
+
+void PostgreSQLBase::openDatabase(const char *connect_string)
+{
+	if (db)
+		throw std::logic_error("Database already open");
+
+	db = PQconnectdb(connect_string);
+	if (PQstatus(db) != CONNECTION_OK) {
+		throw std::runtime_error(std::string("PostgreSQL database error: ") +
+			PQerrorMessage(db)
+		);
+	}
+}
+
+PGresult *PostgreSQLBase::checkResults(PGresult *res, bool clear)
+{
+	ExecStatusType statusType = PQresultStatus(res);
+
+	switch (statusType) {
+	case PGRES_COMMAND_OK:
+	case PGRES_TUPLES_OK:
+		break;
+	case PGRES_FATAL_ERROR:
+		throw std::runtime_error(
+			std::string("PostgreSQL database error: ") +
+			PQresultErrorMessage(res)
+		);
+	default:
+		throw std::runtime_error(
+			std::string("Unhandled PostgreSQL result code ") +
+			std::to_string(statusType)
+		);
+	}
+
+	if (clear)
+		PQclear(res);
+	return res;
+}
+
+PGresult *PostgreSQLBase::execPrepared(
+	const char *stmtName, const int paramsNumber,
+	const void **params,
+	const int *paramsLengths, const int *paramsFormats,
+	bool clear)
+{
+	return checkResults(PQexecPrepared(db, stmtName, paramsNumber,
+		(const char* const*) params, paramsLengths, paramsFormats,
+		1 /* binary output */), clear
+	);
+}
+
+/* DBPostgreSQL */
 
 DBPostgreSQL::DBPostgreSQL(const std::string &mapdir)
 {
@@ -16,21 +76,15 @@ DBPostgreSQL::DBPostgreSQL(const std::string &mapdir)
 		throw std::runtime_error("Failed to read world.mt");
 	std::string connect_string = read_setting("pgsql_connection", ifs);
 	ifs.close();
-	db = PQconnectdb(connect_string.c_str());
 
-	if (PQstatus(db) != CONNECTION_OK) {
+	openDatabase(connect_string.c_str());
-		throw std::runtime_error(std::string(
-			"PostgreSQL database error: ") +
-			PQerrorMessage(db)
-		);
-	}
 
 	prepareStatement(
 		"get_block_pos",
-		"SELECT posX::int4, posY::int4, posZ::int4 FROM blocks WHERE"
+		"SELECT posX::int4, posZ::int4 FROM blocks WHERE"
 		" (posX BETWEEN $1::int4 AND $2::int4) AND"
 		" (posY BETWEEN $3::int4 AND $4::int4) AND"
-		" (posZ BETWEEN $5::int4 AND $6::int4)"
+		" (posZ BETWEEN $5::int4 AND $6::int4) GROUP BY posX, posZ"
 	);
 	prepareStatement(
 		"get_blocks",
@@ -54,13 +108,12 @@ DBPostgreSQL::~DBPostgreSQL()
 	try {
 		checkResults(PQexec(db, "COMMIT;"));
 	} catch (const std::exception& caught) {
-		std::cerr << "could not finalize: " << caught.what() << std::endl;
+		errorstream << "could not finalize: " << caught.what() << std::endl;
 	}
-	PQfinish(db);
 }
 
 
-std::vector<BlockPos> DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max)
+std::vector<BlockPos> DBPostgreSQL::getBlockPosXZ(BlockPos min, BlockPos max)
 {
 	int32_t const x1 = htonl(min.x);
 	int32_t const x2 = htonl(max.x - 1);
@@ -83,11 +136,14 @@ std::vector<BlockPos> DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max)
 	std::vector<BlockPos> positions;
 	positions.reserve(numrows);
 
-	for (int row = 0; row < numrows; ++row)
+	BlockPos pos;
-		positions.emplace_back(pg_to_blockpos(results, row, 0));
+	for (int row = 0; row < numrows; ++row) {
+		pos.x = pg_binary_to_int(results, row, 0);
+		pos.z = pg_binary_to_int(results, row, 1);
+		positions.push_back(pos);
+	}
 
 	PQclear(results);
-
 	return positions;
 }
 
@@ -166,61 +222,8 @@ void DBPostgreSQL::getBlocksByPos(BlockList &blocks,
 	}
 }
 
-
-PGresult *DBPostgreSQL::checkResults(PGresult *res, bool clear)
-{
-	ExecStatusType statusType = PQresultStatus(res);
-
-	switch (statusType) {
-	case PGRES_COMMAND_OK:
-	case PGRES_TUPLES_OK:
-		break;
-	case PGRES_FATAL_ERROR:
-		throw std::runtime_error(
-			std::string("PostgreSQL database error: ") +
-			PQresultErrorMessage(res)
-		);
-	default:
-		throw std::runtime_error(
-			"Unhandled PostgreSQL result code"
-		);
-	}
-
-	if (clear)
-		PQclear(res);
-
-	return res;
-}
-
-void DBPostgreSQL::prepareStatement(const std::string &name, const std::string &sql)
-{
-	checkResults(PQprepare(db, name.c_str(), sql.c_str(), 0, NULL));
-}
-
-PGresult *DBPostgreSQL::execPrepared(
-	const char *stmtName, const int paramsNumber,
-	const void **params,
-	const int *paramsLengths, const int *paramsFormats,
-	bool clear
-)
-{
-	return checkResults(PQexecPrepared(db, stmtName, paramsNumber,
-		(const char* const*) params, paramsLengths, paramsFormats,
-		1 /* binary output */), clear
-	);
-}
-
 int DBPostgreSQL::pg_binary_to_int(PGresult *res, int row, int col)
 {
 	int32_t* raw = reinterpret_cast<int32_t*>(PQgetvalue(res, row, col));
 	return ntohl(*raw);
 }
-
-BlockPos DBPostgreSQL::pg_to_blockpos(PGresult *res, int row, int col)
-{
-	BlockPos result;
-	result.x = pg_binary_to_int(res, row, col);
-	result.y = pg_binary_to_int(res, row, col + 1);
-	result.z = pg_binary_to_int(res, row, col + 2);
-	return result;
-}

src/db-postgresql.h

@@ -3,10 +3,31 @@
 #include "db.h"
 #include <libpq-fe.h>
 
-class DBPostgreSQL : public DB {
+class PostgreSQLBase {
+public:
+	~PostgreSQLBase();
+
+protected:
+	void openDatabase(const char *connect_string);
+
+	PGresult *checkResults(PGresult *res, bool clear = true);
+	void prepareStatement(const std::string &name, const std::string &sql) {
+		checkResults(PQprepare(db, name.c_str(), sql.c_str(), 0, NULL));
+	}
+	PGresult *execPrepared(
+		const char *stmtName, const int paramsNumber,
+		const void **params,
+		const int *paramsLengths = nullptr, const int *paramsFormats = nullptr,
+		bool clear = true
+	);
+
+	PGconn *db = NULL;
+};
+
+class DBPostgreSQL : public DB, PostgreSQLBase {
 public:
 	DBPostgreSQL(const std::string &mapdir);
-	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
+	std::vector<BlockPos> getBlockPosXZ(BlockPos min, BlockPos max) override;
 	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 		int16_t min_y, int16_t max_y) override;
 	void getBlocksByPos(BlockList &blocks,
@@ -15,18 +36,6 @@ public:
 
 	bool preferRangeQueries() const override { return true; }
 
-protected:
-	PGresult *checkResults(PGresult *res, bool clear = true);
-	void prepareStatement(const std::string &name, const std::string &sql);
-	PGresult *execPrepared(
-		const char *stmtName, const int paramsNumber,
-		const void **params,
-		const int *paramsLengths = nullptr, const int *paramsFormats = nullptr,
-		bool clear = true
-	);
-	int pg_binary_to_int(PGresult *res, int row, int col);
-	BlockPos pg_to_blockpos(PGresult *res, int row, int col);
-
 private:
-	PGconn *db;
+	int pg_binary_to_int(PGresult *res, int row, int col);
 };

src/db-redis.cpp

@@ -68,7 +68,7 @@ DBRedis::~DBRedis()
 }
 
 
-std::vector<BlockPos> DBRedis::getBlockPos(BlockPos min, BlockPos max)
+std::vector<BlockPos> DBRedis::getBlockPosXZ(BlockPos min, BlockPos max)
 {
 	std::vector<BlockPos> res;
 	for (const auto &it : posCache) {

src/db-redis.h

@@ -9,7 +9,7 @@
 class DBRedis : public DB {
 public:
 	DBRedis(const std::string &mapdir);
-	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
+	std::vector<BlockPos> getBlockPosXZ(BlockPos min, BlockPos max) override;
 	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 		int16_t min_y, int16_t max_y) override;
 	void getBlocksByPos(BlockList &blocks,

src/db-sqlite3.cpp

@@ -0,0 +1,261 @@
+#include <stdexcept>
+#include <unistd.h> // for usleep
+#include <iostream>
+#include <algorithm>
+#include <cassert>
+
+#include "db-sqlite3.h"
+#include "log.h"
+#include "types.h"
+
+/* SQLite3Base */
+
+#define SQLRES(r, good) check_result(r, good)
+#define SQLOK(r) SQLRES(r, SQLITE_OK)
+
+SQLite3Base::~SQLite3Base()
+{
+	if (db && sqlite3_close(db) != SQLITE_OK) {
+		errorstream << "Error closing SQLite database: "
+			<< sqlite3_errmsg(db) << std::endl;
+	}
+}
+
+void SQLite3Base::openDatabase(const char *path, bool readonly)
+{
+	if (db)
+		throw std::logic_error("Database already open");
+
+	int flags = 0;
+	if (readonly)
+		flags |= SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE;
+	else
+		flags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;
+#ifdef SQLITE_OPEN_EXRESCODE
+	flags |= SQLITE_OPEN_EXRESCODE;
+#endif
+	SQLOK(sqlite3_open_v2(path, &db, flags, 0));
+}
+
+/* DBSQLite3 */
+
+// make sure a row is available. intended to be used outside a loop.
+// compare result to SQLITE_ROW afterwards.
+#define SQLROW1(stmt) \
+	while ((result = sqlite3_step(stmt)) == SQLITE_BUSY) \
+		usleep(10000); /* wait some time and try again */ \
+	if (result != SQLITE_ROW && result != SQLITE_DONE) { \
+		throw std::runtime_error(sqlite3_errmsg(db)); \
+	}
+
+// make sure next row is available. intended to be used in a while(sqlite3_step) loop
+#define SQLROW2() \
+	if (result == SQLITE_BUSY) { \
+		usleep(10000); /* wait some time and try again */ \
+		continue; \
+	} else if (result != SQLITE_ROW) { \
+		throw std::runtime_error(sqlite3_errmsg(db)); \
+	}
+
+DBSQLite3::DBSQLite3(const std::string &mapdir)
+{
+	std::string db_name = mapdir + "map.sqlite";
+
+	openDatabase(db_name.c_str());
+
+	// There's a simple, dumb way to check if we have a new or old database schema.
+	// If we prepare a statement that references columns that don't exist, it will
+	// error right there.
+	int result = prepare(stmt_get_block_pos, "SELECT x, y, z FROM blocks");
+	newFormat = result == SQLITE_OK;
+	verbosestream << "Detected " << (newFormat ? "new" : "old") << " SQLite schema" << std::endl;
+
+	if (newFormat) {
+		SQLOK(prepare(stmt_get_blocks_xz_range,
+				"SELECT y, data FROM blocks WHERE "
+				"x = ? AND z = ? AND y BETWEEN ? AND ?"));
+
+		SQLOK(prepare(stmt_get_block_exact,
+				"SELECT data FROM blocks WHERE x = ? AND y = ? AND z = ?"));
+
+		SQLOK(prepare(stmt_get_block_pos_range,
+				"SELECT x, z FROM blocks WHERE "
+				"x >= ? AND y >= ? AND z >= ? AND "
+				"x < ? AND y < ? AND z < ? GROUP BY x, z"));
+	} else {
+		SQLOK(prepare(stmt_get_blocks_z,
+				"SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ?"));
+
+		SQLOK(prepare(stmt_get_block_exact,
+				"SELECT data FROM blocks WHERE pos = ?"));
+
+		SQLOK(prepare(stmt_get_block_pos,
+				"SELECT pos FROM blocks"));
+
+		SQLOK(prepare(stmt_get_block_pos_range,
+				"SELECT pos FROM blocks WHERE pos BETWEEN ? AND ?"));
+	}
+
+#undef RANGE
+}
+
+
+DBSQLite3::~DBSQLite3()
+{
+	sqlite3_finalize(stmt_get_blocks_z);
+	sqlite3_finalize(stmt_get_blocks_xz_range);
+	sqlite3_finalize(stmt_get_block_pos);
+	sqlite3_finalize(stmt_get_block_pos_range);
+	sqlite3_finalize(stmt_get_block_exact);
+}
+
+
+inline void DBSQLite3::getPosRange(int64_t &min, int64_t &max,
+		int16_t zPos, int16_t zPos2)
+{
+	// Magic numbers!
+	min = encodeBlockPos(BlockPos(0, -2048, zPos));
+	max = encodeBlockPos(BlockPos(0, 2048, zPos2)) - 1;
+}
+
+
+std::vector<BlockPos> DBSQLite3::getBlockPosXZ(BlockPos min, BlockPos max)
+{
+	int result;
+	sqlite3_stmt *stmt;
+
+	if (newFormat) {
+		stmt = stmt_get_block_pos_range;
+		int col = bind_pos(stmt, 1, min);
+		bind_pos(stmt, col, max);
+	} else {
+		// can handle range query on Z axis via SQL
+		if (min.z <= -2048 && max.z >= 2048) {
+			stmt = stmt_get_block_pos;
+		} else {
+			stmt = stmt_get_block_pos_range;
+			int64_t minPos, maxPos;
+			if (min.z < -2048)
+				min.z = -2048;
+			if (max.z > 2048)
+				max.z = 2048;
+			getPosRange(minPos, maxPos, min.z, max.z - 1);
+			SQLOK(sqlite3_bind_int64(stmt, 1, minPos));
+			SQLOK(sqlite3_bind_int64(stmt, 2, maxPos));
+		}
+	}
+
+	std::vector<BlockPos> positions;
+	BlockPos pos;
+	while ((result = sqlite3_step(stmt)) != SQLITE_DONE) {
+		SQLROW2()
+
+		if (newFormat) {
+			pos.x = sqlite3_column_int(stmt, 0);
+			pos.z = sqlite3_column_int(stmt, 1);
+		} else {
+			pos = decodeBlockPos(sqlite3_column_int64(stmt, 0));
+			if (pos.x < min.x || pos.x >= max.x || pos.y < min.y || pos.y >= max.y)
+				continue;
+			// note that we can't try to deduplicate these because the order
+			// of the encoded pos (if sorted) is ZYX.
+		}
+		positions.emplace_back(pos);
+	}
+	SQLOK(sqlite3_reset(stmt));
+	return positions;
+}
+
+
+void DBSQLite3::loadBlockCache(int16_t zPos)
+{
+	int result;
+	blockCache.clear();
+
+	assert(!newFormat);
+
+	int64_t minPos, maxPos;
+	getPosRange(minPos, maxPos, zPos, zPos);
+
+	SQLOK(sqlite3_bind_int64(stmt_get_blocks_z, 1, minPos));
+	SQLOK(sqlite3_bind_int64(stmt_get_blocks_z, 2, maxPos));
+
+	while ((result = sqlite3_step(stmt_get_blocks_z)) != SQLITE_DONE) {
+		SQLROW2()
+
+		int64_t posHash = sqlite3_column_int64(stmt_get_blocks_z, 0);
+		BlockPos pos = decodeBlockPos(posHash);
+		blockCache[pos.x].emplace_back(pos, read_blob(stmt_get_blocks_z, 1));
+	}
+	SQLOK(sqlite3_reset(stmt_get_blocks_z));
+}
+
+
+void DBSQLite3::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+		int16_t min_y, int16_t max_y)
+{
+	// New format: use a real range query
+	if (newFormat) {
+		auto *stmt = stmt_get_blocks_xz_range;
+		SQLOK(sqlite3_bind_int(stmt, 1, x));
+		SQLOK(sqlite3_bind_int(stmt, 2, z));
+		SQLOK(sqlite3_bind_int(stmt, 3, min_y));
+		SQLOK(sqlite3_bind_int(stmt, 4, max_y - 1)); // BETWEEN is inclusive
+
+		int result;
+		while ((result = sqlite3_step(stmt)) != SQLITE_DONE) {
+			SQLROW2()
+
+			BlockPos pos(x, sqlite3_column_int(stmt, 0), z);
+			blocks.emplace_back(pos, read_blob(stmt, 1));
+		}
+		SQLOK(sqlite3_reset(stmt));
+		return;
+	}
+
+	/* Cache the blocks on the given Z coordinate between calls, this only
+	 * works due to order in which the TileGenerator asks for blocks. */
+	if (z != blockCachedZ) {
+		loadBlockCache(z);
+		blockCachedZ = z;
+	}
+
+	auto it = blockCache.find(x);
+	if (it == blockCache.end())
+		return;
+
+	if (it->second.empty()) {
+		/* We have swapped this list before, this is not supposed to happen
+		 * because it's bad for performance. But rather than silently breaking
+		 * do the right thing and load the blocks again. */
+		verbosestream << "suboptimal access pattern for sqlite3 backend?!" << std::endl;
+		loadBlockCache(z);
+	}
+	// Swap lists to avoid copying contents
+	blocks.clear();
+	std::swap(blocks, it->second);
+
+	for (auto it = blocks.begin(); it != blocks.end(); ) {
+		if (it->first.y < min_y || it->first.y >= max_y)
+			it = blocks.erase(it);
+		else
+			it++;
+	}
+}
+
+
+void DBSQLite3::getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions)
+{
+	int result;
+
+	for (auto pos : positions) {
+		bind_pos(stmt_get_block_exact, 1, pos);
+
+		SQLROW1(stmt_get_block_exact)
+		if (result == SQLITE_ROW)
+			blocks.emplace_back(pos, read_blob(stmt_get_block_exact, 0));
+
+		SQLOK(sqlite3_reset(stmt_get_block_exact));
+	}
+}

src/db-sqlite3.h

@@ -0,0 +1,87 @@
+#pragma once
+
+#include "db.h"
+#include <unordered_map>
+#include <sqlite3.h>
+
+class SQLite3Base {
+public:
+	~SQLite3Base();
+
+protected:
+	void openDatabase(const char *path, bool readonly = true);
+
+	// check function result or throw error
+	inline void check_result(int result, int good = SQLITE_OK)
+	{
+		if (result != good)
+			throw std::runtime_error(sqlite3_errmsg(db));
+	}
+
+	// prepare a statement
+	inline int prepare(sqlite3_stmt *&stmt, const char *sql)
+	{
+		return sqlite3_prepare_v2(db, sql, -1, &stmt, NULL);
+	}
+
+	// read string from statement
+	static inline std::string read_str(sqlite3_stmt *stmt, int iCol)
+	{
+		auto *data = reinterpret_cast<const char*>(
+			sqlite3_column_text(stmt, iCol));
+		return std::string(data);
+	}
+
+	// read blob from statement
+	static inline ustring read_blob(sqlite3_stmt *stmt, int iCol)
+	{
+		auto *data = reinterpret_cast<const unsigned char *>(
+			sqlite3_column_blob(stmt, iCol));
+		size_t size = sqlite3_column_bytes(stmt, iCol);
+		return ustring(data, size);
+	}
+
+	sqlite3 *db = NULL;
+};
+
+class DBSQLite3 : public DB, SQLite3Base {
+public:
+	DBSQLite3(const std::string &mapdir);
+	std::vector<BlockPos> getBlockPosXZ(BlockPos min, BlockPos max) override;
+	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+			int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
+	~DBSQLite3() override;
+
+	bool preferRangeQueries() const override { return newFormat; }
+
+private:
+	static inline void getPosRange(int64_t &min, int64_t &max, int16_t zPos,
+			int16_t zPos2);
+	void loadBlockCache(int16_t zPos);
+
+	// bind pos to statement. returns index of next column.
+	inline int bind_pos(sqlite3_stmt *stmt, int iCol, BlockPos pos)
+	{
+		if (newFormat) {
+			sqlite3_bind_int(stmt, iCol, pos.x);
+			sqlite3_bind_int(stmt, iCol + 1, pos.y);
+			sqlite3_bind_int(stmt, iCol + 2, pos.z);
+			return iCol + 3;
+		} else {
+			sqlite3_bind_int64(stmt, iCol, encodeBlockPos(pos));
+			return iCol + 1;
+		}
+	}
+
+	sqlite3_stmt *stmt_get_block_pos = NULL;
+	sqlite3_stmt *stmt_get_block_pos_range = NULL;
+	sqlite3_stmt *stmt_get_blocks_z = NULL;
+	sqlite3_stmt *stmt_get_blocks_xz_range = NULL;
+	sqlite3_stmt *stmt_get_block_exact = NULL;
+
+	bool newFormat = false;
+	int16_t blockCachedZ = -10000;
+	std::unordered_map<int16_t, BlockList> blockCache; // indexed by X
+};

src/db.h

@@ -6,18 +6,15 @@
 #include <utility>
 #include "types.h"
 
-
 struct BlockPos {
-	int16_t x;
+	int16_t x, y, z;
-	int16_t y;
-	int16_t z;
 
-	BlockPos() : x(0), y(0), z(0) {}
+	constexpr BlockPos() : x(0), y(0), z(0) {}
-	explicit BlockPos(int16_t v) : x(v), y(v), z(v) {}
+	explicit constexpr BlockPos(int16_t v) : x(v), y(v), z(v) {}
-	BlockPos(int16_t x, int16_t y, int16_t z) : x(x), y(y), z(z) {}
+	constexpr BlockPos(int16_t x, int16_t y, int16_t z) : x(x), y(y), z(z) {}
 
 	// Implements the inverse ordering so that (2,2,2) < (1,1,1)
-	bool operator < (const BlockPos &p) const
+	inline bool operator<(const BlockPos &p) const
 	{
 		if (z > p.z)
 			return true;
@@ -27,11 +24,7 @@ struct BlockPos {
 			return true;
 		if (y < p.y)
 			return false;
-		if (x > p.x)
+		return x > p.x;
-			return true;
-		if (x < p.x)
-			return false;
-		return false;
 	}
 };
 
@@ -43,29 +36,32 @@ typedef std::list<Block> BlockList;
 class DB {
 protected:
 	// Helpers that implement the hashed positions used by most backends
-	inline int64_t  encodeBlockPos(const BlockPos pos) const;
+	static inline int64_t  encodeBlockPos(const BlockPos pos);
-	inline BlockPos decodeBlockPos(int64_t hash) const;
+	static inline BlockPos decodeBlockPos(int64_t hash);
 
 public:
-	/* Return all block positions inside the range given by min and max,
+	/* Return all unique (X, Z) position pairs inside area given by min and max,
-	 * so that min.x <= x < max.x, ...
+	 * so that min.x <= x < max.x && min.z <= z < max.z
+	 * Note: duplicates are allowed, but results in wasted time.
 	 */
-	virtual std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) = 0;
+	virtual std::vector<BlockPos> getBlockPosXZ(BlockPos min, BlockPos max) = 0;
+
 	/* Read all blocks in column given by x and z
 	 * and inside the given Y range (min_y <= y < max_y) into list
 	 */
 	virtual void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 			int16_t min_y, int16_t max_y) = 0;
+
 	/* Read blocks at given positions into list
 	 */
 	virtual void getBlocksByPos(BlockList &blocks,
 			const std::vector<BlockPos> &positions) = 0;
+
 	/* Can this database efficiently do range queries?
 	 * (for large data sets, more efficient that brute force)
 	 */
 	virtual bool preferRangeQueries() const = 0;
 
-
 	virtual ~DB() {}
 };
 
@@ -98,7 +94,7 @@ static inline int64_t pythonmodulo(int64_t i, int64_t mod)
 }
 
 
-inline int64_t DB::encodeBlockPos(const BlockPos pos) const
+inline int64_t DB::encodeBlockPos(const BlockPos pos)
 {
 	return (uint64_t) pos.z * 0x1000000 +
 		(uint64_t) pos.y * 0x1000 +
@@ -106,7 +102,7 @@ inline int64_t DB::encodeBlockPos(const BlockPos pos) const
 }
 
 
-inline BlockPos DB::decodeBlockPos(int64_t hash) const
+inline BlockPos DB::decodeBlockPos(int64_t hash)
 {
 	BlockPos pos;
 	pos.x = unsigned_to_signed(pythonmodulo(hash, 4096), 2048);

src/log.cpp

@@ -0,0 +1,16 @@
+#include <iostream>
+
+#include "log.h"
+
+StreamProxy errorstream(nullptr);
+StreamProxy verbosestream(nullptr);
+
+void configure_log_streams(bool verbose)
+{
+	errorstream << std::flush;
+	verbosestream << std::flush;
+
+	errorstream = std::cerr.good() ? &std::cerr : nullptr;
+	// std::clog does not automatically flush
+	verbosestream = (verbose && std::clog.good()) ? &std::clog : nullptr;
+}

src/log.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include <iostream>
+#include <utility>
+
+// Forwards to an ostream, optionally
+class StreamProxy {
+public:
+	StreamProxy(std::ostream *os) : m_os(os) {}
+
+	template<typename T>
+	StreamProxy &operator<<(T &&arg)
+	{
+		if (m_os)
+			*m_os << std::forward<T>(arg);
+		return *this;
+	}
+
+	StreamProxy &operator<<(std::ostream &(*func)(std::ostream&))
+	{
+		if (m_os)
+			*m_os << func;
+		return *this;
+	}
+
+private:
+	std::ostream *m_os;
+};
+
+/// Error and warning output, forwards to std::cerr
+extern StreamProxy errorstream;
+/// Verbose output, might forward to std::cerr
+extern StreamProxy verbosestream;
+
+/**
+ * Configure log streams defined in this file.
+ * @param verbose enable verbose output
+ * @note not thread-safe!
+ */
+void configure_log_streams(bool verbose);

src/mapper.cpp

@@ -10,12 +10,14 @@
 #include <stdexcept>
 #include "config.h"
 #include "TileGenerator.h"
+#include "util.h"
+#include "log.h"
 
 static void usage()
 {
-	const std::pair<const char*, const char*> options[] = {
+	static const std::pair<const char*, const char*> options[] = {
-		{"-i/--input", "<world_path>"},
+		{"-i/--input", "<path>"},
-		{"-o/--output", "<output_image.png>"},
+		{"-o/--output", "<path>"},
 		{"--bgcolor", "<color>"},
 		{"--scalecolor", "<color>"},
 		{"--playercolor", "<color>"},
@@ -26,19 +28,20 @@ static void usage()
 		{"--drawalpha", ""},
 		{"--noshading", ""},
 		{"--noemptyimage", ""},
+		{"-v/--verbose", ""},
 		{"--min-y", "<y>"},
 		{"--max-y", "<y>"},
 		{"--backend", "<backend>"},
-		{"--geometry", "x:y+w+h"},
+		{"--geometry", "x:z+w+h"},
 		{"--extent", ""},
-		{"--zoom", "<zoomlevel>"},
+		{"--zoom", "<factor>"},
-		{"--colors", "<colors.txt>"},
+		{"--colors", "<path>"},
 		{"--scales", "[t][b][l][r]"},
 		{"--exhaustive", "never|y|full|auto"},
 		{"--dumpblock", "x,y,z"},
 	};
 	const char *top_text =
-		"minetestmapper -i <world_path> -o <output_image.png> [options]\n"
+		"minetestmapper -i <world_path> -o <output_image> [options]\n"
 		"Generate an overview image of a Luanti map.\n"
 		"\n"
 		"Options:\n";
@@ -55,12 +58,16 @@ static void usage()
 	for (auto s : backends)
 		printf("%s ", s.c_str());
 	printf("\n");
+#ifdef _WIN32
+	printf("See also the full documentation in README.rst\n");
+#else
+	printf("See also the full documentation in minetestmapper(6) or README.rst\n");
+#endif
 }
 
 static inline bool file_exists(const std::string &path)
 {
-	std::ifstream ifs(path);
+	return file_exists(path.c_str());
-	return ifs.is_open();
 }
 
 static inline int stoi(const char *s)
@@ -78,23 +85,24 @@ static std::string search_colors(const std::string &worldpath)
 
 #ifndef _WIN32
 	char *home = std::getenv("HOME");
-	if (home) {
+	if (home && home[0]) {
 		std::string check = std::string(home) + "/.minetest/colors.txt";
 		if (file_exists(check))
 			return check;
 	}
 #endif
 
-	constexpr bool sharedir_valid = !(SHAREDIR[0] == '.' || SHAREDIR[0] == '\0');
+	constexpr bool sharedir_valid = !(SHAREDIR[0] == '.' || !SHAREDIR[0]);
 	if (sharedir_valid && file_exists(SHAREDIR "/colors.txt"))
 		return SHAREDIR "/colors.txt";
 
-	std::cerr << "Warning: Falling back to using colors.txt from current directory." << std::endl;
+	errorstream << "Warning: Falling back to using colors.txt from current directory." << std::endl;
-	return "colors.txt";
+	return "./colors.txt";
 }
 
 int main(int argc, char *argv[])
 {
+	const char *short_options = "hi:o:v";
 	const static struct option long_options[] =
 	{
 		{"help", no_argument, 0, 'h'},
@@ -120,9 +128,12 @@ int main(int argc, char *argv[])
 		{"noemptyimage", no_argument, 0, 'n'},
 		{"exhaustive", required_argument, 0, 'j'},
 		{"dumpblock", required_argument, 0, 'k'},
+		{"verbose", no_argument, 0, 'v'},
 		{0, 0, 0, 0}
 	};
 
+	configure_log_streams(false);
+
 	std::string input;
 	std::string output;
 	std::string colors;
@@ -132,7 +143,7 @@ int main(int argc, char *argv[])
 	TileGenerator generator;
 	while (1) {
 		int option_index;
-		int c = getopt_long(argc, argv, "hi:o:", long_options, &option_index);
+		int c = getopt_long(argc, argv, short_options, long_options, &option_index);
 		if (c == -1)
 			break; // done
 
@@ -192,7 +203,7 @@ int main(int argc, char *argv[])
 					geometry >> x >> c >> y >> w >> h;
 					if (geometry.fail() || c != ':' || w < 1 || h < 1) {
 						usage();
-						exit(1);
+						return 1;
 					}
 					generator.setGeometry(x, y, w, h);
 				}
@@ -220,7 +231,7 @@ int main(int argc, char *argv[])
 				generator.setDontWriteEmpty(true);
 				break;
 			case 'j': {
-					int mode = EXH_AUTO;;
+					int mode = EXH_AUTO;
 					if (!strcmp(optarg, "never"))
 						mode = EXH_NEVER;
 					else if (!strcmp(optarg, "y"))
@@ -236,12 +247,15 @@ int main(int argc, char *argv[])
 				iss >> dumpblock.x >> c >> dumpblock.y >> c2 >> dumpblock.z;
 				if (iss.fail() || c != ',' || c2 != ',') {
 					usage();
-					exit(1);
+					return 1;
 				}
 				break;
 			}
+			case 'v':
+				configure_log_streams(true);
+				break;
 			default:
-				exit(1);
+				return 1;
 		}
 	}
 
@@ -252,7 +266,6 @@ int main(int argc, char *argv[])
 	}
 
 	try {
-
 		if (onlyPrintExtent) {
 			generator.printGeometry(input);
 			return 0;
@@ -267,7 +280,7 @@ int main(int argc, char *argv[])
 		generator.generate(input, output);
 
 	} catch (const std::exception &e) {
-		std::cerr << "Exception: " << e.what() << std::endl;
+		errorstream << "Exception: " << e.what() << std::endl;
 		return 1;
 	}
 	return 0;

src/types.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include <string>
+
+// Define custom char traits since std::char_traits<unsigend char> is not part of C++ standard
+struct uchar_traits : std::char_traits<char>
+{
+	using super = std::char_traits<char>;
+	using char_type = unsigned char;
+
+	static void assign(char_type& c1, const char_type& c2) noexcept {
+		c1 = c2;
+	}
+	static char_type* assign(char_type* ptr, std::size_t count, char_type c2) {
+		return reinterpret_cast<char_type*>(
+			super::assign(reinterpret_cast<char*>(ptr), count, static_cast<char>(c2)));
+	}
+
+	static char_type* move(char_type* dest, const char_type* src, std::size_t count) {
+		return reinterpret_cast<char_type*>(
+			super::move(reinterpret_cast<char*>(dest), reinterpret_cast<const char*>(src), count));
+	}
+
+	static char_type* copy(char_type* dest, const char_type* src, std::size_t count) {
+		return reinterpret_cast<char_type*>(
+			super::copy(reinterpret_cast<char*>(dest), reinterpret_cast<const char*>(src), count));
+	}
+
+	static int compare(const char_type* s1, const char_type* s2, std::size_t count) {
+		return super::compare(reinterpret_cast<const char*>(s1), reinterpret_cast<const char*>(s2), count);
+	}
+
+	static char_type to_char_type(int_type c) noexcept {
+		return static_cast<char_type>(c);
+	}
+};
+
+typedef std::basic_string<unsigned char, uchar_traits> ustring;
+typedef unsigned int uint;
+typedef unsigned char u8;

src/util.cpp

@@ -0,0 +1,76 @@
+#include <stdexcept>
+#include <iostream>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include "util.h"
+
+static std::string trim(const std::string &s)
+{
+	auto isspace = [] (char c) {
+		return c == ' ' || c == '\t' || c == '\r' || c == '\n';
+	};
+
+	size_t front = 0;
+	while (isspace(s[front]))
+		++front;
+	size_t back = s.size() - 1;
+	while (back > front && isspace(s[back]))
+		--back;
+
+	return s.substr(front, back - front + 1);
+}
+
+static bool read_setting(const std::string &name, std::istream &is, std::string &out)
+{
+	char linebuf[512];
+	is.seekg(0);
+	while (is.good()) {
+		is.getline(linebuf, sizeof(linebuf));
+		std::string line(linebuf);
+
+		auto pos = line.find('#');
+		if (pos != std::string::npos)
+			line.erase(pos); // remove comments
+
+		pos = line.find('=');
+		if (pos == std::string::npos)
+			continue;
+		auto key = trim(line.substr(0, pos));
+		if (key != name)
+			continue;
+		out = trim(line.substr(pos+1));
+		return true;
+	}
+	return false;
+}
+
+std::string read_setting(const std::string &name, std::istream &is)
+{
+	std::string ret;
+	if (!read_setting(name, is, ret))
+		throw std::runtime_error(std::string("Setting not found: ") + name);
+	return ret;
+}
+
+std::string read_setting_default(const std::string &name, std::istream &is,
+	const std::string &def)
+{
+	std::string ret;
+	if (!read_setting(name, is, ret))
+		return def;
+	return ret;
+}
+
+bool file_exists(const char *path)
+{
+	struct stat s{};
+	// check for !dir to allow symlinks or such
+	return stat(path, &s) == 0 && (s.st_mode & S_IFDIR) != S_IFDIR;
+}
+
+bool dir_exists(const char *path)
+{
+	struct stat s{};
+	return stat(path, &s) == 0 && (s.st_mode & S_IFDIR) == S_IFDIR;
+}

src/util.h

@@ -3,6 +3,8 @@
 #include <string>
 #include <iostream>
 
+#define ARRLEN(x) (sizeof(x) / sizeof((x)[0]))
+
 template<typename T>
 static inline T mymax(T a, T b)
 {
@@ -19,3 +21,7 @@ std::string read_setting(const std::string &name, std::istream &is);
 
 std::string read_setting_default(const std::string &name, std::istream &is,
 	const std::string &def);
+
+bool file_exists(const char *path);
+
+bool dir_exists(const char *path);

util.cpp

@@ -1,55 +0,0 @@
-#include <stdexcept>
-#include <sstream>
-
-#include "util.h"
-
-static std::string trim(const std::string &s)
-{
-	auto isspace = [] (char c) -> bool { return c == ' ' || c == '\t' || c == '\r' || c == '\n'; };
-
-	size_t front = 0;
-	while (isspace(s[front]))
-		++front;
-	size_t back = s.size() - 1;
-	while (back > front && isspace(s[back]))
-		--back;
-
-	return s.substr(front, back - front + 1);
-}
-
-std::string read_setting(const std::string &name, std::istream &is)
-{
-	char linebuf[512];
-	while (is.good()) {
-		is.getline(linebuf, sizeof(linebuf));
-
-		for (char *p = linebuf; *p; p++) {
-			if(*p != '#')
-				continue;
-			*p = '\0'; // Cut off at the first #
-			break;
-		}
-		std::string line(linebuf);
-
-		auto pos = line.find('=');
-		if (pos == std::string::npos)
-			continue;
-		auto key = trim(line.substr(0, pos));
-		if (key != name)
-			continue;
-		return trim(line.substr(pos+1));
-	}
-	std::ostringstream oss;
-	oss << "Setting '" << name << "' not found";
-	throw std::runtime_error(oss.str());
-}
-
-std::string read_setting_default(const std::string &name, std::istream &is,
-	const std::string &def)
-{
-	try {
-		return read_setting(name, is);
-	} catch(const std::runtime_error &e) {
-		return def;
-	}
-}

util/build-mingw.sh

@@ -1,26 +1,27 @@
 #!/bin/bash -e
 
-[ -z "$CXX" ] && exit 255
-export CC=false # don't need it actually
-
-variant=win32
-[[ "$(basename "$CXX")" == "x86_64-"* ]] && variant=win64
-
 #######
-# this expects unpacked libraries similar to what Luanti's buildbot uses
+# this expects unpacked libraries and a toolchain file like Luanti's buildbot uses
-# $extradlls will typically point to the DLLs for libgcc, libstdc++ and libpng
+# $extradlls will typically contain the compiler-specific DLLs and libpng
+toolchain_file=
 libgd_dir=
 zlib_dir=
 zstd_dir=
 sqlite_dir=
 leveldb_dir=
-extradlls=()
+extradlls=(
+)
 #######
 
-[ -f ./CMakeLists.txt ] || exit 1
+[ -f "$toolchain_file" ] || exit 1
+variant=win32
+grep -q 'CX?X?_COMPILER.*x86_64-' $toolchain_file && variant=win64
+echo "Detected target $variant"
+
+[ -f ./CMakeLists.txt ] || { echo "run from root folder" >&2; exit 1; }
 
 cmake -S . -B build \
-	-DCMAKE_SYSTEM_NAME=Windows \
+	-DCMAKE_TOOLCHAIN_FILE="$toolchain_file" \
 	-DCMAKE_EXE_LINKER_FLAGS="-s" \
 	\
 	-DENABLE_LEVELDB=1 \
@@ -34,7 +35,7 @@ cmake -S . -B build \
 	-DZLIB_INCLUDE_DIR=$zlib_dir/include \
 	-DZLIB_LIBRARY=$zlib_dir/lib/libz.dll.a \
 	-DZSTD_INCLUDE_DIR=$zstd_dir/include \
-	-DZSTD_LIBRARY=$zstd_dir/lib/libzstd.dll.a \
+	-DZSTD_LIBRARY=$zstd_dir/lib/libzstd.dll.a
 
 make -C build -j4
 

util/ci/script.sh

@@ -1,32 +1,29 @@
 #!/bin/bash -e
 
 install_linux_deps() {
-	local pkgs=(cmake libgd-dev libsqlite3-dev libleveldb-dev libpq-dev libhiredis-dev libzstd-dev)
+	local upkgs=(
+		cmake libgd-dev libsqlite3-dev libleveldb-dev libpq-dev
+		libhiredis-dev libzstd-dev
+	)
+	local fpkgs=(
+		cmake gcc-g++ gd-devel sqlite-devel libzstd-devel zlib-ng-devel
+	)
 
+	if command -v dnf; then
+		sudo dnf install --setopt=install_weak_deps=False -y "${fpkgs[@]}"
+	else
 		sudo apt-get update
-	sudo apt-get remove -y 'libgd3' nginx || : # ????
+		sudo apt-get install -y --no-install-recommends "${upkgs[@]}"
-	sudo apt-get install -y --no-install-recommends "${pkgs[@]}" "$@"
+	fi
 }
 
 run_build() {
 	local args=(
 		-DCMAKE_BUILD_TYPE=Debug
-		-DENABLE_LEVELDB=1 -DENABLE_POSTGRESQL=1 -DENABLE_REDIS=1
+		-DENABLE_LEVELDB=ON -DENABLE_POSTGRESQL=ON -DENABLE_REDIS=ON
 	)
 	[[ "$CXX" == clang* ]] && args+=(-DCMAKE_CXX_FLAGS="-fsanitize=address")
 	cmake . "${args[@]}"
 
 	make -j2
 }
-
-do_functional_test() {
-	mkdir testmap
-	echo "backend = sqlite3" >testmap/world.mt
-	sqlite3 testmap/map.sqlite <<END
-CREATE TABLE blocks(pos INT,data BLOB);
-INSERT INTO blocks(pos, data) VALUES(0, x'$(cat util/ci/test_block)');
-END
-
-	./minetestmapper --noemptyimage -i ./testmap -o map.png
-	file map.png
-}

util/ci/test.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+set -eo pipefail
+mapdir=./testmap
+
+msg () {
+	echo
+	echo "==== $1"
+	echo
+}
+
+# encodes a block position by X, Y, Z (positive numbers only!)
+encodepos () {
+	echo "$(($1 + 0x1000 * $2 + 0x1000000 * $3))"
+}
+
+# create map file with sql statements
+writemap () {
+	rm -rf $mapdir
+	mkdir $mapdir
+	echo "backend = sqlite3" >$mapdir/world.mt
+	echo "default:stone 10 10 10" >$mapdir/colors.txt
+	printf '%s\n' \
+		"CREATE TABLE d(d BLOB);" \
+		"INSERT INTO d VALUES (x'$(cat util/ci/test_block)');" \
+		"$1" \
+		"DROP TABLE d;" | sqlite3 $mapdir/map.sqlite
+}
+
+# check that a non-empty ($1=1) or empty map ($1=0) was written with the args ($2 ...)
+checkmap () {
+	local c=$1
+	shift
+	rm -f map.png
+	./minetestmapper --noemptyimage -v -i ./testmap -o map.png "$@"
+	if [[ $c -eq 1 && ! -f map.png ]]; then
+		echo "Output not generated!"
+		exit 1
+	elif [[ $c -eq 0 && -f map.png ]]; then
+		echo "Output was generated, none expected!"
+		exit 1
+	fi
+	echo "Passed."
+}
+
+# this is missing the indices and primary keys but that doesn't matter
+schema_old="CREATE TABLE blocks(pos INT, data BLOB);"
+schema_new="CREATE TABLE blocks(x INT, y INT, z INT, data BLOB);"
+
+msg "old schema"
+writemap "
+$schema_old
+INSERT INTO blocks SELECT $(encodepos 0 1 0), d FROM d;
+"
+checkmap 1
+
+msg "old schema: Y limit"
+# Note: test data contains a plane at y = 17 an a single node at y = 18
+checkmap 1 --max-y 17
+checkmap 0 --max-y 16
+checkmap 1 --min-y 18
+checkmap 0 --min-y 19
+
+# do this for every strategy
+for exh in never y full; do
+msg "old schema: all limits ($exh)"
+# fill the map with more blocks and then request just a single one to be rendered
+# this will run through internal consistency asserts.
+writemap "
+$schema_old
+INSERT INTO blocks SELECT $(encodepos 2 2 2), d FROM d;
+INSERT INTO blocks SELECT $(encodepos 1 2 2), d FROM d;
+INSERT INTO blocks SELECT $(encodepos 2 1 2), d FROM d;
+INSERT INTO blocks SELECT $(encodepos 2 2 1), d FROM d;
+INSERT INTO blocks SELECT $(encodepos 3 2 2), d FROM d;
+INSERT INTO blocks SELECT $(encodepos 2 3 2), d FROM d;
+INSERT INTO blocks SELECT $(encodepos 2 2 3), d FROM d;
+"
+checkmap 1 --geometry 32:32+16+16 --min-y 32 --max-y $((32+16-1)) --exhaustive $exh
+done
+
+msg "new schema"
+writemap "
+$schema_new
+INSERT INTO blocks SELECT 0, 1, 0, d FROM d;
+"
+checkmap 1
+
+# same as above
+for exh in never y full; do
+msg "new schema: all limits ($exh)"
+writemap "
+$schema_new
+INSERT INTO blocks SELECT 2, 2, 2, d FROM d;
+INSERT INTO blocks SELECT 1, 2, 2, d FROM d;
+INSERT INTO blocks SELECT 2, 1, 2, d FROM d;
+INSERT INTO blocks SELECT 2, 2, 1, d FROM d;
+INSERT INTO blocks SELECT 3, 2, 2, d FROM d;
+INSERT INTO blocks SELECT 2, 3, 2, d FROM d;
+INSERT INTO blocks SELECT 2, 2, 3, d FROM d;
+"
+checkmap 1 --geometry 32:32+16+16 --min-y 32 --max-y $((32+16-1)) --exhaustive $exh
+done
+
+msg "new schema: empty map"
+writemap "$schema_new"
+checkmap 0
+
+msg "drawplayers"
+writemap "
+$schema_new
+INSERT INTO blocks SELECT 0, 0, 0, d FROM d;
+"
+mkdir $mapdir/players
+printf '%s\n' "name = cat" "position = (80,0,80)" >$mapdir/players/cat
+# we can't check that it actually worked, however
+checkmap 1 --drawplayers --zoom 4

util/dumpnodes/init.lua

@@ -2,8 +2,19 @@ local function get_tile(tiles, n)
 	local tile = tiles[n]
 	if type(tile) == 'table' then
 		return tile.name or tile.image
-	end
+	elseif type(tile) == 'string' then
 		return tile
+	end
+end
+
+local function strip_texture(tex)
+	tex = (tex .. '^'):match('%(*(.-)%)*^') -- strip modifiers
+	if tex:find("[combine", 1, true) then
+		tex = tex:match('.-=([^:]-)') -- extract first texture
+	elseif tex:find("[png", 1, true) then
+		return nil -- can't
+	end
+	return tex
 end
 
 local function pairs_s(dict)
@@ -20,7 +31,7 @@ core.register_chatcommand("dumpnodes", {
 	func = function()
 		local ntbl = {}
 		for _, nn in pairs_s(minetest.registered_nodes) do
-			local prefix, name = nn:match('(.*):(.*)')
+			local prefix, name = nn:match('(.-):(.*)')
 			if prefix == nil or name == nil then
 				print("ignored(1): " .. nn)
 			else
@@ -45,14 +56,15 @@ core.register_chatcommand("dumpnodes", {
 					print("ignored(2): " .. nn)
 				else
 					local tex = get_tile(tiles, 1)
-					tex = (tex .. '^'):match('%(*(.-)%)*^') -- strip modifiers
+					tex = tex and strip_texture(tex)
-					if tex:find("[combine", 1, true) then
+					if not tex then
-						tex = tex:match('.-=([^:]-)') -- extract first texture
+						print("ignored(3): " .. nn)
-					end
+					else
 						out:write(nn .. ' ' .. tex .. '\n')
 						n = n + 1
 					end
 				end
+			end
 			out:write('\n')
 		end
 		out:close()