Run CI test also with ASan

shoutout to github for shipping broken OS images

.github/workflows/build.yml

@@ -0,0 +1,63 @@
+name: build
+
+# build on c/cpp changes or workflow changes
+on:
+  push:
+    paths:
+      - '**.[ch]'
+      - '**.cpp'
+      - '**/CMakeLists.txt'
+      - 'util/ci/**'
+      - '.github/workflows/**.yml'
+  pull_request:
+    paths:
+      - '**.[ch]'
+      - '**.cpp'
+      - '**/CMakeLists.txt'
+      - 'util/ci/**'
+      - '.github/workflows/**.yml'
+
+jobs:
+  gcc:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install deps
+        run: |
+          source util/ci/script.sh
+          install_linux_deps
+
+      - name: Build
+        run: |
+          source util/ci/script.sh
+          run_build
+        env:
+          CC: gcc
+          CXX: g++
+
+      - name: Test
+        run: |
+          source util/ci/script.sh
+          do_functional_test
+
+  clang:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install deps
+        run: |
+          source util/ci/script.sh
+          install_linux_deps
+
+      - name: Build
+        run: |
+          source util/ci/script.sh
+          run_build
+        env:
+          CC: clang
+          CXX: clang++
+
+      - name: Test
+        run: |
+          source util/ci/script.sh
+          do_functional_test

.gitignore

@@ -1,10 +1,14 @@
-colors.txt
+*~
+
 minetestmapper
 minetestmapper.exe
+colors.txt
+
 CMakeCache.txt
 CMakeFiles/
-CPack*
+CPack*.cmake
+_CPack_Packages/
+install_manifest.txt
 Makefile
 cmake_install.cmake
 cmake_config.h
-*~

.travis.yml

@@ -1,26 +0,0 @@
-language: cpp
-compiler:
-  - gcc
-  - clang
-sudo: false
-addons:
-  apt:
-    sources:
-      - llvm-toolchain-precise-3.8
-      - ubuntu-toolchain-r-test
-    packages:
-      - cmake
-      - libgd2-noxpm-dev
-      - libsqlite3-dev
-      - p7zip
-      - g++-6
-      - clang-3.8
-before_install:
-  - # Nothing ever works correctly with precise, use a custom libleveldb build
-  - wget http://minetest.kitsunemimi.pw/libleveldb-1.18-ubuntu12.04.7z
-  - 7zr x -olibleveldb libleveldb-1.18-ubuntu12.04.7z
-script: ./util/travis/script.sh
-notifications:
-  email: false
-matrix:
-  fast_finish: true

BlockDecoder.cpp

@@ -0,0 +1,168 @@
+#include <string>
+#include <iostream>
+#include <sstream>
+
+#include "BlockDecoder.h"
+#include "ZlibDecompressor.h"
+
+static inline uint16_t readU16(const unsigned char *data)
+{
+	return data[0] << 8 | data[1];
+}
+
+static inline uint16_t readBlockContent(const unsigned char *mapData,
+	u8 contentWidth, unsigned int datapos)
+{
+	if (contentWidth == 2) {
+		size_t index = datapos << 1;
+		return (mapData[index] << 8) | mapData[index + 1];
+	} else {
+		u8 param = mapData[datapos];
+		if (param <= 0x7f)
+			return param;
+		else
+			return (param << 4) | (mapData[datapos + 0x2000] >> 4);
+	}
+}
+
+BlockDecoder::BlockDecoder()
+{
+	reset();
+}
+
+void BlockDecoder::reset()
+{
+	m_blockAirId = -1;
+	m_blockIgnoreId = -1;
+	m_nameMap.clear();
+
+	m_version = 0;
+	m_contentWidth = 0;
+	m_mapData.clear();
+}
+
+void BlockDecoder::decode(const ustring &datastr)
+{
+	const unsigned char *data = datastr.c_str();
+	size_t length = datastr.length();
+	// TODO: bounds checks
+
+	uint8_t version = data[0];
+	if (version < 22) {
+		std::ostringstream oss;
+		oss << "Unsupported map version " << (int)version;
+		throw std::runtime_error(oss.str());
+	}
+	m_version = version;
+
+	ustring datastr2;
+	if (version >= 29) {
+		// decompress whole block at once
+		m_zstd_decompressor.setData(data, length, 1);
+		datastr2 = m_zstd_decompressor.decompress();
+		data = datastr2.c_str();
+		length = datastr2.size();
+	}
+
+	size_t dataOffset = 0;
+	if (version >= 29)
+		dataOffset = 7;
+	else if (version >= 27)
+		dataOffset = 4;
+	else
+		dataOffset = 2;
+
+	auto decode_mapping = [&] () {
+		dataOffset++; // mapping version
+		uint16_t numMappings = readU16(data + dataOffset);
+		dataOffset += 2;
+		for (int i = 0; i < numMappings; ++i) {
+			uint16_t nodeId = readU16(data + dataOffset);
+			dataOffset += 2;
+			uint16_t nameLen = readU16(data + dataOffset);
+			dataOffset += 2;
+			std::string name(reinterpret_cast<const char *>(data) + dataOffset, nameLen);
+			if (name == "air")
+				m_blockAirId = nodeId;
+			else if (name == "ignore")
+				m_blockIgnoreId = nodeId;
+			else
+				m_nameMap[nodeId] = name;
+			dataOffset += nameLen;
+		}
+	};
+
+	if (version >= 29)
+		decode_mapping();
+
+	uint8_t contentWidth = data[dataOffset];
+	dataOffset++;
+	uint8_t paramsWidth = data[dataOffset];
+	dataOffset++;
+	if (contentWidth != 1 && contentWidth != 2)
+		throw std::runtime_error("unsupported map version (contentWidth)");
+	if (paramsWidth != 2)
+		throw std::runtime_error("unsupported map version (paramsWidth)");
+	m_contentWidth = contentWidth;
+
+	if (version >= 29) {
+		m_mapData.resize((contentWidth + paramsWidth) * 4096);
+		m_mapData.assign(data + dataOffset, m_mapData.size());
+		return; // we have read everything we need and can return early
+	}
+
+	// version < 29
+	ZlibDecompressor decompressor(data, length);
+	decompressor.setSeekPos(dataOffset);
+	m_mapData = decompressor.decompress();
+	decompressor.decompress(); // unused metadata
+	dataOffset = decompressor.seekPos();
+
+	// Skip unused node timers
+	if (version == 23)
+		dataOffset += 1;
+	if (version == 24) {
+		uint8_t ver = data[dataOffset++];
+		if (ver == 1) {
+			uint16_t num = readU16(data + dataOffset);
+			dataOffset += 2;
+			dataOffset += 10 * num;
+		}
+	}
+
+	// Skip unused static objects
+	dataOffset++; // Skip static object version
+	int staticObjectCount = readU16(data + dataOffset);
+	dataOffset += 2;
+	for (int i = 0; i < staticObjectCount; ++i) {
+		dataOffset += 13;
+		uint16_t dataSize = readU16(data + dataOffset);
+		dataOffset += dataSize + 2;
+	}
+	dataOffset += 4; // Skip timestamp
+
+	// Read mapping
+	decode_mapping();
+}
+
+bool BlockDecoder::isEmpty() const
+{
+	// only contains ignore and air nodes?
+	return m_nameMap.empty();
+}
+
+const static std::string empty;
+
+const std::string &BlockDecoder::getNode(u8 x, u8 y, u8 z) const
+{
+	unsigned int position = x + (y << 4) + (z << 8);
+	uint16_t content = readBlockContent(m_mapData.c_str(), m_contentWidth, position);
+	if (content == m_blockAirId || content == m_blockIgnoreId)
+		return empty;
+	NameMap::const_iterator it = m_nameMap.find(content);
+	if (it == m_nameMap.end()) {
+		std::cerr << "Skipping node with invalid ID." << std::endl;
+		return empty;
+	}
+	return it->second;
+}

CMakeLists.txt

@@ -1,11 +1,9 @@
-project(minetestmapper CXX)
-cmake_minimum_required(VERSION 2.6)
-cmake_policy(SET CMP0003 NEW)
-set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
+cmake_minimum_required(VERSION 3.5)
 
-set(VERSION_MAJOR 1)
-set(VERSION_MINOR 0)
-set(VERSION_STRING "${VERSION_MAJOR}.${VERSION_MINOR}")
+project(minetestmapper
+	VERSION 1.0
+	LANGUAGES CXX
+)
 
 # Stuff & Paths
 
@@ -13,17 +11,18 @@ if(NOT CMAKE_BUILD_TYPE)
 	set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build." FORCE)
 endif()
 
-set(CMAKE_CXX_FLAGS_RELEASE "-O3 -Wall")
-set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g2 -Wall")
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 if(WIN32)
 	set(SHAREDIR ".")
 	set(BINDIR ".")
 	set(DOCDIR ".")
 else()
-	set(SHAREDIR "${CMAKE_INSTALL_PREFIX}/share/minetest") # an extra dir. for just one file doesn't seem useful
-	set(BINDIR "${CMAKE_INSTALL_PREFIX}/bin")
-	set(DOCDIR "${CMAKE_INSTALL_PREFIX}/share/doc/${PROJECT_NAME}")
+	set(SHAREDIR "share/minetest") # reuse Minetest share dir
+	set(BINDIR "bin")
+	set(DOCDIR "share/doc/${PROJECT_NAME}")
+	set(MANDIR "share/man")
 endif()
 
 set(CUSTOM_SHAREDIR "" CACHE STRING "Directory to install data files into")
@@ -44,6 +43,7 @@ if(NOT CUSTOM_DOCDIR STREQUAL "")
         message(STATUS "Using DOCDIR=${DOCDIR}")
 endif()
 
+set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
 
 # Libraries: gd
 
@@ -57,116 +57,145 @@ endif(NOT LIBGD_LIBRARY OR NOT LIBGD_INCLUDE_DIR)
 
 # Libraries: zlib
 
-find_library(ZLIB_LIBRARY z)
-find_path(ZLIB_INCLUDE_DIR zlib.h)
-message (STATUS "zlib library: ${ZLIB_LIBRARY}")
-message (STATUS "zlib headers: ${ZLIB_INCLUDE_DIR}")
-if(NOT ZLIB_LIBRARY OR NOT ZLIB_INCLUDE_DIR)
-	message(FATAL_ERROR "zlib not found!")
-endif(NOT ZLIB_LIBRARY OR NOT ZLIB_INCLUDE_DIR)
+find_package(ZLIB REQUIRED)
 
-find_package(PkgConfig)
-include(FindPackageHandleStandardArgs)
+# Libraries: zstd
+
+find_package(Zstd REQUIRED)
 
 # Libraries: sqlite3
 
 find_library(SQLITE3_LIBRARY sqlite3)
-find_path(SQLITE3_INCLUDE_DIR zlib.h)
+find_path(SQLITE3_INCLUDE_DIR sqlite3.h)
 message (STATUS "sqlite3 library: ${SQLITE3_LIBRARY}")
 message (STATUS "sqlite3 headers: ${SQLITE3_INCLUDE_DIR}")
 if(NOT SQLITE3_LIBRARY OR NOT SQLITE3_INCLUDE_DIR)
 	message(FATAL_ERROR "sqlite3 not found!")
 endif(NOT SQLITE3_LIBRARY OR NOT SQLITE3_INCLUDE_DIR)
 
+# Libraries: postgresql
+
+option(ENABLE_POSTGRESQL "Enable PostgreSQL backend" TRUE)
+set(USE_POSTGRESQL FALSE)
+
+if(ENABLE_POSTGRESQL)
+	if(CMAKE_VERSION VERSION_LESS "3.20")
+		find_package(PostgreSQL QUIET)
+		# Before CMake 3.20 FindPostgreSQL.cmake always looked for server includes
+		# but we don't need them, so continue anyway if only those are missing.
+		if(PostgreSQL_INCLUDE_DIR AND PostgreSQL_LIBRARY)
+			set(PostgreSQL_FOUND TRUE)
+			set(PostgreSQL_INCLUDE_DIRS ${PostgreSQL_INCLUDE_DIR})
+			set(PostgreSQL_LIBRARIES ${PostgreSQL_LIBRARY})
+		endif()
+	else()
+		find_package(PostgreSQL)
+	endif()
+
+	if(PostgreSQL_FOUND)
+		set(USE_POSTGRESQL TRUE)
+		message(STATUS "PostgreSQL backend enabled")
+		# This variable is case sensitive, don't try to change it to POSTGRESQL_INCLUDE_DIR
+		message(STATUS "PostgreSQL includes: ${PostgreSQL_INCLUDE_DIRS}")
+		include_directories(${PostgreSQL_INCLUDE_DIRS})
+	else()
+		message(STATUS "PostgreSQL not found!")
+		set(PostgreSQL_LIBRARIES "")
+	endif()
+endif(ENABLE_POSTGRESQL)
+
 # Libraries: leveldb
 
-set(USE_LEVELDB 0)
-
-OPTION(ENABLE_LEVELDB "Enable LevelDB backend")
+OPTION(ENABLE_LEVELDB "Enable LevelDB backend" TRUE)
+set(USE_LEVELDB FALSE)
 
 if(ENABLE_LEVELDB)
 	find_library(LEVELDB_LIBRARY leveldb)
-	find_path(LEVELDB_INCLUDE_DIR db.h PATH_SUFFIXES leveldb)
+	find_path(LEVELDB_INCLUDE_DIR leveldb/db.h)
 	message (STATUS "LevelDB library: ${LEVELDB_LIBRARY}")
 	message (STATUS "LevelDB headers: ${LEVELDB_INCLUDE_DIR}")
 	if(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR)
-		set(USE_LEVELDB 1)
+		set(USE_LEVELDB TRUE)
 		message(STATUS "LevelDB backend enabled")
 		include_directories(${LEVELDB_INCLUDE_DIR})
-	else(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR)
-		set(USE_LEVELDB 0)
+	else()
 		message(STATUS "LevelDB not found!")
-	endif(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR)
+		set(LEVELDB_LIBRARY "")
+	endif()
 endif(ENABLE_LEVELDB)
 
 # Libraries: redis
 
-set(USE_REDIS 0)
-
-OPTION(ENABLE_REDIS "Enable redis backend")
+OPTION(ENABLE_REDIS "Enable redis backend" TRUE)
+set(USE_REDIS FALSE)
 
 if(ENABLE_REDIS)
 	find_library(REDIS_LIBRARY hiredis)
-	find_path(REDIS_INCLUDE_DIR hiredis.h PATH_SUFFIXES hiredis)
+	find_path(REDIS_INCLUDE_DIR hiredis/hiredis.h)
 	message (STATUS "redis library: ${REDIS_LIBRARY}")
 	message (STATUS "redis headers: ${REDIS_INCLUDE_DIR}")
 	if(REDIS_LIBRARY AND REDIS_INCLUDE_DIR)
-		set(USE_REDIS 1)
+		set(USE_REDIS TRUE)
 		message(STATUS "redis backend enabled")
 		include_directories(${REDIS_INCLUDE_DIR})
-	else(REDIS_LIBRARY AND REDIS_INCLUDE_DIR)
-		set(USE_REDIS 0)
+	else()
 		message(STATUS "redis not found!")
-	endif(REDIS_LIBRARY AND REDIS_INCLUDE_DIR)
+		set(REDIS_LIBRARY "")
+	endif()
 endif(ENABLE_REDIS)
 
 # Compiling & Linking
 
 include_directories(
 	"${PROJECT_BINARY_DIR}"
-	"${CMAKE_CURRENT_SOURCE_DIR}"
+	"${CMAKE_CURRENT_SOURCE_DIR}/include"
 	"${CMAKE_CURRENT_BINARY_DIR}"
 	${SQLITE3_INCLUDE_DIR}
 	${LIBGD_INCLUDE_DIR}
 	${ZLIB_INCLUDE_DIR}
+	${ZSTD_INCLUDE_DIR}
 )
 
 configure_file(
-	"${PROJECT_SOURCE_DIR}/cmake_config.h.in"
+	"${PROJECT_SOURCE_DIR}/include/cmake_config.h.in"
 	"${PROJECT_BINARY_DIR}/cmake_config.h"
 )
-add_definitions ( -DUSE_CMAKE_CONFIG_H )
+add_definitions(-DUSE_CMAKE_CONFIG_H)
 
-set(mapper_SRCS
+if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang)$")
+	set(CMAKE_CXX_FLAGS_RELEASE "-O2")
+	set(CMAKE_CXX_FLAGS_DEBUG "-Og -g2")
+	add_compile_options(-Wall -pipe)
+endif()
+if(CMAKE_BUILD_TYPE STREQUAL "Release")
+	add_definitions(-DNDEBUG)
+endif()
+
+add_executable(minetestmapper
+	BlockDecoder.cpp
 	PixelAttributes.cpp
 	PlayerAttributes.cpp
 	TileGenerator.cpp
 	ZlibDecompressor.cpp
+	ZstdDecompressor.cpp
 	Image.cpp
 	mapper.cpp
 	util.cpp
 	db-sqlite3.cpp
-)
-
-if(USE_LEVELDB)
-	set(mapper_SRCS ${mapper_SRCS} db-leveldb.cpp)
-endif(USE_LEVELDB)
-
-if(USE_REDIS)
-	set(mapper_SRCS ${mapper_SRCS} db-redis.cpp)
-endif(USE_REDIS)
-
-add_executable(minetestmapper
-	${mapper_SRCS}
+	$<$<BOOL:${USE_POSTGRESQL}>:db-postgresql.cpp>
+	$<$<BOOL:${USE_LEVELDB}>:db-leveldb.cpp>
+	$<$<BOOL:${USE_REDIS}>:db-redis.cpp>
 )
 
 target_link_libraries(
 	minetestmapper
 	${SQLITE3_LIBRARY}
+	${PostgreSQL_LIBRARIES}
 	${LEVELDB_LIBRARY}
 	${REDIS_LIBRARY}
 	${LIBGD_LIBRARY}
 	${ZLIB_LIBRARY}
+	${ZSTD_LIBRARY}
 )
 
 # Installing & Packaging
@@ -176,18 +205,19 @@ install(FILES "AUTHORS" DESTINATION "${DOCDIR}")
 install(FILES "COPYING" DESTINATION "${DOCDIR}")
 install(FILES "README.rst" DESTINATION "${DOCDIR}")
 install(FILES "colors.txt" DESTINATION "${SHAREDIR}")
+if(UNIX)
+	install(FILES "minetestmapper.6" DESTINATION "${MANDIR}/man6")
+endif()
 
 set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Overview mapper for Minetest")
-set(CPACK_PACKAGE_VERSION_MAJOR ${VERSION_MAJOR})
-set(CPACK_PACKAGE_VERSION_MINOR ${VERSION_MINOR})
 set(CPACK_PACKAGE_VENDOR "celeron55")
 set(CPACK_PACKAGE_CONTACT "Perttu Ahola <celeron55@gmail.com>")
 
 if(WIN32)
-	set(CPACK_PACKAGE_FILE_NAME "${PROJECT_NAME}-${VERSION_STRING}-win32")
+	set(CPACK_PACKAGE_FILE_NAME "${PROJECT_NAME}-${PROJECT_VERSION}-win32")
 	set(CPACK_GENERATOR ZIP)
 else()
-	set(CPACK_PACKAGE_FILE_NAME "${PROJECT_NAME}-${VERSION_STRING}-linux")
+	set(CPACK_PACKAGE_FILE_NAME "${PROJECT_NAME}-${PROJECT_VERSION}-linux")
 	set(CPACK_GENERATOR TGZ)
 	set(CPACK_SOURCE_GENERATOR TGZ)
 endif()

Image.cpp

@@ -10,40 +10,53 @@
 #include "Image.h"
 
 #ifndef NDEBUG
-#define SIZECHECK(x, y) do { \
-		if((x) < 0 || (x) >= m_width) \
-			throw std::out_of_range("sizecheck x"); \
-		if((y) < 0 || (y) >= m_height) \
-			throw std::out_of_range("sizecheck y"); \
-	} while(0)
+#define SIZECHECK(x, y) check_bounds((x), (y), m_width, m_height)
 #else
 #define SIZECHECK(x, y) do {} while(0)
 #endif
 
 // ARGB but with inverted alpha
 
-static inline int color2int(Color c)
+static inline int color2int(const Color &c)
 {
-	u8 a = 255 - c.a;
+	u8 a = (255 - c.a) * gdAlphaMax / 255;
 	return (a << 24) | (c.r << 16) | (c.g << 8) | c.b;
 }
 
 static inline Color int2color(int c)
 {
 	Color c2;
-	u8 a;
 	c2.b = c & 0xff;
 	c2.g = (c >> 8) & 0xff;
 	c2.r = (c >> 16) & 0xff;
-	a = (c >> 24) & 0xff;
-	c2.a = 255 - a;
+	u8 a = (c >> 24) & 0xff;
+	c2.a = 255 - (a*255 / gdAlphaMax);
 	return c2;
 }
 
+#ifndef NDEBUG
+static inline void check_bounds(int x, int y, int width, int height)
+{
+	if(x < 0 || x >= width) {
+		std::ostringstream oss;
+		oss << "Access outside image bounds (x), 0 < "
+			<< x << " < " << width << " is false.";
+		throw std::out_of_range(oss.str());
+	}
+	if(y < 0 || y >= height) {
+		std::ostringstream oss;
+		oss << "Access outside image bounds (y), 0 < "
+			<< y << " < " << height << " is false.";
+		throw std::out_of_range(oss.str());
+	}
+}
+#endif
+
 
 Image::Image(int width, int height) :
-	m_width(width), m_height(height), m_image(NULL)
+	m_width(width), m_height(height), m_image(nullptr)
 {
+	SIZECHECK(0, 0);
 	m_image = gdImageCreateTrueColor(m_width, m_height);
 }
 
@@ -60,10 +73,7 @@ void Image::setPixel(int x, int y, const Color &c)
 
 Color Image::getPixel(int x, int y)
 {
-#ifndef NDEBUG
-	if(x < 0 || x > m_width || y < 0 || y > m_height)
-		throw std::out_of_range("sizecheck");
-#endif
+	SIZECHECK(x, y);
 	return int2color(m_image->tpixels[y][x]);
 }
 
@@ -95,12 +105,22 @@ void Image::drawCircle(int x, int y, int diameter, const Color &c)
 
 void Image::save(const std::string &filename)
 {
+#if (GD_MAJOR_VERSION == 2 && GD_MINOR_VERSION == 1 && GD_RELEASE_VERSION >= 1) || (GD_MAJOR_VERSION == 2 && GD_MINOR_VERSION > 1) || GD_MAJOR_VERSION > 2
+	const char *f = filename.c_str();
+	if (gdSupportsFileType(f, 1) == GD_FALSE)
+		throw std::runtime_error("Image format not supported by gd");
+	if (gdImageFile(m_image, f) == GD_FALSE)
+		throw std::runtime_error("Error saving image");
+#else
+	if (filename.compare(filename.length() - 4, 4, ".png") != 0)
+		throw std::runtime_error("Only PNG is supported");
 	FILE *f = fopen(filename.c_str(), "wb");
-	if(!f) {
+	if (!f) {
 		std::ostringstream oss;
-		oss << "Error writing image file: " << std::strerror(errno);
+		oss << "Error opening image file: " << std::strerror(errno);
 		throw std::runtime_error(oss.str());
 	}
-	gdImagePng(m_image, f); // other formats?
+	gdImagePng(m_image, f);
 	fclose(f);
+#endif
 }

PixelAttributes.cpp

@@ -1,23 +1,12 @@
-/*
- * =====================================================================
- *        Version:  1.0
- *        Created:  25.08.2012 10:55:27
- *         Author:  Miroslav Bendík
- *        Company:  LinuxOS.sk
- * =====================================================================
- */
-
-#include <cstdlib>
 #include <cstring>
-#include "PixelAttributes.h"
 
-using namespace std;
+#include "PixelAttributes.h"
 
 PixelAttributes::PixelAttributes():
 	m_width(0)
 {
 	for (size_t i = 0; i < LineCount; ++i) {
-		m_pixelAttributes[i] = 0;
+		m_pixelAttributes[i] = nullptr;
 	}
 }
 
@@ -47,9 +36,9 @@ void PixelAttributes::scroll()
 void PixelAttributes::freeAttributes()
 {
 	for (size_t i = 0; i < LineCount; ++i) {
-		if (m_pixelAttributes[i] != 0) {
+		if (m_pixelAttributes[i] != nullptr) {
 			delete[] m_pixelAttributes[i];
-			m_pixelAttributes[i] = 0;
+			m_pixelAttributes[i] = nullptr;
 		}
 	}
 }

PixelAttributes.h

@@ -1,50 +0,0 @@
-/*
- * =====================================================================
- *        Version:  1.0
- *        Created:  25.08.2012 10:55:29
- *         Author:  Miroslav Bendík
- *        Company:  LinuxOS.sk
- * =====================================================================
- */
-
-#ifndef PIXELATTRIBUTES_H_ADZ35GYF
-#define PIXELATTRIBUTES_H_ADZ35GYF
-
-#include <limits>
-#include <stdint.h>
-#include "config.h"
-
-struct PixelAttribute {
-	PixelAttribute(): height(std::numeric_limits<int>::min()), thickness(0) {};
-	int height;
-	uint8_t thickness;
-	inline bool valid_height() {
-		return height != std::numeric_limits<int>::min();
-	}
-};
-
-class PixelAttributes
-{
-public:
-	PixelAttributes();
-	virtual ~PixelAttributes();
-	void setWidth(int width);
-	void scroll();
-	inline PixelAttribute &attribute(int z, int x) { return m_pixelAttributes[z + 1][x + 1]; };
-
-private:
-	void freeAttributes();
-
-private:
-	enum Line {
-		FirstLine = 0,
-		LastLine = BLOCK_SIZE,
-		EmptyLine = BLOCK_SIZE + 1,
-		LineCount = BLOCK_SIZE + 2
-	};
-	PixelAttribute *m_pixelAttributes[BLOCK_SIZE + 2]; // 1px gradient + empty
-	int m_width;
-};
-
-#endif /* end of include guard: PIXELATTRIBUTES_H_ADZ35GYF */
-

PlayerAttributes.cpp

@@ -1,73 +1,131 @@
-/*
- * =====================================================================
- *        Version:  1.0
- *        Created:  01.09.2012 14:38:05
- *         Author:  Miroslav Bendík
- *        Company:  LinuxOS.sk
- * =====================================================================
- */
-
-#include <dirent.h>
 #include <fstream>
 #include <sstream>
+#include <stdexcept>
+#include <dirent.h>
+#include <unistd.h> // for usleep
+#include <sqlite3.h>
+
 #include "config.h"
 #include "PlayerAttributes.h"
+#include "util.h"
 
-using namespace std;
-
-PlayerAttributes::PlayerAttributes(const std::string &sourceDirectory)
+PlayerAttributes::PlayerAttributes(const std::string &worldDir)
 {
+	std::ifstream ifs(worldDir + "world.mt");
+	if (!ifs.good())
+		throw std::runtime_error("Failed to read world.mt");
+	std::string backend = read_setting_default("player_backend", ifs, "files");
+	ifs.close();
 
-	string playersPath = sourceDirectory + "players";
+	if (backend == "files")
+		readFiles(worldDir + "players");
+	else if (backend == "sqlite3")
+		readSqlite(worldDir + "players.sqlite");
+	else
+		throw std::runtime_error(std::string("Unknown player backend: ") + backend);
+}
+
+void PlayerAttributes::readFiles(const std::string &playersPath)
+{
 	DIR *dir;
 	dir = opendir (playersPath.c_str());
-	if (dir == NULL) {
+	if (!dir)
 		return;
-	}
 
 	struct dirent *ent;
 	while ((ent = readdir (dir)) != NULL) {
-		if (ent->d_name[0] == '.') {
+		if (ent->d_name[0] == '.')
 			continue;
-		}
 
-		string path = playersPath + PATH_SEPARATOR + ent->d_name;
+		std::ifstream in(playersPath + PATH_SEPARATOR + ent->d_name);
+		if (!in.good())
+			continue;
+
+		std::string name, position;
+		name = read_setting("name", in);
+		in.seekg(0);
+		position = read_setting("position", in);
 
-		ifstream in;
-		in.open(path.c_str(), ifstream::in);
-		string buffer;
-		string name;
-		string position;
-		while (getline(in, buffer)) {
-			if (buffer.find("name = ") == 0) {
-				name = buffer.substr(7);
-			}
-			else if (buffer.find("position = ") == 0) {
-				position = buffer.substr(12, buffer.length() - 13);
-			}
-		}
-		char comma;
 		Player player;
-		istringstream positionStream(position, istringstream::in);
-		positionStream >> player.x;
-		positionStream >> comma;
-		positionStream >> player.y;
-		positionStream >> comma;
-		positionStream >> player.z;
+		std::istringstream iss(position);
+		char tmp;
+		iss >> tmp; // '('
+		iss >> player.x;
+		iss >> tmp; // ','
+		iss >> player.y;
+		iss >> tmp; // ','
+		iss >> player.z;
+		iss >> tmp; // ')'
+		if (tmp != ')')
+			continue;
 		player.name = name;
 
+		player.x /= 10.0f;
+		player.y /= 10.0f;
+		player.z /= 10.0f;
+
 		m_players.push_back(player);
 	}
+
 	closedir(dir);
 }
 
-PlayerAttributes::Players::iterator PlayerAttributes::begin()
+/**********/
+
+#define SQLRES(f, good) \
+	result = (sqlite3_##f); \
+	if (result != good) { \
+		throw std::runtime_error(sqlite3_errmsg(db));\
+	}
+#define SQLOK(f) SQLRES(f, SQLITE_OK)
+
+void PlayerAttributes::readSqlite(const std::string &db_name)
 {
-	return m_players.begin();
+	int result;
+	sqlite3 *db;
+	sqlite3_stmt *stmt_get_player_pos;
+
+	SQLOK(open_v2(db_name.c_str(), &db, SQLITE_OPEN_READONLY |
+			SQLITE_OPEN_PRIVATECACHE, 0))
+
+	SQLOK(prepare_v2(db,
+			"SELECT name, posX, posY, posZ FROM player",
+		-1, &stmt_get_player_pos, NULL))
+
+	while ((result = sqlite3_step(stmt_get_player_pos)) != SQLITE_DONE) {
+		if (result == SQLITE_BUSY) { // Wait some time and try again
+			usleep(10000);
+		} else if (result != SQLITE_ROW) {
+			throw std::runtime_error(sqlite3_errmsg(db));
+		}
+
+		Player player;
+		const unsigned char *name_ = sqlite3_column_text(stmt_get_player_pos, 0);
+		player.name = reinterpret_cast<const char*>(name_);
+		player.x = sqlite3_column_double(stmt_get_player_pos, 1);
+		player.y = sqlite3_column_double(stmt_get_player_pos, 2);
+		player.z = sqlite3_column_double(stmt_get_player_pos, 3);
+
+		player.x /= 10.0f;
+		player.y /= 10.0f;
+		player.z /= 10.0f;
+
+		m_players.push_back(player);
+	}
+
+	sqlite3_finalize(stmt_get_player_pos);
+	sqlite3_close(db);
 }
 
-PlayerAttributes::Players::iterator PlayerAttributes::end()
+/**********/
+
+PlayerAttributes::Players::const_iterator PlayerAttributes::begin() const
 {
-	return m_players.end();
+	return m_players.cbegin();
+}
+
+PlayerAttributes::Players::const_iterator PlayerAttributes::end() const
+{
+	return m_players.cend();
 }
 

PlayerAttributes.h

@@ -1,38 +0,0 @@
-/*
- * =====================================================================
- *        Version:  1.0
- *        Created:  01.09.2012 14:38:08
- *         Author:  Miroslav Bendík
- *        Company:  LinuxOS.sk
- * =====================================================================
- */
-
-#ifndef PLAYERATTRIBUTES_H_D7THWFVV
-#define PLAYERATTRIBUTES_H_D7THWFVV
-
-#include <list>
-#include <string>
-
-struct Player
-{
-	std::string name;
-	double x;
-	double y;
-	double z;
-}; /* -----  end of struct Player  ----- */
-
-class PlayerAttributes
-{
-public:
-	typedef std::list<Player> Players;
-
-	PlayerAttributes(const std::string &sourceDirectory);
-	Players::iterator begin();
-	Players::iterator end();
-
-private:
-	Players m_players;
-}; /* -----  end of class PlayerAttributes  ----- */
-
-#endif /* end of include guard: PLAYERATTRIBUTES_H_D7THWFVV */
-

README.rst

@@ -1,24 +1,49 @@
 Minetest Mapper C++
 ===================
 
-.. image:: https://travis-ci.org/minetest/minetestmapper.svg?branch=master
-    :target: https://travis-ci.org/minetest/minetestmapper
+.. image:: https://github.com/minetest/minetestmapper/workflows/build/badge.svg
+    :target: https://github.com/minetest/minetestmapper/actions/workflows/build.yml
 
-A port of minetestmapper.py to C++ from https://github.com/minetest/minetest/tree/master/util.
-This version is both faster and provides more features than the now deprecated Python script.
+Minetestmapper generates an overview image from a Minetest map.
+
+A port of minetestmapper.py to C++ from https://github.com/minetest/minetest/tree/0.4.17/util.
+This version is both faster and provides more features than the now obsolete Python script.
+
+Minetestmapper ships with a colors.txt file for Minetest Game, if you use a different game or have
+many mods installed you should generate a matching colors.txt for better results.
+The `generate_colorstxt.py script 
+<./util/generate_colorstxt.py>`_ in the util folder exists for this purpose, detailed instructions can be found within.
 
 Requirements
 ------------
 
+* C++ compiler, zlib, zstd
 * libgd
 * sqlite3
-* leveldb (optional, set ENABLE_LEVELDB=1 in CMake to enable leveldb support)
-* hiredis (optional, set ENABLE_REDIS=1 in CMake to enable redis support)
+* LevelDB (optional)
+* hiredis (optional)
+* Postgres libraries (optional)
 
-e.g. on Debian:
-^^^^^^^^^^^^^^^
+on Debian/Ubuntu:
+^^^^^^^^^^^^^^^^^
 
-	sudo apt-get install libgd-dev libsqlite3-dev libleveldb-dev libhiredis-dev
+``sudo apt install cmake libgd-dev libhiredis-dev libleveldb-dev libpq-dev libsqlite3-dev zlib1g-dev libzstd-dev``
+
+on openSUSE:
+^^^^^^^^^^^^
+
+``sudo zypper install gd-devel hiredis-devel leveldb-devel postgresql-devel sqlite3-devel zlib-devel libzstd-devel``
+
+for Windows:
+^^^^^^^^^^^^
+Minetestmapper for Windows can be downloaded `from the Releases section
+<https://github.com/minetest/minetestmapper/releases>`_.
+
+After extracting the archive, it can be invoked from cmd.exe:
+::
+
+	cd C:\Users\yourname\Desktop\example\path
+	minetestmapper.exe --help
 
 Compilation
 -----------
@@ -26,7 +51,7 @@ Compilation
 ::
 
     cmake . -DENABLE_LEVELDB=1
-    make -j2
+    make -j$(nproc)
 
 Usage
 -----
@@ -43,32 +68,38 @@ Parameters
 ^^^^^^^^^^
 
 bgcolor:
-    Background color of image, e.g. ``--bgcolor #ffffff``
+    Background color of image, e.g. ``--bgcolor '#ffffff'``
 
 scalecolor:
-    Color of scale, e.g. ``--scalecolor #000000``
+    Color of scale marks and text, e.g. ``--scalecolor '#000000'``
 
 playercolor:
-    Color of player indicators, e.g. ``--playercolor #ff0000``
+    Color of player indicators, e.g. ``--playercolor '#ff0000'``
 
 origincolor:
-    Color of origin indicator, e.g. ``--origincolor #ff0000``
+    Color of origin indicator, e.g. ``--origincolor '#ff0000'``
 
 drawscale:
-    Draw tick marks, ``--drawscale``
+    Draw scale(s) with tick marks and numbers, ``--drawscale``
 
 drawplayers:
-    Draw player indicators, ``--drawplayers``
+    Draw player indicators with name, ``--drawplayers``
 
 draworigin:
     Draw origin indicator, ``--draworigin``
 
 drawalpha:
-    Allow nodes to be drawn with transparency, ``--drawalpha``
+    Allow nodes to be drawn with transparency (e.g. water), ``--drawalpha``
+
+extent:
+    Don't output any imagery, just print the extent of the full map, ``--extent``
 
 noshading:
     Don't draw shading on nodes, ``--noshading``
 
+noemptyimage:
+    Don't output anything when the image would be empty, ``--noemptyimage``
+
 min-y:
     Don't draw nodes below this y value, e.g. ``--min-y -25``
 
@@ -76,16 +107,21 @@ max-y:
     Don't draw nodes above this y value, e.g. ``--max-y 75``
 
 backend:
-    Use specific map backend, supported: *sqlite3*, *leveldb*, *redis*, e.g. ``--backend leveldb``
+    Override auto-detected map backend; supported: *sqlite3*, *leveldb*, *redis*, *postgresql*, e.g. ``--backend leveldb``
 
 geometry:
-    Limit area to specific geometry (*x:y+w+h* where x and y specify the lower left corner), e.g. ``--geometry -800:-800+1600+1600``
+    Limit area to specific geometry (*x:z+w+h* where x and z specify the lower left corner), e.g. ``--geometry -800:-800+1600+1600``
 
 zoom:
-    "Zoom" the image by using more than one pixel per node, e.g. ``--zoom 4``
+    Apply zoom to drawn nodes by enlarging them to n*n squares, e.g. ``--zoom 4``
 
 colors:
-    Forcefully set path to colors.txt file (it's autodetected otherwise), e.g. ``--colors ../minetest/mycolors.txt``
+    Override auto-detected path to colors.txt, e.g. ``--colors ../minetest/mycolors.txt``
 
 scales:
     Draw scales on specified image edges (letters *t b l r* meaning top, bottom, left and right), e.g. ``--scales tbr``
+
+exhaustive:
+    | Select if database should be traversed exhaustively or using range queries, available: *never*, *y*, *full*, *auto*
+    | Defaults to *auto*. You shouldn't need to change this, but doing so can improve rendering times on large maps.
+    | For these optimizations to work it is important that you set ``min-y`` and ``max-y`` when you don't care about the world below e.g. -60 and above 1000 nodes.

TileGenerator.h

@@ -1,138 +0,0 @@
-#ifndef TILEGENERATOR_HEADER
-#define TILEGENERATOR_HEADER
-
-#include <gd.h>
-#include <iosfwd>
-#include <list>
-#include <config.h>
-#if __cplusplus >= 201103L
-#include <unordered_map>
-#include <unordered_set>
-#else
-#include <map>
-#include <set>
-#endif
-#include <stdint.h>
-#include <string>
-#include "PixelAttributes.h"
-#include "Image.h"
-#include "db.h"
-#include "types.h"
-
-enum {
-	SCALE_TOP = (1 << 0),
-	SCALE_BOTTOM = (1 << 1),
-	SCALE_LEFT = (1 << 2),
-	SCALE_RIGHT = (1 << 3),
-};
-
-struct ColorEntry {
-	ColorEntry(): r(0), g(0), b(0), a(0), t(0) {};
-	ColorEntry(uint8_t r, uint8_t g, uint8_t b, uint8_t a, uint8_t t): r(r), g(g), b(b), a(a), t(t) {};
-	inline Color to_color() const { return Color(r, g, b, a); }
-	uint8_t r;
-	uint8_t g;
-	uint8_t b;
-	uint8_t a;
-	uint8_t t;
-};
-
-
-class TileGenerator
-{
-private:
-#if __cplusplus >= 201103L
-	typedef std::unordered_map<std::string, ColorEntry> ColorMap;
-	typedef std::unordered_map<int, std::string> NameMap;
-	typedef std::unordered_set<std::string> NameSet;
-#else
-	typedef std::map<std::string, ColorEntry> ColorMap;
-	typedef std::map<int, std::string> NameMap;
-	typedef std::set<std::string> NameSet;
-#endif
-
-public:
-	TileGenerator();
-	~TileGenerator();
-	void setBgColor(const std::string &bgColor);
-	void setScaleColor(const std::string &scaleColor);
-	void setOriginColor(const std::string &originColor);
-	void setPlayerColor(const std::string &playerColor); Color parseColor(const std::string &color);
-	void setDrawOrigin(bool drawOrigin);
-	void setDrawPlayers(bool drawPlayers);
-	void setDrawScale(bool drawScale);
-	void setDrawAlpha(bool drawAlpha);
-	void setShading(bool shading);
-	void setGeometry(int x, int y, int w, int h);
-	void setMinY(int y);
-	void setMaxY(int y);
-	void parseColorsFile(const std::string &fileName);
-	void setBackend(std::string backend);
-	void generate(const std::string &input, const std::string &output);
-	void setZoom(int zoom);
-	void setScales(uint flags);
-
-private:
-	void parseColorsStream(std::istream &in);
-	void openDb(const std::string &input);
-	void closeDatabase();
-	void loadBlocks();
-	void createImage();
-	void renderMap();
-	std::list<int> getZValueList() const;
-	void renderMapBlock(const ustring &mapBlock, const BlockPos &pos, int version);
-	void renderMapBlockBottom(const BlockPos &pos);
-	void renderShading(int zPos);
-	void renderScale();
-	void renderOrigin();
-	void renderPlayers(const std::string &inputPath);
-	void writeImage(const std::string &output);
-	void printUnknown();
-	int getImageX(int val) const;
-	int getImageY(int val) const;
-	void setZoomed(int x, int y, Color color);
-
-private:
-	Color m_bgColor;
-	Color m_scaleColor;
-	Color m_originColor;
-	Color m_playerColor;
-	bool m_drawOrigin;
-	bool m_drawPlayers;
-	bool m_drawScale;
-	bool m_drawAlpha;
-	bool m_shading;
-	std::string m_backend;
-	int m_xBorder, m_yBorder;
-
-	DB *m_db;
-	Image *m_image;
-	PixelAttributes m_blockPixelAttributes;
-	int m_xMin;
-	int m_xMax;
-	int m_zMin;
-	int m_zMax;
-	int m_yMin;
-	int m_yMax;
-	int m_geomX;
-	int m_geomY;
-	int m_geomX2;
-	int m_geomY2;
-	int m_mapWidth;
-	int m_mapHeight;
-	std::list<std::pair<int, int> > m_positions;
-	NameMap m_nameMap;
-	ColorMap m_colorMap;
-	uint16_t m_readPixels[16];
-	uint16_t m_readInfo[16];
-	NameSet m_unknownNodes;
-	Color m_color[16][16];
-	uint8_t m_thickness[16][16];
-
-	int m_blockAirId;
-	int m_blockIgnoreId;
-	int m_zoom;
-	uint m_scales;
-}; // class TileGenerator
-
-#endif // TILEGENERATOR_HEADER

ZlibDecompressor.cpp

@@ -1,17 +1,8 @@
-/*
- * =====================================================================
- *        Version:  1.0
- *        Created:  18.09.2012 10:20:47
- *         Author:  Miroslav Bendík
- *        Company:  LinuxOS.sk
- * =====================================================================
- */
-
 #include <zlib.h>
 #include <stdint.h>
 #include "ZlibDecompressor.h"
 
-ZlibDecompressor::ZlibDecompressor(const unsigned char *data, std::size_t size):
+ZlibDecompressor::ZlibDecompressor(const u8 *data, size_t size):
 	m_data(data),
 	m_seekPos(0),
 	m_size(size)
@@ -22,12 +13,12 @@ ZlibDecompressor::~ZlibDecompressor()
 {
 }
 
-void ZlibDecompressor::setSeekPos(std::size_t seekPos)
+void ZlibDecompressor::setSeekPos(size_t seekPos)
 {
 	m_seekPos = seekPos;
 }
 
-std::size_t ZlibDecompressor::seekPos() const
+size_t ZlibDecompressor::seekPos() const
 {
 	return m_seekPos;
 }
@@ -35,36 +26,43 @@ std::size_t ZlibDecompressor::seekPos() const
 ustring ZlibDecompressor::decompress()
 {
 	const unsigned char *data = m_data + m_seekPos;
-	const std::size_t size = m_size - m_seekPos;
+	const size_t size = m_size - m_seekPos;
 
 	ustring buffer;
-	const size_t BUFSIZE = 128 * 1024;
-	uint8_t temp_buffer[BUFSIZE];
+	constexpr size_t BUFSIZE = 32 * 1024;
 
 	z_stream strm;
 	strm.zalloc = Z_NULL;
 	strm.zfree = Z_NULL;
 	strm.opaque = Z_NULL;
 	strm.next_in = Z_NULL;
-	strm.avail_in = size;
+	strm.avail_in = 0;
 
-	if (inflateInit(&strm) != Z_OK) {
+	if (inflateInit(&strm) != Z_OK)
 		throw DecompressError();
-	}
 
 	strm.next_in = const_cast<unsigned char *>(data);
+	strm.avail_in = size;
+	buffer.resize(BUFSIZE);
+	strm.next_out = &buffer[0];
+	strm.avail_out = BUFSIZE;
+
 	int ret = 0;
 	do {
-		strm.avail_out = BUFSIZE;
-		strm.next_out = temp_buffer;
 		ret = inflate(&strm, Z_NO_FLUSH);
-		buffer += ustring(reinterpret_cast<unsigned char *>(temp_buffer), BUFSIZE - strm.avail_out);
+		if (strm.avail_out == 0) {
+			const auto off = buffer.size();
+			buffer.reserve(off + BUFSIZE);
+			strm.next_out = &buffer[off];
+			strm.avail_out = BUFSIZE;
+		}
 	} while (ret == Z_OK);
-	if (ret != Z_STREAM_END) {
+	if (ret != Z_STREAM_END)
 		throw DecompressError();
-	}
+
 	m_seekPos += strm.next_in - data;
-	(void)inflateEnd(&strm);
+	buffer.resize(buffer.size() - strm.avail_out);
+	(void) inflateEnd(&strm);
 
 	return buffer;
 }

ZlibDecompressor.h

@@ -1,37 +0,0 @@
-/*
- * =====================================================================
- *        Version:  1.0
- *        Created:  18.09.2012 10:20:51
- *         Author:  Miroslav Bendík
- *        Company:  LinuxOS.sk
- * =====================================================================
- */
-
-#ifndef ZLIBDECOMPRESSOR_H_ZQL1PN8Q
-#define ZLIBDECOMPRESSOR_H_ZQL1PN8Q
-
-#include <cstdlib>
-#include <string>
-#include "types.h"
-
-
-class ZlibDecompressor
-{
-public:
-	class DecompressError {
-	};
-
-	ZlibDecompressor(const unsigned char *data, std::size_t size);
-	~ZlibDecompressor();
-	void setSeekPos(std::size_t seekPos);
-	std::size_t seekPos() const;
-	ustring decompress();
-
-private:
-	const unsigned char *m_data;
-	std::size_t m_seekPos;
-	std::size_t m_size;
-}; /* -----  end of class ZlibDecompressor  ----- */
-
-#endif /* end of include guard: ZLIBDECOMPRESSOR_H_ZQL1PN8Q */
-

ZstdDecompressor.cpp

@@ -0,0 +1,58 @@
+#include <zstd.h>
+#include "ZstdDecompressor.h"
+
+ZstdDecompressor::ZstdDecompressor():
+	m_data(nullptr),
+	m_seekPos(0),
+	m_size(0)
+{
+	m_stream = ZSTD_createDStream();
+}
+
+ZstdDecompressor::~ZstdDecompressor()
+{
+	ZSTD_freeDStream(reinterpret_cast<ZSTD_DStream*>(m_stream));
+}
+
+void ZstdDecompressor::setData(const u8 *data, size_t size, size_t seekPos)
+{
+	m_data = data;
+	m_seekPos = seekPos;
+	m_size = size;
+}
+
+std::size_t ZstdDecompressor::seekPos() const
+{
+	return m_seekPos;
+}
+
+ustring ZstdDecompressor::decompress()
+{
+	ZSTD_DStream *stream = reinterpret_cast<ZSTD_DStream*>(m_stream);
+	ZSTD_inBuffer inbuf = { m_data, m_size, m_seekPos };
+
+	ustring buffer;
+	constexpr size_t BUFSIZE = 32 * 1024;
+
+	buffer.resize(BUFSIZE);
+	ZSTD_outBuffer outbuf = { &buffer[0], buffer.size(), 0 };
+
+	ZSTD_initDStream(stream);
+
+	size_t ret;
+	do {
+		ret = ZSTD_decompressStream(stream, &outbuf, &inbuf);
+		if (outbuf.size == outbuf.pos) {
+			outbuf.size += BUFSIZE;
+			buffer.resize(outbuf.size);
+			outbuf.dst = &buffer[0];
+		}
+		if (ret && ZSTD_isError(ret))
+			throw DecompressError();
+	} while (ret != 0);
+
+	m_seekPos = inbuf.pos;
+	buffer.resize(outbuf.pos);
+
+	return buffer;
+}

autogenerating-colors.txt

@@ -1,135 +0,0 @@
-==FILE== mods/dumpnodes/init.lua
-local function nd_get_tiles(nd)
-	if nd.tiles then
-		return nd.tiles
-	elseif nd.tile_images then
-		return nd.tile_images
-	end
-	return nil
-end
-
-local function pairs_s(dict)
-	local keys = {}
-	for k in pairs(dict) do
-		table.insert(keys, k)
-	end
-	table.sort(keys)
-	return ipairs(keys)
-end
-
-minetest.register_chatcommand("dumpnodes", {
-	params = "",
-	description = "",
-	func = function(plname, param)
-		local n = 0
-		local ntbl = {}
-		for _, nn in pairs_s(minetest.registered_nodes) do
-			local nd = minetest.registered_nodes[nn]
-			local prefix, name = nn:match('(.*):(.*)')
-			if prefix == nil or name == nil or prefix == '' or name == '' then
-				print("ignored(1): " .. nn)
-			else
-				if ntbl[prefix] == nil then
-					ntbl[prefix] = {}
-				end
-				ntbl[prefix][name] = nd
-			end
-		end
-		local out, err = io.open('nodes.txt', 'wb')
-		if not out then
-			return true, "io.open(): " .. err
-		end
-		for _, prefix in pairs_s(ntbl) do
-			local nodes = ntbl[prefix]
-			out:write('# ' .. prefix .. '\n')
-			for _, name in pairs_s(nodes) do
-				local nd = nodes[name]
-				if nd.drawtype ~= 'airlike' and nd_get_tiles(nd) ~= nil then
-					local tl = nd_get_tiles(nd)[1]
-					if type(tl) == 'table' then
-						tl = tl.name
-					end
-					tl = (tl .. '^'):match('(.-)^')
-					out:write(prefix .. ':' .. name .. ' ' .. tl .. '\n')
-					n = n + 1
-				else
-					print("ignored(2): " .. prefix .. ':' .. name)
-				end
-			end
-			out:write('\n')
-		end
-		out:close()
-		return true, n .. " nodes dumped."
-	end,
-})
-==FILE== avgcolor.py
-#!/usr/bin/env python
-import sys
-from PIL import Image
-
-def tsum(a, b):
-	return tuple(sum(e) for e in zip(a, b))
-
-if len(sys.argv) < 2:
-	print("Prints average color (RGB) of input image")
-	print("Usage: %s <input>" % sys.argv[0])
-	exit(1)
-
-inp = Image.open(sys.argv[1])
-inp = inp.convert('RGBA')
-ind = inp.load()
-
-cl = (0, 0, 0)
-counted = 0
-for x in range(inp.size[0]):
-	for y in range(inp.size[1]):
-		px = ind[x, y]
-		if px[3] < 128:
-			continue
-		cl = tsum(cl, px[:3])
-		counted = counted + 1
-
-if counted == 0:
-	sys.stderr.write("didn't find avg. color for %s\n" % sys.argv[1])
-	print("0 0 0")
-	exit(0)
-
-cl = tuple(int(n / counted) for n in cl)
-print("%d %d %d" % cl)
-==SCRIPT==
-#!/bin/bash -e
-
-AVGCOLOR_PATH=/path/to/avgcolor.py
-MTGAME_PATH=/path/to/minetest_game
-NODESTXT_PATH=./nodes.txt
-COLORSTXT_PATH=./colors.txt
-
-while read -r line; do
-	set -- junk $line
-	shift
-	if [[ -z "$1" || $1 == "#" ]]; then
-		echo $line
-		continue
-	fi
-	tex=$(find $MTGAME_PATH -type f -name "$2")
-	if [ -z "$tex" ]; then
-		echo "skip $1: texture not found" >&2
-		continue
-	fi
-	echo $1 $(python $AVGCOLOR_PATH "$tex")
-	echo "ok $1" >&2
-done < $NODESTXT_PATH > $COLORSTXT_PATH
-# Use nicer colors for water and lava:
-sed -re 's/^default:((river_)?water_(flowing|source)) [0-9 ]+$/default:\1 39 66 106 128 224/g' $COLORSTXT_PATH -i
-sed -re 's/^default:(lava_(flowing|source)) [0-9 ]+$/default:\1 255 100 0/g' $COLORSTXT_PATH -i
-# Add transparency to glass nodes:
-sed -re 's/^default:(.*glass) ([0-9 ]+)$/default:\1 \2 64 16/g' $COLORSTXT_PATH -i
-sed -re 's/^doors:(.*glass[^ ]*) ([0-9 ]+)$/doors:\1 \2 64 16/g' $COLORSTXT_PATH -i
-# Fix xpanes color:
-sed -re 's/^xpanes:((pane|bar)(_flat)?) [0-9 ]+$/xpanes:\1 194 194 227 64 16/g' $COLORSTXT_PATH -i
-==INSTRUCTIONS==
-1) Make sure avgcolors.py outputs the usage instructions
-2) Add the dumpnodes mod to Minetest
-3) Create a world and load dumpnodes & all mods you want to generate colors for
-4) Execute /dumpnodes ingame
-5) Run the script to generate colors.txt (make sure to replace /path/to/... with the actual paths)

cmake/FindZstd.cmake

@@ -0,0 +1,24 @@
+mark_as_advanced(ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+find_path(ZSTD_INCLUDE_DIR NAMES zstd.h)
+
+find_library(ZSTD_LIBRARY NAMES zstd)
+
+if(ZSTD_INCLUDE_DIR AND ZSTD_LIBRARY)
+	# Check that the API we use exists
+	include(CheckSymbolExists)
+	unset(HAVE_ZSTD_INITDSTREAM CACHE)
+	set(CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
+	set(CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
+	check_symbol_exists(ZSTD_initDStream zstd.h HAVE_ZSTD_INITDSTREAM)
+	unset(CMAKE_REQUIRED_INCLUDES)
+	unset(CMAKE_REQUIRED_LIBRARIES)
+
+	if(NOT HAVE_ZSTD_INITDSTREAM)
+		unset(ZSTD_INCLUDE_DIR CACHE)
+		unset(ZSTD_LIBRARY CACHE)
+	endif()
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Zstd DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR)

colors.txt

@@ -1,290 +1,456 @@
 # beds
 beds:bed_bottom 130 3 3
-beds:bed_top 178 116 116
-beds:fancy_bed_bottom 135 23 14
-beds:fancy_bed_top 172 112 103
+beds:bed_top 185 162 163
+beds:fancy_bed_bottom 136 49 28
+beds:fancy_bed_top 179 153 148
 
 # bones
-bones:bones 86 86 86
+bones:bones 117 117 117
+
+# butterflies
 
 # carts
-carts:brakerail 129 110 81
-carts:powerrail 154 136 81
-carts:rail 143 123 90
+carts:brakerail 150 121 102
+carts:powerrail 160 145 102
+carts:rail 146 128 108
 
 # default
-default:acacia_bush_leaves 90 124 55
-default:acacia_bush_stem 84 76 69
-default:acacia_leaves 108 147 67
-default:acacia_sapling 87 116 61
-default:acacia_tree 188 109 90
-default:acacia_wood 146 60 37
-default:apple 145 20 9
-default:aspen_leaves 66 89 38
-default:aspen_sapling 82 110 43
-default:aspen_tree 218 197 166
-default:aspen_wood 209 198 169
-default:bookshelf 128 99 55
-default:brick 117 71 69
-default:bronzeblock 185 110 15
-default:bush_leaves 34 52 29
-default:bush_stem 45 33 23
-default:cactus 52 116 15
-default:chest 140 108 65
-default:chest_locked 140 108 65
-default:clay 182 182 182
+default:acacia_bush_leaves 109 133 87
+default:acacia_bush_sapling 85 121 61
+default:acacia_bush_stem 84 77 70
+default:acacia_leaves 126 153 101
+default:acacia_sapling 87 120 64
+default:acacia_tree 195 119 97
+default:acacia_wood 150 61 39
+default:apple 161 34 19
+default:aspen_leaves 72 105 29
+default:aspen_sapling 85 123 45
+default:aspen_tree 218 198 168
+default:aspen_wood 210 199 170
+default:blueberry_bush_leaves 63 99 22
+default:blueberry_bush_leaves_with_berries 63 99 22
+default:blueberry_bush_sapling 81 112 33
+default:bookshelf 131 102 57
+default:brick 123 99 95
+default:bronzeblock 186 111 15
+default:bush_leaves 35 55 29
+default:bush_sapling 66 64 40
+default:bush_stem 46 34 24
+default:cactus 70 119 52
+default:cave_ice 168 206 247
+default:chest 149 115 69
+default:chest_locked 149 115 69
+default:chest_locked_open 149 115 69
+default:chest_open 149 115 69
+default:clay 183 183 183
 default:cloud 255 255 255
-default:coalblock 57 57 57
-default:cobble 88 84 82
-default:copperblock 192 126 63
-default:coral_brown 139 104 72
-default:coral_orange 191 62 12
-default:coral_skeleton 235 230 214
-default:desert_cobble 146 95 76
+default:coalblock 58 58 58
+default:cobble 89 86 84
+default:copperblock 193 126 65
+default:coral_brown 146 113 77
+default:coral_cyan 235 230 215
+default:coral_green 235 230 215
+default:coral_orange 197 68 17
+default:coral_pink 235 230 215
+default:coral_skeleton 235 230 215
+default:desert_cobble 110 67 50
 default:desert_sand 206 165 98
-default:desert_stone 129 79 60
-default:desert_stone_block 130 79 60
-default:desert_stonebrick 129 79 60
-default:diamondblock 135 217 223
-default:dirt 95 64 39
-default:dirt_with_dry_grass 187 148 77
-default:dirt_with_grass 66 112 31
-default:dirt_with_grass_footsteps 66 112 31
-default:dirt_with_snow 223 224 236
+default:desert_sandstone 195 152 92
+default:desert_sandstone_block 193 152 94
+default:desert_sandstone_brick 191 151 95
+default:desert_stone 130 79 61
+default:desert_stone_block 131 80 61
+default:desert_stonebrick 131 80 61
+default:diamondblock 140 218 223
+default:dirt 97 67 43
+default:dirt_with_coniferous_litter 109 90 71
+default:dirt_with_dry_grass 187 148 78
+default:dirt_with_grass 64 111 26
+default:dirt_with_grass_footsteps 64 111 26
+default:dirt_with_rainforest_litter 76 39 10
+default:dirt_with_snow 225 225 238
+default:dry_dirt 178 136 90
+default:dry_dirt_with_dry_grass 187 148 78
 default:dry_grass_1 208 172 87
 default:dry_grass_2 210 174 87
 default:dry_grass_3 210 174 87
 default:dry_grass_4 211 175 88
-default:dry_grass_5 213 178 92
-default:dry_shrub 101 66 17
-default:fence_acacia_wood 147 60 38
+default:dry_grass_5 214 178 92
+default:dry_shrub 103 67 18
+default:emergent_jungle_sapling 51 40 16
+default:fence_acacia_wood 151 62 39
 default:fence_aspen_wood 210 199 170
-default:fence_junglewood 54 37 11
-default:fence_pine_wood 221 184 129
-default:fence_wood 129 100 55
-default:furnace 100 96 94
-default:furnace_active 100 96 94
-default:glass 194 194 227 64 16
-default:goldblock 230 201 29
-default:grass_1 97 138 53
-default:grass_2 95 136 54
-default:grass_3 91 133 52
-default:grass_4 86 130 45
-default:grass_5 83 124 47
-default:gravel 131 131 131
-default:ice 167 206 247
-default:junglegrass 63 105 25
-default:jungleleaves 20 28 15
-default:junglesapling 48 38 14
-default:jungletree 105 76 41
-default:junglewood 54 37 11
-default:ladder_steel 126 126 126
-default:ladder_wood 123 90 34
+default:fence_junglewood 57 39 14
+default:fence_pine_wood 221 185 131
+default:fence_rail_acacia_wood 150 61 39
+default:fence_rail_aspen_wood 209 198 170
+default:fence_rail_junglewood 56 39 14
+default:fence_rail_pine_wood 221 184 130
+default:fence_rail_wood 131 102 57
+default:fence_wood 132 103 57
+default:fern_1 85 118 51
+default:fern_2 90 123 53
+default:fern_3 91 125 54
+default:furnace 101 98 96
+default:furnace_active 101 98 96
+default:glass 247 247 247 64 16
+default:goldblock 231 203 35
+default:grass_1 100 140 54
+default:grass_2 98 139 55
+default:grass_3 94 136 53
+default:grass_4 89 133 48
+default:grass_5 86 126 48
+default:gravel 132 132 132
+default:ice 168 206 247
+default:junglegrass 67 110 28
+default:jungleleaves 22 31 16
+default:junglesapling 51 39 15
+default:jungletree 121 97 62
+default:junglewood 56 39 14
+default:ladder_steel 132 132 132
+default:ladder_wood 125 93 43
+default:large_cactus_seedling 67 107 52
 default:lava_flowing 255 100 0
 default:lava_source 255 100 0
-default:leaves 34 52 29
-default:mese 220 220 0
-default:meselamp 211 213 139
-default:mossycobble 86 90 68
-default:obsidian 19 21 24
-default:obsidian_block 20 22 25
-default:obsidian_glass 19 21 23 64 16
-default:obsidianbrick 20 22 24
-default:papyrus 94 132 33
-default:pine_needles 13 36 21
-default:pine_sapling 27 48 25
-default:pine_tree 182 155 124
-default:pine_wood 221 184 128
+default:leaves 36 55 29
+default:marram_grass_1 113 139 96
+default:marram_grass_2 102 131 90
+default:marram_grass_3 99 130 88
+default:mese 222 222 0
+default:mese_post_light 132 103 57
+default:mese_post_light_acacia_wood 151 62 39
+default:mese_post_light_aspen_wood 210 199 170
+default:mese_post_light_junglewood 57 39 14
+default:mese_post_light_pine_wood 221 185 131
+default:meselamp 213 215 143
+default:mossycobble 88 91 73
+default:obsidian 21 24 29
+default:obsidian_block 23 25 30
+default:obsidian_glass 20 23 27 64 16
+default:obsidianbrick 23 25 29
+default:papyrus 97 134 38
+default:permafrost 71 66 61
+default:permafrost_with_moss 108 150 51
+default:permafrost_with_stones 71 66 61
+default:pine_bush_needles 16 50 19
+default:pine_bush_sapling 58 51 40
+default:pine_bush_stem 73 62 53
+default:pine_needles 16 50 19
+default:pine_sapling 41 48 26
+default:pine_tree 191 165 132
+default:pine_wood 221 185 130
 default:river_water_flowing 39 66 106 128 224
 default:river_water_source 39 66 106 128 224
 default:sand 214 207 158
-default:sandstone 197 193 143
-default:sandstone_block 195 190 141
-default:sandstonebrick 193 189 140
-default:sapling 65 59 40
-default:sign_wall_steel 144 144 144
-default:sign_wall_wood 145 101 64
+default:sand_with_kelp 214 207 158
+default:sandstone 198 193 143
+default:sandstone_block 195 191 142
+default:sandstonebrick 194 190 141
+default:sapling 67 63 41
+default:sign_wall_steel 147 147 147
+default:sign_wall_wood 148 103 66
 default:silver_sand 193 191 179
-default:snow 223 224 236
-default:snowblock 223 224 236
-default:steelblock 194 194 194
+default:silver_sandstone 195 192 181
+default:silver_sandstone_block 192 190 180
+default:silver_sandstone_brick 191 189 179
+default:snow 225 225 238
+default:snowblock 225 225 238
+default:steelblock 195 195 195
 default:stone 97 94 93
-default:stone_block 99 96 95
+default:stone_block 100 97 96
 default:stone_with_coal 97 94 93
 default:stone_with_copper 97 94 93
 default:stone_with_diamond 97 94 93
 default:stone_with_gold 97 94 93
 default:stone_with_iron 97 94 93
 default:stone_with_mese 97 94 93
-default:stonebrick 99 96 95
-default:torch 120 98 67
-default:torch_ceiling 120 98 67
-default:torch_wall 120 98 67
-default:tree 164 131 88
+default:stone_with_tin 97 94 93
+default:stonebrick 102 99 98
+default:tinblock 150 150 150
+default:torch 141 123 93
+default:torch_ceiling 141 123 93
+default:torch_wall 141 123 93
+default:tree 179 145 99
 default:water_flowing 39 66 106 128 224
 default:water_source 39 66 106 128 224
-default:wood 128 99 55
+default:wood 131 102 57
 
 # doors
-doors:door_glass_a 184 184 216 64 16
-doors:door_glass_b 184 184 216 64 16
-doors:door_obsidian_glass_a 27 28 29 64 16
-doors:door_obsidian_glass_b 27 28 29 64 16
-doors:door_steel_a 201 201 201
-doors:door_steel_b 201 201 201
-doors:door_wood_a 87 67 35
-doors:door_wood_b 87 67 35
-doors:gate_acacia_wood_closed 146 60 37
-doors:gate_acacia_wood_open 146 60 37
-doors:gate_aspen_wood_closed 209 198 169
-doors:gate_aspen_wood_open 209 198 169
-doors:gate_junglewood_closed 54 37 11
-doors:gate_junglewood_open 54 37 11
-doors:gate_pine_wood_closed 221 184 128
-doors:gate_pine_wood_open 221 184 128
-doors:gate_wood_closed 128 99 55
-doors:gate_wood_open 128 99 55
-doors:hidden 0 0 0
-doors:trapdoor 128 99 50
-doors:trapdoor_open 225 217 206
-doors:trapdoor_steel 199 199 199
-doors:trapdoor_steel_open 206 206 206
+doors:door_glass_a 245 245 245 64 16
+doors:door_glass_b 245 245 245 64 16
+doors:door_glass_c 245 245 245 64 16
+doors:door_glass_d 245 245 245 64 16
+doors:door_obsidian_glass_a 48 49 50 64 16
+doors:door_obsidian_glass_b 48 49 50 64 16
+doors:door_obsidian_glass_c 48 49 50 64 16
+doors:door_obsidian_glass_d 48 49 50 64 16
+doors:door_steel_a 203 203 203
+doors:door_steel_b 203 203 203
+doors:door_steel_c 203 203 203
+doors:door_steel_d 203 203 203
+doors:door_wood_a 89 68 37
+doors:door_wood_b 89 68 37
+doors:door_wood_c 89 68 37
+doors:door_wood_d 89 68 37
+doors:gate_acacia_wood_closed 150 61 39
+doors:gate_acacia_wood_open 150 61 39
+doors:gate_aspen_wood_closed 210 199 170
+doors:gate_aspen_wood_open 210 199 170
+doors:gate_junglewood_closed 56 39 14
+doors:gate_junglewood_open 56 39 14
+doors:gate_pine_wood_closed 221 185 130
+doors:gate_pine_wood_open 221 185 130
+doors:gate_wood_closed 131 102 57
+doors:gate_wood_open 131 102 57
+doors:trapdoor 130 100 51
+doors:trapdoor_open 68 53 30
+doors:trapdoor_steel 200 200 200
+doors:trapdoor_steel_open 97 97 97
 
 # farming
-farming:cotton_1 88 116 39
-farming:cotton_2 87 116 38
-farming:cotton_3 97 120 41
-farming:cotton_4 106 113 47
-farming:cotton_5 114 104 53
-farming:cotton_6 119 94 59
-farming:cotton_7 92 69 37
-farming:cotton_8 110 91 61
-farming:desert_sand_soil 159 131 70
-farming:desert_sand_soil_wet 119 98 52
-farming:seed_cotton 90 85 57
-farming:seed_wheat 175 159 93
-farming:soil 95 64 39
-farming:soil_wet 95 64 39
-farming:straw 211 182 67
-farming:wheat_1 107 174 32
-farming:wheat_2 133 175 50
-farming:wheat_3 162 182 82
-farming:wheat_4 169 187 93
-farming:wheat_5 169 178 94
-farming:wheat_6 172 176 85
-farming:wheat_7 192 181 81
-farming:wheat_8 186 161 35
+farming:cotton_1 89 117 39
+farming:cotton_2 89 116 38
+farming:cotton_3 99 121 41
+farming:cotton_4 108 114 47
+farming:cotton_5 116 105 53
+farming:cotton_6 121 95 59
+farming:cotton_7 94 70 37
+farming:cotton_8 122 108 93
+farming:cotton_wild 111 111 101
+farming:desert_sand_soil 161 132 72
+farming:desert_sand_soil_wet 120 99 53
+farming:dry_soil 178 136 90
+farming:dry_soil_wet 178 136 90
+farming:seed_cotton 92 87 60
+farming:seed_wheat 177 161 96
+farming:soil 97 67 43
+farming:soil_wet 97 67 43
+farming:straw 212 184 68
+farming:wheat_1 110 175 36
+farming:wheat_2 136 177 53
+farming:wheat_3 163 182 84
+farming:wheat_4 170 188 95
+farming:wheat_5 171 179 97
+farming:wheat_6 173 177 87
+farming:wheat_7 193 181 83
+farming:wheat_8 187 162 40
 
 # fire
-fire:basic_flame 218 120 36
-fire:permanent_flame 218 120 36
+fire:basic_flame 223 136 44
+fire:permanent_flame 223 136 44
+
+# fireflies
+fireflies:firefly_bottle 191 194 202
 
 # flowers
-flowers:dandelion_white 178 176 140
-flowers:dandelion_yellow 178 148 24
-flowers:geranium 72 87 155
-flowers:mushroom_brown 106 79 72
-flowers:mushroom_red 192 85 85
-flowers:rose 118 48 24
-flowers:tulip 130 99 36
-flowers:viola 106 60 159
-flowers:waterlily 102 158 61
-
-# nyancat
-nyancat:nyancat 198 111 167
-nyancat:nyancat_rainbow 123 100 94
+flowers:chrysanthemum_green 118 152 44
+flowers:dandelion_white 199 191 176
+flowers:dandelion_yellow 212 167 31
+flowers:geranium 77 91 168
+flowers:mushroom_brown 109 84 78
+flowers:mushroom_red 195 102 102
+flowers:rose 130 68 33
+flowers:tulip 156 101 44
+flowers:tulip_black 78 120 72
+flowers:viola 115 69 184
+flowers:waterlily 107 160 68
+flowers:waterlily_waving 107 160 68
 
 # stairs
-stairs:slab_acacia_wood 146 60 37
-stairs:slab_aspen_wood 209 198 169
-stairs:slab_brick 117 71 69
-stairs:slab_bronzeblock 185 110 15
-stairs:slab_cobble 88 84 82
-stairs:slab_copperblock 192 126 63
-stairs:slab_desert_cobble 146 95 76
-stairs:slab_desert_stone 129 79 60
-stairs:slab_desert_stone_block 130 79 60
-stairs:slab_desert_stonebrick 129 79 60
-stairs:slab_goldblock 230 201 29
-stairs:slab_junglewood 54 37 11
-stairs:slab_mossycobble 86 90 68
-stairs:slab_obsidian 19 21 24
-stairs:slab_obsidian_block 20 22 25
-stairs:slab_obsidianbrick 20 22 24
-stairs:slab_pine_wood 221 184 128
-stairs:slab_sandstone 197 193 143
-stairs:slab_sandstone_block 195 190 141
-stairs:slab_sandstonebrick 193 189 140
-stairs:slab_steelblock 194 194 194
+stairs:slab_acacia_wood 150 61 39
+stairs:slab_aspen_wood 210 199 170
+stairs:slab_brick 123 99 95
+stairs:slab_bronzeblock 186 111 15
+stairs:slab_cobble 89 86 84
+stairs:slab_copperblock 193 126 65
+stairs:slab_desert_cobble 110 67 50
+stairs:slab_desert_sandstone 195 152 92
+stairs:slab_desert_sandstone_block 193 152 94
+stairs:slab_desert_sandstone_brick 191 151 95
+stairs:slab_desert_stone 130 79 61
+stairs:slab_desert_stone_block 131 80 61
+stairs:slab_desert_stonebrick 131 80 61
+stairs:slab_glass 247 247 247
+stairs:slab_goldblock 231 203 35
+stairs:slab_ice 168 206 247
+stairs:slab_junglewood 56 39 14
+stairs:slab_mossycobble 88 91 73
+stairs:slab_obsidian 21 24 29
+stairs:slab_obsidian_block 23 25 30
+stairs:slab_obsidian_glass 20 23 27
+stairs:slab_obsidianbrick 23 25 29
+stairs:slab_pine_wood 221 185 130
+stairs:slab_sandstone 198 193 143
+stairs:slab_sandstone_block 195 191 142
+stairs:slab_sandstonebrick 194 190 141
+stairs:slab_silver_sandstone 195 192 181
+stairs:slab_silver_sandstone_block 192 190 180
+stairs:slab_silver_sandstone_brick 191 189 179
+stairs:slab_snowblock 225 225 238
+stairs:slab_steelblock 195 195 195
 stairs:slab_stone 97 94 93
-stairs:slab_stone_block 99 96 95
-stairs:slab_stonebrick 99 96 95
-stairs:slab_straw 211 182 67
-stairs:slab_wood 128 99 55
-stairs:stair_acacia_wood 146 60 37
-stairs:stair_aspen_wood 209 198 169
-stairs:stair_brick 117 71 69
-stairs:stair_bronzeblock 185 110 15
-stairs:stair_cobble 88 84 82
-stairs:stair_copperblock 192 126 63
-stairs:stair_desert_cobble 146 95 76
-stairs:stair_desert_stone 129 79 60
-stairs:stair_desert_stone_block 130 79 60
-stairs:stair_desert_stonebrick 129 79 60
-stairs:stair_goldblock 230 201 29
-stairs:stair_junglewood 54 37 11
-stairs:stair_mossycobble 86 90 68
-stairs:stair_obsidian 19 21 24
-stairs:stair_obsidian_block 20 22 25
-stairs:stair_obsidianbrick 20 22 24
-stairs:stair_pine_wood 221 184 128
-stairs:stair_sandstone 197 193 143
-stairs:stair_sandstone_block 195 190 141
-stairs:stair_sandstonebrick 193 189 140
-stairs:stair_steelblock 194 194 194
+stairs:slab_stone_block 100 97 96
+stairs:slab_stonebrick 102 99 98
+stairs:slab_straw 212 184 68
+stairs:slab_tinblock 150 150 150
+stairs:slab_wood 131 102 57
+stairs:stair_acacia_wood 150 61 39
+stairs:stair_aspen_wood 210 199 170
+stairs:stair_brick 123 99 95
+stairs:stair_bronzeblock 186 111 15
+stairs:stair_cobble 89 86 84
+stairs:stair_copperblock 193 126 65
+stairs:stair_desert_cobble 110 67 50
+stairs:stair_desert_sandstone 195 152 92
+stairs:stair_desert_sandstone_block 193 152 94
+stairs:stair_desert_sandstone_brick 191 151 95
+stairs:stair_desert_stone 130 79 61
+stairs:stair_desert_stone_block 131 80 61
+stairs:stair_desert_stonebrick 131 80 61
+stairs:stair_glass 249 249 249
+stairs:stair_goldblock 231 203 35
+stairs:stair_ice 168 206 247
+stairs:stair_inner_acacia_wood 150 61 39
+stairs:stair_inner_aspen_wood 210 199 170
+stairs:stair_inner_brick 123 99 95
+stairs:stair_inner_bronzeblock 186 111 15
+stairs:stair_inner_cobble 89 86 84
+stairs:stair_inner_copperblock 193 126 65
+stairs:stair_inner_desert_cobble 110 67 50
+stairs:stair_inner_desert_sandstone 195 152 92
+stairs:stair_inner_desert_sandstone_block 193 152 94
+stairs:stair_inner_desert_sandstone_brick 191 151 95
+stairs:stair_inner_desert_stone 130 79 61
+stairs:stair_inner_desert_stone_block 131 80 61
+stairs:stair_inner_desert_stonebrick 131 80 61
+stairs:stair_inner_glass 250 250 250
+stairs:stair_inner_goldblock 231 203 35
+stairs:stair_inner_ice 168 206 247
+stairs:stair_inner_junglewood 56 39 14
+stairs:stair_inner_mossycobble 88 91 73
+stairs:stair_inner_obsidian 21 24 29
+stairs:stair_inner_obsidian_block 23 25 30
+stairs:stair_inner_obsidian_glass 20 22 27
+stairs:stair_inner_obsidianbrick 23 25 29
+stairs:stair_inner_pine_wood 221 185 130
+stairs:stair_inner_sandstone 198 193 143
+stairs:stair_inner_sandstone_block 195 191 142
+stairs:stair_inner_sandstonebrick 194 190 141
+stairs:stair_inner_silver_sandstone 195 192 181
+stairs:stair_inner_silver_sandstone_block 192 190 180
+stairs:stair_inner_silver_sandstone_brick 191 189 179
+stairs:stair_inner_snowblock 225 225 238
+stairs:stair_inner_steelblock 195 195 195
+stairs:stair_inner_stone 97 94 93
+stairs:stair_inner_stone_block 100 97 96
+stairs:stair_inner_stonebrick 102 99 98
+stairs:stair_inner_straw 212 184 68
+stairs:stair_inner_tinblock 150 150 150
+stairs:stair_inner_wood 131 102 57
+stairs:stair_junglewood 56 39 14
+stairs:stair_mossycobble 88 91 73
+stairs:stair_obsidian 21 24 29
+stairs:stair_obsidian_block 23 25 30
+stairs:stair_obsidian_glass 20 22 27
+stairs:stair_obsidianbrick 23 25 29
+stairs:stair_outer_acacia_wood 150 61 39
+stairs:stair_outer_aspen_wood 210 199 170
+stairs:stair_outer_brick 123 99 95
+stairs:stair_outer_bronzeblock 186 111 15
+stairs:stair_outer_cobble 89 86 84
+stairs:stair_outer_copperblock 193 126 65
+stairs:stair_outer_desert_cobble 110 67 50
+stairs:stair_outer_desert_sandstone 195 152 92
+stairs:stair_outer_desert_sandstone_block 193 152 94
+stairs:stair_outer_desert_sandstone_brick 191 151 95
+stairs:stair_outer_desert_stone 130 79 61
+stairs:stair_outer_desert_stone_block 131 80 61
+stairs:stair_outer_desert_stonebrick 131 80 61
+stairs:stair_outer_glass 250 250 250
+stairs:stair_outer_goldblock 231 203 35
+stairs:stair_outer_ice 168 206 247
+stairs:stair_outer_junglewood 56 39 14
+stairs:stair_outer_mossycobble 88 91 73
+stairs:stair_outer_obsidian 21 24 29
+stairs:stair_outer_obsidian_block 23 25 30
+stairs:stair_outer_obsidian_glass 20 22 27
+stairs:stair_outer_obsidianbrick 23 25 29
+stairs:stair_outer_pine_wood 221 185 130
+stairs:stair_outer_sandstone 198 193 143
+stairs:stair_outer_sandstone_block 195 191 142
+stairs:stair_outer_sandstonebrick 194 190 141
+stairs:stair_outer_silver_sandstone 195 192 181
+stairs:stair_outer_silver_sandstone_block 192 190 180
+stairs:stair_outer_silver_sandstone_brick 191 189 179
+stairs:stair_outer_snowblock 225 225 238
+stairs:stair_outer_steelblock 195 195 195
+stairs:stair_outer_stone 97 94 93
+stairs:stair_outer_stone_block 100 97 96
+stairs:stair_outer_stonebrick 102 99 98
+stairs:stair_outer_straw 212 184 68
+stairs:stair_outer_tinblock 150 150 150
+stairs:stair_outer_wood 131 102 57
+stairs:stair_pine_wood 221 185 130
+stairs:stair_sandstone 198 193 143
+stairs:stair_sandstone_block 195 191 142
+stairs:stair_sandstonebrick 194 190 141
+stairs:stair_silver_sandstone 195 192 181
+stairs:stair_silver_sandstone_block 192 190 180
+stairs:stair_silver_sandstone_brick 191 189 179
+stairs:stair_snowblock 225 225 238
+stairs:stair_steelblock 195 195 195
 stairs:stair_stone 97 94 93
-stairs:stair_stone_block 99 96 95
-stairs:stair_stonebrick 99 96 95
-stairs:stair_straw 211 182 67
-stairs:stair_wood 128 99 55
+stairs:stair_stone_block 100 97 96
+stairs:stair_stonebrick 102 99 98
+stairs:stair_straw 212 184 68
+stairs:stair_tinblock 150 150 150
+stairs:stair_wood 131 102 57
 
 # tnt
-tnt:gunpowder 6 6 6
-tnt:gunpowder_burning 112 103 2
-tnt:tnt 181 0 0
-tnt:tnt_burning 190 8 0
+tnt:gunpowder 12 12 12
+tnt:gunpowder_burning 156 143 7
+tnt:tnt 196 0 0
+tnt:tnt_burning 201 41 0
 
 # vessels
-vessels:drinking_glass 206 214 228
-vessels:glass_bottle 188 190 201
-vessels:shelf 128 99 55
-vessels:steel_bottle 193 192 191
+vessels:drinking_glass 207 214 228
+vessels:glass_bottle 189 192 204
+vessels:shelf 131 102 57
+vessels:steel_bottle 194 193 193
 
 # walls
-walls:cobble 88 84 82
-walls:desertcobble 146 95 76
-walls:mossycobble 86 90 68
+walls:cobble 89 86 84
+walls:desertcobble 110 67 50
+walls:mossycobble 88 91 73
 
 # wool
-wool:black 29 29 29
-wool:blue 0 72 145
-wool:brown 86 42 0
-wool:cyan 0 130 139
-wool:dark_green 32 102 0
-wool:dark_grey 59 59 59
-wool:green 92 216 28
-wool:grey 131 131 131
-wool:magenta 200 3 109
-wool:orange 213 81 21
-wool:pink 255 131 131
-wool:red 168 17 17
-wool:violet 92 2 169
+wool:black 30 30 30
+wool:blue 0 73 146
+wool:brown 88 44 0
+wool:cyan 0 132 140
+wool:dark_green 33 103 0
+wool:dark_grey 60 60 60
+wool:green 93 218 28
+wool:grey 133 133 133
+wool:magenta 201 3 112
+wool:orange 214 83 22
+wool:pink 255 133 133
+wool:red 170 18 18
+wool:violet 93 5 169
 wool:white 220 220 220
-wool:yellow 254 225 15
+wool:yellow 254 226 16
 
 # xpanes
-xpanes:bar 194 194 227 64 16
-xpanes:bar_flat 194 194 227 64 16
-xpanes:pane 194 194 227 64 16
-xpanes:pane_flat 194 194 227 64 16
+xpanes:bar 114 114 114 64 16
+xpanes:bar_flat 114 114 114 64 16
+xpanes:door_steel_bar_a 133 133 133 64 16
+xpanes:door_steel_bar_b 133 133 133 64 16
+xpanes:door_steel_bar_c 133 133 133 64 16
+xpanes:door_steel_bar_d 133 133 133 64 16
+xpanes:obsidian_pane 16 17 18 64 16
+xpanes:obsidian_pane_flat 16 17 18 64 16
+xpanes:pane 249 249 249 64 16
+xpanes:pane_flat 249 249 249 64 16
+xpanes:trapdoor_steel_bar 127 127 127 64 16
+xpanes:trapdoor_steel_bar_open 77 77 77 64 16
 

db-leveldb.cpp

@@ -5,13 +5,12 @@
 
 static inline int64_t stoi64(const std::string &s)
 {
-	std::stringstream tmp(s);
+	std::istringstream tmp(s);
 	int64_t t;
 	tmp >> t;
 	return t;
 }
 
-
 static inline std::string i64tos(int64_t i)
 {
 	std::ostringstream os;
@@ -19,6 +18,7 @@ static inline std::string i64tos(int64_t i)
 	return os.str();
 }
 
+
 DBLevelDB::DBLevelDB(const std::string &mapdir)
 {
 	leveldb::Options options;
@@ -28,6 +28,9 @@ DBLevelDB::DBLevelDB(const std::string &mapdir)
 		throw std::runtime_error(std::string("Failed to open Database: ") + status.ToString());
 	}
 
+	/* LevelDB is a dumb key-value store, so the only optimization we can do
+	 * is to cache the block positions that exist in the db.
+	 */
 	loadPosCache();
 }
 
@@ -38,9 +41,21 @@ DBLevelDB::~DBLevelDB()
 }
 
 
-std::vector<BlockPos> DBLevelDB::getBlockPos()
+std::vector<BlockPos> DBLevelDB::getBlockPos(BlockPos min, BlockPos max)
 {
-	return posCache;
+	std::vector<BlockPos> res;
+	for (const auto &it : posCache) {
+		if (it.first < min.z || it.first >= max.z)
+			continue;
+		for (auto pos2 : it.second) {
+			if (pos2.first < min.x || pos2.first >= max.x)
+				continue;
+			if (pos2.second < min.y || pos2.second >= max.y)
+				continue;
+			res.emplace_back(pos2.first, pos2.second, it.first);
+		}
+	}
+	return res;
 }
 
 
@@ -49,26 +64,51 @@ void DBLevelDB::loadPosCache()
 	leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions());
 	for (it->SeekToFirst(); it->Valid(); it->Next()) {
 		int64_t posHash = stoi64(it->key().ToString());
-		posCache.push_back(decodeBlockPos(posHash));
+		BlockPos pos = decodeBlockPos(posHash);
+
+		posCache[pos.z].emplace_back(pos.x, pos.y);
 	}
 	delete it;
 }
 
 
-void DBLevelDB::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
+void DBLevelDB::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+		int16_t min_y, int16_t max_y)
 {
 	std::string datastr;
 	leveldb::Status status;
 
-	for (std::vector<BlockPos>::iterator it = posCache.begin(); it != posCache.end(); ++it) {
-		if (it->z != zPos) {
+	auto it = posCache.find(z);
+	if (it == posCache.cend())
+		return;
+	for (auto pos2 : it->second) {
+		if (pos2.first != x)
 			continue;
-		}
-		status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(*it)), &datastr);
+		if (pos2.second < min_y || pos2.second >= max_y)
+			continue;
+
+		BlockPos pos(x, pos2.second, z);
+		status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(pos)), &datastr);
 		if (status.ok()) {
-			Block b(*it, ustring((const unsigned char *) datastr.data(), datastr.size()));
-			blocks[b.first.x].push_back(b);
+			blocks.emplace_back(
+				pos, ustring((unsigned char *) datastr.data(), datastr.size())
+			);
 		}
 	}
 }
 
+void DBLevelDB::getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions)
+{
+	std::string datastr;
+	leveldb::Status status;
+
+	for (auto pos : positions) {
+		status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(pos)), &datastr);
+		if (status.ok()) {
+			blocks.emplace_back(
+				pos, ustring((unsigned char *) datastr.data(), datastr.size())
+			);
+		}
+	}
+}

db-leveldb.h

@@ -1,21 +0,0 @@
-#ifndef DB_LEVELDB_HEADER
-#define DB_LEVELDB_HEADER
-
-#include "db.h"
-#include <leveldb/db.h>
-
-class DBLevelDB : public DB {
-public:
-	DBLevelDB(const std::string &mapdir);
-	virtual std::vector<BlockPos> getBlockPos();
-	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
-	virtual ~DBLevelDB();
-private:
-	void loadPosCache();
-
-	std::vector<BlockPos> posCache;
-
-	leveldb::DB *db;
-};
-
-#endif // DB_LEVELDB_HEADER

db-postgresql.cpp

@@ -0,0 +1,226 @@
+#include <stdexcept>
+#include <iostream>
+#include <fstream>
+#include <cstdlib>
+#include <arpa/inet.h>
+#include "db-postgresql.h"
+#include "util.h"
+#include "types.h"
+
+#define ARRLEN(x) (sizeof(x) / sizeof((x)[0]))
+
+DBPostgreSQL::DBPostgreSQL(const std::string &mapdir)
+{
+	std::ifstream ifs(mapdir + "world.mt");
+	if (!ifs.good())
+		throw std::runtime_error("Failed to read world.mt");
+	std::string connect_string = read_setting("pgsql_connection", ifs);
+	ifs.close();
+	db = PQconnectdb(connect_string.c_str());
+
+	if (PQstatus(db) != CONNECTION_OK) {
+		throw std::runtime_error(std::string(
+			"PostgreSQL database error: ") +
+			PQerrorMessage(db)
+		);
+	}
+
+	prepareStatement(
+		"get_block_pos",
+		"SELECT posX::int4, posY::int4, posZ::int4 FROM blocks WHERE"
+		" (posX BETWEEN $1::int4 AND $2::int4) AND"
+		" (posY BETWEEN $3::int4 AND $4::int4) AND"
+		" (posZ BETWEEN $5::int4 AND $6::int4)"
+	);
+	prepareStatement(
+		"get_blocks",
+		"SELECT posY::int4, data FROM blocks WHERE"
+		" posX = $1::int4 AND posZ = $2::int4"
+		" AND (posY BETWEEN $3::int4 AND $4::int4)"
+	);
+	prepareStatement(
+		"get_block_exact",
+		"SELECT data FROM blocks WHERE"
+		" posX = $1::int4 AND posY = $2::int4 AND posZ = $3::int4"
+	);
+
+	checkResults(PQexec(db, "START TRANSACTION;"));
+	checkResults(PQexec(db, "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;"));
+}
+
+
+DBPostgreSQL::~DBPostgreSQL()
+{
+	try {
+		checkResults(PQexec(db, "COMMIT;"));
+	} catch (const std::exception& caught) {
+		std::cerr << "could not finalize: " << caught.what() << std::endl;
+	}
+	PQfinish(db);
+}
+
+
+std::vector<BlockPos> DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max)
+{
+	int32_t const x1 = htonl(min.x);
+	int32_t const x2 = htonl(max.x - 1);
+	int32_t const y1 = htonl(min.y);
+	int32_t const y2 = htonl(max.y - 1);
+	int32_t const z1 = htonl(min.z);
+	int32_t const z2 = htonl(max.z - 1);
+
+	const void *args[] = { &x1, &x2, &y1, &y2, &z1, &z2 };
+	const int argLen[] = { 4, 4, 4, 4, 4, 4 };
+	const int argFmt[] = { 1, 1, 1, 1, 1, 1 };
+
+	PGresult *results = execPrepared(
+		"get_block_pos", ARRLEN(args), args,
+		argLen, argFmt, false
+	);
+
+	int numrows = PQntuples(results);
+
+	std::vector<BlockPos> positions;
+	positions.reserve(numrows);
+
+	for (int row = 0; row < numrows; ++row)
+		positions.emplace_back(pg_to_blockpos(results, row, 0));
+
+	PQclear(results);
+
+	return positions;
+}
+
+
+void DBPostgreSQL::getBlocksOnXZ(BlockList &blocks, int16_t xPos, int16_t zPos,
+		int16_t min_y, int16_t max_y)
+{
+	int32_t const x = htonl(xPos);
+	int32_t const z = htonl(zPos);
+	int32_t const y1 = htonl(min_y);
+	int32_t const y2 = htonl(max_y - 1);
+
+	const void *args[] = { &x, &z, &y1, &y2 };
+	const int argLen[] = { 4, 4, 4, 4 };
+	const int argFmt[] = { 1, 1, 1, 1 };
+
+	PGresult *results = execPrepared(
+		"get_blocks", ARRLEN(args), args,
+		argLen, argFmt, false
+	);
+
+	int numrows = PQntuples(results);
+
+	for (int row = 0; row < numrows; ++row) {
+		BlockPos position;
+		position.x = xPos;
+		position.y = pg_binary_to_int(results, row, 0);
+		position.z = zPos;
+		blocks.emplace_back(
+			position,
+			ustring(
+				reinterpret_cast<unsigned char*>(
+					PQgetvalue(results, row, 1)
+				),
+				PQgetlength(results, row, 1)
+			)
+		);
+	}
+
+	PQclear(results);
+}
+
+
+void DBPostgreSQL::getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions)
+{
+	int32_t x, y, z;
+
+	const void *args[] = { &x, &y, &z };
+	const int argLen[] = { 4, 4, 4 };
+	const int argFmt[] = { 1, 1, 1 };
+
+	for (auto pos : positions) {
+		x = htonl(pos.x);
+		y = htonl(pos.y);
+		z = htonl(pos.z);
+
+		PGresult *results = execPrepared(
+			"get_block_exact", ARRLEN(args), args,
+			argLen, argFmt, false
+		);
+
+		if (PQntuples(results) > 0) {
+			blocks.emplace_back(
+				pos,
+				ustring(
+					reinterpret_cast<unsigned char*>(
+						PQgetvalue(results, 0, 0)
+					),
+					PQgetlength(results, 0, 0)
+				)
+			);
+		}
+
+		PQclear(results);
+	}
+}
+
+
+PGresult *DBPostgreSQL::checkResults(PGresult *res, bool clear)
+{
+	ExecStatusType statusType = PQresultStatus(res);
+
+	switch (statusType) {
+	case PGRES_COMMAND_OK:
+	case PGRES_TUPLES_OK:
+		break;
+	case PGRES_FATAL_ERROR:
+		throw std::runtime_error(
+			std::string("PostgreSQL database error: ") +
+			PQresultErrorMessage(res)
+		);
+	default:
+		throw std::runtime_error(
+			"Unhandled PostgreSQL result code"
+		);
+	}
+
+	if (clear)
+		PQclear(res);
+
+	return res;
+}
+
+void DBPostgreSQL::prepareStatement(const std::string &name, const std::string &sql)
+{
+	checkResults(PQprepare(db, name.c_str(), sql.c_str(), 0, NULL));
+}
+
+PGresult *DBPostgreSQL::execPrepared(
+	const char *stmtName, const int paramsNumber,
+	const void **params,
+	const int *paramsLengths, const int *paramsFormats,
+	bool clear
+)
+{
+	return checkResults(PQexecPrepared(db, stmtName, paramsNumber,
+		(const char* const*) params, paramsLengths, paramsFormats,
+		1 /* binary output */), clear
+	);
+}
+
+int DBPostgreSQL::pg_binary_to_int(PGresult *res, int row, int col)
+{
+	int32_t* raw = reinterpret_cast<int32_t*>(PQgetvalue(res, row, col));
+	return ntohl(*raw);
+}
+
+BlockPos DBPostgreSQL::pg_to_blockpos(PGresult *res, int row, int col)
+{
+	BlockPos result;
+	result.x = pg_binary_to_int(res, row, col);
+	result.y = pg_binary_to_int(res, row, col + 1);
+	result.z = pg_binary_to_int(res, row, col + 2);
+	return result;
+}

db-redis.cpp

@@ -20,7 +20,6 @@ static inline int64_t stoi64(const std::string &s)
 	return t;
 }
 
-
 static inline std::string i64tos(int64_t i)
 {
 	std::ostringstream os;
@@ -28,40 +27,37 @@ static inline std::string i64tos(int64_t i)
 	return os.str();
 }
 
-std::string get_setting_default(std::string name, std::istream &is, const std::string def)
-{
-	try {
-		return get_setting(name, is);
-	} catch(std::runtime_error e) {
-		return def;
-	}
-}
 
 DBRedis::DBRedis(const std::string &mapdir)
 {
-	std::ifstream ifs((mapdir + "/world.mt").c_str());
-	if(!ifs.good())
+	std::ifstream ifs(mapdir + "world.mt");
+	if (!ifs.good())
 		throw std::runtime_error("Failed to read world.mt");
 	std::string tmp;
-	try {
-		tmp = get_setting("redis_address", ifs);
-		ifs.seekg(0);
-		hash = get_setting("redis_hash", ifs);
-		ifs.seekg(0);
-	} catch(std::runtime_error e) {
-		throw std::runtime_error("Set redis_address and redis_hash in world.mt to use the redis backend");
+
+	tmp = read_setting("redis_address", ifs);
+	ifs.seekg(0);
+	hash = read_setting("redis_hash", ifs);
+	ifs.seekg(0);
+
+	if (tmp.find('/') != std::string::npos) {
+		ctx = redisConnectUnix(tmp.c_str());
+	} else {
+		int port = stoi64(read_setting_default("redis_port", ifs, "6379"));
+		ctx = redisConnect(tmp.c_str(), port);
 	}
-	const char *addr = tmp.c_str();
-	int port = stoi64(get_setting_default("redis_port", ifs, "6379"));
-	ctx = redisConnect(addr, port);
-	if(!ctx)
+
+	if (!ctx) {
 		throw std::runtime_error("Cannot allocate redis context");
-	else if(ctx->err) {
+	} else if (ctx->err) {
 		std::string err = std::string("Connection error: ") + ctx->errstr;
 		redisFree(ctx);
 		throw std::runtime_error(err);
 	}
 
+	/* Redis is just a key-value store, so the only optimization we can do
+	 * is to cache the block positions that exist in the db.
+	 */
 	loadPosCache();
 }
 
@@ -72,14 +68,27 @@ DBRedis::~DBRedis()
 }
 
 
-std::vector<BlockPos> DBRedis::getBlockPos()
+std::vector<BlockPos> DBRedis::getBlockPos(BlockPos min, BlockPos max)
 {
-	return posCache;
+	std::vector<BlockPos> res;
+	for (const auto &it : posCache) {
+		if (it.first < min.z || it.first >= max.z)
+			continue;
+		for (auto pos2 : it.second) {
+			if (pos2.first < min.x || pos2.first >= max.x)
+				continue;
+			if (pos2.second < min.y || pos2.second >= max.y)
+				continue;
+			res.emplace_back(pos2.first, pos2.second, it.first);
+		}
+	}
+	return res;
 }
 
 
-std::string DBRedis::replyTypeStr(int type) {
-	switch(type) {
+const char *DBRedis::replyTypeStr(int type)
+{
+	switch (type) {
 		case REDIS_REPLY_STATUS:
 			return "REDIS_REPLY_STATUS";
 		case REDIS_REPLY_ERROR:
@@ -93,7 +102,7 @@ std::string DBRedis::replyTypeStr(int type) {
 		case REDIS_REPLY_ARRAY:
 			return "REDIS_REPLY_ARRAY";
 		default:
-			return "unknown";
+			return "(unknown)";
 	}
 }
 
@@ -102,89 +111,96 @@ void DBRedis::loadPosCache()
 {
 	redisReply *reply;
 	reply = (redisReply*) redisCommand(ctx, "HKEYS %s", hash.c_str());
-	if(!reply)
+	if (!reply)
 		throw std::runtime_error("Redis command HKEYS failed");
-	if(reply->type != REDIS_REPLY_ARRAY)
+	if (reply->type != REDIS_REPLY_ARRAY)
 		REPLY_TYPE_ERR(reply, "HKEYS reply");
-	for(size_t i = 0; i < reply->elements; i++) {
-		if(reply->element[i]->type != REDIS_REPLY_STRING)
+	for (size_t i = 0; i < reply->elements; i++) {
+		if (reply->element[i]->type != REDIS_REPLY_STRING)
 			REPLY_TYPE_ERR(reply->element[i], "HKEYS subreply");
-		posCache.push_back(decodeBlockPos(stoi64(reply->element[i]->str)));
+		BlockPos pos = decodeBlockPos(stoi64(reply->element[i]->str));
+		posCache[pos.z].emplace_back(pos.x, pos.y);
 	}
 
 	freeReplyObject(reply);
 }
 
 
-void DBRedis::HMGET(const std::vector<BlockPos> &positions, std::vector<ustring> *result)
+void DBRedis::HMGET(const std::vector<BlockPos> &positions,
+	std::function<void(std::size_t, ustring)> result)
 {
 	const char *argv[DB_REDIS_HMGET_NUMFIELDS + 2];
 	argv[0] = "HMGET";
 	argv[1] = hash.c_str();
 
-	std::vector<BlockPos>::const_iterator position = positions.begin();
-	std::size_t remaining = positions.size();
+	auto position = positions.begin();
+	size_t remaining = positions.size();
+	size_t abs_i = 0;
 	while (remaining > 0) {
-		const std::size_t batch_size =
-			(remaining > DB_REDIS_HMGET_NUMFIELDS) ? DB_REDIS_HMGET_NUMFIELDS : remaining;
+		const size_t batch_size = mymin<size_t>(DB_REDIS_HMGET_NUMFIELDS, remaining);
 
 		redisReply *reply;
 		{
 			// storage to preserve validity of .c_str()
 			std::string keys[batch_size];
-			for (std::size_t i = 0; i < batch_size; ++i) {
+			for (size_t i = 0; i < batch_size; ++i) {
 				keys[i] = i64tos(encodeBlockPos(*position++));
 				argv[i+2] = keys[i].c_str();
 			}
 			reply = (redisReply*) redisCommandArgv(ctx, batch_size + 2, argv, NULL);
 		}
 
-		if(!reply)
+		if (!reply)
 			throw std::runtime_error("Redis command HMGET failed");
-		if (reply->type != REDIS_REPLY_ARRAY) {
-			freeReplyObject(reply);
-			REPLY_TYPE_ERR(reply, "HKEYS subreply");
-		}
+		if (reply->type != REDIS_REPLY_ARRAY)
+			REPLY_TYPE_ERR(reply, "HMGET reply");
 		if (reply->elements != batch_size) {
 			freeReplyObject(reply);
 			throw std::runtime_error("HMGET wrong number of elements");
 		}
-		for (std::size_t i = 0; i < batch_size; ++i) {
+		for (size_t i = 0; i < reply->elements; ++i) {
 			redisReply *subreply = reply->element[i];
-			if(!subreply)
-				throw std::runtime_error("Redis command HMGET failed");
-			if (subreply->type != REDIS_REPLY_STRING) {
-				freeReplyObject(reply);
-				REPLY_TYPE_ERR(reply, "HKEYS subreply");
-			}
-			if (subreply->len == 0) {
-				freeReplyObject(reply);
+			if (subreply->type == REDIS_REPLY_NIL)
+				continue;
+			else if (subreply->type != REDIS_REPLY_STRING)
+				REPLY_TYPE_ERR(subreply, "HMGET subreply");
+			if (subreply->len == 0)
 				throw std::runtime_error("HMGET empty string");
-			}
-			result->push_back(ustring((const unsigned char *) subreply->str, subreply->len));
+			result(abs_i + i, ustring(
+				reinterpret_cast<const unsigned char*>(subreply->str),
+				subreply->len
+			));
 		}
 		freeReplyObject(reply);
+
+		abs_i += batch_size;
 		remaining -= batch_size;
 	}
 }
 
 
-void DBRedis::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
+void DBRedis::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+		int16_t min_y, int16_t max_y)
 {
-	std::vector<BlockPos> z_positions;
-	for (std::vector<BlockPos>::const_iterator it = posCache.begin(); it != posCache.end(); ++it) {
-		if (it->z != zPos) {
-			continue;
-		}
-		z_positions.push_back(*it);
-	}
-	std::vector<ustring> z_blocks;
-	HMGET(z_positions, &z_blocks);
+	auto it = posCache.find(z);
+	if (it == posCache.cend())
+		return;
 
-	std::vector<ustring>::const_iterator z_block = z_blocks.begin();
-	for (std::vector<BlockPos>::const_iterator pos = z_positions.begin();
-			pos != z_positions.end();
-			++pos, ++z_block) {
-		blocks[pos->x].push_back(Block(*pos, *z_block));
+	std::vector<BlockPos> positions;
+	for (auto pos2 : it->second) {
+		if (pos2.first == x && pos2.second >= min_y && pos2.second < max_y)
+			positions.emplace_back(x, pos2.second, z);
 	}
+
+	getBlocksByPos(blocks, positions);
+}
+
+
+void DBRedis::getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions)
+{
+	auto result = [&] (std::size_t i, ustring data) {
+		blocks.emplace_back(positions[i], std::move(data));
+	};
+	HMGET(positions, result);
 }

db-redis.h

@@ -1,25 +0,0 @@
-#ifndef DB_REDIS_HEADER
-#define DB_REDIS_HEADER
-
-#include "db.h"
-#include <hiredis.h>
-
-class DBRedis : public DB {
-public:
-	DBRedis(const std::string &mapdir);
-	virtual std::vector<BlockPos> getBlockPos();
-	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
-	virtual ~DBRedis();
-private:
-	static std::string replyTypeStr(int type);
-
-	void loadPosCache();
-	void HMGET(const std::vector<BlockPos> &positions, std::vector<ustring> *result);
-
-	std::vector<BlockPos> posCache;
-
-	redisContext *ctx;
-	std::string hash;
-};
-
-#endif // DB_REDIS_HEADER

db-sqlite3.cpp

@@ -1,6 +1,8 @@
 #include <stdexcept>
 #include <unistd.h> // for usleep
 #include <iostream>
+#include <algorithm>
+#include <time.h>
 #include "db-sqlite3.h"
 #include "types.h"
 
@@ -11,7 +13,6 @@
 	}
 #define SQLOK(f) SQLRES(f, SQLITE_OK)
 
-
 DBSQLite3::DBSQLite3(const std::string &mapdir)
 {
 	int result;
@@ -24,9 +25,17 @@ DBSQLite3::DBSQLite3(const std::string &mapdir)
 			"SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ?",
 		-1, &stmt_get_blocks_z, NULL))
 
+	SQLOK(prepare_v2(db,
+			"SELECT data FROM blocks WHERE pos = ?",
+		-1, &stmt_get_block_exact, NULL))
+
 	SQLOK(prepare_v2(db,
 			"SELECT pos FROM blocks",
 		-1, &stmt_get_block_pos, NULL))
+
+	SQLOK(prepare_v2(db,
+			"SELECT pos FROM blocks WHERE pos BETWEEN ? AND ?",
+		-1, &stmt_get_block_pos_z, NULL))
 }
 
 
@@ -34,59 +43,154 @@ DBSQLite3::~DBSQLite3()
 {
 	sqlite3_finalize(stmt_get_blocks_z);
 	sqlite3_finalize(stmt_get_block_pos);
+	sqlite3_finalize(stmt_get_block_pos_z);
+	sqlite3_finalize(stmt_get_block_exact);
 
 	if (sqlite3_close(db) != SQLITE_OK) {
 		std::cerr << "Error closing SQLite database." << std::endl;
 	};
 }
 
-std::vector<BlockPos> DBSQLite3::getBlockPos()
+
+inline void DBSQLite3::getPosRange(int64_t &min, int64_t &max, int16_t zPos,
+		int16_t zPos2) const
+{
+	/* The range of block positions is [-2048, 2047], which turns into [0, 4095]
+	 * when casted to unsigned. This didn't actually help me understand the
+	 * numbers below, but I wanted to write it down.
+	 */
+
+	// Magic numbers!
+	min = encodeBlockPos(BlockPos(0, -2048, zPos));
+	max = encodeBlockPos(BlockPos(0, 2048, zPos2)) - 1;
+}
+
+
+std::vector<BlockPos> DBSQLite3::getBlockPos(BlockPos min, BlockPos max)
 {
 	int result;
+	sqlite3_stmt *stmt;
+
+	if(min.z <= -2048 && max.z >= 2048) {
+		stmt = stmt_get_block_pos;
+	} else {
+		stmt = stmt_get_block_pos_z;
+		int64_t minPos, maxPos;
+		if (min.z < -2048)
+			min.z = -2048;
+		if (max.z > 2048)
+			max.z = 2048;
+		getPosRange(minPos, maxPos, min.z, max.z - 1);
+		SQLOK(bind_int64(stmt, 1, minPos))
+		SQLOK(bind_int64(stmt, 2, maxPos))
+	}
+
 	std::vector<BlockPos> positions;
-	while ((result = sqlite3_step(stmt_get_block_pos)) != SQLITE_DONE) {
-		if (result == SQLITE_ROW) {
-			int64_t posHash = sqlite3_column_int64(stmt_get_block_pos, 0);
-			positions.push_back(decodeBlockPos(posHash));
-		} else if (result == SQLITE_BUSY) { // Wait some time and try again
+	while ((result = sqlite3_step(stmt)) != SQLITE_DONE) {
+		if (result == SQLITE_BUSY) { // Wait some time and try again
 			usleep(10000);
-		} else {
+		} else if (result != SQLITE_ROW) {
 			throw std::runtime_error(sqlite3_errmsg(db));
 		}
+
+		int64_t posHash = sqlite3_column_int64(stmt, 0);
+		BlockPos pos = decodeBlockPos(posHash);
+		if(pos.x >= min.x && pos.x < max.x && pos.y >= min.y && pos.y < max.y)
+			positions.emplace_back(pos);
 	}
-	SQLOK(reset(stmt_get_block_pos));
+	SQLOK(reset(stmt));
 	return positions;
 }
 
 
-void DBSQLite3::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
+void DBSQLite3::loadBlockCache(int16_t zPos)
 {
 	int result;
+	blockCache.clear();
 
-	// Magic numbers!
-	int64_t minPos = encodeBlockPos(BlockPos(0, -2048, zPos));
-	int64_t maxPos = encodeBlockPos(BlockPos(0, 2048, zPos)) - 1;
+	int64_t minPos, maxPos;
+	getPosRange(minPos, maxPos, zPos, zPos);
 
 	SQLOK(bind_int64(stmt_get_blocks_z, 1, minPos));
 	SQLOK(bind_int64(stmt_get_blocks_z, 2, maxPos));
 
 	while ((result = sqlite3_step(stmt_get_blocks_z)) != SQLITE_DONE) {
-		if (result == SQLITE_ROW) {
-			int64_t posHash = sqlite3_column_int64(stmt_get_blocks_z, 0);
-			const unsigned char *data = reinterpret_cast<const unsigned char *>(
-					sqlite3_column_blob(stmt_get_blocks_z, 1));
-			size_t size = sqlite3_column_bytes(stmt_get_blocks_z, 1);
-			Block b(decodeBlockPos(posHash), ustring(data, size));
-			blocks[b.first.x].push_back(b);
-		} else if (result == SQLITE_BUSY) { // Wait some time and try again
+		if (result == SQLITE_BUSY) { // Wait some time and try again
 			usleep(10000);
-		} else {
+		} else if (result != SQLITE_ROW) {
 			throw std::runtime_error(sqlite3_errmsg(db));
 		}
+
+		int64_t posHash = sqlite3_column_int64(stmt_get_blocks_z, 0);
+		BlockPos pos = decodeBlockPos(posHash);
+		const unsigned char *data = reinterpret_cast<const unsigned char *>(
+				sqlite3_column_blob(stmt_get_blocks_z, 1));
+		size_t size = sqlite3_column_bytes(stmt_get_blocks_z, 1);
+		blockCache[pos.x].emplace_back(pos, ustring(data, size));
 	}
-	SQLOK(reset(stmt_get_blocks_z));
+	SQLOK(reset(stmt_get_blocks_z))
 }
 
-#undef SQLRES
-#undef SQLOK
 
+void DBSQLite3::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+		int16_t min_y, int16_t max_y)
+{
+	/* Cache the blocks on the given Z coordinate between calls, this only
+	 * works due to order in which the TileGenerator asks for blocks. */
+	if (z != blockCachedZ) {
+		loadBlockCache(z);
+		blockCachedZ = z;
+	}
+
+	auto it = blockCache.find(x);
+	if (it == blockCache.end())
+		return;
+
+	if (it->second.empty()) {
+		/* We have swapped this list before, this is not supposed to happen
+		 * because it's bad for performance. But rather than silently breaking
+		 * do the right thing and load the blocks again. */
+#ifndef NDEBUG
+		std::cerr << "Warning: suboptimal access pattern for sqlite3 backend" << std::endl;
+#endif
+		loadBlockCache(z);
+	}
+	// Swap lists to avoid copying contents
+	blocks.clear();
+	std::swap(blocks, it->second);
+
+	for (auto it = blocks.begin(); it != blocks.end(); ) {
+		if (it->first.y < min_y || it->first.y >= max_y)
+			it = blocks.erase(it);
+		else
+			it++;
+	}
+}
+
+
+void DBSQLite3::getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions)
+{
+	int result;
+
+	for (auto pos : positions) {
+		int64_t dbPos = encodeBlockPos(pos);
+		SQLOK(bind_int64(stmt_get_block_exact, 1, dbPos));
+
+		while ((result = sqlite3_step(stmt_get_block_exact)) == SQLITE_BUSY) {
+			usleep(10000); // Wait some time and try again
+		}
+		if (result == SQLITE_DONE) {
+			// no data
+		} else if (result != SQLITE_ROW) {
+			throw std::runtime_error(sqlite3_errmsg(db));
+		} else {
+			const unsigned char *data = reinterpret_cast<const unsigned char *>(
+					sqlite3_column_blob(stmt_get_block_exact, 0));
+			size_t size = sqlite3_column_bytes(stmt_get_block_exact, 0);
+			blocks.emplace_back(pos, ustring(data, size));
+		}
+
+		SQLOK(reset(stmt_get_block_exact))
+	}
+}

db-sqlite3.h

@@ -1,20 +0,0 @@
-#ifndef _DB_SQLITE3_H
-#define _DB_SQLITE3_H
-
-#include "db.h"
-#include <sqlite3.h>
-
-class DBSQLite3 : public DB {
-public:
-	DBSQLite3(const std::string &mapdir);
-	virtual std::vector<BlockPos> getBlockPos();
-	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
-	virtual ~DBSQLite3();
-private:
-	sqlite3 *db;
-
-	sqlite3_stmt *stmt_get_block_pos;
-	sqlite3_stmt *stmt_get_blocks_z;
-};
-
-#endif // _DB_SQLITE3_H

include/BlockDecoder.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <cstdint>
+#include <unordered_map>
+#include "types.h"
+#include <ZstdDecompressor.h>
+
+class BlockDecoder {
+public:
+	BlockDecoder();
+
+	void reset();
+	void decode(const ustring &data);
+	bool isEmpty() const;
+	// returns "" for air, ignore and invalid nodes
+	const std::string &getNode(u8 x, u8 y, u8 z) const;
+
+private:
+	typedef std::unordered_map<uint16_t, std::string> NameMap;
+	NameMap m_nameMap;
+	uint16_t m_blockAirId, m_blockIgnoreId;
+
+	u8 m_version, m_contentWidth;
+	ustring m_mapData;
+
+	// one instance for performance
+	ZstdDecompressor m_zstd_decompressor;
+};

include/Image.h

@@ -1,5 +1,4 @@
-#ifndef IMAGE_HEADER
-#define IMAGE_HEADER
+#pragma once
 
 #include "types.h"
 #include <string>
@@ -9,6 +8,7 @@ struct Color {
 	Color() : r(0), g(0), b(0), a(0) {};
 	Color(u8 r, u8 g, u8 b) : r(r), g(g), b(b), a(255) {};
 	Color(u8 r, u8 g, u8 b, u8 a) : r(r), g(g), b(b), a(a) {};
+
 	u8 r, g, b, a;
 };
 
@@ -17,6 +17,9 @@ public:
 	Image(int width, int height);
 	~Image();
 
+	Image(const Image&) = delete;
+	Image& operator=(const Image&) = delete;
+
 	void setPixel(int x, int y, const Color &c);
 	Color getPixel(int x, int y);
 	void drawLine(int x1, int y1, int x2, int y2, const Color &c);
@@ -26,10 +29,6 @@ public:
 	void save(const std::string &filename);
 
 private:
-	Image(const Image&);
-
 	int m_width, m_height;
 	gdImagePtr m_image;
 };
-
-#endif // IMAGE_HEADER

include/PixelAttributes.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <climits>
+#include <cstdint>
+
+#define BLOCK_SIZE 16
+
+struct PixelAttribute {
+	PixelAttribute() : height(INT16_MIN), thickness(0) {};
+
+	int16_t height;
+	uint8_t thickness;
+
+	inline bool valid_height() const {
+		return height != INT16_MIN;
+	}
+};
+
+class PixelAttributes
+{
+public:
+	PixelAttributes();
+	virtual ~PixelAttributes();
+
+	void setWidth(int width);
+	void scroll();
+
+	inline PixelAttribute &attribute(int z, int x) {
+		return m_pixelAttributes[z + 1][x + 1];
+	};
+
+private:
+	void freeAttributes();
+
+private:
+	enum Line {
+		FirstLine = 0,
+		LastLine = BLOCK_SIZE,
+		EmptyLine = BLOCK_SIZE + 1,
+		LineCount = BLOCK_SIZE + 2
+	};
+	PixelAttribute *m_pixelAttributes[BLOCK_SIZE + 2]; // 1px gradient + empty
+	int m_width;
+};

include/PlayerAttributes.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <list>
+#include <string>
+
+struct Player
+{
+	std::string name;
+	float x, y, z;
+};
+
+class PlayerAttributes
+{
+public:
+	typedef std::list<Player> Players;
+
+	PlayerAttributes(const std::string &worldDir);
+	Players::const_iterator begin() const;
+	Players::const_iterator end() const;
+
+private:
+	void readFiles(const std::string &playersPath);
+	void readSqlite(const std::string &db_name);
+
+	Players m_players;
+};

include/TileGenerator.h

@@ -0,0 +1,167 @@
+#pragma once
+
+#include <iostream>
+#include <map>
+#include <set>
+#include <unordered_map>
+#include <cstdint>
+#include <string>
+
+#include "PixelAttributes.h"
+#include "Image.h"
+#include "db.h"
+#include "types.h"
+
+class BlockDecoder;
+class Image;
+
+enum {
+	SCALE_TOP = (1 << 0),
+	SCALE_BOTTOM = (1 << 1),
+	SCALE_LEFT = (1 << 2),
+	SCALE_RIGHT = (1 << 3),
+};
+
+enum {
+	EXH_NEVER, // Always use range queries
+	EXH_Y,     // Exhaustively search Y space, range queries for X/Z
+	EXH_FULL,  // Exhaustively search entire requested geometry
+	EXH_AUTO,  // Automatically pick one of the previous modes
+};
+
+struct ColorEntry {
+	ColorEntry() : r(0), g(0), b(0), a(0), t(0) {};
+	ColorEntry(uint8_t r, uint8_t g, uint8_t b, uint8_t a, uint8_t t) :
+		r(r), g(g), b(b), a(a), t(t) {};
+	inline Color toColor() const { return Color(r, g, b, a); }
+	uint8_t r, g, b, a; // Red, Green, Blue, Alpha
+	uint8_t t; // "thickness" value
+};
+
+struct BitmapThing { // 16x16 bitmap
+	inline void reset() {
+		for (int i = 0; i < 16; ++i)
+			val[i] = 0;
+	}
+	inline bool any_neq(uint16_t v) const {
+		for (int i = 0; i < 16; ++i) {
+			if (val[i] != v)
+				return true;
+		}
+		return false;
+	}
+	inline bool any() const { return any_neq(0); }
+	inline bool full() const { return !any_neq(0xffff); }
+	inline void set(unsigned int x, unsigned int z) {
+		val[z] |= (1 << x);
+	}
+	inline bool get(unsigned int x, unsigned int z) const {
+		return !!(val[z] & (1 << x));
+	}
+
+	uint16_t val[16];
+};
+
+
+class TileGenerator
+{
+private:
+	typedef std::unordered_map<std::string, ColorEntry> ColorMap;
+
+public:
+	TileGenerator();
+	~TileGenerator();
+	void setBgColor(const std::string &bgColor);
+	void setScaleColor(const std::string &scaleColor);
+	void setOriginColor(const std::string &originColor);
+	void setPlayerColor(const std::string &playerColor);
+	void setDrawOrigin(bool drawOrigin);
+	void setDrawPlayers(bool drawPlayers);
+	void setDrawScale(bool drawScale);
+	void setDrawAlpha(bool drawAlpha);
+	void setShading(bool shading);
+	void setGeometry(int x, int y, int w, int h);
+	void setMinY(int y);
+	void setMaxY(int y);
+	void setExhaustiveSearch(int mode);
+	void parseColorsFile(const std::string &fileName);
+	void setBackend(std::string backend);
+	void setZoom(int zoom);
+	void setScales(uint flags);
+	void setDontWriteEmpty(bool f);
+
+	void generate(const std::string &input, const std::string &output);
+	void printGeometry(const std::string &input);
+	void dumpBlock(const std::string &input, BlockPos pos);
+
+	static std::set<std::string> getSupportedBackends();
+
+private:
+	void parseColorsStream(std::istream &in);
+	void openDb(const std::string &input);
+	void closeDatabase();
+	void loadBlocks();
+	void createImage();
+	void renderMap();
+	void renderMapBlock(const BlockDecoder &blk, const BlockPos &pos);
+	void renderMapBlockBottom(const BlockPos &pos);
+	void renderShading(int zPos);
+	void renderScale();
+	void renderOrigin();
+	void renderPlayers(const std::string &inputPath);
+	void writeImage(const std::string &output);
+	void printUnknown();
+	void reportProgress(size_t count);
+	int getImageX(int val, bool absolute=false) const;
+	int getImageY(int val, bool absolute=false) const;
+	void setZoomed(int x, int y, Color color);
+
+private:
+	Color m_bgColor;
+	Color m_scaleColor;
+	Color m_originColor;
+	Color m_playerColor;
+	bool m_drawOrigin;
+	bool m_drawPlayers;
+	bool m_drawScale;
+	bool m_drawAlpha;
+	bool m_shading;
+	bool m_dontWriteEmpty;
+	std::string m_backend;
+	int m_xBorder, m_yBorder;
+
+	DB *m_db;
+	Image *m_image;
+	PixelAttributes m_blockPixelAttributes;
+	/* smallest/largest seen X or Z block coordinate */
+	int m_xMin;
+	int m_xMax;
+	int m_zMin;
+	int m_zMax;
+	/* Y limits for rendered area (node units) */
+	int m_yMin;
+	int m_yMax;
+	/* limits for rendered area (block units) */
+	int16_t m_geomX;
+	int16_t m_geomY; /* Y in terms of rendered image, Z in the world */
+	int16_t m_geomX2;
+	int16_t m_geomY2;
+
+	int m_mapWidth;
+	int m_mapHeight;
+	int m_exhaustiveSearch;
+	std::set<std::string> m_unknownNodes;
+	bool m_renderedAny;
+	std::map<int16_t, std::set<int16_t>> m_positions; /* indexed by Z, contains X coords */
+	ColorMap m_colorMap;
+	BitmapThing m_readPixels;
+	BitmapThing m_readInfo;
+	Color m_color[16][16];
+	uint8_t m_thickness[16][16];
+
+	int m_zoom;
+	uint m_scales;
+
+	size_t m_progressMax;
+	int m_progressLast; // percentage
+}; // class TileGenerator

include/ZlibDecompressor.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <exception>
+#include "types.h"
+
+class ZlibDecompressor
+{
+public:
+	class DecompressError : std::exception {};
+
+	ZlibDecompressor(const u8 *data, size_t size);
+	~ZlibDecompressor();
+	void setSeekPos(size_t seekPos);
+	size_t seekPos() const;
+	ustring decompress();
+
+private:
+	const u8 *m_data;
+	size_t m_seekPos, m_size;
+};

include/ZstdDecompressor.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <exception>
+#include "types.h"
+
+class ZstdDecompressor
+{
+public:
+	class DecompressError : std::exception {};
+
+	ZstdDecompressor();
+	~ZstdDecompressor();
+	void setData(const u8 *data, size_t size, size_t seekPos);
+	size_t seekPos() const;
+	ustring decompress();
+
+private:
+	void *m_stream; // ZSTD_DStream
+	const u8 *m_data;
+	size_t m_seekPos, m_size;
+};

include/cmake_config.h.in

@@ -3,8 +3,9 @@
 #ifndef CMAKE_CONFIG_H
 #define CMAKE_CONFIG_H
 
-#define USE_LEVELDB @USE_LEVELDB@
-#define USE_REDIS @USE_REDIS@
+#cmakedefine01 USE_POSTGRESQL
+#cmakedefine01 USE_LEVELDB
+#cmakedefine01 USE_REDIS
 
 #define SHAREDIR "@SHAREDIR@"
 

include/config.h

@@ -4,11 +4,10 @@
 #define PATH_SEPARATOR '/'
 #endif
 
-#define BLOCK_SIZE 16
-
 #ifdef USE_CMAKE_CONFIG_H
 #include "cmake_config.h"
 #else
+#define USE_POSTGRESQL 0
 #define USE_LEVELDB 0
 #define USE_REDIS 0
 

include/db-leveldb.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "db.h"
+#include <unordered_map>
+#include <utility>
+#include <leveldb/db.h>
+
+class DBLevelDB : public DB {
+public:
+	DBLevelDB(const std::string &mapdir);
+	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
+	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+			int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
+	~DBLevelDB() override;
+
+	bool preferRangeQueries() const override { return false; }
+
+private:
+	using pos2d = std::pair<int16_t, int16_t>;
+
+	void loadPosCache();
+
+	// indexed by Z, contains all (x,y) position pairs
+	std::unordered_map<int16_t, std::vector<pos2d>> posCache;
+	leveldb::DB *db;
+};

include/db-postgresql.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "db.h"
+#include <libpq-fe.h>
+
+class DBPostgreSQL : public DB {
+public:
+	DBPostgreSQL(const std::string &mapdir);
+	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
+	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+		int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
+	~DBPostgreSQL() override;
+
+	bool preferRangeQueries() const override { return true; }
+
+protected:
+	PGresult *checkResults(PGresult *res, bool clear = true);
+	void prepareStatement(const std::string &name, const std::string &sql);
+	PGresult *execPrepared(
+		const char *stmtName, const int paramsNumber,
+		const void **params,
+		const int *paramsLengths = nullptr, const int *paramsFormats = nullptr,
+		bool clear = true
+	);
+	int pg_binary_to_int(PGresult *res, int row, int col);
+	BlockPos pg_to_blockpos(PGresult *res, int row, int col);
+
+private:
+	PGconn *db;
+};

include/db-redis.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "db.h"
+#include <unordered_map>
+#include <utility>
+#include <functional>
+#include <hiredis/hiredis.h>
+
+class DBRedis : public DB {
+public:
+	DBRedis(const std::string &mapdir);
+	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
+	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+		int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
+	~DBRedis() override;
+
+	bool preferRangeQueries() const override { return false; }
+
+private:
+	using pos2d = std::pair<int16_t, int16_t>;
+	static const char *replyTypeStr(int type);
+
+	void loadPosCache();
+	void HMGET(const std::vector<BlockPos> &positions,
+			std::function<void(std::size_t, ustring)> result);
+
+	// indexed by Z, contains all (x,y) position pairs
+	std::unordered_map<int16_t, std::vector<pos2d>> posCache;
+
+	redisContext *ctx;
+	std::string hash;
+};

include/db-sqlite3.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "db.h"
+#include <unordered_map>
+#include <sqlite3.h>
+
+class DBSQLite3 : public DB {
+public:
+	DBSQLite3(const std::string &mapdir);
+	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
+	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+			int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
+	~DBSQLite3() override;
+
+	bool preferRangeQueries() const override { return false; }
+
+private:
+	inline void getPosRange(int64_t &min, int64_t &max, int16_t zPos,
+			int16_t zPos2) const;
+	void loadBlockCache(int16_t zPos);
+
+	sqlite3 *db;
+
+	sqlite3_stmt *stmt_get_block_pos;
+	sqlite3_stmt *stmt_get_block_pos_z;
+	sqlite3_stmt *stmt_get_blocks_z;
+	sqlite3_stmt *stmt_get_block_exact;
+
+	int16_t blockCachedZ = -10000;
+	std::unordered_map<int16_t, BlockList> blockCache; // indexed by X
+};

include/db.h

@@ -1,43 +1,36 @@
-#ifndef DB_HEADER
-#define DB_HEADER
+#pragma once
 
-#include <stdint.h>
-#include <map>
+#include <cstdint>
 #include <list>
 #include <vector>
-#include <string>
 #include <utility>
 #include "types.h"
 
 
-class BlockPos {
-public:
+struct BlockPos {
 	int16_t x;
 	int16_t y;
 	int16_t z;
 
 	BlockPos() : x(0), y(0), z(0) {}
+	explicit BlockPos(int16_t v) : x(v), y(v), z(v) {}
 	BlockPos(int16_t x, int16_t y, int16_t z) : x(x), y(y), z(z) {}
+
+	// Implements the inverse ordering so that (2,2,2) < (1,1,1)
 	bool operator < (const BlockPos &p) const
 	{
-		if (z > p.z) {
+		if (z > p.z)
 			return true;
-		}
-		if (z < p.z) {
+		if (z < p.z)
 			return false;
-		}
-		if (y > p.y) {
+		if (y > p.y)
 			return true;
-		}
-		if (y < p.y) {
+		if (y < p.y)
 			return false;
-		}
-		if (x > p.x) {
+		if (x > p.x)
 			return true;
-		}
-		if (x < p.x) {
+		if (x < p.x)
 			return false;
-		}
 		return false;
 	}
 };
@@ -49,13 +42,31 @@ typedef std::list<Block> BlockList;
 
 class DB {
 protected:
+	// Helpers that implement the hashed positions used by most backends
 	inline int64_t  encodeBlockPos(const BlockPos pos) const;
 	inline BlockPos decodeBlockPos(int64_t hash) const;
 
 public:
-	virtual std::vector<BlockPos> getBlockPos() = 0;
-	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos) = 0;
-	virtual ~DB() {};
+	/* Return all block positions inside the range given by min and max,
+	 * so that min.x <= x < max.x, ...
+	 */
+	virtual std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) = 0;
+	/* Read all blocks in column given by x and z
+	 * and inside the given Y range (min_y <= y < max_y) into list
+	 */
+	virtual void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+			int16_t min_y, int16_t max_y) = 0;
+	/* Read blocks at given positions into list
+	 */
+	virtual void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) = 0;
+	/* Can this database efficiently do range queries?
+	 * (for large data sets, more efficient that brute force)
+	 */
+	virtual bool preferRangeQueries() const = 0;
+
+
+	virtual ~DB() {}
 };
 
 
@@ -110,4 +121,3 @@ inline BlockPos DB::decodeBlockPos(int64_t hash) const
  * End black magic *
  *******************/
 
-#endif // DB_HEADER

include/util.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <string>
+#include <iostream>
+
+template<typename T>
+static inline T mymax(T a, T b)
+{
+	return (a > b) ? a : b;
+}
+
+template<typename T>
+static inline T mymin(T a, T b)
+{
+	return (a > b) ? b : a;
+}
+
+std::string read_setting(const std::string &name, std::istream &is);
+
+std::string read_setting_default(const std::string &name, std::istream &is,
+	const std::string &def);

mapper.cpp

@@ -1,61 +1,92 @@
+#include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <getopt.h>
 #include <fstream>
 #include <iostream>
-#include <map>
+#include <utility>
 #include <string>
 #include <sstream>
 #include <stdexcept>
-#include "cmake_config.h"
+#include "config.h"
 #include "TileGenerator.h"
 
-void usage()
+static void usage()
 {
-	const char *usage_text = "minetestmapper [options]\n"
-			"  -i/--input <world_path>\n"
-			"  -o/--output <output_image.png>\n"
-			"  --bgcolor <color>\n"
-			"  --scalecolor <color>\n"
-			"  --playercolor <color>\n"
-			"  --origincolor <color>\n"
-			"  --drawscale\n"
-			"  --drawplayers\n"
-			"  --draworigin\n"
-			"  --drawalpha\n"
-			"  --noshading\n"
-			"  --min-y <y>\n"
-			"  --max-y <y>\n"
-			"  --backend <backend>\n"
-			"  --geometry x:y+w+h\n"
-			"  --zoom <zoomlevel>\n"
-			"  --colors <colors.txt>\n"
-			"  --scales [t][b][l][r]\n"
-			"Color format: '#000000'\n";
-	std::cout << usage_text;
+	const std::pair<const char*, const char*> options[] = {
+		{"-i/--input", "<world_path>"},
+		{"-o/--output", "<output_image.png>"},
+		{"--bgcolor", "<color>"},
+		{"--scalecolor", "<color>"},
+		{"--playercolor", "<color>"},
+		{"--origincolor", "<color>"},
+		{"--drawscale", ""},
+		{"--drawplayers", ""},
+		{"--draworigin", ""},
+		{"--drawalpha", ""},
+		{"--noshading", ""},
+		{"--noemptyimage", ""},
+		{"--min-y", "<y>"},
+		{"--max-y", "<y>"},
+		{"--backend", "<backend>"},
+		{"--geometry", "x:y+w+h"},
+		{"--extent", ""},
+		{"--zoom", "<zoomlevel>"},
+		{"--colors", "<colors.txt>"},
+		{"--scales", "[t][b][l][r]"},
+		{"--exhaustive", "never|y|full|auto"},
+		{"--dumpblock", "x,y,z"},
+	};
+	const char *top_text =
+		"minetestmapper -i <world_path> -o <output_image.png> [options]\n"
+		"Generate an overview image of a Minetest map.\n"
+		"\n"
+		"Options:\n";
+	const char *bottom_text =
+		"\n"
+		"Color format: hexadecimal '#RRGGBB', e.g. '#FF0000' = red\n";
+
+	printf("%s", top_text);
+	for (const auto &p : options)
+		printf("  %-18s%s\n", p.first, p.second);
+	printf("%s", bottom_text);
+	auto backends = TileGenerator::getSupportedBackends();
+	printf("Supported backends: ");
+	for (auto s : backends)
+		printf("%s ", s.c_str());
+	printf("\n");
 }
 
-bool file_exists(const std::string &path)
+static inline bool file_exists(const std::string &path)
 {
-	std::ifstream ifs(path.c_str());
+	std::ifstream ifs(path);
 	return ifs.is_open();
 }
 
-std::string search_colors(const std::string &worldpath)
+static inline int stoi(const char *s)
 {
-	if(file_exists(worldpath + "/colors.txt"))
+	std::istringstream iss(s);
+	int ret;
+	iss >> ret;
+	return ret;
+}
+
+static std::string search_colors(const std::string &worldpath)
+{
+	if (file_exists(worldpath + "/colors.txt"))
 		return worldpath + "/colors.txt";
 
 #ifndef _WIN32
 	char *home = std::getenv("HOME");
-	if(home) {
-		std::string check = ((std::string) home) + "/.minetest/colors.txt";
-		if(file_exists(check))
+	if (home) {
+		std::string check = std::string(home) + "/.minetest/colors.txt";
+		if (file_exists(check))
 			return check;
 	}
 #endif
 
-	if(!(SHAREDIR[0] == '.' || SHAREDIR[0] == '\0') && file_exists(SHAREDIR "/colors.txt"))
+	constexpr bool sharedir_valid = !(SHAREDIR[0] == '.' || SHAREDIR[0] == '\0');
+	if (sharedir_valid && file_exists(SHAREDIR "/colors.txt"))
 		return SHAREDIR "/colors.txt";
 
 	std::cerr << "Warning: Falling back to using colors.txt from current directory." << std::endl;
@@ -64,7 +95,7 @@ std::string search_colors(const std::string &worldpath)
 
 int main(int argc, char *argv[])
 {
-	static struct option long_options[] =
+	const static struct option long_options[] =
 	{
 		{"help", no_argument, 0, 'h'},
 		{"input", required_argument, 0, 'i'},
@@ -80,35 +111,35 @@ int main(int argc, char *argv[])
 		{"noshading", no_argument, 0, 'H'},
 		{"backend", required_argument, 0, 'd'},
 		{"geometry", required_argument, 0, 'g'},
+		{"extent", no_argument, 0, 'E'},
 		{"min-y", required_argument, 0, 'a'},
 		{"max-y", required_argument, 0, 'c'},
 		{"zoom", required_argument, 0, 'z'},
 		{"colors", required_argument, 0, 'C'},
 		{"scales", required_argument, 0, 'f'},
+		{"noemptyimage", no_argument, 0, 'n'},
+		{"exhaustive", required_argument, 0, 'j'},
+		{"dumpblock", required_argument, 0, 'k'},
 		{0, 0, 0, 0}
 	};
 
 	std::string input;
 	std::string output;
-	std::string colors = "";
+	std::string colors;
+	bool onlyPrintExtent = false;
+	BlockPos dumpblock(INT16_MIN);
 
 	TileGenerator generator;
-	int option_index = 0;
-	int c = 0;
 	while (1) {
-		c = getopt_long(argc, argv, "hi:o:", long_options, &option_index);
-		if (c == -1) {
-			if (input.empty() || output.empty()) {
-				usage();
-				return 0;
-			}
-			break;
-		}
+		int option_index;
+		int c = getopt_long(argc, argv, "hi:o:", long_options, &option_index);
+		if (c == -1)
+			break; // done
+
 		switch (c) {
 			case 'h':
 				usage();
 				return 0;
-				break;
 			case 'i':
 				input = optarg;
 				break;
@@ -139,31 +170,23 @@ int main(int argc, char *argv[])
 			case 'e':
 				generator.setDrawAlpha(true);
 				break;
+			case 'E':
+				onlyPrintExtent = true;
+				break;
 			case 'H':
 				generator.setShading(false);
 				break;
 			case 'd':
 				generator.setBackend(optarg);
 				break;
-			case 'a': {
-					std::istringstream iss;
-					iss.str(optarg);
-					int miny;
-					iss >> miny;
-					generator.setMinY(miny);
-				}
+			case 'a':
+				generator.setMinY(stoi(optarg));
 				break;
-			case 'c': {
-					std::istringstream iss;
-					iss.str(optarg);
-					int maxy;
-					iss >> maxy;
-					generator.setMaxY(maxy);
-				}
+			case 'c':
+				generator.setMaxY(stoi(optarg));
 				break;
 			case 'g': {
-					std::istringstream geometry;
-					geometry.str(optarg);
+					std::istringstream geometry(optarg);
 					int x, y, w, h;
 					char c;
 					geometry >> x >> c >> y >> w >> h;
@@ -176,38 +199,74 @@ int main(int argc, char *argv[])
 				break;
 			case 'f': {
 					uint flags = 0;
-					if(strchr(optarg, 't') != NULL)
+					if (strchr(optarg, 't'))
 						flags |= SCALE_TOP;
-					if(strchr(optarg, 'b') != NULL)
+					if (strchr(optarg, 'b'))
 						flags |= SCALE_BOTTOM;
-					if(strchr(optarg, 'l') != NULL)
+					if (strchr(optarg, 'l'))
 						flags |= SCALE_LEFT;
-					if(strchr(optarg, 'r') != NULL)
+					if (strchr(optarg, 'r'))
 						flags |= SCALE_RIGHT;
 					generator.setScales(flags);
 				}
 				break;
-			case 'z': {
-					std::istringstream iss;
-					iss.str(optarg);
-					int zoom;
-					iss >> zoom;
-					generator.setZoom(zoom);
-				}
+			case 'z':
+				generator.setZoom(stoi(optarg));
 				break;
 			case 'C':
 				colors = optarg;
 				break;
+			case 'n':
+				generator.setDontWriteEmpty(true);
+				break;
+			case 'j': {
+					int mode = EXH_AUTO;;
+					if (!strcmp(optarg, "never"))
+						mode = EXH_NEVER;
+					else if (!strcmp(optarg, "y"))
+						mode = EXH_Y;
+					else if (!strcmp(optarg, "full"))
+						mode = EXH_FULL;
+					generator.setExhaustiveSearch(mode);
+				}
+				break;
+			case 'k': {
+				std::istringstream iss(optarg);
+				char c, c2;
+				iss >> dumpblock.x >> c >> dumpblock.y >> c2 >> dumpblock.z;
+				if (iss.fail() || c != ',' || c2 != ',') {
+					usage();
+					exit(1);
+				}
+				break;
+			}
 			default:
 				exit(1);
 		}
 	}
-	if(colors == "")
-		colors = search_colors(input);
+
+	const bool need_output = !onlyPrintExtent && dumpblock.x == INT16_MIN;
+	if (input.empty() || (need_output && output.empty())) {
+		usage();
+		return 0;
+	}
+
 	try {
+
+		if (onlyPrintExtent) {
+			generator.printGeometry(input);
+			return 0;
+		} else if (dumpblock.x != INT16_MIN) {
+			generator.dumpBlock(input, dumpblock);
+			return 0;
+		}
+
+		if(colors.empty())
+			colors = search_colors(input);
 		generator.parseColorsFile(colors);
 		generator.generate(input, output);
-	} catch(std::runtime_error e) {
+
+	} catch (const std::exception &e) {
 		std::cerr << "Exception: " << e.what() << std::endl;
 		return 1;
 	}

minetestmapper.6

@@ -0,0 +1,114 @@
+.TH MINETESTMAPPER 6
+.SH NAME
+minetestmapper \- generate an overview image of a Minetest map
+.SH SYNOPSIS
+.B minetestmapper
+\fB\-i\fR \fIworld_path\fR
+\fB\-o\fR \fIoutput_image\fR
+.PP
+See additional optional parameters below.
+.SH DESCRIPTION
+.B minetestmapper
+generates an overview image of a minetest map. This is a port of
+the original minetestmapper.py to C++, that is both faster and
+provides more functionality than the deprecated Python script.
+.SH MANDATORY PARAMETERS
+.TP
+.BR \-i " " \fIworld_path\fR
+Input world path.
+.TP
+.BR \-o " " \fIoutput_image\fR
+Path to output image. (only PNG supported currently)
+.SH OPTIONAL PARAMETERS
+.TP
+.BR \-\-bgcolor " " \fIcolor\fR
+Background color of image, e.g. "--bgcolor #ffffff"
+
+.TP
+.BR \-\-scalecolor " " \fIcolor\fR
+Color of scale, e.g. "--scalecolor #000000"
+
+.TP
+.BR \-\-playercolor " " \fIcolor\fR
+Color of player indicators, e.g. "--playercolor #ff0000"
+
+.TP
+.BR \-\-origincolor " " \fIcolor\fR
+Color of origin indicator, e.g. "--origincolor #ff0000"
+
+.TP
+.BR \-\-drawscale
+Draw tick marks
+
+.TP
+.BR \-\-drawplayers
+Draw player indicators
+
+.TP
+.BR \-\-draworigin
+Draw origin indicator
+
+.TP
+.BR \-\-drawalpha
+Allow nodes to be drawn with transparency
+
+.TP
+.BR \-\-noshading
+Don't draw shading on nodes
+
+.TP
+.BR \-\-noemptyimage
+Don't output anything when the image would be empty.
+
+.TP
+.BR \-\-min-y " " \fInumber\fR
+Don't draw nodes below this y value, e.g. "--min-y -25"
+
+.TP
+.BR \-\-max-y " " \fInumber\fR
+Don't draw nodes above this y value, e.g. "--max-y 75"
+
+.TP
+.BR \-\-backend " " \fIbackend\fR
+Use specific map backend; supported: \fIsqlite3\fP, \fIleveldb\fP, \fIredis\fP, \fIpostgresql\fP, e.g. "--backend leveldb"
+
+.TP
+.BR \-\-geometry " " \fIgeometry\fR
+Limit area to specific geometry (\fIx:y+w+h\fP where x and y specify the lower left corner), e.g. "--geometry -800:-800+1600+1600"
+
+.TP
+.BR \-\-extent
+Don't render the image, just print the extent of the map that would be generated, in the same format as the geometry above.
+
+.TP
+.BR \-\-zoom " " \fIfactor\fR
+Zoom the image by using more than one pixel per node, e.g. "--zoom 4"
+
+.TP
+.BR \-\-colors " " \fIpath\fR
+Forcefully set path to colors.txt file (it's autodetected otherwise), e.g. "--colors ../minetest/mycolors.txt"
+
+.TP
+.BR \-\-scales " " \fIedges\fR
+Draw scales on specified image edges (letters \fIt b l r\fP meaning top, bottom, left and right), e.g. "--scales tbr"
+
+.TP
+.BR \-\-exhaustive " " \fImode\fR
+Select if database should be traversed exhaustively or using range queries, available: \fInever\fP, \fIy\fP, \fIfull\fP, \fIauto\fP
+
+Defaults to \fIauto\fP. You shouldn't need to change this, but doing so can improve rendering times on large maps.
+For these optimizations to work it is important that you set
+.B min-y
+and
+.B max-y
+when you don't care about the world below e.g. -60 and above 1000 nodes.
+
+.TP
+.BR \-\-dumpblock " " \fIpos\fR
+Instead of rendering anything try to load the block at the given position (\fIx,y,z\fR) and print its raw data as hexadecimal.
+
+.SH MORE INFORMATION
+Website: https://github.com/minetest/minetestmapper
+
+.SH MAN PAGE AUTHOR
+Daniel Moerner

util.cpp

@@ -1,56 +1,55 @@
+#include <stdexcept>
+#include <sstream>
+
 #include "util.h"
 
-inline std::string trim(const std::string &s)
+static std::string trim(const std::string &s)
 {
-  size_t front = 0;
-  while(s[front] == ' '    ||
-        s[front] == '\t'   ||
-        s[front] == '\r'   ||
-        s[front] == '\n'
-       )
-    ++front;
+	auto isspace = [] (char c) -> bool { return c == ' ' || c == '\t' || c == '\r' || c == '\n'; };
 
-  size_t back = s.size();
-  while(back > front &&
-        (s[back-1] == ' '  ||
-         s[back-1] == '\t' ||
-         s[back-1] == '\r' ||
-         s[back-1] == '\n'
-        )
-       )
-    --back;
+	size_t front = 0;
+	while (isspace(s[front]))
+		++front;
+	size_t back = s.size() - 1;
+	while (back > front && isspace(s[back]))
+		--back;
 
-  return s.substr(front, back - front);
+	return s.substr(front, back - front + 1);
 }
 
-#define EOFCHECK() \
-   if(is.eof()) \
-    throw std::runtime_error(((std::string) "Setting '") + name + "' not found");
-
-std::string get_setting(std::string name, std::istream &is)
+std::string read_setting(const std::string &name, std::istream &is)
 {
-  char c;
-  char s[256];
-  std::string nm, value;
+	char linebuf[512];
+	while (is.good()) {
+		is.getline(linebuf, sizeof(linebuf));
 
-  next:
-  while((c = is.get()) == ' ' || c == '\t' || c == '\r' || c == '\n')
-    ;
-  EOFCHECK();
-  if(c == '#') // Ignore comments
-    is.ignore(0xffff, '\n');
-  EOFCHECK();
-  s[0] = c; // The current char belongs to the name too
-  is.get(&s[1], 255, '=');
-  is.ignore(1); // Jump over the =
-  EOFCHECK();
-  nm = trim(std::string(s));
-  is.get(s, 256, '\n');
-  value = trim(std::string(s));
-  if(name == nm)
-    return value;
-  else
-    goto next;
+		for (char *p = linebuf; *p; p++) {
+			if(*p != '#')
+				continue;
+			*p = '\0'; // Cut off at the first #
+			break;
+		}
+		std::string line(linebuf);
+
+		auto pos = line.find('=');
+		if (pos == std::string::npos)
+			continue;
+		auto key = trim(line.substr(0, pos));
+		if (key != name)
+			continue;
+		return trim(line.substr(pos+1));
+	}
+	std::ostringstream oss;
+	oss << "Setting '" << name << "' not found";
+	throw std::runtime_error(oss.str());
 }
 
-#undef EOFCHECK
+std::string read_setting_default(const std::string &name, std::istream &is,
+	const std::string &def)
+{
+	try {
+		return read_setting(name, is);
+	} catch(const std::runtime_error &e) {
+		return def;
+	}
+}

util.h

@@ -1,10 +0,0 @@
-#ifndef _UTIL_H
-#define _UTIL_H
-
-#include <string>
-#include <stdexcept>
-#include <fstream>
-
-std::string get_setting(std::string name, std::istream &is);
-
-#endif // _UTIL_H

util/build-mingw.sh

@@ -0,0 +1,56 @@
+#!/bin/bash -e
+
+[ -z "$CXX" ] && exit 255
+export CC=false # don't need it actually
+
+variant=win32
+[[ "$(basename "$CXX")" == "x86_64-"* ]] && variant=win64
+
+#######
+# this expects unpacked libraries similar to what Minetest's buildbot uses
+# $extradlls will typically point to the DLLs for libgcc, libstdc++ and libpng
+libgd_dir=
+zlib_dir=
+zstd_dir=
+sqlite_dir=
+leveldb_dir=
+extradlls=()
+#######
+
+[ -f ./CMakeLists.txt ] || exit 1
+
+cmake -S . -B build \
+	-DCMAKE_SYSTEM_NAME=Windows \
+	-DCMAKE_EXE_LINKER_FLAGS="-s" \
+	\
+	-DENABLE_LEVELDB=1 \
+	\
+	-DLEVELDB_INCLUDE_DIR=$leveldb_dir/include \
+	-DLEVELDB_LIBRARY=$leveldb_dir/lib/libleveldb.dll.a \
+	-DLIBGD_INCLUDE_DIR=$libgd_dir/include \
+	-DLIBGD_LIBRARY=$libgd_dir/lib/libgd.dll.a \
+	-DSQLITE3_INCLUDE_DIR=$sqlite_dir/include \
+	-DSQLITE3_LIBRARY=$sqlite_dir/lib/libsqlite3.dll.a \
+	-DZLIB_INCLUDE_DIR=$zlib_dir/include \
+	-DZLIB_LIBRARY=$zlib_dir/lib/libz.dll.a \
+	-DZSTD_INCLUDE_DIR=$zstd_dir/include \
+	-DZSTD_LIBRARY=$zstd_dir/lib/libzstd.dll.a \
+
+make -C build -j4
+
+mkdir pack
+cp -p \
+	AUTHORS colors.txt COPYING README.rst \
+	build/minetestmapper.exe \
+	$leveldb_dir/bin/libleveldb.dll \
+	$libgd_dir/bin/libgd*.dll \
+	$sqlite_dir/bin/libsqlite*.dll \
+	$zlib_dir/bin/zlib1.dll \
+	$zstd_dir/bin/libzstd.dll \
+	"${extradlls[@]}" \
+	pack/
+zipfile=$PWD/minetestmapper-$variant.zip
+(cd pack; zip -9r "$zipfile" *)
+
+rm -rf build pack
+echo "Done."

util/ci/script.sh

@@ -0,0 +1,32 @@
+#!/bin/bash -e
+
+install_linux_deps() {
+	local pkgs=(cmake libgd-dev libsqlite3-dev libleveldb-dev libpq-dev libhiredis-dev libzstd-dev)
+
+	sudo apt-get update
+	sudo apt-get remove -y 'libgd3' nginx || : # ????
+	sudo apt-get install -y --no-install-recommends "${pkgs[@]}" "$@"
+}
+
+run_build() {
+	local args=(
+		-DCMAKE_BUILD_TYPE=Debug
+		-DENABLE_LEVELDB=1 -DENABLE_POSTGRESQL=1 -DENABLE_REDIS=1
+	)
+	[[ "$CXX" == clang* ]] && args+=(-DCMAKE_CXX_FLAGS="-fsanitize=address")
+	cmake . "${args[@]}"
+
+	make -j2
+}
+
+do_functional_test() {
+	mkdir testmap
+	echo "backend = sqlite3" >testmap/world.mt
+	sqlite3 testmap/map.sqlite <<END
+CREATE TABLE blocks(pos INT,data BLOB);
+INSERT INTO blocks(pos, data) VALUES(0, x'$(cat util/ci/test_block)');
+END
+
+	./minetestmapper --noemptyimage -i ./testmap -o map.png
+	file map.png
+}

util/ci/test_block

@@ -0,0 +1 @@
+1b00ffff020278daedd4c1090000080331dd7f691710faf12589235cb12ae870fca6bffefaebafbffefaebafbffefaebbff7b708fdf1ffd11ffdd11ffdd11ffd01000000000000003836d59f010578da63000000010001000000ffffffff000002000000036169720001000d64656661756c743a73746f6e650a0000

util/dumpnodes/init.lua

@@ -0,0 +1,61 @@
+local function get_tile(tiles, n)
+	local tile = tiles[n]
+	if type(tile) == 'table' then
+		return tile.name or tile.image
+	end
+	return tile
+end
+
+local function pairs_s(dict)
+	local keys = {}
+	for k in pairs(dict) do
+		keys[#keys+1] = k
+	end
+	table.sort(keys)
+	return ipairs(keys)
+end
+
+minetest.register_chatcommand("dumpnodes", {
+	description = "Dump node and texture list for use with minetestmapper",
+	func = function()
+		local ntbl = {}
+		for _, nn in pairs_s(minetest.registered_nodes) do
+			local prefix, name = nn:match('(.*):(.*)')
+			if prefix == nil or name == nil then
+				print("ignored(1): " .. nn)
+			else
+				if ntbl[prefix] == nil then
+					ntbl[prefix] = {}
+				end
+				ntbl[prefix][name] = true
+			end
+		end
+		local out, err = io.open(minetest.get_worldpath() .. "/nodes.txt", 'wb')
+		if not out then
+			return true, err
+		end
+		local n = 0
+		for _, prefix in pairs_s(ntbl) do
+			out:write('# ' .. prefix .. '\n')
+			for _, name in pairs_s(ntbl[prefix]) do
+				local nn = prefix .. ":" .. name
+				local nd = minetest.registered_nodes[nn]
+				local tiles = nd.tiles or nd.tile_images
+				if tiles == nil or nd.drawtype == 'airlike' then
+					print("ignored(2): " .. nn)
+				else
+					local tex = get_tile(tiles, 1)
+					tex = (tex .. '^'):match('%(*(.-)%)*^') -- strip modifiers
+					if tex:find("[combine", 1, true) then
+						tex = tex:match('.-=([^:]-)') -- extract first texture
+					end
+					out:write(nn .. ' ' .. tex .. '\n')
+					n = n + 1
+				end
+			end
+			out:write('\n')
+		end
+		out:close()
+		return true, n .. " nodes dumped."
+	end,
+})

util/dumpnodes/mod.conf

@@ -0,0 +1,2 @@
+name = dumpnodes
+description = minetestmapper development mod (node dumper)

util/generate_colorstxt.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+import sys
+import os.path
+import getopt
+import re
+from math import sqrt
+try:
+	from PIL import Image
+except:
+	print("Could not load image routines, install PIL ('pillow' on pypi)!", file=sys.stderr)
+	exit(1)
+
+############
+############
+# Instructions for generating a colors.txt file for custom games and/or mods:
+# 1) Add the dumpnodes mod to a Minetest world with the chosen game and mods enabled.
+# 2) Join ingame and run the /dumpnodes chat command.
+# 3) Run this script and poin it to the installation path of the game using -g,
+#    the path(s) where mods are stored using -m and the nodes.txt in your world folder.
+#    Example command line:
+#      ./util/generate_colorstxt.py --game /usr/share/minetest/games/minetest_game \
+#        -m ~/.minetest/mods ~/.minetest/worlds/my_world/nodes.txt
+# 4) Copy the resulting colors.txt file to your world folder or to any other places
+#    and use it with minetestmapper's --colors option.
+###########
+###########
+
+# minimal sed syntax, s|match|replace| and /match/d supported
+REPLACEMENTS = [
+	# Delete some nodes that are usually hidden
+	r'/^fireflies:firefly /d',
+	r'/^butterflies:butterfly_/d',
+	# Nicer colors for water and lava
+	r's/^(default:(river_)?water_(flowing|source)) [0-9 ]+$/\1 39 66 106 128 224/',
+	r's/^(default:lava_(flowing|source)) [0-9 ]+$/\1 255 100 0/',
+	# Transparency for glass nodes and panes
+	r's/^(default:.*glass) ([0-9 ]+)$/\1 \2 64 16/',
+	r's/^(doors:.*glass[^ ]*) ([0-9 ]+)$/\1 \2 64 16/',
+	r's/^(xpanes:.*(pane|bar)[^ ]*) ([0-9 ]+)$/\1 \3 64 16/',
+]
+
+def usage():
+	print("Usage: generate_colorstxt.py [options] [input file] [output file]")
+	print("If not specified the input file defaults to ./nodes.txt and the output file to ./colors.txt")
+	print("  -g / --game <folder>\t\tSet path to the game (for textures), required")
+	print("  -m / --mods <folder>\t\tAdd search path for mod textures")
+	print("  --replace <file>\t\tLoad replacements from file (ADVANCED)")
+
+def collect_files(path):
+	dirs = []
+	with os.scandir(path) as it:
+		for entry in it:
+			if entry.name[0] == '.': continue
+			if entry.is_dir():
+				dirs.append(entry.path)
+				continue
+			if entry.is_file() and '.' in entry.name:
+				if entry.name not in textures.keys():
+					textures[entry.name] = entry.path
+	for path2 in dirs:
+		collect_files(path2)
+
+def average_color(filename):
+	inp = Image.open(filename).convert('RGBA')
+	data = inp.load()
+
+	c0, c1, c2 = [], [], []
+	for x in range(inp.size[0]):
+		for y in range(inp.size[1]):
+			px = data[x, y]
+			if px[3] < 128: continue # alpha
+			c0.append(px[0]**2)
+			c1.append(px[1]**2)
+			c2.append(px[2]**2)
+
+	if len(c0) == 0:
+		print(f"didn't find color for '{os.path.basename(filename)}'", file=sys.stderr)
+		return "0 0 0"
+	c0 = sqrt(sum(c0) / len(c0))
+	c1 = sqrt(sum(c1) / len(c1))
+	c2 = sqrt(sum(c2) / len(c2))
+	return "%d %d %d" % (c0, c1, c2)
+
+def apply_sed(line, exprs):
+	for expr in exprs:
+		if expr[0] == '/':
+			if not expr.endswith("/d"): raise ValueError()
+			if re.search(expr[1:-2], line):
+				return ''
+		elif expr[0] == 's':
+			expr = expr.split(expr[1])
+			if len(expr) != 4 or expr[3] != '': raise ValueError()
+			line = re.sub(expr[1], expr[2], line)
+		else:
+			raise ValueError()
+	return line
+#
+
+try:
+	opts, args = getopt.getopt(sys.argv[1:], "hg:m:", ["help", "game=", "mods=", "replace="])
+except getopt.GetoptError as e:
+	print(str(e))
+	exit(1)
+if ('-h', '') in opts or ('--help', '') in opts:
+	usage()
+	exit(0)
+
+input_file = "./nodes.txt"
+output_file = "./colors.txt"
+texturepaths = []
+
+try:
+	gamepath = next(o[1] for o in opts if o[0] in ('-g', '--game'))
+	if not os.path.isdir(os.path.join(gamepath, "mods")):
+		print(f"'{gamepath}' doesn't exist or does not contain a game.", file=sys.stderr)
+		exit(1)
+	texturepaths.append(os.path.join(gamepath, "mods"))
+except StopIteration:
+	print("No game path set but one is required. (see --help)", file=sys.stderr)
+	exit(1)
+
+try:
+	tmp = next(o[1] for o in opts if o[0] == "--replace")
+	REPLACEMENTS.clear()
+	with open(tmp, 'r') as f:
+		for line in f:
+			if not line or line[0] == '#': continue
+			REPLACEMENTS.append(line.strip())
+except StopIteration:
+	pass
+
+for o in opts:
+	if o[0] not in ('-m', '--mods'): continue
+	if not os.path.isdir(o[1]):
+		print(f"Given path '{o[1]}' does not exist.'", file=sys.stderr)
+		exit(1)
+	texturepaths.append(o[1])
+
+if len(args) > 2:
+	print("Too many arguments.", file=sys.stderr)
+	exit(1)
+if len(args) > 1:
+	output_file = args[1]
+if len(args) > 0:
+	input_file = args[0]
+
+if not os.path.exists(input_file) or os.path.isdir(input_file):
+	print(f"Input file '{input_file}' does not exist.", file=sys.stderr)
+	exit(1)
+
+#
+
+print(f"Collecting textures from {len(texturepaths)} path(s)... ", end="", flush=True)
+textures = {}
+for path in texturepaths:
+	collect_files(path)
+print("done")
+
+print("Processing nodes...")
+fin = open(input_file, 'r')
+fout = open(output_file, 'w')
+n = 0
+for line in fin:
+	line = line.rstrip('\r\n')
+	if not line or line[0] == '#':
+		fout.write(line + '\n')
+		continue
+	node, tex = line.split(" ")
+	if not tex or tex == "blank.png":
+		continue
+	elif tex not in textures.keys():
+		print(f"skip {node} texture not found")
+		continue
+	color = average_color(textures[tex])
+	line = f"{node} {color}"
+	#print(f"ok {node}")
+	line = apply_sed(line, REPLACEMENTS)
+	if line:
+		fout.write(line + '\n')
+		n += 1
+fin.close()
+fout.close()
+print(f"Done, {n} entries written.")

util/travis/script.sh

@@ -1,17 +0,0 @@
-#!/bin/bash -e
-
-CXX=g++-6
-[ $CC == "clang" ] && CXX=clang-3.8
-export CXX
-
-
-mkdir -p travisbuild
-cd travisbuild
-
-cmake \
-	-DENABLE_LEVELDB=1 \
-	-DLEVELDB_LIBRARY=../libleveldb/lib/libleveldb.so \
-	-DLEVELDB_INCLUDE_DIR=../libleveldb/include \
-	..
-
-make -j2