Optimize database access further by allowing "brute-force" queries instead of listing available blocks

Also adds a heuristic that will enable this behaviour automatically.

README.rst

@@ -106,3 +106,7 @@ colors:
 
 scales:
     Draw scales on specified image edges (letters *t b l r* meaning top, bottom, left and right), e.g. ``--scales tbr``
+
+exhaustive:
+    Select if database should be traversed exhaustively or using range queries, available: *never*, *y*, *full*, *auto*
+    Defaults to *auto*. You shouldn't need to change this, but doing so can improve rendering times on large maps.

TileGenerator.cpp

@@ -96,6 +96,7 @@ TileGenerator::TileGenerator():
 	m_geomY(-2048),
 	m_geomX2(2048),
 	m_geomY2(2048),
+	m_exhaustiveSearch(EXH_AUTO),
 	m_zoom(1),
 	m_scales(SCALE_LEFT | SCALE_TOP)
 {
@@ -206,6 +207,11 @@ void TileGenerator::setMaxY(int y)
 		std::swap(m_yMin, m_yMax);
 }
 
+void TileGenerator::setExhaustiveSearch(int mode)
+{
+	m_exhaustiveSearch = mode;
+}
+
 void TileGenerator::parseColorsFile(const std::string &fileName)
 {
 	ifstream in;
@@ -222,6 +228,7 @@ void TileGenerator::printGeometry(const std::string &input)
 		input_path += PATH_SEPARATOR;
 	}
 
+	setExhaustiveSearch(EXH_NEVER);
 	openDb(input_path);
 	loadBlocks();
 
@@ -247,6 +254,8 @@ void TileGenerator::generate(const std::string &input, const std::string &output
 		input_path += PATH_SEPARATOR;
 	}
 
+	if (m_dontWriteEmpty) // FIXME: possible too, just needs to be done differently
+		setExhaustiveSearch(EXH_NEVER);
 	openDb(input_path);
 	loadBlocks();
 
@@ -329,6 +338,35 @@ void TileGenerator::openDb(const std::string &input)
 #endif
 	else
 		throw std::runtime_error(((std::string) "Unknown map backend: ") + backend);
+
+	// Determine how we're going to traverse the database (heuristic)
+	if (m_exhaustiveSearch == EXH_AUTO) {
+		using u64 = uint64_t;
+		u64 y_range = (m_yMax / 16 + 1) - (m_yMin / 16);
+		u64 blocks = (u64)(m_geomX2 - m_geomX) * y_range * (u64)(m_geomY2 - m_geomY);
+#ifndef NDEBUG
+		std::cout << "Heuristic parameters:"
+			<< " preferRangeQueries()=" << m_db->preferRangeQueries()
+			<< " y_range=" << y_range << " blocks=" << blocks << std::endl;
+#endif
+		if (m_db->preferRangeQueries())
+			m_exhaustiveSearch = EXH_NEVER;
+		else if (blocks < 200000)
+			m_exhaustiveSearch = EXH_FULL;
+		else if (y_range < 600)
+			m_exhaustiveSearch = EXH_Y;
+		else
+			m_exhaustiveSearch = EXH_NEVER;
+	} else if (m_exhaustiveSearch == EXH_FULL || m_exhaustiveSearch == EXH_Y) {
+		if (m_db->preferRangeQueries()) {
+			std::cerr << "Note: The current database backend supports efficient "
+				"range queries, forcing exhaustive search should always result "
+				" in worse performance." << std::endl;
+		}
+	}
+	if (m_exhaustiveSearch == EXH_Y)
+		m_exhaustiveSearch = EXH_NEVER; // (TODO remove when implemented)
+	assert(m_exhaustiveSearch != EXH_AUTO);
 }
 
 void TileGenerator::closeDatabase()
@@ -340,37 +378,40 @@ void TileGenerator::closeDatabase()
 void TileGenerator::loadBlocks()
 {
 	const int16_t yMax = m_yMax / 16 + 1;
-	std::vector<BlockPos> vec = m_db->getBlockPos(
-		BlockPos(m_geomX, m_yMin / 16, m_geomY),
-		BlockPos(m_geomX2, yMax, m_geomY2)
-	);
 
-	for (auto pos : vec) {
+	if (m_exhaustiveSearch == EXH_NEVER || m_exhaustiveSearch == EXH_Y) {
-		assert(pos.x >= m_geomX && pos.x < m_geomX2);
+		std::vector<BlockPos> vec = m_db->getBlockPos(
-		assert(pos.y >= m_yMin / 16 && pos.y < yMax);
+			BlockPos(m_geomX, m_yMin / 16, m_geomY),
-		assert(pos.z >= m_geomY && pos.z < m_geomY2);
+			BlockPos(m_geomX2, yMax, m_geomY2)
+		);
 
-		// Adjust minimum and maximum positions to the nearest block
+		for (auto pos : vec) {
-		if (pos.x < m_xMin)
+			assert(pos.x >= m_geomX && pos.x < m_geomX2);
-			m_xMin = pos.x;
+			assert(pos.y >= m_yMin / 16 && pos.y < yMax);
-		if (pos.x > m_xMax)
+			assert(pos.z >= m_geomY && pos.z < m_geomY2);
-			m_xMax = pos.x;
 
-		if (pos.z < m_zMin)
+			// Adjust minimum and maximum positions to the nearest block
-			m_zMin = pos.z;
+			if (pos.x < m_xMin)
-		if (pos.z > m_zMax)
+				m_xMin = pos.x;
-			m_zMax = pos.z;
+			if (pos.x > m_xMax)
+				m_xMax = pos.x;
 
-		m_positions[pos.z].emplace(pos.x);
+			if (pos.z < m_zMin)
-	}
+				m_zMin = pos.z;
+			if (pos.z > m_zMax)
+				m_zMax = pos.z;
+
+			m_positions[pos.z].emplace(pos.x);
+		}
 
 #ifndef NDEBUG
-	int count = 0;
+		int count = 0;
-	for (const auto &it : m_positions)
+		for (const auto &it : m_positions)
-		count += it.second.size();
+			count += it.second.size();
-	std::cout << "Loaded " << count
+		std::cout << "Loaded " << count
-		<< " positions (across Z: " << m_positions.size() << ") for rendering" << std::endl;
+			<< " positions (across Z: " << m_positions.size() << ") for rendering" << std::endl;
 #endif
+	}
 }
 
 void TileGenerator::createImage()
@@ -422,44 +463,76 @@ void TileGenerator::renderMap()
 	BlockDecoder blk;
 	const int16_t yMax = m_yMax / 16 + 1;
 
-	for (auto it = m_positions.rbegin(); it != m_positions.rend(); ++it) {
+	auto renderSingle = [&] (int16_t xPos, int16_t zPos, BlockList &blockStack) {
-		int16_t zPos = it->first;
+		m_readPixels.reset();
-		for (auto it2 = it->second.rbegin(); it2 != it->second.rend(); ++it2) {
+		m_readInfo.reset();
-			int16_t xPos = *it2;
+		for (int i = 0; i < 16; i++) {
-
+			for (int j = 0; j < 16; j++) {
-			m_readPixels.reset();
+				m_color[i][j] = m_bgColor; // This will be drawn by renderMapBlockBottom() for y-rows with only 'air', 'ignore' or unknown nodes if --drawalpha is used
-			m_readInfo.reset();
+				m_color[i][j].a = 0; // ..but set alpha to 0 to tell renderMapBlock() not to use this color to mix a shade
-			for (int i = 0; i < 16; i++) {
+				m_thickness[i][j] = 0;
-				for (int j = 0; j < 16; j++) {
-					m_color[i][j] = m_bgColor; // This will be drawn by renderMapBlockBottom() for y-rows with only 'air', 'ignore' or unknown nodes if --drawalpha is used
-					m_color[i][j].a = 0; // ..but set alpha to 0 to tell renderMapBlock() not to use this color to mix a shade
-					m_thickness[i][j] = 0;
-				}
 			}
-
-			BlockList blockStack;
-			m_db->getBlocksOnXZ(blockStack, xPos, zPos, m_yMin / 16, yMax);
-			blockStack.sort();
-			for (const auto &it : blockStack) {
-				const BlockPos pos = it.first;
-				assert(pos.x == xPos && pos.z == zPos);
-				assert(pos.y >= m_yMin / 16 && pos.y < yMax);
-
-				blk.reset();
-				blk.decode(it.second);
-				if (blk.isEmpty())
-					continue;
-				renderMapBlock(blk, pos);
-
-				// Exit out if all pixels for this MapBlock are covered
-				if (m_readPixels.full())
-					break;
-			}
-			if (!m_readPixels.full())
-				renderMapBlockBottom(blockStack.begin()->first);
 		}
+
+		for (const auto &it : blockStack) {
+			const BlockPos pos = it.first;
+			assert(pos.x == xPos && pos.z == zPos);
+			assert(pos.y >= m_yMin / 16 && pos.y < yMax);
+
+			blk.reset();
+			blk.decode(it.second);
+			if (blk.isEmpty())
+				continue;
+			renderMapBlock(blk, pos);
+
+			// Exit out if all pixels for this MapBlock are covered
+			if (m_readPixels.full())
+				break;
+		}
+		if (!m_readPixels.full())
+			renderMapBlockBottom(blockStack.begin()->first);
+	};
+	auto postRenderRow = [&] (int16_t zPos) {
 		if (m_shading)
 			renderShading(zPos);
+	};
+
+	if (m_exhaustiveSearch == EXH_NEVER) {
+		for (auto it = m_positions.rbegin(); it != m_positions.rend(); ++it) {
+			int16_t zPos = it->first;
+			for (auto it2 = it->second.rbegin(); it2 != it->second.rend(); ++it2) {
+				int16_t xPos = *it2;
+
+				BlockList blockStack;
+				m_db->getBlocksOnXZ(blockStack, xPos, zPos, m_yMin / 16, yMax);
+				blockStack.sort();
+
+				renderSingle(xPos, zPos, blockStack);
+			}
+			postRenderRow(zPos);
+		}
+	} else if (m_exhaustiveSearch == EXH_FULL) {
+#ifndef NDEBUG
+		std::cerr << "Exhaustively searching "
+			<< (m_geomX2 - m_geomX) << "x" << (yMax - (m_yMin / 16)) << "x"
+			<< (m_geomY2 - m_geomY) << " blocks" << std::endl;
+#endif
+		std::vector<BlockPos> positions;
+		positions.reserve(yMax - (m_yMin / 16));
+		for (int16_t zPos = m_geomY2 - 1; zPos >= m_geomY; zPos--) {
+			for (int16_t xPos = m_geomX2 - 1; xPos >= m_geomX; xPos--) {
+				positions.clear();
+				for (int16_t yPos = m_yMin / 16; yPos < yMax; yPos++)
+					positions.emplace_back(xPos, yPos, zPos);
+
+				BlockList blockStack;
+				m_db->getBlocksByPos(blockStack, positions);
+				blockStack.sort();
+
+				renderSingle(xPos, zPos, blockStack);
+			}
+			postRenderRow(zPos);
+		}
 	}
 }
 

db-leveldb.cpp

@@ -96,3 +96,19 @@ void DBLevelDB::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 		}
 	}
 }
+
+void DBLevelDB::getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions)
+{
+	std::string datastr;
+	leveldb::Status status;
+
+	for (auto pos : positions) {
+		status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(pos)), &datastr);
+		if (status.ok()) {
+			blocks.emplace_back(
+				pos, ustring((unsigned char *) datastr.data(), datastr.size())
+			);
+		}
+	}
+}

db-postgresql.cpp

@@ -38,6 +38,11 @@ DBPostgreSQL::DBPostgreSQL(const std::string &mapdir)
 		" posX = $1::int4 AND posZ = $2::int4"
 		" AND (posY BETWEEN $3::int4 AND $4::int4)"
 	);
+	prepareStatement(
+		"get_block_exact",
+		"SELECT data FROM blocks WHERE"
+		" posX = $1::int4 AND posY = $2::int4 AND posZ = $3::int4"
+	);
 
 	checkResults(PQexec(db, "START TRANSACTION;"));
 	checkResults(PQexec(db, "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;"));
@@ -48,12 +53,13 @@ DBPostgreSQL::~DBPostgreSQL()
 {
 	try {
 		checkResults(PQexec(db, "COMMIT;"));
-	} catch (std::exception& caught) {
+	} catch (const std::exception& caught) {
 		std::cerr << "could not finalize: " << caught.what() << std::endl;
 	}
 	PQfinish(db);
 }
 
+
 std::vector<BlockPos> DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max)
 {
 	int32_t const x1 = htonl(min.x);
@@ -124,6 +130,43 @@ void DBPostgreSQL::getBlocksOnXZ(BlockList &blocks, int16_t xPos, int16_t zPos,
 	PQclear(results);
 }
 
+
+void DBPostgreSQL::getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions)
+{
+	int32_t x, y, z;
+
+	const void *args[] = { &x, &y, &z };
+	const int argLen[] = { 4, 4, 4 };
+	const int argFmt[] = { 1, 1, 1 };
+
+	for (auto pos : positions) {
+		x = htonl(pos.x);
+		y = htonl(pos.y);
+		z = htonl(pos.z);
+
+		PGresult *results = execPrepared(
+			"get_block_exact", ARRLEN(args), args,
+			argLen, argFmt, false
+		);
+
+		if (PQntuples(results) > 0) {
+			blocks.emplace_back(
+				pos,
+				ustring(
+					reinterpret_cast<unsigned char*>(
+						PQgetvalue(results, 0, 0)
+					),
+					PQgetlength(results, 0, 0)
+				)
+			);
+		}
+
+		PQclear(results);
+	}
+}
+
+
 PGresult *DBPostgreSQL::checkResults(PGresult *res, bool clear)
 {
 	ExecStatusType statusType = PQresultStatus(res);

db-redis.cpp

@@ -82,7 +82,7 @@ std::vector<BlockPos> DBRedis::getBlockPos(BlockPos min, BlockPos max)
 }
 
 
-std::string DBRedis::replyTypeStr(int type) {
+const char *DBRedis::replyTypeStr(int type) {
 	switch(type) {
 		case REDIS_REPLY_STATUS:
 			return "REDIS_REPLY_STATUS";
@@ -121,7 +121,8 @@ void DBRedis::loadPosCache()
 }
 
 
-void DBRedis::HMGET(const std::vector<BlockPos> &positions, std::vector<ustring> *result)
+void DBRedis::HMGET(const std::vector<BlockPos> &positions,
+	std::function<void(std::size_t, ustring)> result)
 {
 	const char *argv[DB_REDIS_HMGET_NUMFIELDS + 2];
 	argv[0] = "HMGET";
@@ -146,27 +147,21 @@ void DBRedis::HMGET(const std::vector<BlockPos> &positions, std::vector<ustring>
 
 		if(!reply)
 			throw std::runtime_error("Redis command HMGET failed");
-		if (reply->type != REDIS_REPLY_ARRAY) {
+		if (reply->type != REDIS_REPLY_ARRAY)
-			freeReplyObject(reply);
+			REPLY_TYPE_ERR(reply, "HMGET reply");
-			REPLY_TYPE_ERR(reply, "HKEYS subreply");
-		}
 		if (reply->elements != batch_size) {
 			freeReplyObject(reply);
 			throw std::runtime_error("HMGET wrong number of elements");
 		}
-		for (std::size_t i = 0; i < batch_size; ++i) {
+		for (std::size_t i = 0; i < reply->elements; ++i) {
 			redisReply *subreply = reply->element[i];
-			if(!subreply)
+			if (subreply->type == REDIS_REPLY_NIL)
-				throw std::runtime_error("Redis command HMGET failed");
+				continue;
-			if (subreply->type != REDIS_REPLY_STRING) {
+			else if (subreply->type != REDIS_REPLY_STRING)
-				freeReplyObject(reply);
+				REPLY_TYPE_ERR(subreply, "HMGET subreply");
-				REPLY_TYPE_ERR(reply, "HKEYS subreply");
+			if (subreply->len == 0)
-			}
-			if (subreply->len == 0) {
-				freeReplyObject(reply);
 				throw std::runtime_error("HMGET empty string");
-			}
+			result(i, ustring((const unsigned char *) subreply->str, subreply->len));
-			result->emplace_back((const unsigned char *) subreply->str, subreply->len);
 		}
 		freeReplyObject(reply);
 		remaining -= batch_size;
@@ -187,10 +182,15 @@ void DBRedis::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 			positions.emplace_back(x, pos2.second, z);
 	}
 
-	std::vector<ustring> db_blocks;
+	getBlocksByPos(blocks, positions);
-	HMGET(positions, &db_blocks);
+}
-
+
-	auto block = db_blocks.cbegin();
+
-	for (auto pos = positions.cbegin(); pos != positions.cend(); ++pos, ++block)
+void DBRedis::getBlocksByPos(BlockList &blocks,
-		blocks.emplace_back(*pos, *block);
+			const std::vector<BlockPos> &positions)
+{
+	auto result = [&] (std::size_t i, ustring data) {
+		blocks.emplace_back(positions[i], std::move(data));
+	};
+	HMGET(positions, result);
 }

db-sqlite3.cpp

@@ -25,6 +25,10 @@ DBSQLite3::DBSQLite3(const std::string &mapdir)
 			"SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ?",
 		-1, &stmt_get_blocks_z, NULL))
 
+	SQLOK(prepare_v2(db,
+			"SELECT data FROM blocks WHERE pos = ?",
+		-1, &stmt_get_block_exact, NULL))
+
 	SQLOK(prepare_v2(db,
 			"SELECT pos FROM blocks",
 		-1, &stmt_get_block_pos, NULL))
@@ -40,6 +44,7 @@ DBSQLite3::~DBSQLite3()
 	sqlite3_finalize(stmt_get_blocks_z);
 	sqlite3_finalize(stmt_get_block_pos);
 	sqlite3_finalize(stmt_get_block_pos_z);
+	sqlite3_finalize(stmt_get_block_exact);
 
 	if (sqlite3_close(db) != SQLITE_OK) {
 		std::cerr << "Error closing SQLite database." << std::endl;
@@ -161,3 +166,31 @@ void DBSQLite3::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 			it++;
 	}
 }
+
+
+void DBSQLite3::getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions)
+{
+	int result;
+
+	for (auto pos : positions) {
+		int64_t dbPos = encodeBlockPos(pos);
+		SQLOK(bind_int64(stmt_get_block_exact, 1, dbPos));
+
+		while ((result = sqlite3_step(stmt_get_block_exact)) == SQLITE_BUSY) {
+			usleep(10000); // Wait some time and try again
+		}
+		if (result == SQLITE_DONE) {
+			// no data
+		} else if (result != SQLITE_ROW) {
+			throw std::runtime_error(sqlite3_errmsg(db));
+		} else {
+			const unsigned char *data = reinterpret_cast<const unsigned char *>(
+					sqlite3_column_blob(stmt_get_block_exact, 0));
+			size_t size = sqlite3_column_bytes(stmt_get_block_exact, 0);
+			blocks.emplace_back(pos, ustring(data, size));
+		}
+
+		SQLOK(reset(stmt_get_block_exact))
+	}
+}

include/TileGenerator.h

@@ -23,6 +23,13 @@ enum {
 	SCALE_RIGHT = (1 << 3),
 };
 
+enum {
+	EXH_NEVER, // Always use range queries
+	EXH_Y,     // Exhaustively search Y space, range queries for X/Z (future TODO)
+	EXH_FULL,  // Exhaustively search entire requested geometry
+	EXH_AUTO,  // Automatically pick one of the previous modes
+};
+
 struct ColorEntry {
 	ColorEntry(): r(0), g(0), b(0), a(0), t(0) {};
 	ColorEntry(uint8_t r, uint8_t g, uint8_t b, uint8_t a, uint8_t t): r(r), g(g), b(b), a(a), t(t) {};
@@ -75,6 +82,7 @@ public:
 	void setGeometry(int x, int y, int w, int h);
 	void setMinY(int y);
 	void setMaxY(int y);
+	void setExhaustiveSearch(int mode);
 	void parseColorsFile(const std::string &fileName);
 	void setBackend(std::string backend);
 	void generate(const std::string &input, const std::string &output);
@@ -135,6 +143,7 @@ private:
 	/* */
 	int m_mapWidth;
 	int m_mapHeight;
+	int m_exhaustiveSearch;
 	std::map<int16_t, std::set<int16_t>> m_positions; /* indexed by Z, contains X coords */
 	ColorMap m_colorMap;
 	BitmapThing m_readPixels;

include/db-leveldb.h

@@ -12,8 +12,12 @@ public:
 	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
 	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 			int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
 	~DBLevelDB() override;
 
+	bool preferRangeQueries() const override { return false; }
+
 private:
 	using pos2d = std::pair<int16_t, int16_t>;
 

include/db-postgresql.h

@@ -10,8 +10,12 @@ public:
 	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
 	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 		int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
 	~DBPostgreSQL() override;
 
+	bool preferRangeQueries() const override { return true; }
+
 protected:
 	PGresult *checkResults(PGresult *res, bool clear = true);
 	void prepareStatement(const std::string &name, const std::string &sql);

include/db-redis.h

@@ -4,6 +4,7 @@
 #include "db.h"
 #include <unordered_map>
 #include <utility>
+#include <functional>
 #include <hiredis/hiredis.h>
 
 class DBRedis : public DB {
@@ -12,14 +13,19 @@ public:
 	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
 	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 		int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
 	~DBRedis() override;
 
+	bool preferRangeQueries() const override { return false; }
+
 private:
 	using pos2d = std::pair<int16_t, int16_t>;
-	static std::string replyTypeStr(int type);
+	static const char *replyTypeStr(int type);
 
 	void loadPosCache();
-	void HMGET(const std::vector<BlockPos> &positions, std::vector<ustring> *result);
+	void HMGET(const std::vector<BlockPos> &positions,
+			std::function<void(std::size_t, ustring)> result);
 
 	// indexed by Z, contains all (x,y) position pairs
 	std::unordered_map<int16_t, std::vector<pos2d>> posCache;

include/db-sqlite3.h

@@ -11,8 +11,12 @@ public:
 	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
 	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 			int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
 	~DBSQLite3() override;
 
+	bool preferRangeQueries() const override { return false; }
+
 private:
 	inline void getPosRange(int64_t &min, int64_t &max, int16_t zPos,
 			int16_t zPos2) const;
@@ -23,6 +27,7 @@ private:
 	sqlite3_stmt *stmt_get_block_pos;
 	sqlite3_stmt *stmt_get_block_pos_z;
 	sqlite3_stmt *stmt_get_blocks_z;
+	sqlite3_stmt *stmt_get_block_exact;
 
 	int16_t blockCachedZ = -10000;
 	std::unordered_map<int16_t, BlockList> blockCache; // indexed by X

include/db.h

@@ -52,11 +52,21 @@ public:
 	 * so that min.x <= x < max.x, ...
 	 */
 	virtual std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) = 0;
-	/* Return all blocks in column given by x and z
+	/* Read all blocks in column given by x and z
-	 * and inside the given Y range (min_y <= y < max_y)
+	 * and inside the given Y range (min_y <= y < max_y) into list
 	 */
 	virtual void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 			int16_t min_y, int16_t max_y) = 0;
+	/* Read blocks at given positions into list
+	 */
+	virtual void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) = 0;
+	/* Can this database efficiently do range queries?
+	 * (for large data sets, more efficient that brute force)
+	 */
+	virtual bool preferRangeQueries() const = 0;
+
+
 	virtual ~DB() {}
 };
 

mapper.cpp

@@ -33,6 +33,7 @@ static void usage()
 			"  --zoom <zoomlevel>\n"
 			"  --colors <colors.txt>\n"
 			"  --scales [t][b][l][r]\n"
+			"  --exhaustive never|y|full|auto\n"
 			"Color format: '#000000'\n";
 	std::cout << usage_text;
 }
@@ -90,6 +91,7 @@ int main(int argc, char *argv[])
 		{"colors", required_argument, 0, 'C'},
 		{"scales", required_argument, 0, 'f'},
 		{"noemptyimage", no_argument, 0, 'n'},
+		{"exhaustive", required_argument, 0, 'j'},
 		{0, 0, 0, 0}
 	};
 
@@ -201,6 +203,19 @@ int main(int argc, char *argv[])
 			case 'n':
 				generator.setDontWriteEmpty(true);
 				break;
+			case 'j': {
+					int mode;
+					if (!strcmp(optarg, "never"))
+						mode = EXH_NEVER;
+					else if (!strcmp(optarg, "y"))
+						mode = EXH_Y;
+					else if (!strcmp(optarg, "full"))
+						mode = EXH_FULL;
+					else
+						mode = EXH_AUTO;
+					generator.setExhaustiveSearch(mode);
+				}
+				break;
 			default:
 				exit(1);
 		}

minetestmapper.6

@@ -70,11 +70,11 @@ Don't draw nodes above this y value, e.g. "--max-y 75"
 
 .TP
 .BR \-\-backend " " \fIbackend\fR
-Use specific map backend; supported: *sqlite3*, *leveldb*, *redis*, *postgresql*, e.g. "--backend leveldb"
+Use specific map backend; supported: \fIsqlite3\fP, \fIleveldb\fP, \fIredis\fP, \fIpostgresql\fP, e.g. "--backend leveldb"
 
 .TP
 .BR \-\-geometry " " \fIgeometry\fR
-Limit area to specific geometry (*x:y+w+h* where x and y specify the lower left corner), e.g. "--geometry -800:-800+1600+1600"
+Limit area to specific geometry (\fIx:y+w+h\fP where x and y specify the lower left corner), e.g. "--geometry -800:-800+1600+1600"
 
 .TP
 .BR \-\-extent " " \fIextent\fR
@@ -90,7 +90,13 @@ Forcefully set path to colors.txt file (it's autodetected otherwise), e.g. "--co
 
 .TP
 .BR \-\-scales " " \fIedges\fR
-Draw scales on specified image edges (letters *t b l r* meaning top, bottom, left and right), e.g. "--scales tbr"
+Draw scales on specified image edges (letters \fIt b l r\fP meaning top, bottom, left and right), e.g. "--scales tbr"
+
+.TP
+.BR \-\-exhaustive " \fImode\fR
+Select if database should be traversed exhaustively or using range queries, available: \fInever\fP, \fIy\fP, \fIfull\fP, \fIauto\fP
+
+Defaults to \fIauto\fP. You shouldn't need to change this, but doing so can improve rendering times on large maps.
 
 .SH MORE INFORMATION
 Website: https://github.com/minetest/minetestmapper