Optimize database access further by allowing "brute-force" queries instead of listing available blocks

Also adds a heuristic that will enable this behaviour automatically.

include/TileGenerator.h

@@ -23,6 +23,13 @@ enum {
 	SCALE_RIGHT = (1 << 3),
 };
 
+enum {
+	EXH_NEVER, // Always use range queries
+	EXH_Y,     // Exhaustively search Y space, range queries for X/Z (future TODO)
+	EXH_FULL,  // Exhaustively search entire requested geometry
+	EXH_AUTO,  // Automatically pick one of the previous modes
+};
+
 struct ColorEntry {
 	ColorEntry(): r(0), g(0), b(0), a(0), t(0) {};
 	ColorEntry(uint8_t r, uint8_t g, uint8_t b, uint8_t a, uint8_t t): r(r), g(g), b(b), a(a), t(t) {};
@@ -75,6 +82,7 @@ public:
 	void setGeometry(int x, int y, int w, int h);
 	void setMinY(int y);
 	void setMaxY(int y);
+	void setExhaustiveSearch(int mode);
 	void parseColorsFile(const std::string &fileName);
 	void setBackend(std::string backend);
 	void generate(const std::string &input, const std::string &output);
@@ -135,6 +143,7 @@ private:
 	/* */
 	int m_mapWidth;
 	int m_mapHeight;
+	int m_exhaustiveSearch;
 	std::map<int16_t, std::set<int16_t>> m_positions; /* indexed by Z, contains X coords */
 	ColorMap m_colorMap;
 	BitmapThing m_readPixels;

include/db-leveldb.h

@@ -12,8 +12,12 @@ public:
 	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
 	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 			int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
 	~DBLevelDB() override;
 
+	bool preferRangeQueries() const override { return false; }
+
 private:
 	using pos2d = std::pair<int16_t, int16_t>;
 

include/db-postgresql.h

@@ -10,8 +10,12 @@ public:
 	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
 	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 		int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
 	~DBPostgreSQL() override;
 
+	bool preferRangeQueries() const override { return true; }
+
 protected:
 	PGresult *checkResults(PGresult *res, bool clear = true);
 	void prepareStatement(const std::string &name, const std::string &sql);

include/db-redis.h

@@ -4,6 +4,7 @@
 #include "db.h"
 #include <unordered_map>
 #include <utility>
+#include <functional>
 #include <hiredis/hiredis.h>
 
 class DBRedis : public DB {
@@ -12,14 +13,19 @@ public:
 	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
 	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 		int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
 	~DBRedis() override;
 
+	bool preferRangeQueries() const override { return false; }
+
 private:
 	using pos2d = std::pair<int16_t, int16_t>;
-	static std::string replyTypeStr(int type);
+	static const char *replyTypeStr(int type);
 
 	void loadPosCache();
-	void HMGET(const std::vector<BlockPos> &positions, std::vector<ustring> *result);
+	void HMGET(const std::vector<BlockPos> &positions,
+			std::function<void(std::size_t, ustring)> result);
 
 	// indexed by Z, contains all (x,y) position pairs
 	std::unordered_map<int16_t, std::vector<pos2d>> posCache;

include/db-sqlite3.h

@@ -11,8 +11,12 @@ public:
 	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
 	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 			int16_t min_y, int16_t max_y) override;
+	void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) override;
 	~DBSQLite3() override;
 
+	bool preferRangeQueries() const override { return false; }
+
 private:
 	inline void getPosRange(int64_t &min, int64_t &max, int16_t zPos,
 			int16_t zPos2) const;
@@ -23,6 +27,7 @@ private:
 	sqlite3_stmt *stmt_get_block_pos;
 	sqlite3_stmt *stmt_get_block_pos_z;
 	sqlite3_stmt *stmt_get_blocks_z;
+	sqlite3_stmt *stmt_get_block_exact;
 
 	int16_t blockCachedZ = -10000;
 	std::unordered_map<int16_t, BlockList> blockCache; // indexed by X

include/db.h

@@ -52,11 +52,21 @@ public:
 	 * so that min.x <= x < max.x, ...
 	 */
 	virtual std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) = 0;
-	/* Return all blocks in column given by x and z
-	 * and inside the given Y range (min_y <= y < max_y)
+	/* Read all blocks in column given by x and z
+	 * and inside the given Y range (min_y <= y < max_y) into list
 	 */
 	virtual void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
 			int16_t min_y, int16_t max_y) = 0;
+	/* Read blocks at given positions into list
+	 */
+	virtual void getBlocksByPos(BlockList &blocks,
+			const std::vector<BlockPos> &positions) = 0;
+	/* Can this database efficiently do range queries?
+	 * (for large data sets, more efficient that brute force)
+	 */
+	virtual bool preferRangeQueries() const = 0;
+
+
 	virtual ~DB() {}
 };