Significantly optimize LevelDB database backend

TileGenerator.cpp

@@ -17,6 +17,7 @@
 #include <stdexcept>
 #include <cerrno>
 #include <cstring>
+#include <vector>
 #include "config.h"
 #include "PlayerAttributes.h"
 #include "TileGenerator.h"
@@ -308,17 +309,17 @@ void TileGenerator::openDb(const std::string &input)
 void TileGenerator::loadBlocks()
 {
 	std::vector<int64_t> vec = m_db->getBlockPos();
-	for(unsigned int i = 0; i < vec.size(); i++) {
-		BlockPos pos = decodeBlockPos(vec[i]);
+	for(std::vector<int64_t>::iterator it = vec.begin(); it != vec.end(); ++it) {
+		BlockPos pos = decodeBlockPos(*it);
+		// Check that it's in geometry (from --geometry option)
 		if (pos.x < m_geomX || pos.x > m_geomX2 || pos.z < m_geomY || pos.z > m_geomY2) {
 			continue;
 		}
-		if (pos.y < m_yMin * 16) {
-			continue;
-		}
-		if (pos.y > m_yMax * 16) {
+		// Check that it's between --miny and --maxy
+		if (pos.y < m_yMin * 16 || pos.y > m_yMax * 16) {
 			continue;
 		}
+		// Adjust minimum and maximum positions to the nearest block
 		if (pos.x < m_xMin) {
 			m_xMin = pos.x;
 		}

db-leveldb.cpp

@@ -17,46 +17,56 @@ inline std::string i64tos(int64_t i) {
 
 DBLevelDB::DBLevelDB(const std::string &mapdir) {
 	leveldb::Options options;
+	posCacheLoaded = false;
 	options.create_if_missing = false;
-	leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &m_db);
+	leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &db);
 	if(!status.ok())
 		throw std::runtime_error("Failed to open Database");
 }
 
 DBLevelDB::~DBLevelDB() {
-	delete m_db;
+	delete db;
 }
 
 std::vector<int64_t> DBLevelDB::getBlockPos() {
-	std::vector<int64_t> vec;
-	std::set<int64_t> s;
-	leveldb::Iterator* it = m_db->NewIterator(leveldb::ReadOptions());
-	for (it->SeekToFirst(); it->Valid(); it->Next()) {
-		vec.push_back(stoi64(it->key().ToString()));
-		s.insert(stoi64(it->key().ToString()));
-	}
-	delete it;
-	m_bpcache = s;
-	return vec;
+	loadPosCache();
+	return posCache;
 }
 
-DBBlockList DBLevelDB::getBlocksOnZ(int zPos)
-{
+void DBLevelDB::loadPosCache() {
+	if (posCacheLoaded) {
+		return;
+	}
+
+	leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions());
+	for (it->SeekToFirst(); it->Valid(); it->Next()) {
+		posCache.push_back(stoi64(it->key().ToString()));
+	}
+	delete it;
+	posCacheLoaded = true;
+}
+
+DBBlockList DBLevelDB::getBlocksOnZ(int zPos) {
 	DBBlockList blocks;
 	std::string datastr;
 	leveldb::Status status;
 
-	int64_t psMin;
-	int64_t psMax;
-	psMin = (zPos * 16777216l) - 0x800000;
-	psMax = (zPos * 16777216l) + 0x7fffff;
+	int64_t psMin = (zPos * 16777216L) - 0x800000;
+	int64_t psMax = (zPos * 16777216L) + 0x7fffff;
 
-	for(int64_t i = psMin; i <= psMax; i++) { // FIXME: This is still very very inefficent (even with m_bpcache)
-		if(m_bpcache.find(i) == m_bpcache.end())
+	for (std::vector<int64_t>::iterator it = posCache.begin(); it != posCache.end(); ++it) {
+		int64_t i = *it;
+		if (i < psMin || i > psMax) {
 			continue;
-		status = m_db->Get(leveldb::ReadOptions(), i64tos(i), &datastr);
-		if(status.ok())
-			blocks.push_back( DBBlock( i, std::basic_string<unsigned char>( (const unsigned char*) datastr.c_str(), datastr.size() ) ) );
+		}
+		status = db->Get(leveldb::ReadOptions(), i64tos(i), &datastr);
+		if (status.ok()) {
+			blocks.push_back(
+				DBBlock(i,
+					std::basic_string<unsigned char>((const unsigned char*) datastr.data(), datastr.size())
+				)
+			);
+		}
 	}
 
 	return blocks;

db-leveldb.h

@@ -3,7 +3,6 @@
 
 #include "db.h"
 #include <leveldb/db.h>
-#include <set>
 
 class DBLevelDB : public DB {
 public:
@@ -12,8 +11,12 @@ public:
 	virtual DBBlockList getBlocksOnZ(int zPos);
 	~DBLevelDB();
 private:
-	leveldb::DB *m_db;
-	std::set<int64_t> m_bpcache;
+	void loadPosCache();
+
+	leveldb::DB *db;
+
+	bool posCacheLoaded;
+	std::vector<int64_t> posCache;
 };
 
 #endif // _DB_LEVELDB_H

db-sqlite3.cpp

@@ -5,7 +5,7 @@
 #define SQLRES(f, good) \
 	result = (sqlite3_##f);\
 	if (result != good) {\
-		throw std::runtime_error(sqlite3_errmsg(m_db));\
+		throw std::runtime_error(sqlite3_errmsg(db));\
 	}
 #define SQLOK(f) SQLRES(f, SQLITE_OK)
 
@@ -13,13 +13,13 @@ DBSQLite3::DBSQLite3(const std::string &mapdir) {
 	int result;
 	std::string db_name = mapdir + "map.sqlite";
 
-	SQLOK(open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0))
+	SQLOK(open_v2(db_name.c_str(), &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0))
 
-	SQLOK(prepare_v2(m_db,
+	SQLOK(prepare_v2(db,
 			"SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)",
 		-1, &stmt_get_blocks, NULL))
 
-	SQLOK(prepare_v2(m_db,
+	SQLOK(prepare_v2(db,
 			"SELECT pos FROM blocks",
 		-1, &stmt_get_block_pos, NULL))
 }
@@ -29,12 +29,12 @@ DBSQLite3::~DBSQLite3() {
 	SQLOK(finalize(stmt_get_blocks));
 	SQLOK(finalize(stmt_get_block_pos));
 
-	SQLOK(close(m_db));
+	SQLOK(close(db));
 }
 
 std::vector<int64_t> DBSQLite3::getBlockPos() {
 	std::vector<int64_t> vec;
-	int result = 0;
+	int result;
 	while ((result = sqlite3_step(stmt_get_block_pos)) != SQLITE_DONE) {
 		if (result == SQLITE_ROW) {
 			int64_t blockpos = sqlite3_column_int64(stmt_get_block_pos, 0);
@@ -42,9 +42,10 @@ std::vector<int64_t> DBSQLite3::getBlockPos() {
 		} else if (result == SQLITE_BUSY) { // Wait some time and try again
 			usleep(10000);
 		} else {
-			throw std::runtime_error(sqlite3_errmsg(m_db));
+			throw std::runtime_error(sqlite3_errmsg(db));
 		}
 	}
+	SQLOK(reset(stmt_get_block_pos));
 	return vec;
 }
 
@@ -52,15 +53,13 @@ DBBlockList DBSQLite3::getBlocksOnZ(int zPos)
 {
 	DBBlockList blocks;
 
-	sqlite3_int64 psMin;
-	sqlite3_int64 psMax;
+	int64_t psMin = (static_cast<sqlite3_int64>(zPos) * 16777216L) - 0x800000;
+	int64_t psMax = (static_cast<sqlite3_int64>(zPos) * 16777216L) + 0x7fffff;
 
-	psMin = (static_cast<sqlite3_int64>(zPos) * 16777216L) - 0x800000;
-	psMax = (static_cast<sqlite3_int64>(zPos) * 16777216L) + 0x7fffff;
 	sqlite3_bind_int64(stmt_get_blocks, 1, psMin);
 	sqlite3_bind_int64(stmt_get_blocks, 2, psMax);
 
-	int result = 0;
+	int result;
 	while ((result = sqlite3_step(stmt_get_blocks)) != SQLITE_DONE) {
 		if (result == SQLITE_ROW) {
 			int64_t blocknum = sqlite3_column_int64(stmt_get_blocks, 0);
@@ -70,7 +69,7 @@ DBBlockList DBSQLite3::getBlocksOnZ(int zPos)
 		} else if (result == SQLITE_BUSY) { // Wait some time and try again
 			usleep(10000);
 		} else {
-			throw std::runtime_error(sqlite3_errmsg(m_db));
+			throw std::runtime_error(sqlite3_errmsg(db));
 		}
 	}
 	SQLOK(reset(stmt_get_blocks));

db-sqlite3.h

@@ -11,7 +11,7 @@ public:
 	virtual DBBlockList getBlocksOnZ(int zPos);
 	~DBSQLite3();
 private:
-	sqlite3 *m_db;
+	sqlite3 *db;
 
 	sqlite3_stmt *stmt_get_block_pos;
 	sqlite3_stmt *stmt_get_blocks;

db.h

@@ -7,14 +7,13 @@
 #include <string>
 #include <utility>
 
-// we cannot use ... char>> here because mingw-gcc is f**king retarded (caring about whitespace and shit)
 typedef std::pair<int64_t, std::basic_string<unsigned char> > DBBlock;
 typedef std::list<DBBlock> DBBlockList;
 
 class DB {
 public:
-	virtual std::vector<int64_t> getBlockPos()=0;
-	virtual DBBlockList getBlocksOnZ(int zPos)=0;
+	virtual std::vector<int64_t> getBlockPos() = 0;
+	virtual DBBlockList getBlocksOnZ(int zPos) = 0;
 };
 
 #endif // _DB_H