From 5905c34ec083d23df8fda928c0d4865d81d53770 Mon Sep 17 00:00:00 2001 From: ShadowNinja Date: Fri, 28 Mar 2014 16:47:19 -0400 Subject: [PATCH] Significantly optimize LevelDB database backend --- TileGenerator.cpp | 13 ++++++----- db-leveldb.cpp | 56 ++++++++++++++++++++++++++++------------------- db-leveldb.h | 9 +++++--- db-sqlite3.cpp | 25 ++++++++++----------- db-sqlite3.h | 2 +- db.h | 5 ++--- 6 files changed, 61 insertions(+), 49 deletions(-) diff --git a/TileGenerator.cpp b/TileGenerator.cpp index 7167a34..45327ab 100644 --- a/TileGenerator.cpp +++ b/TileGenerator.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include "config.h" #include "PlayerAttributes.h" #include "TileGenerator.h" @@ -308,17 +309,17 @@ void TileGenerator::openDb(const std::string &input) void TileGenerator::loadBlocks() { std::vector vec = m_db->getBlockPos(); - for(unsigned int i = 0; i < vec.size(); i++) { - BlockPos pos = decodeBlockPos(vec[i]); + for(std::vector::iterator it = vec.begin(); it != vec.end(); ++it) { + BlockPos pos = decodeBlockPos(*it); + // Check that it's in geometry (from --geometry option) if (pos.x < m_geomX || pos.x > m_geomX2 || pos.z < m_geomY || pos.z > m_geomY2) { continue; } - if (pos.y < m_yMin * 16) { - continue; - } - if (pos.y > m_yMax * 16) { + // Check that it's between --miny and --maxy + if (pos.y < m_yMin * 16 || pos.y > m_yMax * 16) { continue; } + // Adjust minimum and maximum positions to the nearest block if (pos.x < m_xMin) { m_xMin = pos.x; } diff --git a/db-leveldb.cpp b/db-leveldb.cpp index 5e98233..18d105c 100644 --- a/db-leveldb.cpp +++ b/db-leveldb.cpp @@ -17,46 +17,56 @@ inline std::string i64tos(int64_t i) { DBLevelDB::DBLevelDB(const std::string &mapdir) { leveldb::Options options; + posCacheLoaded = false; options.create_if_missing = false; - leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &m_db); + leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &db); if(!status.ok()) throw std::runtime_error("Failed to open Database"); } DBLevelDB::~DBLevelDB() { - delete m_db; + delete db; } std::vector DBLevelDB::getBlockPos() { - std::vector vec; - std::set s; - leveldb::Iterator* it = m_db->NewIterator(leveldb::ReadOptions()); - for (it->SeekToFirst(); it->Valid(); it->Next()) { - vec.push_back(stoi64(it->key().ToString())); - s.insert(stoi64(it->key().ToString())); - } - delete it; - m_bpcache = s; - return vec; + loadPosCache(); + return posCache; } -DBBlockList DBLevelDB::getBlocksOnZ(int zPos) -{ +void DBLevelDB::loadPosCache() { + if (posCacheLoaded) { + return; + } + + leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions()); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + posCache.push_back(stoi64(it->key().ToString())); + } + delete it; + posCacheLoaded = true; +} + +DBBlockList DBLevelDB::getBlocksOnZ(int zPos) { DBBlockList blocks; std::string datastr; leveldb::Status status; - int64_t psMin; - int64_t psMax; - psMin = (zPos * 16777216l) - 0x800000; - psMax = (zPos * 16777216l) + 0x7fffff; + int64_t psMin = (zPos * 16777216L) - 0x800000; + int64_t psMax = (zPos * 16777216L) + 0x7fffff; - for(int64_t i = psMin; i <= psMax; i++) { // FIXME: This is still very very inefficent (even with m_bpcache) - if(m_bpcache.find(i) == m_bpcache.end()) + for (std::vector::iterator it = posCache.begin(); it != posCache.end(); ++it) { + int64_t i = *it; + if (i < psMin || i > psMax) { continue; - status = m_db->Get(leveldb::ReadOptions(), i64tos(i), &datastr); - if(status.ok()) - blocks.push_back( DBBlock( i, std::basic_string( (const unsigned char*) datastr.c_str(), datastr.size() ) ) ); + } + status = db->Get(leveldb::ReadOptions(), i64tos(i), &datastr); + if (status.ok()) { + blocks.push_back( + DBBlock(i, + std::basic_string((const unsigned char*) datastr.data(), datastr.size()) + ) + ); + } } return blocks; diff --git a/db-leveldb.h b/db-leveldb.h index 7b9f97e..bf0002f 100644 --- a/db-leveldb.h +++ b/db-leveldb.h @@ -3,7 +3,6 @@ #include "db.h" #include -#include class DBLevelDB : public DB { public: @@ -12,8 +11,12 @@ public: virtual DBBlockList getBlocksOnZ(int zPos); ~DBLevelDB(); private: - leveldb::DB *m_db; - std::set m_bpcache; + void loadPosCache(); + + leveldb::DB *db; + + bool posCacheLoaded; + std::vector posCache; }; #endif // _DB_LEVELDB_H diff --git a/db-sqlite3.cpp b/db-sqlite3.cpp index ad7460e..966f450 100644 --- a/db-sqlite3.cpp +++ b/db-sqlite3.cpp @@ -5,7 +5,7 @@ #define SQLRES(f, good) \ result = (sqlite3_##f);\ if (result != good) {\ - throw std::runtime_error(sqlite3_errmsg(m_db));\ + throw std::runtime_error(sqlite3_errmsg(db));\ } #define SQLOK(f) SQLRES(f, SQLITE_OK) @@ -13,13 +13,13 @@ DBSQLite3::DBSQLite3(const std::string &mapdir) { int result; std::string db_name = mapdir + "map.sqlite"; - SQLOK(open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0)) + SQLOK(open_v2(db_name.c_str(), &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0)) - SQLOK(prepare_v2(m_db, + SQLOK(prepare_v2(db, "SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)", -1, &stmt_get_blocks, NULL)) - SQLOK(prepare_v2(m_db, + SQLOK(prepare_v2(db, "SELECT pos FROM blocks", -1, &stmt_get_block_pos, NULL)) } @@ -29,12 +29,12 @@ DBSQLite3::~DBSQLite3() { SQLOK(finalize(stmt_get_blocks)); SQLOK(finalize(stmt_get_block_pos)); - SQLOK(close(m_db)); + SQLOK(close(db)); } std::vector DBSQLite3::getBlockPos() { std::vector vec; - int result = 0; + int result; while ((result = sqlite3_step(stmt_get_block_pos)) != SQLITE_DONE) { if (result == SQLITE_ROW) { int64_t blockpos = sqlite3_column_int64(stmt_get_block_pos, 0); @@ -42,9 +42,10 @@ std::vector DBSQLite3::getBlockPos() { } else if (result == SQLITE_BUSY) { // Wait some time and try again usleep(10000); } else { - throw std::runtime_error(sqlite3_errmsg(m_db)); + throw std::runtime_error(sqlite3_errmsg(db)); } } + SQLOK(reset(stmt_get_block_pos)); return vec; } @@ -52,15 +53,13 @@ DBBlockList DBSQLite3::getBlocksOnZ(int zPos) { DBBlockList blocks; - sqlite3_int64 psMin; - sqlite3_int64 psMax; + int64_t psMin = (static_cast(zPos) * 16777216L) - 0x800000; + int64_t psMax = (static_cast(zPos) * 16777216L) + 0x7fffff; - psMin = (static_cast(zPos) * 16777216L) - 0x800000; - psMax = (static_cast(zPos) * 16777216L) + 0x7fffff; sqlite3_bind_int64(stmt_get_blocks, 1, psMin); sqlite3_bind_int64(stmt_get_blocks, 2, psMax); - int result = 0; + int result; while ((result = sqlite3_step(stmt_get_blocks)) != SQLITE_DONE) { if (result == SQLITE_ROW) { int64_t blocknum = sqlite3_column_int64(stmt_get_blocks, 0); @@ -70,7 +69,7 @@ DBBlockList DBSQLite3::getBlocksOnZ(int zPos) } else if (result == SQLITE_BUSY) { // Wait some time and try again usleep(10000); } else { - throw std::runtime_error(sqlite3_errmsg(m_db)); + throw std::runtime_error(sqlite3_errmsg(db)); } } SQLOK(reset(stmt_get_blocks)); diff --git a/db-sqlite3.h b/db-sqlite3.h index f3b355b..38b447a 100644 --- a/db-sqlite3.h +++ b/db-sqlite3.h @@ -11,7 +11,7 @@ public: virtual DBBlockList getBlocksOnZ(int zPos); ~DBSQLite3(); private: - sqlite3 *m_db; + sqlite3 *db; sqlite3_stmt *stmt_get_block_pos; sqlite3_stmt *stmt_get_blocks; diff --git a/db.h b/db.h index 505c743..329e409 100644 --- a/db.h +++ b/db.h @@ -7,14 +7,13 @@ #include #include -// we cannot use ... char>> here because mingw-gcc is f**king retarded (caring about whitespace and shit) typedef std::pair > DBBlock; typedef std::list DBBlockList; class DB { public: - virtual std::vector getBlockPos()=0; - virtual DBBlockList getBlocksOnZ(int zPos)=0; + virtual std::vector getBlockPos() = 0; + virtual DBBlockList getBlocksOnZ(int zPos) = 0; }; #endif // _DB_H