Significantly optimize LevelDB database backend

This commit is contained in:
ShadowNinja 2014-03-28 16:47:19 -04:00
parent 6e565e93d1
commit 5905c34ec0
6 changed files with 61 additions and 49 deletions

View File

@ -17,6 +17,7 @@
#include <stdexcept>
#include <cerrno>
#include <cstring>
#include <vector>
#include "config.h"
#include "PlayerAttributes.h"
#include "TileGenerator.h"
@ -308,17 +309,17 @@ void TileGenerator::openDb(const std::string &input)
void TileGenerator::loadBlocks()
{
std::vector<int64_t> vec = m_db->getBlockPos();
for(unsigned int i = 0; i < vec.size(); i++) {
BlockPos pos = decodeBlockPos(vec[i]);
for(std::vector<int64_t>::iterator it = vec.begin(); it != vec.end(); ++it) {
BlockPos pos = decodeBlockPos(*it);
// Check that it's in geometry (from --geometry option)
if (pos.x < m_geomX || pos.x > m_geomX2 || pos.z < m_geomY || pos.z > m_geomY2) {
continue;
}
if (pos.y < m_yMin * 16) {
continue;
}
if (pos.y > m_yMax * 16) {
// Check that it's between --miny and --maxy
if (pos.y < m_yMin * 16 || pos.y > m_yMax * 16) {
continue;
}
// Adjust minimum and maximum positions to the nearest block
if (pos.x < m_xMin) {
m_xMin = pos.x;
}

View File

@ -17,46 +17,56 @@ inline std::string i64tos(int64_t i) {
DBLevelDB::DBLevelDB(const std::string &mapdir) {
leveldb::Options options;
posCacheLoaded = false;
options.create_if_missing = false;
leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &m_db);
leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &db);
if(!status.ok())
throw std::runtime_error("Failed to open Database");
}
DBLevelDB::~DBLevelDB() {
delete m_db;
delete db;
}
std::vector<int64_t> DBLevelDB::getBlockPos() {
std::vector<int64_t> vec;
std::set<int64_t> s;
leveldb::Iterator* it = m_db->NewIterator(leveldb::ReadOptions());
for (it->SeekToFirst(); it->Valid(); it->Next()) {
vec.push_back(stoi64(it->key().ToString()));
s.insert(stoi64(it->key().ToString()));
}
delete it;
m_bpcache = s;
return vec;
loadPosCache();
return posCache;
}
DBBlockList DBLevelDB::getBlocksOnZ(int zPos)
{
void DBLevelDB::loadPosCache() {
if (posCacheLoaded) {
return;
}
leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions());
for (it->SeekToFirst(); it->Valid(); it->Next()) {
posCache.push_back(stoi64(it->key().ToString()));
}
delete it;
posCacheLoaded = true;
}
DBBlockList DBLevelDB::getBlocksOnZ(int zPos) {
DBBlockList blocks;
std::string datastr;
leveldb::Status status;
int64_t psMin;
int64_t psMax;
psMin = (zPos * 16777216l) - 0x800000;
psMax = (zPos * 16777216l) + 0x7fffff;
int64_t psMin = (zPos * 16777216L) - 0x800000;
int64_t psMax = (zPos * 16777216L) + 0x7fffff;
for(int64_t i = psMin; i <= psMax; i++) { // FIXME: This is still very very inefficent (even with m_bpcache)
if(m_bpcache.find(i) == m_bpcache.end())
for (std::vector<int64_t>::iterator it = posCache.begin(); it != posCache.end(); ++it) {
int64_t i = *it;
if (i < psMin || i > psMax) {
continue;
status = m_db->Get(leveldb::ReadOptions(), i64tos(i), &datastr);
if(status.ok())
blocks.push_back( DBBlock( i, std::basic_string<unsigned char>( (const unsigned char*) datastr.c_str(), datastr.size() ) ) );
}
status = db->Get(leveldb::ReadOptions(), i64tos(i), &datastr);
if (status.ok()) {
blocks.push_back(
DBBlock(i,
std::basic_string<unsigned char>((const unsigned char*) datastr.data(), datastr.size())
)
);
}
}
return blocks;

View File

@ -3,7 +3,6 @@
#include "db.h"
#include <leveldb/db.h>
#include <set>
class DBLevelDB : public DB {
public:
@ -12,8 +11,12 @@ public:
virtual DBBlockList getBlocksOnZ(int zPos);
~DBLevelDB();
private:
leveldb::DB *m_db;
std::set<int64_t> m_bpcache;
void loadPosCache();
leveldb::DB *db;
bool posCacheLoaded;
std::vector<int64_t> posCache;
};
#endif // _DB_LEVELDB_H

View File

@ -5,7 +5,7 @@
#define SQLRES(f, good) \
result = (sqlite3_##f);\
if (result != good) {\
throw std::runtime_error(sqlite3_errmsg(m_db));\
throw std::runtime_error(sqlite3_errmsg(db));\
}
#define SQLOK(f) SQLRES(f, SQLITE_OK)
@ -13,13 +13,13 @@ DBSQLite3::DBSQLite3(const std::string &mapdir) {
int result;
std::string db_name = mapdir + "map.sqlite";
SQLOK(open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0))
SQLOK(open_v2(db_name.c_str(), &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0))
SQLOK(prepare_v2(m_db,
SQLOK(prepare_v2(db,
"SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)",
-1, &stmt_get_blocks, NULL))
SQLOK(prepare_v2(m_db,
SQLOK(prepare_v2(db,
"SELECT pos FROM blocks",
-1, &stmt_get_block_pos, NULL))
}
@ -29,12 +29,12 @@ DBSQLite3::~DBSQLite3() {
SQLOK(finalize(stmt_get_blocks));
SQLOK(finalize(stmt_get_block_pos));
SQLOK(close(m_db));
SQLOK(close(db));
}
std::vector<int64_t> DBSQLite3::getBlockPos() {
std::vector<int64_t> vec;
int result = 0;
int result;
while ((result = sqlite3_step(stmt_get_block_pos)) != SQLITE_DONE) {
if (result == SQLITE_ROW) {
int64_t blockpos = sqlite3_column_int64(stmt_get_block_pos, 0);
@ -42,9 +42,10 @@ std::vector<int64_t> DBSQLite3::getBlockPos() {
} else if (result == SQLITE_BUSY) { // Wait some time and try again
usleep(10000);
} else {
throw std::runtime_error(sqlite3_errmsg(m_db));
throw std::runtime_error(sqlite3_errmsg(db));
}
}
SQLOK(reset(stmt_get_block_pos));
return vec;
}
@ -52,15 +53,13 @@ DBBlockList DBSQLite3::getBlocksOnZ(int zPos)
{
DBBlockList blocks;
sqlite3_int64 psMin;
sqlite3_int64 psMax;
int64_t psMin = (static_cast<sqlite3_int64>(zPos) * 16777216L) - 0x800000;
int64_t psMax = (static_cast<sqlite3_int64>(zPos) * 16777216L) + 0x7fffff;
psMin = (static_cast<sqlite3_int64>(zPos) * 16777216L) - 0x800000;
psMax = (static_cast<sqlite3_int64>(zPos) * 16777216L) + 0x7fffff;
sqlite3_bind_int64(stmt_get_blocks, 1, psMin);
sqlite3_bind_int64(stmt_get_blocks, 2, psMax);
int result = 0;
int result;
while ((result = sqlite3_step(stmt_get_blocks)) != SQLITE_DONE) {
if (result == SQLITE_ROW) {
int64_t blocknum = sqlite3_column_int64(stmt_get_blocks, 0);
@ -70,7 +69,7 @@ DBBlockList DBSQLite3::getBlocksOnZ(int zPos)
} else if (result == SQLITE_BUSY) { // Wait some time and try again
usleep(10000);
} else {
throw std::runtime_error(sqlite3_errmsg(m_db));
throw std::runtime_error(sqlite3_errmsg(db));
}
}
SQLOK(reset(stmt_get_blocks));

View File

@ -11,7 +11,7 @@ public:
virtual DBBlockList getBlocksOnZ(int zPos);
~DBSQLite3();
private:
sqlite3 *m_db;
sqlite3 *db;
sqlite3_stmt *stmt_get_block_pos;
sqlite3_stmt *stmt_get_blocks;

1
db.h
View File

@ -7,7 +7,6 @@
#include <string>
#include <utility>
// we cannot use ... char>> here because mingw-gcc is f**king retarded (caring about whitespace and shit)
typedef std::pair<int64_t, std::basic_string<unsigned char> > DBBlock;
typedef std::list<DBBlock> DBBlockList;