From 393d7e2ac066b02d89397a40c6ed7431756355e5 Mon Sep 17 00:00:00 2001 From: Sfan5 Date: Wed, 5 Mar 2014 21:41:27 +0100 Subject: [PATCH] LevelDB support --- .gitignore | 2 +- CMakeLists.txt | 35 +++++++++-- README.rst | 3 + TileGenerator.cpp | 147 +++++++++++++++++++--------------------------- TileGenerator.h | 10 ++-- cmake_config.h.in | 9 +++ config.h | 6 ++ db-leveldb.cpp | 64 ++++++++++++++++++++ db-leveldb.h | 19 ++++++ db-sqlite3.cpp | 71 ++++++++++++++++++++++ db-sqlite3.h | 17 ++++++ db.h | 18 ++++++ mapper.cpp | 7 +++ 13 files changed, 310 insertions(+), 98 deletions(-) create mode 100644 cmake_config.h.in create mode 100644 db-leveldb.cpp create mode 100644 db-leveldb.h create mode 100644 db-sqlite3.cpp create mode 100644 db-sqlite3.h create mode 100644 db.h diff --git a/.gitignore b/.gitignore index 7e43bc5..4807413 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,5 @@ CMakeCache.txt CMakeFiles/ Makefile cmake_install.cmake - +cmake_config.h *~ diff --git a/CMakeLists.txt b/CMakeLists.txt index b5b6e08..a861bbc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,25 @@ else (LIBSQLITE3_INCLUDE_DIR) message(FATAL_ERROR "Could not find sqlite3") endif (LIBSQLITE3_INCLUDE_DIR) +set(USE_LEVELDB 0) + +OPTION(ENABLE_LEVELDB "Enable LevelDB backend") + +if(ENABLE_LEVELDB) + find_library(LEVELDB_LIBRARY leveldb) + find_path(LEVELDB_INCLUDE_DIR db.h PATH_SUFFIXES leveldb) + message (STATUS "LevelDB library: ${LEVELDB_LIBRARY}") + message (STATUS "LevelDB headers: ${LEVELDB_INCLUDE_DIR}") + if(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR) + set(USE_LEVELDB 1) + message(STATUS "LevelDB backend enabled") + include_directories(${LEVELDB_INCLUDE_DIR}) + else(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR) + set(USE_LEVELDB 0) + message(STATUS "LevelDB not found!") + endif(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR) +endif(ENABLE_LEVELDB) + find_program(XXD_EXECUTABLE xxd) if (XXD_EXECUTABLE) @@ -51,12 +70,11 @@ include_directories( ${LIBSQLITE3_INCLUDE_DIRS} ) -set(mapper_HDRS - PixelAttributes.h - PlayerAttributes.h - TileGenerator.h - ZlibDecompressor.h +configure_file( + "${PROJECT_SOURCE_DIR}/cmake_config.h.in" + "${PROJECT_BINARY_DIR}/cmake_config.h" ) +add_definitions ( -DUSE_CMAKE_CONFIG_H ) set(mapper_SRCS PixelAttributes.cpp @@ -64,16 +82,21 @@ set(mapper_SRCS TileGenerator.cpp ZlibDecompressor.cpp mapper.cpp + db-sqlite3.cpp ) +if(USE_LEVELDB) + set(mapper_SRCS ${mapper_SRCS} db-leveldb.cpp) +endif(USE_LEVELDB) + add_executable(minetestmapper - ${mapper_HDRS} ${mapper_SRCS} ) target_link_libraries( minetestmapper ${LIBSQLITE3_LIBRARIES} + ${LEVELDB_LIBRARY} gd z ) diff --git a/README.rst b/README.rst index dff4a4a..e9d0e64 100644 --- a/README.rst +++ b/README.rst @@ -62,6 +62,9 @@ min-y: max-y: Don't draw nodes above this y value, `--max-y 75` +backend: + Use specific map backend, supported: sqlite3, leveldb, `--backend leveldb` + geometry: Limit area to specific geometry, `--geometry -800:-800+1600+1600` diff --git a/TileGenerator.cpp b/TileGenerator.cpp index ccf7559..67be16d 100644 --- a/TileGenerator.cpp +++ b/TileGenerator.cpp @@ -19,10 +19,14 @@ #include "TileGenerator.h" #include "ZlibDecompressor.h" #include "colors.h" +#include "db-sqlite3.h" +#if USE_LEVELDB +#include "db-leveldb.h" +#endif using namespace std; -static inline sqlite3_int64 pythonmodulo(sqlite3_int64 i, sqlite3_int64 mod) +static inline int64_t pythonmodulo(int64_t i, int64_t mod) { if (i >= 0) { return i % mod; @@ -96,7 +100,7 @@ TileGenerator::TileGenerator(): m_drawScale(false), m_shading(true), m_border(0), - m_db(0), + m_backend("sqlite3"), m_image(0), m_xMin(INT_MAX), m_xMax(INT_MIN), @@ -116,10 +120,6 @@ TileGenerator::TileGenerator(): TileGenerator::~TileGenerator() { - if (m_db != 0) { - sqlite3_close(m_db); - m_db = 0; - } } void TileGenerator::setBgColor(const std::string &bgColor) @@ -235,6 +235,11 @@ void TileGenerator::parseColorsFile(const std::string &fileName) parseColorsStream(in); } +void TileGenerator::setBackend(std::string backend) +{ + m_backend = backend; +} + void TileGenerator::generate(const std::string &input, const std::string &output) { string input_path = input; @@ -287,59 +292,49 @@ void TileGenerator::parseColorsStream(std::istream &in) void TileGenerator::openDb(const std::string &input) { - string db_name = input + "map.sqlite"; - if (sqlite3_open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0) != SQLITE_OK) { - throw std::runtime_error(std::string(sqlite3_errmsg(m_db)) + ", Database file: " + db_name); - } + if(m_backend == "sqlite3") + m_db = new DBSQLite3(input); +#if USE_LEVELDB + if(m_backend == "leveldb") + m_db = new DBLevelDB(input); +#endif + else + throw std::runtime_error(((std::string) "Unknown map backend: ") + m_backend); } void TileGenerator::loadBlocks() { - sqlite3_stmt *statement; - string sql = "SELECT pos FROM blocks"; - if (sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &statement, 0) == SQLITE_OK) { - int result = 0; - while (true) { - result = sqlite3_step(statement); - if(result == SQLITE_ROW) { - sqlite3_int64 blocknum = sqlite3_column_int64(statement, 0); - BlockPos pos = decodeBlockPos(blocknum); - if (pos.x < m_geomX || pos.x >= m_geomX2 || pos.z < m_geomY || pos.z >= m_geomY2) { - continue; - } - if (pos.y < m_yMin) { - continue; - } - if (pos.y > m_yMax) { - continue; - } - if (pos.x < m_xMin) { - m_xMin = pos.x; - } - if (pos.x > m_xMax) { - m_xMax = pos.x; - } - if (pos.z < m_zMin) { - m_zMin = pos.z; - } - if (pos.z > m_zMax) { - m_zMax = pos.z; - } - m_positions.push_back(std::pair(pos.x, pos.z)); - } - else { - break; - } + std::vector vec = m_db->getBlockPos(); + for(unsigned int i = 0; i < vec.size(); i++) { + BlockPos pos = decodeBlockPos(vec[i]); + if (pos.x < m_geomX || pos.x >= m_geomX2 || pos.z < m_geomY || pos.z >= m_geomY2) { + continue; } + if (pos.y < m_yMin) { + continue; + } + if (pos.y > m_yMax) { + continue; + } + if (pos.x < m_xMin) { + m_xMin = pos.x; + } + if (pos.x > m_xMax) { + m_xMax = pos.x; + } + if (pos.z < m_zMin) { + m_zMin = pos.z; + } + if (pos.z > m_zMax) { + m_zMax = pos.z; + } + m_positions.push_back(std::pair(pos.x, pos.z)); m_positions.sort(); m_positions.unique(); } - else { - throw std::runtime_error("Failed to get list of MapBlocks"); - } } -inline BlockPos TileGenerator::decodeBlockPos(sqlite3_int64 blockId) const +inline BlockPos TileGenerator::decodeBlockPos(int64_t blockId) const { BlockPos pos; pos.x = unsignedToSigned(pythonmodulo(blockId, 4096), 2048); @@ -360,18 +355,27 @@ void TileGenerator::createImage() gdImageFilledRectangle(m_image, 0, 0, m_mapWidth + m_border - 1, m_mapHeight + m_border -1, rgb2int(m_bgColor.r, m_bgColor.g, m_bgColor.b)); } +std::map TileGenerator::getBlocksOnZ(int zPos) +{ + DBBlockList in = m_db->getBlocksOnZ(zPos); + std::map out; + for(DBBlockList::const_iterator it = in.begin(); it != in.end(); ++it) { + Block b = Block(decodeBlockPos(it->first), it->second); + if(out.find(b.first.x) == out.end()) { + BlockList bl; + out[b.first.x] = bl; + } + out[b.first.x].push_back(b); + } + return out; +} + void TileGenerator::renderMap() { - sqlite3_stmt *statement; - string sql = "SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)"; - if (sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &statement, 0) != SQLITE_OK) { - throw std::runtime_error("Failed to get MapBlock"); - } - std::list zlist = getZValueList(); for (std::list::iterator zPosition = zlist.begin(); zPosition != zlist.end(); ++zPosition) { int zPos = *zPosition; - map blocks = getBlocksOnZ(zPos, statement); + std::map blocks = getBlocksOnZ(zPos); for (std::list >::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) { if (position->second != zPos) { continue; @@ -620,37 +624,6 @@ inline std::list TileGenerator::getZValueList() const return zlist; } -std::map TileGenerator::getBlocksOnZ(int zPos, sqlite3_stmt *statement) const -{ - map blocks; - - sqlite3_int64 psMin; - sqlite3_int64 psMax; - - psMin = (static_cast(zPos) * 16777216l) - 0x800000; - psMax = (static_cast(zPos) * 16777216l) + 0x7fffff; - sqlite3_bind_int64(statement, 1, psMin); - sqlite3_bind_int64(statement, 2, psMax); - - int result = 0; - while (true) { - result = sqlite3_step(statement); - if(result == SQLITE_ROW) { - sqlite3_int64 blocknum = sqlite3_column_int64(statement, 0); - const unsigned char *data = reinterpret_cast(sqlite3_column_blob(statement, 1)); - int size = sqlite3_column_bytes(statement, 1); - BlockPos pos = decodeBlockPos(blocknum); - blocks[pos.x].push_back(Block(pos, unsigned_string(data, size))); - } - else { - break; - } - } - sqlite3_reset(statement); - - return blocks; -} - void TileGenerator::writeImage(const std::string &output) { FILE *out; diff --git a/TileGenerator.h b/TileGenerator.h index 2f6ddc9..905f636 100644 --- a/TileGenerator.h +++ b/TileGenerator.h @@ -15,10 +15,10 @@ #include #include #include -#include #include #include #include "PixelAttributes.h" +#include "db.h" struct Color { Color(): r(255), g(255), b(255) {}; @@ -80,17 +80,18 @@ public: void setMinY(int y); void setMaxY(int y); void parseColorsFile(const std::string &fileName); + void setBackend(std::string backend); void generate(const std::string &input, const std::string &output); private: void parseColorsStream(std::istream &in); void openDb(const std::string &input); void loadBlocks(); - BlockPos decodeBlockPos(sqlite3_int64 blockId) const; + BlockPos decodeBlockPos(int64_t blockId) const; void createImage(); void renderMap(); std::list getZValueList() const; - std::map getBlocksOnZ(int zPos, sqlite3_stmt *statement) const; + std::map getBlocksOnZ(int zPos); void renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version); void renderShading(int zPos); void renderScale(); @@ -111,8 +112,9 @@ private: bool m_drawScale; bool m_shading; int m_border; + std::string m_backend; - sqlite3 *m_db; + DB *m_db; gdImagePtr m_image; PixelAttributes m_blockPixelAttributes; int m_xMin; diff --git a/cmake_config.h.in b/cmake_config.h.in new file mode 100644 index 0000000..6d1699e --- /dev/null +++ b/cmake_config.h.in @@ -0,0 +1,9 @@ +// Filled in by the build system + +#ifndef CMAKE_CONFIG_H +#define CMAKE_CONFIG_H + +#define USE_LEVELDB @USE_LEVELDB@ + +#endif + diff --git a/config.h b/config.h index 7ef8d1c..f7536a2 100644 --- a/config.h +++ b/config.h @@ -14,3 +14,9 @@ #endif #define BLOCK_SIZE 16 + +#ifdef USE_CMAKE_CONFIG_H +#include "cmake_config.h" +#else +#define USE_LEVELDB 0 +#endif diff --git a/db-leveldb.cpp b/db-leveldb.cpp new file mode 100644 index 0000000..5e98233 --- /dev/null +++ b/db-leveldb.cpp @@ -0,0 +1,64 @@ +#include "db-leveldb.h" +#include +#include + +inline int64_t stoi64(const std::string &s) { + std::stringstream tmp(s); + long long t; + tmp >> t; + return t; +} + +inline std::string i64tos(int64_t i) { + std::ostringstream o; + o< DBLevelDB::getBlockPos() { + std::vector vec; + std::set s; + leveldb::Iterator* it = m_db->NewIterator(leveldb::ReadOptions()); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + vec.push_back(stoi64(it->key().ToString())); + s.insert(stoi64(it->key().ToString())); + } + delete it; + m_bpcache = s; + return vec; +} + +DBBlockList DBLevelDB::getBlocksOnZ(int zPos) +{ + DBBlockList blocks; + std::string datastr; + leveldb::Status status; + + int64_t psMin; + int64_t psMax; + psMin = (zPos * 16777216l) - 0x800000; + psMax = (zPos * 16777216l) + 0x7fffff; + + for(int64_t i = psMin; i <= psMax; i++) { // FIXME: This is still very very inefficent (even with m_bpcache) + if(m_bpcache.find(i) == m_bpcache.end()) + continue; + status = m_db->Get(leveldb::ReadOptions(), i64tos(i), &datastr); + if(status.ok()) + blocks.push_back( DBBlock( i, std::basic_string( (const unsigned char*) datastr.c_str(), datastr.size() ) ) ); + } + + return blocks; +} + diff --git a/db-leveldb.h b/db-leveldb.h new file mode 100644 index 0000000..7b9f97e --- /dev/null +++ b/db-leveldb.h @@ -0,0 +1,19 @@ +#ifndef _DB_LEVELDB_H +#define _DB_LEVELDB_H + +#include "db.h" +#include +#include + +class DBLevelDB : public DB { +public: + DBLevelDB(const std::string &mapdir); + virtual std::vector getBlockPos(); + virtual DBBlockList getBlocksOnZ(int zPos); + ~DBLevelDB(); +private: + leveldb::DB *m_db; + std::set m_bpcache; +}; + +#endif // _DB_LEVELDB_H diff --git a/db-sqlite3.cpp b/db-sqlite3.cpp new file mode 100644 index 0000000..9adfe5f --- /dev/null +++ b/db-sqlite3.cpp @@ -0,0 +1,71 @@ +#include "db-sqlite3.h" +#include + +DBSQLite3::DBSQLite3(const std::string &mapdir) { + + std::string db_name = mapdir + "map.sqlite"; + if (sqlite3_open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0) != SQLITE_OK) { + throw std::runtime_error(std::string(sqlite3_errmsg(m_db)) + ", Database file: " + db_name); + } +} + +DBSQLite3::~DBSQLite3() { + sqlite3_close_v2(m_db); +} + +std::vector DBSQLite3::getBlockPos() { + std::vector vec; + sqlite3_stmt *statement; + std::string sql = "SELECT pos FROM blocks"; + if (sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &statement, 0) == SQLITE_OK) { + int result = 0; + while (true) { + result = sqlite3_step(statement); + if(result == SQLITE_ROW) { + sqlite3_int64 blocknum = sqlite3_column_int64(statement, 0); + vec.push_back(blocknum); + } + else + break; + } + } else { + throw std::runtime_error("Failed to get list of MapBlocks"); + } + return vec; +} + +DBBlockList DBSQLite3::getBlocksOnZ(int zPos) +{ + sqlite3_stmt *statement; + std::string sql = "SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)"; + if (sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &statement, 0) != SQLITE_OK) { + throw std::runtime_error("Failed to prepare statement"); + } + DBBlockList blocks; + + sqlite3_int64 psMin; + sqlite3_int64 psMax; + + psMin = (static_cast(zPos) * 16777216l) - 0x800000; + psMax = (static_cast(zPos) * 16777216l) + 0x7fffff; + sqlite3_bind_int64(statement, 1, psMin); + sqlite3_bind_int64(statement, 2, psMax); + + int result = 0; + while (true) { + result = sqlite3_step(statement); + if(result == SQLITE_ROW) { + sqlite3_int64 blocknum = sqlite3_column_int64(statement, 0); + const unsigned char *data = reinterpret_cast(sqlite3_column_blob(statement, 1)); + int size = sqlite3_column_bytes(statement, 1); + blocks.push_back(DBBlock(blocknum, std::basic_string(data, size))); + } + else { + break; + } + } + sqlite3_reset(statement); + + return blocks; +} + diff --git a/db-sqlite3.h b/db-sqlite3.h new file mode 100644 index 0000000..1af5d25 --- /dev/null +++ b/db-sqlite3.h @@ -0,0 +1,17 @@ +#ifndef _DB_SQLITE3_H +#define _DB_SQLITE3_H + +#include "db.h" +#include + +class DBSQLite3 : public DB { +public: + DBSQLite3(const std::string &mapdir); + virtual std::vector getBlockPos(); + virtual DBBlockList getBlocksOnZ(int zPos); + ~DBSQLite3(); +private: + sqlite3 *m_db; +}; + +#endif // _DB_SQLITE3_H diff --git a/db.h b/db.h new file mode 100644 index 0000000..159c22a --- /dev/null +++ b/db.h @@ -0,0 +1,18 @@ +#ifndef _DB_H +#define _DB_H + +#include +#include +#include +#include + +typedef std::pair> DBBlock; +typedef std::list DBBlockList; + +class DB { +public: + virtual std::vector getBlockPos()=0; + virtual DBBlockList getBlocksOnZ(int zPos)=0; +}; + +#endif // _DB_H diff --git a/mapper.cpp b/mapper.cpp index f73d192..91d81ce 100644 --- a/mapper.cpp +++ b/mapper.cpp @@ -30,6 +30,9 @@ void usage() " --drawplayers\n" " --draworigin\n" " --noshading\n" + " --min-y \n" + " --max-y \n" + " --backend \n" " --geometry x:y+w+h\n" "Color format: '#000000'\n"; std::cout << usage_text; @@ -53,6 +56,7 @@ int main(int argc, char *argv[]) {"geometry", required_argument, 0, 'g'}, {"min-y", required_argument, 0, 'a'}, {"max-y", required_argument, 0, 'c'}, + {"backend", required_argument, 0, 'd'}, }; string input; @@ -135,6 +139,9 @@ int main(int argc, char *argv[]) generator.setGeometry(x, y, w, h); } break; + case 'd': + generator.setBackend(optarg); + break; default: exit(1); }