minetest
/
minetestmapper
mirror of https://github.com/minetest/minetestmapper.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

LevelDB support

This commit is contained in:
Sfan5 2014-03-05 21:41:27 +01:00
parent 15444ff8f6
commit 393d7e2ac0
13 changed files with 310 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -6,5 +6,5 @@ CMakeCache.txt
CMakeFiles/
Makefile
cmake_install.cmake
cmake_config.h
*~

35
CMakeLists.txt
View File

@ -28,6 +28,25 @@ else (LIBSQLITE3_INCLUDE_DIR)
message(FATAL_ERROR "Could not find sqlite3")
endif (LIBSQLITE3_INCLUDE_DIR)
set(USE_LEVELDB 0)
OPTION(ENABLE_LEVELDB "Enable LevelDB backend")
if(ENABLE_LEVELDB)
find_library(LEVELDB_LIBRARY leveldb)
find_path(LEVELDB_INCLUDE_DIR db.h PATH_SUFFIXES leveldb)
message (STATUS "LevelDB library: ${LEVELDB_LIBRARY}")
message (STATUS "LevelDB headers: ${LEVELDB_INCLUDE_DIR}")
if(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR)
set(USE_LEVELDB 1)
message(STATUS "LevelDB backend enabled")
include_directories(${LEVELDB_INCLUDE_DIR})
else(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR)
set(USE_LEVELDB 0)
message(STATUS "LevelDB not found!")
endif(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR)
endif(ENABLE_LEVELDB)
find_program(XXD_EXECUTABLE xxd)
if (XXD_EXECUTABLE)
@ -51,12 +70,11 @@ include_directories(
${LIBSQLITE3_INCLUDE_DIRS}
)
set(mapper_HDRS
PixelAttributes.h
PlayerAttributes.h
TileGenerator.h
ZlibDecompressor.h
configure_file(
"${PROJECT_SOURCE_DIR}/cmake_config.h.in"
"${PROJECT_BINARY_DIR}/cmake_config.h"
)
add_definitions ( -DUSE_CMAKE_CONFIG_H )
set(mapper_SRCS
PixelAttributes.cpp
@ -64,16 +82,21 @@ set(mapper_SRCS
TileGenerator.cpp
ZlibDecompressor.cpp
mapper.cpp
db-sqlite3.cpp
)
if(USE_LEVELDB)
set(mapper_SRCS ${mapper_SRCS} db-leveldb.cpp)
endif(USE_LEVELDB)
add_executable(minetestmapper
${mapper_HDRS}
${mapper_SRCS}
)
target_link_libraries(
minetestmapper
${LIBSQLITE3_LIBRARIES}
${LEVELDB_LIBRARY}
gd
z
)

3
README.rst
View File

@ -62,6 +62,9 @@ min-y:
max-y:
Don't draw nodes above this y value, `--max-y 75`
backend:
Use specific map backend, supported: sqlite3, leveldb, `--backend leveldb`
geometry:
Limit area to specific geometry, `--geometry -800:-800+1600+1600`

147
TileGenerator.cpp
View File

@ -19,10 +19,14 @@
#include "TileGenerator.h"
#include "ZlibDecompressor.h"
#include "colors.h"
#include "db-sqlite3.h"
#if USE_LEVELDB
#include "db-leveldb.h"
#endif
using namespace std;
static inline sqlite3_int64 pythonmodulo(sqlite3_int64 i, sqlite3_int64 mod)
static inline int64_t pythonmodulo(int64_t i, int64_t mod)
{
if (i >= 0) {
return i % mod;
@ -96,7 +100,7 @@ TileGenerator::TileGenerator():
m_drawScale(false),
m_shading(true),
m_border(0),
m_db(0),
m_backend("sqlite3"),
m_image(0),
m_xMin(INT_MAX),
m_xMax(INT_MIN),
@ -116,10 +120,6 @@ TileGenerator::TileGenerator():
TileGenerator::~TileGenerator()
{
if (m_db != 0) {
sqlite3_close(m_db);
m_db = 0;
}
}
void TileGenerator::setBgColor(const std::string &bgColor)
@ -235,6 +235,11 @@ void TileGenerator::parseColorsFile(const std::string &fileName)
parseColorsStream(in);
}
void TileGenerator::setBackend(std::string backend)
{
m_backend = backend;
}
void TileGenerator::generate(const std::string &input, const std::string &output)
{
string input_path = input;
@ -287,59 +292,49 @@ void TileGenerator::parseColorsStream(std::istream &in)
void TileGenerator::openDb(const std::string &input)
{
string db_name = input + "map.sqlite";
if (sqlite3_open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0) != SQLITE_OK) {
throw std::runtime_error(std::string(sqlite3_errmsg(m_db)) + ", Database file: " + db_name);
}
if(m_backend == "sqlite3")
m_db = new DBSQLite3(input);
#if USE_LEVELDB
if(m_backend == "leveldb")
m_db = new DBLevelDB(input);
#endif
else
throw std::runtime_error(((std::string) "Unknown map backend: ") + m_backend);
}
void TileGenerator::loadBlocks()
{
sqlite3_stmt *statement;
string sql = "SELECT pos FROM blocks";
if (sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &statement, 0) == SQLITE_OK) {
int result = 0;
while (true) {
result = sqlite3_step(statement);
if(result == SQLITE_ROW) {
sqlite3_int64 blocknum = sqlite3_column_int64(statement, 0);
BlockPos pos = decodeBlockPos(blocknum);
if (pos.x < m_geomX || pos.x >= m_geomX2 || pos.z < m_geomY || pos.z >= m_geomY2) {
continue;
}
if (pos.y < m_yMin) {
continue;
}
if (pos.y > m_yMax) {
continue;
}
if (pos.x < m_xMin) {
m_xMin = pos.x;
}
if (pos.x > m_xMax) {
m_xMax = pos.x;
}
if (pos.z < m_zMin) {
m_zMin = pos.z;
}
if (pos.z > m_zMax) {
m_zMax = pos.z;
}
m_positions.push_back(std::pair<int, int>(pos.x, pos.z));
}
else {
break;
}
std::vector<int64_t> vec = m_db->getBlockPos();
for(unsigned int i = 0; i < vec.size(); i++) {
BlockPos pos = decodeBlockPos(vec[i]);
if (pos.x < m_geomX || pos.x >= m_geomX2 || pos.z < m_geomY || pos.z >= m_geomY2) {
continue;
}
if (pos.y < m_yMin) {
continue;
}
if (pos.y > m_yMax) {
continue;
}
if (pos.x < m_xMin) {
m_xMin = pos.x;
}
if (pos.x > m_xMax) {
m_xMax = pos.x;
}
if (pos.z < m_zMin) {
m_zMin = pos.z;
}
if (pos.z > m_zMax) {
m_zMax = pos.z;
}
m_positions.push_back(std::pair<int, int>(pos.x, pos.z));
m_positions.sort();
m_positions.unique();
}
else {
throw std::runtime_error("Failed to get list of MapBlocks");
}
}
inline BlockPos TileGenerator::decodeBlockPos(sqlite3_int64 blockId) const
inline BlockPos TileGenerator::decodeBlockPos(int64_t blockId) const
{
BlockPos pos;
pos.x = unsignedToSigned(pythonmodulo(blockId, 4096), 2048);
@ -360,18 +355,27 @@ void TileGenerator::createImage()
gdImageFilledRectangle(m_image, 0, 0, m_mapWidth + m_border - 1, m_mapHeight + m_border -1, rgb2int(m_bgColor.r, m_bgColor.g, m_bgColor.b));
}
std::map<int, TileGenerator::BlockList> TileGenerator::getBlocksOnZ(int zPos)
{
DBBlockList in = m_db->getBlocksOnZ(zPos);
std::map<int, BlockList> out;
for(DBBlockList::const_iterator it = in.begin(); it != in.end(); ++it) {
Block b = Block(decodeBlockPos(it->first), it->second);
if(out.find(b.first.x) == out.end()) {
BlockList bl;
out[b.first.x] = bl;
}
out[b.first.x].push_back(b);
}
return out;
}
void TileGenerator::renderMap()
{
sqlite3_stmt *statement;
string sql = "SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)";
if (sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &statement, 0) != SQLITE_OK) {
throw std::runtime_error("Failed to get MapBlock");
}
std::list<int> zlist = getZValueList();
for (std::list<int>::iterator zPosition = zlist.begin(); zPosition != zlist.end(); ++zPosition) {
int zPos = *zPosition;
map<int, BlockList> blocks = getBlocksOnZ(zPos, statement);
std::map<int, BlockList> blocks = getBlocksOnZ(zPos);
for (std::list<std::pair<int, int> >::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
if (position->second != zPos) {
continue;
@ -620,37 +624,6 @@ inline std::list<int> TileGenerator::getZValueList() const
return zlist;
}
std::map<int, TileGenerator::BlockList> TileGenerator::getBlocksOnZ(int zPos, sqlite3_stmt *statement) const
{
map<int, BlockList> blocks;
sqlite3_int64 psMin;
sqlite3_int64 psMax;
psMin = (static_cast<sqlite3_int64>(zPos) * 16777216l) - 0x800000;
psMax = (static_cast<sqlite3_int64>(zPos) * 16777216l) + 0x7fffff;
sqlite3_bind_int64(statement, 1, psMin);
sqlite3_bind_int64(statement, 2, psMax);
int result = 0;
while (true) {
result = sqlite3_step(statement);
if(result == SQLITE_ROW) {
sqlite3_int64 blocknum = sqlite3_column_int64(statement, 0);
const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(statement, 1));
int size = sqlite3_column_bytes(statement, 1);
BlockPos pos = decodeBlockPos(blocknum);
blocks[pos.x].push_back(Block(pos, unsigned_string(data, size)));
}
else {
break;
}
}
sqlite3_reset(statement);
return blocks;
}
void TileGenerator::writeImage(const std::string &output)
{
FILE *out;

10
TileGenerator.h
View File

@ -15,10 +15,10 @@
#include <list>
#include <map>
#include <set>
#include <sqlite3.h>
#include <stdint.h>
#include <string>
#include "PixelAttributes.h"
#include "db.h"
struct Color {
Color(): r(255), g(255), b(255) {};
@ -80,17 +80,18 @@ public:
void setMinY(int y);
void setMaxY(int y);
void parseColorsFile(const std::string &fileName);
void setBackend(std::string backend);
void generate(const std::string &input, const std::string &output);
private:
void parseColorsStream(std::istream &in);
void openDb(const std::string &input);
void loadBlocks();
BlockPos decodeBlockPos(sqlite3_int64 blockId) const;
BlockPos decodeBlockPos(int64_t blockId) const;
void createImage();
void renderMap();
std::list<int> getZValueList() const;
std::map<int, BlockList> getBlocksOnZ(int zPos, sqlite3_stmt *statement) const;
std::map<int, BlockList> getBlocksOnZ(int zPos);
void renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version);
void renderShading(int zPos);
void renderScale();
@ -111,8 +112,9 @@ private:
bool m_drawScale;
bool m_shading;
int m_border;
std::string m_backend;
sqlite3 *m_db;
DB *m_db;
gdImagePtr m_image;
PixelAttributes m_blockPixelAttributes;
int m_xMin;

9
cmake_config.h.in Normal file
View File

@ -0,0 +1,9 @@
// Filled in by the build system
#ifndef CMAKE_CONFIG_H
#define CMAKE_CONFIG_H
#define USE_LEVELDB @USE_LEVELDB@
#endif

6
config.h
View File

@ -14,3 +14,9 @@
#endif
#define BLOCK_SIZE 16
#ifdef USE_CMAKE_CONFIG_H
#include "cmake_config.h"
#else
#define USE_LEVELDB 0
#endif

64
db-leveldb.cpp Normal file
View File

@ -0,0 +1,64 @@
#include "db-leveldb.h"
#include <stdexcept>
#include <sstream>
inline int64_t stoi64(const std::string &s) {
std::stringstream tmp(s);
long long t;
tmp >> t;
return t;
}
inline std::string i64tos(int64_t i) {
std::ostringstream o;
o<<i;
return o.str();
}
DBLevelDB::DBLevelDB(const std::string &mapdir) {
leveldb::Options options;
options.create_if_missing = false;
leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &m_db);
if(!status.ok())
throw std::runtime_error("Failed to open Database");
}
DBLevelDB::~DBLevelDB() {
delete m_db;
}
std::vector<int64_t> DBLevelDB::getBlockPos() {
std::vector<int64_t> vec;
std::set<int64_t> s;
leveldb::Iterator* it = m_db->NewIterator(leveldb::ReadOptions());
for (it->SeekToFirst(); it->Valid(); it->Next()) {
vec.push_back(stoi64(it->key().ToString()));
s.insert(stoi64(it->key().ToString()));
}
delete it;
m_bpcache = s;
return vec;
}
DBBlockList DBLevelDB::getBlocksOnZ(int zPos)
{
DBBlockList blocks;
std::string datastr;
leveldb::Status status;
int64_t psMin;
int64_t psMax;
psMin = (zPos * 16777216l) - 0x800000;
psMax = (zPos * 16777216l) + 0x7fffff;
for(int64_t i = psMin; i <= psMax; i++) { // FIXME: This is still very very inefficent (even with m_bpcache)
if(m_bpcache.find(i) == m_bpcache.end())
continue;
status = m_db->Get(leveldb::ReadOptions(), i64tos(i), &datastr);
if(status.ok())
blocks.push_back( DBBlock( i, std::basic_string<unsigned char>( (const unsigned char*) datastr.c_str(), datastr.size() ) ) );
}
return blocks;
}

19
db-leveldb.h Normal file
View File

@ -0,0 +1,19 @@
#ifndef _DB_LEVELDB_H
#define _DB_LEVELDB_H
#include "db.h"
#include <leveldb/db.h>
#include <set>
class DBLevelDB : public DB {
public:
DBLevelDB(const std::string &mapdir);
virtual std::vector<int64_t> getBlockPos();
virtual DBBlockList getBlocksOnZ(int zPos);
~DBLevelDB();
private:
leveldb::DB *m_db;
std::set<int64_t> m_bpcache;
};
#endif // _DB_LEVELDB_H

71
db-sqlite3.cpp Normal file
View File

@ -0,0 +1,71 @@
#include "db-sqlite3.h"
#include <stdexcept>
DBSQLite3::DBSQLite3(const std::string &mapdir) {
std::string db_name = mapdir + "map.sqlite";
if (sqlite3_open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0) != SQLITE_OK) {
throw std::runtime_error(std::string(sqlite3_errmsg(m_db)) + ", Database file: " + db_name);
}
}
DBSQLite3::~DBSQLite3() {
sqlite3_close_v2(m_db);
}
std::vector<int64_t> DBSQLite3::getBlockPos() {
std::vector<int64_t> vec;
sqlite3_stmt *statement;
std::string sql = "SELECT pos FROM blocks";
if (sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &statement, 0) == SQLITE_OK) {
int result = 0;
while (true) {
result = sqlite3_step(statement);
if(result == SQLITE_ROW) {
sqlite3_int64 blocknum = sqlite3_column_int64(statement, 0);
vec.push_back(blocknum);
}
else
break;
}
} else {
throw std::runtime_error("Failed to get list of MapBlocks");
}
return vec;
}
DBBlockList DBSQLite3::getBlocksOnZ(int zPos)
{
sqlite3_stmt *statement;
std::string sql = "SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)";
if (sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &statement, 0) != SQLITE_OK) {
throw std::runtime_error("Failed to prepare statement");
}
DBBlockList blocks;
sqlite3_int64 psMin;
sqlite3_int64 psMax;
psMin = (static_cast<sqlite3_int64>(zPos) * 16777216l) - 0x800000;
psMax = (static_cast<sqlite3_int64>(zPos) * 16777216l) + 0x7fffff;
sqlite3_bind_int64(statement, 1, psMin);
sqlite3_bind_int64(statement, 2, psMax);
int result = 0;
while (true) {
result = sqlite3_step(statement);
if(result == SQLITE_ROW) {
sqlite3_int64 blocknum = sqlite3_column_int64(statement, 0);
const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(statement, 1));
int size = sqlite3_column_bytes(statement, 1);
blocks.push_back(DBBlock(blocknum, std::basic_string<unsigned char>(data, size)));
}
else {
break;
}
}
sqlite3_reset(statement);
return blocks;
}

17
db-sqlite3.h Normal file
View File

@ -0,0 +1,17 @@
#ifndef _DB_SQLITE3_H
#define _DB_SQLITE3_H
#include "db.h"
#include <sqlite3.h>
class DBSQLite3 : public DB {
public:
DBSQLite3(const std::string &mapdir);
virtual std::vector<int64_t> getBlockPos();
virtual DBBlockList getBlocksOnZ(int zPos);
~DBSQLite3();
private:
sqlite3 *m_db;
};
#endif // _DB_SQLITE3_H

18
db.h Normal file
View File

@ -0,0 +1,18 @@
#ifndef _DB_H
#define _DB_H
#include <cstdint>
#include <vector>
#include <list>
#include <string>
typedef std::pair<int64_t, std::basic_string<unsigned char>> DBBlock;
typedef std::list<DBBlock> DBBlockList;
class DB {
public:
virtual std::vector<int64_t> getBlockPos()=0;
virtual DBBlockList getBlocksOnZ(int zPos)=0;
};
#endif // _DB_H

7
mapper.cpp
View File

@ -30,6 +30,9 @@ void usage()
" --drawplayers\n"
" --draworigin\n"
" --noshading\n"
" --min-y <y>\n"
" --max-y <y>\n"
" --backend <sqlite3/leveldb>\n"
" --geometry x:y+w+h\n"
"Color format: '#000000'\n";
std::cout << usage_text;
@ -53,6 +56,7 @@ int main(int argc, char *argv[])
{"geometry", required_argument, 0, 'g'},
{"min-y", required_argument, 0, 'a'},
{"max-y", required_argument, 0, 'c'},
{"backend", required_argument, 0, 'd'},
};
string input;
@ -135,6 +139,9 @@ int main(int argc, char *argv[])
generator.setGeometry(x, y, w, h);
}
break;
case 'd':
generator.setBackend(optarg);
break;
default:
exit(1);
}