minetest
/
minetestmapper
mirror of https://github.com/minetest/minetestmapper.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Don't use a temporary table for block positions 

This also moves database functions and data structures to db.h
This commit is contained in:
ShadowNinja 2014-04-19 02:13:34 -04:00
parent 3460dabae0
commit 22d3e401f8
10 changed files with 206 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
TileGenerator.cpp
View File

@ -1,12 +1,3 @@
/*
* =====================================================================
* Version: 1.0
* Created: 23.08.2012 12:35:53
* Author: Miroslav Bendík
* Company: LinuxOS.sk
* =====================================================================
*/
#include <cstdio>
#include <cstdlib>
#include <climits>
@ -29,26 +20,6 @@
using namespace std;
static inline int64_t pythonmodulo(int64_t i, int64_t mod)
{
if (i >= 0) {
return i % mod;
}
else {
return mod - ((-i) % mod);
}
}
static inline int unsignedToSigned(long i, long max_positive)
{
if (i < max_positive) {
return i;
}
else {
return i - 2l * max_positive;
}
}
static inline uint16_t readU16(const unsigned char *data)
{
return data[0] << 8 | data[1];
@ -334,9 +305,9 @@ void TileGenerator::openDb(const std::string &input)
void TileGenerator::loadBlocks()
{
std::vector<int64_t> vec = m_db->getBlockPos();
for(std::vector<int64_t>::iterator it = vec.begin(); it != vec.end(); ++it) {
BlockPos pos = decodeBlockPos(*it);
std::vector<BlockPos> vec = m_db->getBlockPos();
for (std::vector<BlockPos>::iterator it = vec.begin(); it != vec.end(); ++it) {
BlockPos pos = *it;
// Check that it's in geometry (from --geometry option)
if (pos.x < m_geomX || pos.x > m_geomX2 || pos.z < m_geomY || pos.z > m_geomY2) {
continue;
@ -364,17 +335,6 @@ void TileGenerator::loadBlocks()
m_positions.unique();
}
inline BlockPos TileGenerator::decodeBlockPos(int64_t blockId) const
{
BlockPos pos;
pos.x = unsignedToSigned(pythonmodulo(blockId, 4096), 2048);
blockId = (blockId - pos.x) / 4096;
pos.y = unsignedToSigned(pythonmodulo(blockId, 4096), 2048);
blockId = (blockId - pos.y) / 4096;
pos.z = unsignedToSigned(pythonmodulo(blockId, 4096), 2048);
return pos;
}
void TileGenerator::createImage()
{
m_mapWidth = (m_xMax - m_xMin + 1) * 16;
@ -385,27 +345,13 @@ void TileGenerator::createImage()
gdImageFilledRectangle(m_image, 0, 0, m_mapWidth + m_border - 1, m_mapHeight + m_border -1, color2int(m_bgColor));
}
std::map<int, TileGenerator::BlockList> TileGenerator::getBlocksOnZ(int zPos)
{
DBBlockList in = m_db->getBlocksOnZ(zPos);
std::map<int, BlockList> out;
for(DBBlockList::const_iterator it = in.begin(); it != in.end(); ++it) {
Block b = Block(decodeBlockPos(it->first), it->second);
if(out.find(b.first.x) == out.end()) {
BlockList bl;
out[b.first.x] = bl;
}
out[b.first.x].push_back(b);
}
return out;
}
void TileGenerator::renderMap()
{
std::list<int> zlist = getZValueList();
for (std::list<int>::iterator zPosition = zlist.begin(); zPosition != zlist.end(); ++zPosition) {
int zPos = *zPosition;
std::map<int, BlockList> blocks = getBlocksOnZ(zPos);
std::map<int16_t, BlockList> blocks;
m_db->getBlocksOnZ(blocks, zPos);
for (std::list<std::pair<int, int> >::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
if (position->second != zPos) {
continue;
@ -436,8 +382,8 @@ void TileGenerator::renderMap()
ZlibDecompressor decompressor(data, length);
decompressor.setSeekPos(dataOffset);
ZlibDecompressor::string mapData = decompressor.decompress();
ZlibDecompressor::string mapMetadata = decompressor.decompress();
ustring mapData = decompressor.decompress();
ustring mapMetadata = decompressor.decompress();
dataOffset = decompressor.seekPos();
// Skip unused data
@ -519,7 +465,7 @@ void TileGenerator::renderMap()
}
}
inline void TileGenerator::renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version)
inline void TileGenerator::renderMapBlock(const ustring &mapBlock, const BlockPos &pos, int version)
{
int xBegin = (pos.x - m_xMin) * 16;
int zBegin = (m_zMax - pos.z) * 16;

54
TileGenerator.h
View File

@ -1,14 +1,5 @@
/*
* =====================================================================
* Version: 1.0
* Created: 23.08.2012 12:35:59
* Author: Miroslav Bendík
* Company: LinuxOS.sk
* =====================================================================
*/
#ifndef TILEGENERATOR_H_JJNUCARH
#define TILEGENERATOR_H_JJNUCARH
#ifndef TILEGENERATOR_HEADER
#define TILEGENERATOR_HEADER
#include <gd.h>
#include <iosfwd>
@ -19,6 +10,7 @@
#include <string>
#include "PixelAttributes.h"
#include "db.h"
#include "types.h"
struct Color {
Color(): r(0xFF), g(0xFF), b(0xFF), a(0) {};
@ -42,42 +34,10 @@ struct ColorEntry {
};
struct BlockPos {
int x;
int y;
int z;
bool operator<(const BlockPos& p) const
{
if (z > p.z) {
return true;
}
if (z < p.z) {
return false;
}
if (y > p.y) {
return true;
}
if (y < p.y) {
return false;
}
if (x > p.x) {
return true;
}
if (x < p.x) {
return false;
}
return false;
}
};
class TileGenerator
{
private:
typedef std::basic_string<unsigned char> unsigned_string;
typedef std::map<std::string, ColorEntry> ColorMap;
typedef std::pair<BlockPos, unsigned_string> Block;
typedef std::list<Block> BlockList;
public:
TileGenerator();
@ -102,12 +62,10 @@ private:
void parseColorsStream(std::istream &in);
void openDb(const std::string &input);
void loadBlocks();
BlockPos decodeBlockPos(int64_t blockId) const;
void createImage();
void renderMap();
std::list<int> getZValueList() const;
std::map<int, BlockList> getBlocksOnZ(int zPos);
void renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version);
void renderMapBlock(const ustring &mapBlock, const BlockPos &pos, int version);
void renderShading(int zPos);
void renderScale();
void renderOrigin();
@ -153,7 +111,7 @@ private:
int m_blockAirId;
int m_blockIgnoreId;
}; /* ----- end of class TileGenerator ----- */
}; // class TileGenerator
#endif /* end of include guard: TILEGENERATOR_H_JJNUCARH */
#endif // TILEGENERATOR_HEADER

6
ZlibDecompressor.cpp
View File

@ -32,12 +32,12 @@ std::size_t ZlibDecompressor::seekPos() const
return m_seekPos;
}
ZlibDecompressor::string ZlibDecompressor::decompress()
ustring ZlibDecompressor::decompress()
{
const unsigned char *data = m_data + m_seekPos;
const std::size_t size = m_size - m_seekPos;
string buffer;
ustring buffer;
const size_t BUFSIZE = 128 * 1024;
uint8_t temp_buffer[BUFSIZE];
@ -58,7 +58,7 @@ ZlibDecompressor::string ZlibDecompressor::decompress()
strm.avail_out = BUFSIZE;
strm.next_out = temp_buffer;
ret = inflate(&strm, Z_NO_FLUSH);
buffer += string(reinterpret_cast<unsigned char *>(temp_buffer), BUFSIZE - strm.avail_out);
buffer += ustring(reinterpret_cast<unsigned char *>(temp_buffer), BUFSIZE - strm.avail_out);
} while (ret == Z_OK);
if (ret != Z_STREAM_END) {
throw DecompressError();

4
ZlibDecompressor.h
View File

@ -12,12 +12,12 @@
#include <cstdlib>
#include <string>
#include "types.h"
class ZlibDecompressor
{
public:
typedef std::basic_string<unsigned char> string;
class DecompressError {
};
@ -25,7 +25,7 @@ public:
~ZlibDecompressor();
void setSeekPos(std::size_t seekPos);
std::size_t seekPos() const;
string decompress();
ustring decompress();
private:
const unsigned char *m_data;

70
db-leveldb.cpp
View File

@ -1,74 +1,74 @@
#include "db-leveldb.h"
#include <stdexcept>
#include <sstream>
#include "db-leveldb.h"
#include "types.h"
inline int64_t stoi64(const std::string &s) {
static inline int64_t stoi64(const std::string &s)
{
std::stringstream tmp(s);
long long t;
int64_t t;
tmp >> t;
return t;
}
inline std::string i64tos(int64_t i) {
std::ostringstream o;
o<<i;
return o.str();
static inline std::string i64tos(int64_t i)
{
std::ostringstream os;
os << i;
return os.str();
}
DBLevelDB::DBLevelDB(const std::string &mapdir) {
DBLevelDB::DBLevelDB(const std::string &mapdir)
{
leveldb::Options options;
posCacheLoaded = false;
options.create_if_missing = false;
leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &db);
if(!status.ok())
if (!status.ok()) {
throw std::runtime_error("Failed to open Database");
}
loadPosCache();
}
DBLevelDB::~DBLevelDB() {
DBLevelDB::~DBLevelDB()
{
delete db;
}
std::vector<int64_t> DBLevelDB::getBlockPos() {
loadPosCache();
std::vector<BlockPos> DBLevelDB::getBlockPos()
{
return posCache;
}
void DBLevelDB::loadPosCache() {
if (posCacheLoaded) {
return;
}
void DBLevelDB::loadPosCache()
{
leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions());
for (it->SeekToFirst(); it->Valid(); it->Next()) {
posCache.push_back(stoi64(it->key().ToString()));
int64_t posHash = stoi64(it->key().ToString());
posCache.push_back(decodeBlockPos(posHash));
}
delete it;
posCacheLoaded = true;
}
DBBlockList DBLevelDB::getBlocksOnZ(int zPos) {
DBBlockList blocks;
void DBLevelDB::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
{
std::string datastr;
leveldb::Status status;
int64_t psMin = (zPos * 16777216L) - 0x800000;
int64_t psMax = (zPos * 16777216L) + 0x7fffff;
for (std::vector<int64_t>::iterator it = posCache.begin(); it != posCache.end(); ++it) {
int64_t i = *it;
if (i < psMin || i > psMax) {
for (std::vector<BlockPos>::iterator it = posCache.begin(); it != posCache.end(); ++it) {
if (it->z != zPos) {
continue;
}
status = db->Get(leveldb::ReadOptions(), i64tos(i), &datastr);
status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(*it)), &datastr);
if (status.ok()) {
blocks.push_back(
DBBlock(i,
std::basic_string<unsigned char>((const unsigned char*) datastr.data(), datastr.size())
)
);
Block b(*it, ustring((const unsigned char *) datastr.data(), datastr.size()));
blocks[b.first.x].push_back(b);
}
}
return blocks;
}

15
db-leveldb.h
View File

@ -1,5 +1,5 @@
#ifndef _DB_LEVELDB_H
#define _DB_LEVELDB_H
#ifndef DB_LEVELDB_HEADER
#define DB_LEVELDB_HEADER
#include "db.h"
#include <leveldb/db.h>
@ -7,16 +7,15 @@
class DBLevelDB : public DB {
public:
DBLevelDB(const std::string &mapdir);
virtual std::vector<int64_t> getBlockPos();
virtual DBBlockList getBlocksOnZ(int zPos);
virtual std::vector<BlockPos> getBlockPos();
virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
~DBLevelDB();
private:
void loadPosCache();
leveldb::DB *db;
std::vector<BlockPos> posCache;
bool posCacheLoaded;
std::vector<int64_t> posCache;
leveldb::DB *db;
};
#endif // _DB_LEVELDB_H
#endif // DB_LEVELDB_HEADER

66
db-sqlite3.cpp
View File

@ -1,6 +1,7 @@
#include "db-sqlite3.h"
#include <stdexcept>
#include <unistd.h> // for usleep
#include "db-sqlite3.h"
#include "types.h"
#define SQLRES(f, good) \
result = (sqlite3_##f);\
@ -9,36 +10,42 @@
}
#define SQLOK(f) SQLRES(f, SQLITE_OK)
DBSQLite3::DBSQLite3(const std::string &mapdir) {
DBSQLite3::DBSQLite3(const std::string &mapdir)
{
int result;
std::string db_name = mapdir + "map.sqlite";
SQLOK(open_v2(db_name.c_str(), &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0))
SQLOK(open_v2(db_name.c_str(), &db, SQLITE_OPEN_READONLY |
SQLITE_OPEN_PRIVATECACHE, 0))
SQLOK(prepare_v2(db,
"SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)",
-1, &stmt_get_blocks, NULL))
"SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ?",
-1, &stmt_get_blocks_z, NULL))
SQLOK(prepare_v2(db,
"SELECT pos FROM blocks",
-1, &stmt_get_block_pos, NULL))
}
DBSQLite3::~DBSQLite3() {
DBSQLite3::~DBSQLite3()
{
int result;
SQLOK(finalize(stmt_get_blocks));
SQLOK(finalize(stmt_get_blocks_z));
SQLOK(finalize(stmt_get_block_pos));
SQLOK(close(db));
}
std::vector<int64_t> DBSQLite3::getBlockPos() {
std::vector<int64_t> vec;
std::vector<BlockPos> DBSQLite3::getBlockPos()
{
int result;
std::vector<BlockPos> positions;
while ((result = sqlite3_step(stmt_get_block_pos)) != SQLITE_DONE) {
if (result == SQLITE_ROW) {
int64_t blockpos = sqlite3_column_int64(stmt_get_block_pos, 0);
vec.push_back(blockpos);
int64_t posHash = sqlite3_column_int64(stmt_get_block_pos, 0);
positions.push_back(decodeBlockPos(posHash));
} else if (result == SQLITE_BUSY) { // Wait some time and try again
usleep(10000);
} else {
@ -46,35 +53,36 @@ std::vector<int64_t> DBSQLite3::getBlockPos() {
}
}
SQLOK(reset(stmt_get_block_pos));
return vec;
return positions;
}
DBBlockList DBSQLite3::getBlocksOnZ(int zPos)
void DBSQLite3::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
{
DBBlockList blocks;
int64_t psMin = (static_cast<sqlite3_int64>(zPos) * 16777216L) - 0x800000;
int64_t psMax = (static_cast<sqlite3_int64>(zPos) * 16777216L) + 0x7fffff;
sqlite3_bind_int64(stmt_get_blocks, 1, psMin);
sqlite3_bind_int64(stmt_get_blocks, 2, psMax);
int result;
while ((result = sqlite3_step(stmt_get_blocks)) != SQLITE_DONE) {
// Magic numbers!
int64_t minPos = (zPos * 0x1000000) - 0x800000;
int64_t maxPos = (zPos * 0x1000000) + 0x7FFFFF;
SQLOK(bind_int64(stmt_get_blocks_z, 1, minPos));
SQLOK(bind_int64(stmt_get_blocks_z, 2, maxPos));
while ((result = sqlite3_step(stmt_get_blocks_z)) != SQLITE_DONE) {
if (result == SQLITE_ROW) {
int64_t blocknum = sqlite3_column_int64(stmt_get_blocks, 0);
const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(stmt_get_blocks, 1));
int size = sqlite3_column_bytes(stmt_get_blocks, 1);
blocks.push_back(DBBlock(blocknum, std::basic_string<unsigned char>(data, size)));
int64_t posHash = sqlite3_column_int64(stmt_get_blocks_z, 0);
const unsigned char *data = reinterpret_cast<const unsigned char *>(
sqlite3_column_blob(stmt_get_blocks_z, 1));
size_t size = sqlite3_column_bytes(stmt_get_blocks_z, 1);
Block b(decodeBlockPos(posHash), ustring(data, size));
blocks[b.first.x].push_back(b);
} else if (result == SQLITE_BUSY) { // Wait some time and try again
usleep(10000);
} else {
throw std::runtime_error(sqlite3_errmsg(db));
}
}
SQLOK(reset(stmt_get_blocks));
return blocks;
SQLOK(reset(stmt_get_blocks_z));
}
#undef SQLRES

6
db-sqlite3.h
View File

@ -7,14 +7,14 @@
class DBSQLite3 : public DB {
public:
DBSQLite3(const std::string &mapdir);
virtual std::vector<int64_t> getBlockPos();
virtual DBBlockList getBlocksOnZ(int zPos);
virtual std::vector<BlockPos> getBlockPos();
virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
~DBSQLite3();
private:
sqlite3 *db;
sqlite3_stmt *stmt_get_block_pos;
sqlite3_stmt *stmt_get_blocks;
sqlite3_stmt *stmt_get_blocks_z;
};
#endif // _DB_SQLITE3_H

109
db.h
View File

@ -1,19 +1,110 @@
#ifndef _DB_H
#define _DB_H
#ifndef DB_HEADER
#define DB_HEADER
#include <stdint.h>
#include <vector>
#include <map>
#include <list>
#include <vector>
#include <string>
#include <utility>
#include "types.h"
typedef std::pair<int64_t, std::basic_string<unsigned char> > DBBlock;
typedef std::list<DBBlock> DBBlockList;
class DB {
class BlockPos {
public:
virtual std::vector<int64_t> getBlockPos() = 0;
virtual DBBlockList getBlocksOnZ(int zPos) = 0;
int16_t x;
int16_t y;
int16_t z;
bool operator < (const BlockPos &p) const
{
if (z > p.z) {
return true;
}
if (z < p.z) {
return false;
}
if (y > p.y) {
return true;
}
if (y < p.y) {
return false;
}
if (x > p.x) {
return true;
}
if (x < p.x) {
return false;
}
return false;
}
};
#endif // _DB_H
typedef std::pair<BlockPos, ustring> Block;
typedef std::list<Block> BlockList;
class DB {
protected:
inline int64_t encodeBlockPos(const BlockPos pos) const;
inline BlockPos decodeBlockPos(int64_t hash) const;
public:
virtual std::vector<BlockPos> getBlockPos() = 0;
virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos) = 0;
};
/****************
* Black magic! *
****************
* The position hashing is seriously messed up,
* and is a lot more complicated than it looks.
*/
static inline int16_t unsigned_to_signed(uint16_t i, uint16_t max_positive)
{
if (i < max_positive) {
return i;
} else {
return i - (max_positive * 2);
}
}
// Modulo of a negative number does not work consistently in C
static inline int64_t pythonmodulo(int64_t i, int64_t mod)
{
if (i >= 0) {
return i % mod;
}
return mod - ((-i) % mod);
}
inline int64_t DB::encodeBlockPos(const BlockPos pos) const
{
return (uint64_t) pos.z * 0x1000000 +
(uint64_t) pos.y * 0x1000 +
(uint64_t) pos.x;
}
inline BlockPos DB::decodeBlockPos(int64_t hash) const
{
BlockPos pos;
pos.x = unsigned_to_signed(pythonmodulo(hash, 4096), 2048);
hash = (hash - pos.x) / 4096;
pos.y = unsigned_to_signed(pythonmodulo(hash, 4096), 2048);
hash = (hash - pos.y) / 4096;
pos.z = unsigned_to_signed(pythonmodulo(hash, 4096), 2048);
return pos;
}
/*******************
* End black magic *
*******************/
#endif // DB_HEADER

5
types.h Normal file
View File

@ -0,0 +1,5 @@
#include <string>
typedef std::basic_string<unsigned char> ustring;