Don't use a temporary table for block positions

This also moves database functions and data structures to db.h
This commit is contained in:
ShadowNinja 2014-04-19 02:13:34 -04:00
parent 3460dabae0
commit 22d3e401f8
10 changed files with 206 additions and 199 deletions

View File

@ -1,12 +1,3 @@
/*
* =====================================================================
* Version: 1.0
* Created: 23.08.2012 12:35:53
* Author: Miroslav Bendík
* Company: LinuxOS.sk
* =====================================================================
*/
#include <cstdio> #include <cstdio>
#include <cstdlib> #include <cstdlib>
#include <climits> #include <climits>
@ -29,26 +20,6 @@
using namespace std; using namespace std;
static inline int64_t pythonmodulo(int64_t i, int64_t mod)
{
if (i >= 0) {
return i % mod;
}
else {
return mod - ((-i) % mod);
}
}
static inline int unsignedToSigned(long i, long max_positive)
{
if (i < max_positive) {
return i;
}
else {
return i - 2l * max_positive;
}
}
static inline uint16_t readU16(const unsigned char *data) static inline uint16_t readU16(const unsigned char *data)
{ {
return data[0] << 8 | data[1]; return data[0] << 8 | data[1];
@ -334,9 +305,9 @@ void TileGenerator::openDb(const std::string &input)
void TileGenerator::loadBlocks() void TileGenerator::loadBlocks()
{ {
std::vector<int64_t> vec = m_db->getBlockPos(); std::vector<BlockPos> vec = m_db->getBlockPos();
for(std::vector<int64_t>::iterator it = vec.begin(); it != vec.end(); ++it) { for (std::vector<BlockPos>::iterator it = vec.begin(); it != vec.end(); ++it) {
BlockPos pos = decodeBlockPos(*it); BlockPos pos = *it;
// Check that it's in geometry (from --geometry option) // Check that it's in geometry (from --geometry option)
if (pos.x < m_geomX || pos.x > m_geomX2 || pos.z < m_geomY || pos.z > m_geomY2) { if (pos.x < m_geomX || pos.x > m_geomX2 || pos.z < m_geomY || pos.z > m_geomY2) {
continue; continue;
@ -364,17 +335,6 @@ void TileGenerator::loadBlocks()
m_positions.unique(); m_positions.unique();
} }
inline BlockPos TileGenerator::decodeBlockPos(int64_t blockId) const
{
BlockPos pos;
pos.x = unsignedToSigned(pythonmodulo(blockId, 4096), 2048);
blockId = (blockId - pos.x) / 4096;
pos.y = unsignedToSigned(pythonmodulo(blockId, 4096), 2048);
blockId = (blockId - pos.y) / 4096;
pos.z = unsignedToSigned(pythonmodulo(blockId, 4096), 2048);
return pos;
}
void TileGenerator::createImage() void TileGenerator::createImage()
{ {
m_mapWidth = (m_xMax - m_xMin + 1) * 16; m_mapWidth = (m_xMax - m_xMin + 1) * 16;
@ -385,27 +345,13 @@ void TileGenerator::createImage()
gdImageFilledRectangle(m_image, 0, 0, m_mapWidth + m_border - 1, m_mapHeight + m_border -1, color2int(m_bgColor)); gdImageFilledRectangle(m_image, 0, 0, m_mapWidth + m_border - 1, m_mapHeight + m_border -1, color2int(m_bgColor));
} }
std::map<int, TileGenerator::BlockList> TileGenerator::getBlocksOnZ(int zPos)
{
DBBlockList in = m_db->getBlocksOnZ(zPos);
std::map<int, BlockList> out;
for(DBBlockList::const_iterator it = in.begin(); it != in.end(); ++it) {
Block b = Block(decodeBlockPos(it->first), it->second);
if(out.find(b.first.x) == out.end()) {
BlockList bl;
out[b.first.x] = bl;
}
out[b.first.x].push_back(b);
}
return out;
}
void TileGenerator::renderMap() void TileGenerator::renderMap()
{ {
std::list<int> zlist = getZValueList(); std::list<int> zlist = getZValueList();
for (std::list<int>::iterator zPosition = zlist.begin(); zPosition != zlist.end(); ++zPosition) { for (std::list<int>::iterator zPosition = zlist.begin(); zPosition != zlist.end(); ++zPosition) {
int zPos = *zPosition; int zPos = *zPosition;
std::map<int, BlockList> blocks = getBlocksOnZ(zPos); std::map<int16_t, BlockList> blocks;
m_db->getBlocksOnZ(blocks, zPos);
for (std::list<std::pair<int, int> >::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) { for (std::list<std::pair<int, int> >::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
if (position->second != zPos) { if (position->second != zPos) {
continue; continue;
@ -436,8 +382,8 @@ void TileGenerator::renderMap()
ZlibDecompressor decompressor(data, length); ZlibDecompressor decompressor(data, length);
decompressor.setSeekPos(dataOffset); decompressor.setSeekPos(dataOffset);
ZlibDecompressor::string mapData = decompressor.decompress(); ustring mapData = decompressor.decompress();
ZlibDecompressor::string mapMetadata = decompressor.decompress(); ustring mapMetadata = decompressor.decompress();
dataOffset = decompressor.seekPos(); dataOffset = decompressor.seekPos();
// Skip unused data // Skip unused data
@ -519,7 +465,7 @@ void TileGenerator::renderMap()
} }
} }
inline void TileGenerator::renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version) inline void TileGenerator::renderMapBlock(const ustring &mapBlock, const BlockPos &pos, int version)
{ {
int xBegin = (pos.x - m_xMin) * 16; int xBegin = (pos.x - m_xMin) * 16;
int zBegin = (m_zMax - pos.z) * 16; int zBegin = (m_zMax - pos.z) * 16;

View File

@ -1,14 +1,5 @@
/* #ifndef TILEGENERATOR_HEADER
* ===================================================================== #define TILEGENERATOR_HEADER
* Version: 1.0
* Created: 23.08.2012 12:35:59
* Author: Miroslav Bendík
* Company: LinuxOS.sk
* =====================================================================
*/
#ifndef TILEGENERATOR_H_JJNUCARH
#define TILEGENERATOR_H_JJNUCARH
#include <gd.h> #include <gd.h>
#include <iosfwd> #include <iosfwd>
@ -19,6 +10,7 @@
#include <string> #include <string>
#include "PixelAttributes.h" #include "PixelAttributes.h"
#include "db.h" #include "db.h"
#include "types.h"
struct Color { struct Color {
Color(): r(0xFF), g(0xFF), b(0xFF), a(0) {}; Color(): r(0xFF), g(0xFF), b(0xFF), a(0) {};
@ -42,42 +34,10 @@ struct ColorEntry {
}; };
struct BlockPos {
int x;
int y;
int z;
bool operator<(const BlockPos& p) const
{
if (z > p.z) {
return true;
}
if (z < p.z) {
return false;
}
if (y > p.y) {
return true;
}
if (y < p.y) {
return false;
}
if (x > p.x) {
return true;
}
if (x < p.x) {
return false;
}
return false;
}
};
class TileGenerator class TileGenerator
{ {
private: private:
typedef std::basic_string<unsigned char> unsigned_string;
typedef std::map<std::string, ColorEntry> ColorMap; typedef std::map<std::string, ColorEntry> ColorMap;
typedef std::pair<BlockPos, unsigned_string> Block;
typedef std::list<Block> BlockList;
public: public:
TileGenerator(); TileGenerator();
@ -102,12 +62,10 @@ private:
void parseColorsStream(std::istream &in); void parseColorsStream(std::istream &in);
void openDb(const std::string &input); void openDb(const std::string &input);
void loadBlocks(); void loadBlocks();
BlockPos decodeBlockPos(int64_t blockId) const;
void createImage(); void createImage();
void renderMap(); void renderMap();
std::list<int> getZValueList() const; std::list<int> getZValueList() const;
std::map<int, BlockList> getBlocksOnZ(int zPos); void renderMapBlock(const ustring &mapBlock, const BlockPos &pos, int version);
void renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version);
void renderShading(int zPos); void renderShading(int zPos);
void renderScale(); void renderScale();
void renderOrigin(); void renderOrigin();
@ -153,7 +111,7 @@ private:
int m_blockAirId; int m_blockAirId;
int m_blockIgnoreId; int m_blockIgnoreId;
}; /* ----- end of class TileGenerator ----- */ }; // class TileGenerator
#endif /* end of include guard: TILEGENERATOR_H_JJNUCARH */ #endif // TILEGENERATOR_HEADER

View File

@ -32,12 +32,12 @@ std::size_t ZlibDecompressor::seekPos() const
return m_seekPos; return m_seekPos;
} }
ZlibDecompressor::string ZlibDecompressor::decompress() ustring ZlibDecompressor::decompress()
{ {
const unsigned char *data = m_data + m_seekPos; const unsigned char *data = m_data + m_seekPos;
const std::size_t size = m_size - m_seekPos; const std::size_t size = m_size - m_seekPos;
string buffer; ustring buffer;
const size_t BUFSIZE = 128 * 1024; const size_t BUFSIZE = 128 * 1024;
uint8_t temp_buffer[BUFSIZE]; uint8_t temp_buffer[BUFSIZE];
@ -58,7 +58,7 @@ ZlibDecompressor::string ZlibDecompressor::decompress()
strm.avail_out = BUFSIZE; strm.avail_out = BUFSIZE;
strm.next_out = temp_buffer; strm.next_out = temp_buffer;
ret = inflate(&strm, Z_NO_FLUSH); ret = inflate(&strm, Z_NO_FLUSH);
buffer += string(reinterpret_cast<unsigned char *>(temp_buffer), BUFSIZE - strm.avail_out); buffer += ustring(reinterpret_cast<unsigned char *>(temp_buffer), BUFSIZE - strm.avail_out);
} while (ret == Z_OK); } while (ret == Z_OK);
if (ret != Z_STREAM_END) { if (ret != Z_STREAM_END) {
throw DecompressError(); throw DecompressError();

View File

@ -12,12 +12,12 @@
#include <cstdlib> #include <cstdlib>
#include <string> #include <string>
#include "types.h"
class ZlibDecompressor class ZlibDecompressor
{ {
public: public:
typedef std::basic_string<unsigned char> string;
class DecompressError { class DecompressError {
}; };
@ -25,7 +25,7 @@ public:
~ZlibDecompressor(); ~ZlibDecompressor();
void setSeekPos(std::size_t seekPos); void setSeekPos(std::size_t seekPos);
std::size_t seekPos() const; std::size_t seekPos() const;
string decompress(); ustring decompress();
private: private:
const unsigned char *m_data; const unsigned char *m_data;

View File

@ -1,74 +1,74 @@
#include "db-leveldb.h"
#include <stdexcept> #include <stdexcept>
#include <sstream> #include <sstream>
#include "db-leveldb.h"
#include "types.h"
inline int64_t stoi64(const std::string &s) { static inline int64_t stoi64(const std::string &s)
{
std::stringstream tmp(s); std::stringstream tmp(s);
long long t; int64_t t;
tmp >> t; tmp >> t;
return t; return t;
} }
inline std::string i64tos(int64_t i) {
std::ostringstream o; static inline std::string i64tos(int64_t i)
o<<i; {
return o.str(); std::ostringstream os;
os << i;
return os.str();
} }
DBLevelDB::DBLevelDB(const std::string &mapdir) { DBLevelDB::DBLevelDB(const std::string &mapdir)
{
leveldb::Options options; leveldb::Options options;
posCacheLoaded = false;
options.create_if_missing = false; options.create_if_missing = false;
leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &db); leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &db);
if(!status.ok()) if (!status.ok()) {
throw std::runtime_error("Failed to open Database"); throw std::runtime_error("Failed to open Database");
}
loadPosCache();
} }
DBLevelDB::~DBLevelDB() {
DBLevelDB::~DBLevelDB()
{
delete db; delete db;
} }
std::vector<int64_t> DBLevelDB::getBlockPos() {
loadPosCache(); std::vector<BlockPos> DBLevelDB::getBlockPos()
{
return posCache; return posCache;
} }
void DBLevelDB::loadPosCache() {
if (posCacheLoaded) {
return;
}
void DBLevelDB::loadPosCache()
{
leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions()); leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions());
for (it->SeekToFirst(); it->Valid(); it->Next()) { for (it->SeekToFirst(); it->Valid(); it->Next()) {
posCache.push_back(stoi64(it->key().ToString())); int64_t posHash = stoi64(it->key().ToString());
posCache.push_back(decodeBlockPos(posHash));
} }
delete it; delete it;
posCacheLoaded = true;
} }
DBBlockList DBLevelDB::getBlocksOnZ(int zPos) {
DBBlockList blocks; void DBLevelDB::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
{
std::string datastr; std::string datastr;
leveldb::Status status; leveldb::Status status;
int64_t psMin = (zPos * 16777216L) - 0x800000; for (std::vector<BlockPos>::iterator it = posCache.begin(); it != posCache.end(); ++it) {
int64_t psMax = (zPos * 16777216L) + 0x7fffff; if (it->z != zPos) {
for (std::vector<int64_t>::iterator it = posCache.begin(); it != posCache.end(); ++it) {
int64_t i = *it;
if (i < psMin || i > psMax) {
continue; continue;
} }
status = db->Get(leveldb::ReadOptions(), i64tos(i), &datastr); status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(*it)), &datastr);
if (status.ok()) { if (status.ok()) {
blocks.push_back( Block b(*it, ustring((const unsigned char *) datastr.data(), datastr.size()));
DBBlock(i, blocks[b.first.x].push_back(b);
std::basic_string<unsigned char>((const unsigned char*) datastr.data(), datastr.size())
)
);
} }
} }
return blocks;
} }

View File

@ -1,5 +1,5 @@
#ifndef _DB_LEVELDB_H #ifndef DB_LEVELDB_HEADER
#define _DB_LEVELDB_H #define DB_LEVELDB_HEADER
#include "db.h" #include "db.h"
#include <leveldb/db.h> #include <leveldb/db.h>
@ -7,16 +7,15 @@
class DBLevelDB : public DB { class DBLevelDB : public DB {
public: public:
DBLevelDB(const std::string &mapdir); DBLevelDB(const std::string &mapdir);
virtual std::vector<int64_t> getBlockPos(); virtual std::vector<BlockPos> getBlockPos();
virtual DBBlockList getBlocksOnZ(int zPos); virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
~DBLevelDB(); ~DBLevelDB();
private: private:
void loadPosCache(); void loadPosCache();
leveldb::DB *db; std::vector<BlockPos> posCache;
bool posCacheLoaded; leveldb::DB *db;
std::vector<int64_t> posCache;
}; };
#endif // _DB_LEVELDB_H #endif // DB_LEVELDB_HEADER

View File

@ -1,6 +1,7 @@
#include "db-sqlite3.h"
#include <stdexcept> #include <stdexcept>
#include <unistd.h> // for usleep #include <unistd.h> // for usleep
#include "db-sqlite3.h"
#include "types.h"
#define SQLRES(f, good) \ #define SQLRES(f, good) \
result = (sqlite3_##f);\ result = (sqlite3_##f);\
@ -9,36 +10,42 @@
} }
#define SQLOK(f) SQLRES(f, SQLITE_OK) #define SQLOK(f) SQLRES(f, SQLITE_OK)
DBSQLite3::DBSQLite3(const std::string &mapdir) {
DBSQLite3::DBSQLite3(const std::string &mapdir)
{
int result; int result;
std::string db_name = mapdir + "map.sqlite"; std::string db_name = mapdir + "map.sqlite";
SQLOK(open_v2(db_name.c_str(), &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0)) SQLOK(open_v2(db_name.c_str(), &db, SQLITE_OPEN_READONLY |
SQLITE_OPEN_PRIVATECACHE, 0))
SQLOK(prepare_v2(db, SQLOK(prepare_v2(db,
"SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)", "SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ?",
-1, &stmt_get_blocks, NULL)) -1, &stmt_get_blocks_z, NULL))
SQLOK(prepare_v2(db, SQLOK(prepare_v2(db,
"SELECT pos FROM blocks", "SELECT pos FROM blocks",
-1, &stmt_get_block_pos, NULL)) -1, &stmt_get_block_pos, NULL))
} }
DBSQLite3::~DBSQLite3() {
DBSQLite3::~DBSQLite3()
{
int result; int result;
SQLOK(finalize(stmt_get_blocks)); SQLOK(finalize(stmt_get_blocks_z));
SQLOK(finalize(stmt_get_block_pos)); SQLOK(finalize(stmt_get_block_pos));
SQLOK(close(db)); SQLOK(close(db));
} }
std::vector<int64_t> DBSQLite3::getBlockPos() { std::vector<BlockPos> DBSQLite3::getBlockPos()
std::vector<int64_t> vec; {
int result; int result;
std::vector<BlockPos> positions;
while ((result = sqlite3_step(stmt_get_block_pos)) != SQLITE_DONE) { while ((result = sqlite3_step(stmt_get_block_pos)) != SQLITE_DONE) {
if (result == SQLITE_ROW) { if (result == SQLITE_ROW) {
int64_t blockpos = sqlite3_column_int64(stmt_get_block_pos, 0); int64_t posHash = sqlite3_column_int64(stmt_get_block_pos, 0);
vec.push_back(blockpos); positions.push_back(decodeBlockPos(posHash));
} else if (result == SQLITE_BUSY) { // Wait some time and try again } else if (result == SQLITE_BUSY) { // Wait some time and try again
usleep(10000); usleep(10000);
} else { } else {
@ -46,35 +53,36 @@ std::vector<int64_t> DBSQLite3::getBlockPos() {
} }
} }
SQLOK(reset(stmt_get_block_pos)); SQLOK(reset(stmt_get_block_pos));
return vec; return positions;
} }
DBBlockList DBSQLite3::getBlocksOnZ(int zPos)
void DBSQLite3::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
{ {
DBBlockList blocks;
int64_t psMin = (static_cast<sqlite3_int64>(zPos) * 16777216L) - 0x800000;
int64_t psMax = (static_cast<sqlite3_int64>(zPos) * 16777216L) + 0x7fffff;
sqlite3_bind_int64(stmt_get_blocks, 1, psMin);
sqlite3_bind_int64(stmt_get_blocks, 2, psMax);
int result; int result;
while ((result = sqlite3_step(stmt_get_blocks)) != SQLITE_DONE) {
// Magic numbers!
int64_t minPos = (zPos * 0x1000000) - 0x800000;
int64_t maxPos = (zPos * 0x1000000) + 0x7FFFFF;
SQLOK(bind_int64(stmt_get_blocks_z, 1, minPos));
SQLOK(bind_int64(stmt_get_blocks_z, 2, maxPos));
while ((result = sqlite3_step(stmt_get_blocks_z)) != SQLITE_DONE) {
if (result == SQLITE_ROW) { if (result == SQLITE_ROW) {
int64_t blocknum = sqlite3_column_int64(stmt_get_blocks, 0); int64_t posHash = sqlite3_column_int64(stmt_get_blocks_z, 0);
const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(stmt_get_blocks, 1)); const unsigned char *data = reinterpret_cast<const unsigned char *>(
int size = sqlite3_column_bytes(stmt_get_blocks, 1); sqlite3_column_blob(stmt_get_blocks_z, 1));
blocks.push_back(DBBlock(blocknum, std::basic_string<unsigned char>(data, size))); size_t size = sqlite3_column_bytes(stmt_get_blocks_z, 1);
Block b(decodeBlockPos(posHash), ustring(data, size));
blocks[b.first.x].push_back(b);
} else if (result == SQLITE_BUSY) { // Wait some time and try again } else if (result == SQLITE_BUSY) { // Wait some time and try again
usleep(10000); usleep(10000);
} else { } else {
throw std::runtime_error(sqlite3_errmsg(db)); throw std::runtime_error(sqlite3_errmsg(db));
} }
} }
SQLOK(reset(stmt_get_blocks)); SQLOK(reset(stmt_get_blocks_z));
return blocks;
} }
#undef SQLRES #undef SQLRES

View File

@ -7,14 +7,14 @@
class DBSQLite3 : public DB { class DBSQLite3 : public DB {
public: public:
DBSQLite3(const std::string &mapdir); DBSQLite3(const std::string &mapdir);
virtual std::vector<int64_t> getBlockPos(); virtual std::vector<BlockPos> getBlockPos();
virtual DBBlockList getBlocksOnZ(int zPos); virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
~DBSQLite3(); ~DBSQLite3();
private: private:
sqlite3 *db; sqlite3 *db;
sqlite3_stmt *stmt_get_block_pos; sqlite3_stmt *stmt_get_block_pos;
sqlite3_stmt *stmt_get_blocks; sqlite3_stmt *stmt_get_blocks_z;
}; };
#endif // _DB_SQLITE3_H #endif // _DB_SQLITE3_H

109
db.h
View File

@ -1,19 +1,110 @@
#ifndef _DB_H #ifndef DB_HEADER
#define _DB_H #define DB_HEADER
#include <stdint.h> #include <stdint.h>
#include <vector> #include <map>
#include <list> #include <list>
#include <vector>
#include <string> #include <string>
#include <utility> #include <utility>
#include "types.h"
typedef std::pair<int64_t, std::basic_string<unsigned char> > DBBlock;
typedef std::list<DBBlock> DBBlockList;
class DB { class BlockPos {
public: public:
virtual std::vector<int64_t> getBlockPos() = 0; int16_t x;
virtual DBBlockList getBlocksOnZ(int zPos) = 0; int16_t y;
int16_t z;
bool operator < (const BlockPos &p) const
{
if (z > p.z) {
return true;
}
if (z < p.z) {
return false;
}
if (y > p.y) {
return true;
}
if (y < p.y) {
return false;
}
if (x > p.x) {
return true;
}
if (x < p.x) {
return false;
}
return false;
}
}; };
#endif // _DB_H
typedef std::pair<BlockPos, ustring> Block;
typedef std::list<Block> BlockList;
class DB {
protected:
inline int64_t encodeBlockPos(const BlockPos pos) const;
inline BlockPos decodeBlockPos(int64_t hash) const;
public:
virtual std::vector<BlockPos> getBlockPos() = 0;
virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos) = 0;
};
/****************
* Black magic! *
****************
* The position hashing is seriously messed up,
* and is a lot more complicated than it looks.
*/
static inline int16_t unsigned_to_signed(uint16_t i, uint16_t max_positive)
{
if (i < max_positive) {
return i;
} else {
return i - (max_positive * 2);
}
}
// Modulo of a negative number does not work consistently in C
static inline int64_t pythonmodulo(int64_t i, int64_t mod)
{
if (i >= 0) {
return i % mod;
}
return mod - ((-i) % mod);
}
inline int64_t DB::encodeBlockPos(const BlockPos pos) const
{
return (uint64_t) pos.z * 0x1000000 +
(uint64_t) pos.y * 0x1000 +
(uint64_t) pos.x;
}
inline BlockPos DB::decodeBlockPos(int64_t hash) const
{
BlockPos pos;
pos.x = unsigned_to_signed(pythonmodulo(hash, 4096), 2048);
hash = (hash - pos.x) / 4096;
pos.y = unsigned_to_signed(pythonmodulo(hash, 4096), 2048);
hash = (hash - pos.y) / 4096;
pos.z = unsigned_to_signed(pythonmodulo(hash, 4096), 2048);
return pos;
}
/*******************
* End black magic *
*******************/
#endif // DB_HEADER

5
types.h Normal file
View File

@ -0,0 +1,5 @@
#include <string>
typedef std::basic_string<unsigned char> ustring;