diff --git a/.gitignore b/.gitignore index 5098a54..96485bf 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ Makefile cmake_install.cmake cmake_config.h compile_commands.json +.vscode/ diff --git a/src/TileGenerator.cpp b/src/TileGenerator.cpp index 2822b80..0f814bc 100644 --- a/src/TileGenerator.cpp +++ b/src/TileGenerator.cpp @@ -464,26 +464,20 @@ void TileGenerator::loadBlocks() const int16_t yMin = mod16(m_yMin); if (m_exhaustiveSearch == EXH_NEVER || m_exhaustiveSearch == EXH_Y) { - std::vector vec = m_db->getBlockPos( + std::vector vec = m_db->getBlockPosXZ( BlockPos(m_geomX, yMin, m_geomY), BlockPos(m_geomX2, yMax, m_geomY2) ); for (auto pos : vec) { assert(pos.x >= m_geomX && pos.x < m_geomX2); - assert(pos.y >= yMin && pos.y < yMax); assert(pos.z >= m_geomY && pos.z < m_geomY2); // Adjust minimum and maximum positions to the nearest block - if (pos.x < m_xMin) - m_xMin = pos.x; - if (pos.x > m_xMax) - m_xMax = pos.x; - - if (pos.z < m_zMin) - m_zMin = pos.z; - if (pos.z > m_zMax) - m_zMax = pos.z; + m_xMin = mymin(m_xMin, pos.x); + m_xMax = mymax(m_xMax, pos.x); + m_zMin = mymin(m_zMin, pos.z); + m_zMax = mymax(m_zMax, pos.z); m_positions[pos.z].emplace(pos.x); } diff --git a/src/db-leveldb.cpp b/src/db-leveldb.cpp index 7a31ca0..9d7091c 100644 --- a/src/db-leveldb.cpp +++ b/src/db-leveldb.cpp @@ -1,5 +1,6 @@ #include #include +#include #include "db-leveldb.h" #include "types.h" @@ -18,6 +19,12 @@ static inline std::string i64tos(int64_t i) return os.str(); } +// finds the first position in the list where it.x >= x +#define lower_bound_x(container, find_x) \ + std::lower_bound((container).begin(), (container).end(), (find_x), \ + [] (const vec2 &left, int16_t right) { \ + return left.x < right; \ + }) DBLevelDB::DBLevelDB(const std::string &mapdir) { @@ -25,7 +32,7 @@ DBLevelDB::DBLevelDB(const std::string &mapdir) options.create_if_missing = false; leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &db); if (!status.ok()) { - throw std::runtime_error(std::string("Failed to open Database: ") + status.ToString()); + throw std::runtime_error(std::string("Failed to open database: ") + status.ToString()); } /* LevelDB is a dumb key-value store, so the only optimization we can do @@ -41,18 +48,24 @@ DBLevelDB::~DBLevelDB() } -std::vector DBLevelDB::getBlockPos(BlockPos min, BlockPos max) +std::vector DBLevelDB::getBlockPosXZ(BlockPos min, BlockPos max) { std::vector res; for (const auto &it : posCache) { - if (it.first < min.z || it.first >= max.z) + const int16_t zpos = it.first; + if (zpos < min.z || zpos >= max.z) continue; - for (auto pos2 : it.second) { - if (pos2.first < min.x || pos2.first >= max.x) + auto it2 = lower_bound_x(it.second, min.x); + for (; it2 != it.second.end(); it2++) { + const auto &pos2 = *it2; + if (pos2.x >= max.x) + break; // went past + if (pos2.y < min.y || pos2.y >= max.y) continue; - if (pos2.second < min.y || pos2.second >= max.y) + // skip duplicates + if (!res.empty() && res.back().x == pos2.x && res.back().z == zpos) continue; - res.emplace_back(pos2.first, pos2.second, it.first); + res.emplace_back(pos2.x, pos2.y, zpos); } } return res; @@ -61,7 +74,7 @@ std::vector DBLevelDB::getBlockPos(BlockPos min, BlockPos max) void DBLevelDB::loadPosCache() { - leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions()); + leveldb::Iterator *it = db->NewIterator(leveldb::ReadOptions()); for (it->SeekToFirst(); it->Valid(); it->Next()) { int64_t posHash = stoi64(it->key().ToString()); BlockPos pos = decodeBlockPos(posHash); @@ -69,6 +82,9 @@ void DBLevelDB::loadPosCache() posCache[pos.z].emplace_back(pos.x, pos.y); } delete it; + + for (auto &it : posCache) + std::sort(it.second.begin(), it.second.end()); } @@ -81,13 +97,18 @@ void DBLevelDB::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z, auto it = posCache.find(z); if (it == posCache.cend()) return; - for (auto pos2 : it->second) { - if (pos2.first != x) - continue; - if (pos2.second < min_y || pos2.second >= max_y) + auto it2 = lower_bound_x(it->second, x); + if (it2 == it->second.end() || it2->x != x) + return; + // it2 is now pointing to a contigous part where it2->x == x + for (; it2 != it->second.end(); it2++) { + const auto &pos2 = *it2; + if (pos2.x != x) + break; // went past + if (pos2.y < min_y || pos2.y >= max_y) continue; - BlockPos pos(x, pos2.second, z); + BlockPos pos(x, pos2.y, z); status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(pos)), &datastr); if (status.ok()) { blocks.emplace_back( diff --git a/src/db-leveldb.h b/src/db-leveldb.h index f742dfe..7c9e498 100644 --- a/src/db-leveldb.h +++ b/src/db-leveldb.h @@ -8,7 +8,7 @@ class DBLevelDB : public DB { public: DBLevelDB(const std::string &mapdir); - std::vector getBlockPos(BlockPos min, BlockPos max) override; + std::vector getBlockPosXZ(BlockPos min, BlockPos max) override; void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z, int16_t min_y, int16_t max_y) override; void getBlocksByPos(BlockList &blocks, @@ -18,11 +18,24 @@ public: bool preferRangeQueries() const override { return false; } private: - using pos2d = std::pair; + struct vec2 { + int16_t x, y; + constexpr vec2() : x(0), y(0) {} + constexpr vec2(int16_t x, int16_t y) : x(x), y(y) {} + + inline bool operator<(const vec2 &p) const + { + if (x < p.x) + return true; + if (x > p.x) + return false; + return y < p.y; + } + }; void loadPosCache(); // indexed by Z, contains all (x,y) position pairs - std::unordered_map> posCache; - leveldb::DB *db; + std::unordered_map> posCache; + leveldb::DB *db = NULL; }; diff --git a/src/db-postgresql.cpp b/src/db-postgresql.cpp index 0eb18f0..db6e84c 100644 --- a/src/db-postgresql.cpp +++ b/src/db-postgresql.cpp @@ -27,10 +27,10 @@ DBPostgreSQL::DBPostgreSQL(const std::string &mapdir) prepareStatement( "get_block_pos", - "SELECT posX::int4, posY::int4, posZ::int4 FROM blocks WHERE" + "SELECT posX::int4, posZ::int4 FROM blocks WHERE" " (posX BETWEEN $1::int4 AND $2::int4) AND" " (posY BETWEEN $3::int4 AND $4::int4) AND" - " (posZ BETWEEN $5::int4 AND $6::int4)" + " (posZ BETWEEN $5::int4 AND $6::int4) GROUP BY posX, posZ" ); prepareStatement( "get_blocks", @@ -60,7 +60,7 @@ DBPostgreSQL::~DBPostgreSQL() } -std::vector DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max) +std::vector DBPostgreSQL::getBlockPosXZ(BlockPos min, BlockPos max) { int32_t const x1 = htonl(min.x); int32_t const x2 = htonl(max.x - 1); @@ -83,11 +83,14 @@ std::vector DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max) std::vector positions; positions.reserve(numrows); - for (int row = 0; row < numrows; ++row) - positions.emplace_back(pg_to_blockpos(results, row, 0)); + BlockPos pos; + for (int row = 0; row < numrows; ++row) { + pos.x = pg_binary_to_int(results, row, 0); + pos.z = pg_binary_to_int(results, row, 1); + positions.push_back(pos); + } PQclear(results); - return positions; } @@ -215,12 +218,3 @@ int DBPostgreSQL::pg_binary_to_int(PGresult *res, int row, int col) int32_t* raw = reinterpret_cast(PQgetvalue(res, row, col)); return ntohl(*raw); } - -BlockPos DBPostgreSQL::pg_to_blockpos(PGresult *res, int row, int col) -{ - BlockPos result; - result.x = pg_binary_to_int(res, row, col); - result.y = pg_binary_to_int(res, row, col + 1); - result.z = pg_binary_to_int(res, row, col + 2); - return result; -} diff --git a/src/db-postgresql.h b/src/db-postgresql.h index 4f74df7..3f29e55 100644 --- a/src/db-postgresql.h +++ b/src/db-postgresql.h @@ -6,7 +6,7 @@ class DBPostgreSQL : public DB { public: DBPostgreSQL(const std::string &mapdir); - std::vector getBlockPos(BlockPos min, BlockPos max) override; + std::vector getBlockPosXZ(BlockPos min, BlockPos max) override; void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z, int16_t min_y, int16_t max_y) override; void getBlocksByPos(BlockList &blocks, @@ -25,7 +25,6 @@ protected: bool clear = true ); int pg_binary_to_int(PGresult *res, int row, int col); - BlockPos pg_to_blockpos(PGresult *res, int row, int col); private: PGconn *db; diff --git a/src/db-redis.cpp b/src/db-redis.cpp index bf89b80..6430924 100644 --- a/src/db-redis.cpp +++ b/src/db-redis.cpp @@ -68,7 +68,7 @@ DBRedis::~DBRedis() } -std::vector DBRedis::getBlockPos(BlockPos min, BlockPos max) +std::vector DBRedis::getBlockPosXZ(BlockPos min, BlockPos max) { std::vector res; for (const auto &it : posCache) { diff --git a/src/db-redis.h b/src/db-redis.h index 3375768..0b7258f 100644 --- a/src/db-redis.h +++ b/src/db-redis.h @@ -9,7 +9,7 @@ class DBRedis : public DB { public: DBRedis(const std::string &mapdir); - std::vector getBlockPos(BlockPos min, BlockPos max) override; + std::vector getBlockPosXZ(BlockPos min, BlockPos max) override; void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z, int16_t min_y, int16_t max_y) override; void getBlocksByPos(BlockList &blocks, diff --git a/src/db-sqlite3.cpp b/src/db-sqlite3.cpp index 52cc2c3..e824e29 100644 --- a/src/db-sqlite3.cpp +++ b/src/db-sqlite3.cpp @@ -79,9 +79,9 @@ DBSQLite3::DBSQLite3(const std::string &mapdir) "SELECT data FROM blocks WHERE x = ? AND y = ? AND z = ?")); SQLOK(prepare(stmt_get_block_pos_range, - "SELECT x, y, z FROM blocks WHERE " + "SELECT x, z FROM blocks WHERE " "x >= ? AND y >= ? AND z >= ? AND " - "x < ? AND y < ? AND z < ?")); + "x < ? AND y < ? AND z < ? GROUP BY x, z")); } else { SQLOK(prepare(stmt_get_blocks_z, "SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ?")); @@ -119,7 +119,7 @@ inline void DBSQLite3::getPosRange(int64_t &min, int64_t &max, } -std::vector DBSQLite3::getBlockPos(BlockPos min, BlockPos max) +std::vector DBSQLite3::getBlockPosXZ(BlockPos min, BlockPos max) { int result; sqlite3_stmt *stmt; @@ -152,12 +152,13 @@ std::vector DBSQLite3::getBlockPos(BlockPos min, BlockPos max) if (newFormat) { pos.x = sqlite3_column_int(stmt, 0); - pos.y = sqlite3_column_int(stmt, 1); - pos.z = sqlite3_column_int(stmt, 2); + pos.z = sqlite3_column_int(stmt, 1); } else { pos = decodeBlockPos(sqlite3_column_int64(stmt, 0)); if (pos.x < min.x || pos.x >= max.x || pos.y < min.y || pos.y >= max.y) continue; + // note that we can't try to deduplicate these because the order + // of the encoded pos (if sorted) is ZYX. } positions.emplace_back(pos); } diff --git a/src/db-sqlite3.h b/src/db-sqlite3.h index f72ef3a..ecc87cf 100644 --- a/src/db-sqlite3.h +++ b/src/db-sqlite3.h @@ -47,7 +47,7 @@ protected: class DBSQLite3 : public DB, SQLite3Base { public: DBSQLite3(const std::string &mapdir); - std::vector getBlockPos(BlockPos min, BlockPos max) override; + std::vector getBlockPosXZ(BlockPos min, BlockPos max) override; void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z, int16_t min_y, int16_t max_y) override; void getBlocksByPos(BlockList &blocks, diff --git a/src/db.h b/src/db.h index d0c5bf0..8e81ca5 100644 --- a/src/db.h +++ b/src/db.h @@ -6,18 +6,15 @@ #include #include "types.h" - struct BlockPos { - int16_t x; - int16_t y; - int16_t z; + int16_t x, y, z; - BlockPos() : x(0), y(0), z(0) {} - explicit BlockPos(int16_t v) : x(v), y(v), z(v) {} - BlockPos(int16_t x, int16_t y, int16_t z) : x(x), y(y), z(z) {} + constexpr BlockPos() : x(0), y(0), z(0) {} + explicit constexpr BlockPos(int16_t v) : x(v), y(v), z(v) {} + constexpr BlockPos(int16_t x, int16_t y, int16_t z) : x(x), y(y), z(z) {} // Implements the inverse ordering so that (2,2,2) < (1,1,1) - bool operator < (const BlockPos &p) const + inline bool operator<(const BlockPos &p) const { if (z > p.z) return true; @@ -27,11 +24,7 @@ struct BlockPos { return true; if (y < p.y) return false; - if (x > p.x) - return true; - if (x < p.x) - return false; - return false; + return x > p.x; } }; @@ -47,25 +40,28 @@ protected: static inline BlockPos decodeBlockPos(int64_t hash); public: - /* Return all block positions inside the range given by min and max, - * so that min.x <= x < max.x, ... + /* Return all unique (X, Z) position pairs inside area given by min and max, + * so that min.x <= x < max.x && min.z <= z < max.z + * Note: duplicates are allowed, but results in wasted time. */ - virtual std::vector getBlockPos(BlockPos min, BlockPos max) = 0; + virtual std::vector getBlockPosXZ(BlockPos min, BlockPos max) = 0; + /* Read all blocks in column given by x and z * and inside the given Y range (min_y <= y < max_y) into list */ virtual void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z, int16_t min_y, int16_t max_y) = 0; + /* Read blocks at given positions into list */ virtual void getBlocksByPos(BlockList &blocks, const std::vector &positions) = 0; + /* Can this database efficiently do range queries? * (for large data sets, more efficient that brute force) */ virtual bool preferRangeQueries() const = 0; - virtual ~DB() {} };