diff --git a/cmd/mtredisalize/sqlite.go b/cmd/mtredisalize/sqlite.go index 1e7ea63..4929f29 100644 --- a/cmd/mtredisalize/sqlite.go +++ b/cmd/mtredisalize/sqlite.go @@ -17,13 +17,18 @@ import ( var globalLock sync.RWMutex const ( - fetchSQL = "SELECT data FROM blocks WHERE pos = ?" - existsSQL = "SELECT 1 FROM blocks WHERE pos = ?" - updateSQL = "UPDATE blocks SET data = ? WHERE pos = ?" - insertSQL = "INSERT INTO blocks (pos, data) VALUES (?, ?)" - countSQL = "SELECT count(*) FROM blocks" - keysSQL = "SELECT pos FROM blocks" - rangeSQL = "SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ? ORDER BY pos" + fetchSQL = "SELECT data FROM blocks WHERE pos = ?" + existsSQL = "SELECT 1 FROM blocks WHERE pos = ?" + updateSQL = "UPDATE blocks SET data = ? WHERE pos = ?" + insertSQL = "INSERT INTO blocks (pos, data) VALUES (?, ?)" + countSQL = "SELECT count(*) FROM blocks" + keysSQL = "SELECT pos FROM blocks" + rangeSQL = "SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ? ORDER BY pos" + rangeDuffSQL = "SELECT pos, data FROM blocks WHERE " + + "pos BETWEEN ? AND ? OR " + + "pos BETWEEN ? AND ? OR " + + "pos BETWEEN ? AND ? OR " + + "pos BETWEEN ? AND ?" ) type SqliteBackend struct { @@ -32,6 +37,7 @@ type SqliteBackend struct { decoder common.KeyDecoder changeTracker *ChangeTracker interleaved bool + coverage *common.Coverage3D existsStmt *sql.Stmt fetchStmt *sql.Stmt insertStmt *sql.Stmt @@ -99,7 +105,14 @@ func NewSqliteBackend( return } - if res.rangeStmt, err = res.db.Prepare(rangeSQL); err != nil { + var rS string + if interleaved { + rS = rangeSQL + } else { + rS = rangeDuffSQL + } + + if res.rangeStmt, err = res.db.Prepare(rS); err != nil { res.closeAll() return } @@ -112,10 +125,38 @@ func NewSqliteBackend( res.decoder = common.DecodeStringFromBytes } + if !interleaved { + if err = res.buildCoverage(); err != nil { + return + } + } + sqlb = &res return } +func (sb *SqliteBackend) buildCoverage() (err error) { + log.Println("INFO: Start building coverage index (this may take some time)...") + sb.coverage = common.NewCoverage3D() + + var rows *sql.Rows + if rows, err = sb.keysStmt.Query(); err != nil { + return + } + defer rows.Close() + + for rows.Next() { + var key int64 + if err = rows.Scan(&key); err != nil { + return + } + sb.coverage.Insert(common.PlainToCoord(key)) + } + err = rows.Err() + log.Println("INFO: Finished building coverage index.") + return +} + func closeStmt(stmt **sql.Stmt) error { s := *stmt if s != nil { @@ -212,6 +253,10 @@ func (ss *SqliteSession) Store(hash, key, value []byte) (exists bool, err error) } // This technically too early because this done in transactions // which are commited (and possible fail) later. + if ss.backend.coverage != nil { + ss.backend.coverage.Insert(common.PlainToCoord(pos)) + } + if ss.backend.changeTracker != nil { ss.backend.changeTracker.BlockChanged(key) } @@ -376,6 +421,52 @@ func (ss *SqliteSession) interleavedSpatialQuery(first, second []byte, done chan return } +type duffStmt struct { + stmt *sql.Stmt + counter int + params [8]int64 +} + +func (ds *duffStmt) push(a, b int64) bool { + ds.params[ds.counter] = a + ds.counter++ + ds.params[ds.counter] = b + ds.counter++ + return ds.counter > 7 +} + +func (ds *duffStmt) Query() (*sql.Rows, error) { + c := ds.counter + ds.counter = 0 + switch c { + case 8: + return ds.stmt.Query( + ds.params[0], ds.params[1], + ds.params[2], ds.params[3], + ds.params[4], ds.params[5], + ds.params[0], ds.params[1]) + case 6: + return ds.stmt.Query( + ds.params[0], ds.params[1], + ds.params[2], ds.params[3], + ds.params[4], ds.params[5], + ds.params[0], ds.params[1]) + case 4: + return ds.stmt.Query( + ds.params[0], ds.params[1], + ds.params[2], ds.params[3], + ds.params[0], ds.params[1], + ds.params[0], ds.params[1]) + case 2: + return ds.stmt.Query( + ds.params[0], ds.params[1], + ds.params[0], ds.params[1], + ds.params[0], ds.params[1], + ds.params[0], ds.params[1]) + } + return nil, nil +} + func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struct{}) (blocks chan Block, err error) { var ( @@ -399,49 +490,73 @@ func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struc go func() { defer globalLock.RUnlock() defer close(blocks) - rangeStmt := ss.txStmt(ss.backend.rangeStmt) + rangeStmt := duffStmt{stmt: ss.txStmt(ss.backend.rangeStmt)} - a, b := common.Coord{X: c1.X}, common.Coord{X: c2.X} + send := func(rows *sql.Rows, err error) bool { + if rows == nil { + return true + } + if err != nil { + log.Printf("Error in range query: %s\n", err) + return false + } + defer rows.Close() - for a.Z = c1.Z; a.Z <= c2.Z; a.Z++ { - b.Z = a.Z - for a.Y = c1.Y; a.Y <= c2.Y; a.Y++ { - b.Y = a.Y - var ( - err error - rows *sql.Rows - ) - // Ordering should not be necessary. - from, to := order(common.CoordToPlain(a), common.CoordToPlain(b)) - if rows, err = rangeStmt.Query(from, to); err != nil { - log.Printf("Error in range query: %s\n", err) - return + for rows.Next() { + var key int64 + var data []byte + if err = rows.Scan(&key, &data); err != nil { + log.Printf("Error in scanning row: %s\n", err) + return false } - for rows.Next() { - var key int64 - var data []byte - if err = rows.Scan(&key, &data); err != nil { - log.Printf("Error in scanning row: %s\n", err) - break - } - var encodedKey []byte - if encodedKey, err = common.EncodeStringToBytes(key); err != nil { - log.Printf("Key encoding failed: %s\n", err) - break - } - select { - case blocks <- Block{Key: encodedKey, Data: data}: - case <-done: - rows.Close() + var encodedKey []byte + if encodedKey, err = common.EncodeStringToBytes(key); err != nil { + log.Printf("Key encoding failed: %s\n", err) + return false + } + select { + case blocks <- Block{Key: encodedKey, Data: data}: + case <-done: + return false + } + } + if err = rows.Err(); err != nil { + log.Printf("Error in range query: %s\n", err) + return false + } + return true + } + + if ss.backend.coverage == nil { + a, b := common.Coord{X: c1.X}, common.Coord{X: c2.X} + for a.Y = c2.Y; a.Y >= c1.Y; a.Y-- { + b.Y = a.Y + for a.Z = c1.Z; a.Z <= c2.Z; a.Z++ { + b.Z = a.Z + // Ordering should not be necessary. + from, to := order(common.CoordToPlain(a), common.CoordToPlain(b)) + if rangeStmt.push(from, to) && !send(rangeStmt.Query()) { return } } - if err = rows.Err(); err != nil { - log.Printf("Error in range query: %s\n", err) + } + } else { + var a, b common.Coord + + for _, r := range ss.backend.coverage.Query(c1, c2) { + a.Z, b.Z = int16(r.Z), int16(r.Z) + a.X, b.X = int16(r.X1), int16(r.X2) + // log.Printf("y1 y2 x1 x2 z: %d %d, %d %d, %d\n", r.Y1, r.Y2, r.X1, r.X2, r.Z) + for y := r.Y2; y >= r.Y1; y-- { + a.Y, b.Y = int16(y), int16(y) + from, to := order(common.CoordToPlain(a), common.CoordToPlain(b)) + if rangeStmt.push(from, to) && !send(rangeStmt.Query()) { + return + } } - rows.Close() } } + send(rangeStmt.Query()) }() return diff --git a/common/coords.go b/common/coords.go index 25af789..6aea9dc 100644 --- a/common/coords.go +++ b/common/coords.go @@ -45,20 +45,6 @@ func (c Coord) String() string { return fmt.Sprintf("(%d, %d, %d)", c.X, c.Y, c.Z) } -func minComponent(a, b int16) int16 { - if a < b { - return a - } - return b -} - -func maxComponent(a, b int16) int16 { - if a > b { - return a - } - return b -} - func clipComponent(x int16) int16 { if x < minValue { return minValue @@ -78,16 +64,16 @@ func ClipCoord(c Coord) Coord { func MinCoord(a, b Coord) Coord { return Coord{ - X: minComponent(a.X, b.X), - Y: minComponent(a.Y, b.Y), - Z: minComponent(a.Z, b.Z)} + X: min16(a.X, b.X), + Y: min16(a.Y, b.Y), + Z: min16(a.Z, b.Z)} } func MaxCoord(a, b Coord) Coord { return Coord{ - X: maxComponent(a.X, b.X), - Y: maxComponent(a.Y, b.Y), - Z: maxComponent(a.Z, b.Z)} + X: max16(a.X, b.X), + Y: max16(a.Y, b.Y), + Z: max16(a.Z, b.Z)} } // DecodeStringFromBytes constructs a database key out of byte slice. diff --git a/common/coverage.go b/common/coverage.go new file mode 100644 index 0000000..433b380 --- /dev/null +++ b/common/coverage.go @@ -0,0 +1,82 @@ +package common + +import "sync" + +type zRange struct { + y1 int16 + y2 int16 + xRange *Span +} + +type Coverage3D struct { + pool *SpanPool + zRanges map[int16]*zRange + mu sync.RWMutex +} + +type Range struct { + Z int16 + Y1 int16 + Y2 int16 + X1 int16 + X2 int16 +} + +func NewCoverage3D() *Coverage3D { + return &Coverage3D{ + pool: NewSpanPool(), + zRanges: map[int16]*zRange{}} +} + +func (s2d *Coverage3D) Insert(c Coord) { + s2d.mu.Lock() + defer s2d.mu.Unlock() + zr := s2d.zRanges[c.Z] + if zr == nil { + xr := s2d.pool.Alloc() + xr.From = int32(c.X) + xr.To = int32(c.X) + xr.Next = nil + s2d.zRanges[c.Z] = &zRange{ + y1: c.Y, + y2: c.Y, + xRange: xr} + return + } + zr.xRange = s2d.pool.Insert(zr.xRange, int32(c.X), 0) + if c.Y < zr.y1 { + zr.y1 = c.Y + } + if c.Y > zr.y2 { + zr.y2 = c.Y + } +} + +func (s2d *Coverage3D) Query(c1, c2 Coord) []Range { + + c1, c2 = MinCoord(c1, c2), MaxCoord(c1, c2) + + s2d.mu.RLock() + defer s2d.mu.RUnlock() + + r := make([]Range, 0, 32) + for z := c1.Z; z <= c2.Z; z++ { + zr := s2d.zRanges[z] + if zr == nil || c1.Y > zr.y2 || c2.Y < zr.y1 { + continue + } + y1, y2 := max16(c1.Y, zr.y1), min16(c2.Y, zr.y2) + for xr := zr.xRange; xr != nil && xr.From <= int32(c2.X); xr = xr.Next { + if xr.To < int32(c1.X) { + continue + } + r = append(r, Range{ + Z: z, + Y1: y1, + Y2: y2, + X1: max16(c1.X, int16(xr.From)), + X2: min16(c2.X, int16(xr.To))}) + } + } + return r +} diff --git a/common/math.go b/common/math.go new file mode 100644 index 0000000..dc16379 --- /dev/null +++ b/common/math.go @@ -0,0 +1,36 @@ +package common + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func max32(a, b int32) int32 { + if a > b { + return a + } + return b +} + +func min32(a, b int32) int32 { + if a < b { + return a + } + return b +} + +func max16(a, b int16) int16 { + if a > b { + return a + } + return b +} + +func min16(a, b int16) int16 { + if a < b { + return a + } + return b +} diff --git a/common/renderer.go b/common/renderer.go index b2b8e3e..1af1c8d 100644 --- a/common/renderer.go +++ b/common/renderer.go @@ -48,20 +48,6 @@ func (yo *YOrder) Reset() { yo.blocks = yo.blocks[0:0] } -func max(a, b int) int { - if a > b { - return a - } - return b -} - -func max32(a, b int32) int32 { - if a > b { - return a - } - return b -} - func copyData(data []byte) []byte { l := len(data) ndata := make([]byte, l, max(l, 8*1024))