From 1921b8211b45a59382cab1e2f11d41a24b781989 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 21 Jul 2015 22:01:10 +0200 Subject: [PATCH 1/9] Some little experiment with a coverage index to speed up spatial queries if backend has no Z order support. --- cmd/mtredisalize/sqlite.go | 199 +++++++++++++++++++++++++++++-------- common/coords.go | 26 ++--- common/coverage.go | 82 +++++++++++++++ common/math.go | 36 +++++++ common/renderer.go | 14 --- 5 files changed, 281 insertions(+), 76 deletions(-) create mode 100644 common/coverage.go create mode 100644 common/math.go diff --git a/cmd/mtredisalize/sqlite.go b/cmd/mtredisalize/sqlite.go index 1e7ea63..4929f29 100644 --- a/cmd/mtredisalize/sqlite.go +++ b/cmd/mtredisalize/sqlite.go @@ -17,13 +17,18 @@ import ( var globalLock sync.RWMutex const ( - fetchSQL = "SELECT data FROM blocks WHERE pos = ?" - existsSQL = "SELECT 1 FROM blocks WHERE pos = ?" - updateSQL = "UPDATE blocks SET data = ? WHERE pos = ?" - insertSQL = "INSERT INTO blocks (pos, data) VALUES (?, ?)" - countSQL = "SELECT count(*) FROM blocks" - keysSQL = "SELECT pos FROM blocks" - rangeSQL = "SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ? ORDER BY pos" + fetchSQL = "SELECT data FROM blocks WHERE pos = ?" + existsSQL = "SELECT 1 FROM blocks WHERE pos = ?" + updateSQL = "UPDATE blocks SET data = ? WHERE pos = ?" + insertSQL = "INSERT INTO blocks (pos, data) VALUES (?, ?)" + countSQL = "SELECT count(*) FROM blocks" + keysSQL = "SELECT pos FROM blocks" + rangeSQL = "SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ? ORDER BY pos" + rangeDuffSQL = "SELECT pos, data FROM blocks WHERE " + + "pos BETWEEN ? AND ? OR " + + "pos BETWEEN ? AND ? OR " + + "pos BETWEEN ? AND ? OR " + + "pos BETWEEN ? AND ?" ) type SqliteBackend struct { @@ -32,6 +37,7 @@ type SqliteBackend struct { decoder common.KeyDecoder changeTracker *ChangeTracker interleaved bool + coverage *common.Coverage3D existsStmt *sql.Stmt fetchStmt *sql.Stmt insertStmt *sql.Stmt @@ -99,7 +105,14 @@ func NewSqliteBackend( return } - if res.rangeStmt, err = res.db.Prepare(rangeSQL); err != nil { + var rS string + if interleaved { + rS = rangeSQL + } else { + rS = rangeDuffSQL + } + + if res.rangeStmt, err = res.db.Prepare(rS); err != nil { res.closeAll() return } @@ -112,10 +125,38 @@ func NewSqliteBackend( res.decoder = common.DecodeStringFromBytes } + if !interleaved { + if err = res.buildCoverage(); err != nil { + return + } + } + sqlb = &res return } +func (sb *SqliteBackend) buildCoverage() (err error) { + log.Println("INFO: Start building coverage index (this may take some time)...") + sb.coverage = common.NewCoverage3D() + + var rows *sql.Rows + if rows, err = sb.keysStmt.Query(); err != nil { + return + } + defer rows.Close() + + for rows.Next() { + var key int64 + if err = rows.Scan(&key); err != nil { + return + } + sb.coverage.Insert(common.PlainToCoord(key)) + } + err = rows.Err() + log.Println("INFO: Finished building coverage index.") + return +} + func closeStmt(stmt **sql.Stmt) error { s := *stmt if s != nil { @@ -212,6 +253,10 @@ func (ss *SqliteSession) Store(hash, key, value []byte) (exists bool, err error) } // This technically too early because this done in transactions // which are commited (and possible fail) later. + if ss.backend.coverage != nil { + ss.backend.coverage.Insert(common.PlainToCoord(pos)) + } + if ss.backend.changeTracker != nil { ss.backend.changeTracker.BlockChanged(key) } @@ -376,6 +421,52 @@ func (ss *SqliteSession) interleavedSpatialQuery(first, second []byte, done chan return } +type duffStmt struct { + stmt *sql.Stmt + counter int + params [8]int64 +} + +func (ds *duffStmt) push(a, b int64) bool { + ds.params[ds.counter] = a + ds.counter++ + ds.params[ds.counter] = b + ds.counter++ + return ds.counter > 7 +} + +func (ds *duffStmt) Query() (*sql.Rows, error) { + c := ds.counter + ds.counter = 0 + switch c { + case 8: + return ds.stmt.Query( + ds.params[0], ds.params[1], + ds.params[2], ds.params[3], + ds.params[4], ds.params[5], + ds.params[0], ds.params[1]) + case 6: + return ds.stmt.Query( + ds.params[0], ds.params[1], + ds.params[2], ds.params[3], + ds.params[4], ds.params[5], + ds.params[0], ds.params[1]) + case 4: + return ds.stmt.Query( + ds.params[0], ds.params[1], + ds.params[2], ds.params[3], + ds.params[0], ds.params[1], + ds.params[0], ds.params[1]) + case 2: + return ds.stmt.Query( + ds.params[0], ds.params[1], + ds.params[0], ds.params[1], + ds.params[0], ds.params[1], + ds.params[0], ds.params[1]) + } + return nil, nil +} + func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struct{}) (blocks chan Block, err error) { var ( @@ -399,49 +490,73 @@ func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struc go func() { defer globalLock.RUnlock() defer close(blocks) - rangeStmt := ss.txStmt(ss.backend.rangeStmt) + rangeStmt := duffStmt{stmt: ss.txStmt(ss.backend.rangeStmt)} - a, b := common.Coord{X: c1.X}, common.Coord{X: c2.X} + send := func(rows *sql.Rows, err error) bool { + if rows == nil { + return true + } + if err != nil { + log.Printf("Error in range query: %s\n", err) + return false + } + defer rows.Close() - for a.Z = c1.Z; a.Z <= c2.Z; a.Z++ { - b.Z = a.Z - for a.Y = c1.Y; a.Y <= c2.Y; a.Y++ { - b.Y = a.Y - var ( - err error - rows *sql.Rows - ) - // Ordering should not be necessary. - from, to := order(common.CoordToPlain(a), common.CoordToPlain(b)) - if rows, err = rangeStmt.Query(from, to); err != nil { - log.Printf("Error in range query: %s\n", err) - return + for rows.Next() { + var key int64 + var data []byte + if err = rows.Scan(&key, &data); err != nil { + log.Printf("Error in scanning row: %s\n", err) + return false } - for rows.Next() { - var key int64 - var data []byte - if err = rows.Scan(&key, &data); err != nil { - log.Printf("Error in scanning row: %s\n", err) - break - } - var encodedKey []byte - if encodedKey, err = common.EncodeStringToBytes(key); err != nil { - log.Printf("Key encoding failed: %s\n", err) - break - } - select { - case blocks <- Block{Key: encodedKey, Data: data}: - case <-done: - rows.Close() + var encodedKey []byte + if encodedKey, err = common.EncodeStringToBytes(key); err != nil { + log.Printf("Key encoding failed: %s\n", err) + return false + } + select { + case blocks <- Block{Key: encodedKey, Data: data}: + case <-done: + return false + } + } + if err = rows.Err(); err != nil { + log.Printf("Error in range query: %s\n", err) + return false + } + return true + } + + if ss.backend.coverage == nil { + a, b := common.Coord{X: c1.X}, common.Coord{X: c2.X} + for a.Y = c2.Y; a.Y >= c1.Y; a.Y-- { + b.Y = a.Y + for a.Z = c1.Z; a.Z <= c2.Z; a.Z++ { + b.Z = a.Z + // Ordering should not be necessary. + from, to := order(common.CoordToPlain(a), common.CoordToPlain(b)) + if rangeStmt.push(from, to) && !send(rangeStmt.Query()) { return } } - if err = rows.Err(); err != nil { - log.Printf("Error in range query: %s\n", err) + } + } else { + var a, b common.Coord + + for _, r := range ss.backend.coverage.Query(c1, c2) { + a.Z, b.Z = int16(r.Z), int16(r.Z) + a.X, b.X = int16(r.X1), int16(r.X2) + // log.Printf("y1 y2 x1 x2 z: %d %d, %d %d, %d\n", r.Y1, r.Y2, r.X1, r.X2, r.Z) + for y := r.Y2; y >= r.Y1; y-- { + a.Y, b.Y = int16(y), int16(y) + from, to := order(common.CoordToPlain(a), common.CoordToPlain(b)) + if rangeStmt.push(from, to) && !send(rangeStmt.Query()) { + return + } } - rows.Close() } } + send(rangeStmt.Query()) }() return diff --git a/common/coords.go b/common/coords.go index 25af789..6aea9dc 100644 --- a/common/coords.go +++ b/common/coords.go @@ -45,20 +45,6 @@ func (c Coord) String() string { return fmt.Sprintf("(%d, %d, %d)", c.X, c.Y, c.Z) } -func minComponent(a, b int16) int16 { - if a < b { - return a - } - return b -} - -func maxComponent(a, b int16) int16 { - if a > b { - return a - } - return b -} - func clipComponent(x int16) int16 { if x < minValue { return minValue @@ -78,16 +64,16 @@ func ClipCoord(c Coord) Coord { func MinCoord(a, b Coord) Coord { return Coord{ - X: minComponent(a.X, b.X), - Y: minComponent(a.Y, b.Y), - Z: minComponent(a.Z, b.Z)} + X: min16(a.X, b.X), + Y: min16(a.Y, b.Y), + Z: min16(a.Z, b.Z)} } func MaxCoord(a, b Coord) Coord { return Coord{ - X: maxComponent(a.X, b.X), - Y: maxComponent(a.Y, b.Y), - Z: maxComponent(a.Z, b.Z)} + X: max16(a.X, b.X), + Y: max16(a.Y, b.Y), + Z: max16(a.Z, b.Z)} } // DecodeStringFromBytes constructs a database key out of byte slice. diff --git a/common/coverage.go b/common/coverage.go new file mode 100644 index 0000000..433b380 --- /dev/null +++ b/common/coverage.go @@ -0,0 +1,82 @@ +package common + +import "sync" + +type zRange struct { + y1 int16 + y2 int16 + xRange *Span +} + +type Coverage3D struct { + pool *SpanPool + zRanges map[int16]*zRange + mu sync.RWMutex +} + +type Range struct { + Z int16 + Y1 int16 + Y2 int16 + X1 int16 + X2 int16 +} + +func NewCoverage3D() *Coverage3D { + return &Coverage3D{ + pool: NewSpanPool(), + zRanges: map[int16]*zRange{}} +} + +func (s2d *Coverage3D) Insert(c Coord) { + s2d.mu.Lock() + defer s2d.mu.Unlock() + zr := s2d.zRanges[c.Z] + if zr == nil { + xr := s2d.pool.Alloc() + xr.From = int32(c.X) + xr.To = int32(c.X) + xr.Next = nil + s2d.zRanges[c.Z] = &zRange{ + y1: c.Y, + y2: c.Y, + xRange: xr} + return + } + zr.xRange = s2d.pool.Insert(zr.xRange, int32(c.X), 0) + if c.Y < zr.y1 { + zr.y1 = c.Y + } + if c.Y > zr.y2 { + zr.y2 = c.Y + } +} + +func (s2d *Coverage3D) Query(c1, c2 Coord) []Range { + + c1, c2 = MinCoord(c1, c2), MaxCoord(c1, c2) + + s2d.mu.RLock() + defer s2d.mu.RUnlock() + + r := make([]Range, 0, 32) + for z := c1.Z; z <= c2.Z; z++ { + zr := s2d.zRanges[z] + if zr == nil || c1.Y > zr.y2 || c2.Y < zr.y1 { + continue + } + y1, y2 := max16(c1.Y, zr.y1), min16(c2.Y, zr.y2) + for xr := zr.xRange; xr != nil && xr.From <= int32(c2.X); xr = xr.Next { + if xr.To < int32(c1.X) { + continue + } + r = append(r, Range{ + Z: z, + Y1: y1, + Y2: y2, + X1: max16(c1.X, int16(xr.From)), + X2: min16(c2.X, int16(xr.To))}) + } + } + return r +} diff --git a/common/math.go b/common/math.go new file mode 100644 index 0000000..dc16379 --- /dev/null +++ b/common/math.go @@ -0,0 +1,36 @@ +package common + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func max32(a, b int32) int32 { + if a > b { + return a + } + return b +} + +func min32(a, b int32) int32 { + if a < b { + return a + } + return b +} + +func max16(a, b int16) int16 { + if a > b { + return a + } + return b +} + +func min16(a, b int16) int16 { + if a < b { + return a + } + return b +} diff --git a/common/renderer.go b/common/renderer.go index b2b8e3e..1af1c8d 100644 --- a/common/renderer.go +++ b/common/renderer.go @@ -48,20 +48,6 @@ func (yo *YOrder) Reset() { yo.blocks = yo.blocks[0:0] } -func max(a, b int) int { - if a > b { - return a - } - return b -} - -func max32(a, b int32) int32 { - if a > b { - return a - } - return b -} - func copyData(data []byte) []byte { l := len(data) ndata := make([]byte, l, max(l, 8*1024)) From fb2b03a14a7195fd542c38aba0620bf4e0b92de2 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 21 Jul 2015 22:09:28 +0200 Subject: [PATCH 2/9] Fix reciever naming in coverage 3d. --- common/coverage.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/common/coverage.go b/common/coverage.go index 433b380..1146aab 100644 --- a/common/coverage.go +++ b/common/coverage.go @@ -28,22 +28,22 @@ func NewCoverage3D() *Coverage3D { zRanges: map[int16]*zRange{}} } -func (s2d *Coverage3D) Insert(c Coord) { - s2d.mu.Lock() - defer s2d.mu.Unlock() - zr := s2d.zRanges[c.Z] +func (c3d *Coverage3D) Insert(c Coord) { + c3d.mu.Lock() + defer c3d.mu.Unlock() + zr := c3d.zRanges[c.Z] if zr == nil { - xr := s2d.pool.Alloc() + xr := c3d.pool.Alloc() xr.From = int32(c.X) xr.To = int32(c.X) xr.Next = nil - s2d.zRanges[c.Z] = &zRange{ + c3d.zRanges[c.Z] = &zRange{ y1: c.Y, y2: c.Y, xRange: xr} return } - zr.xRange = s2d.pool.Insert(zr.xRange, int32(c.X), 0) + zr.xRange = c3d.pool.Insert(zr.xRange, int32(c.X), 0) if c.Y < zr.y1 { zr.y1 = c.Y } @@ -52,16 +52,16 @@ func (s2d *Coverage3D) Insert(c Coord) { } } -func (s2d *Coverage3D) Query(c1, c2 Coord) []Range { +func (c3d *Coverage3D) Query(c1, c2 Coord) []Range { c1, c2 = MinCoord(c1, c2), MaxCoord(c1, c2) - s2d.mu.RLock() - defer s2d.mu.RUnlock() + c3d.mu.RLock() + defer c3d.mu.RUnlock() r := make([]Range, 0, 32) for z := c1.Z; z <= c2.Z; z++ { - zr := s2d.zRanges[z] + zr := c3d.zRanges[z] if zr == nil || c1.Y > zr.y2 || c2.Y < zr.y1 { continue } From e8e4d6afeb1efb4679c8dcac00fe573d62bf85c5 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Wed, 22 Jul 2015 01:11:14 +0200 Subject: [PATCH 3/9] Cache empty tiles to speed up seeding. Enforce Go 1.4 for the ability to compress better. --- README.md | 4 ++-- common/basetilecreator.go | 36 +++++++++++++++++++++++++++--------- common/image.go | 10 ++++++++++ common/renderer.go | 11 +++++++++++ 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 872c419..be590a5 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ on YouTube. A live map of an online world can be viewed [here](http://maps.mt.sha-bang.de/). See [COMPILE](https://bitbucket.org/s_l_teichmann/mtsatellite/src/default/COMPILE.md) how to compile -MTSatellite. Essentially you need Go 1.3 (or higher) and a GNU/Linux system. +MTSatellite. Essentially you need Go 1.4 (or higher) and a GNU/Linux system. See [SETUP](https://bitbucket.org/s_l_teichmann/mtsatellite/src/default/SETUP.md) how to bring MTSatellite to life. @@ -39,4 +39,4 @@ the map has to be pre-rendered with **mtseeder**. This is Free Software under the terms of the MIT license. See [LICENSE](LICENSE) file for details. -(c) 2014 by Sascha L. Teichmann \ No newline at end of file +(c) 2014 by Sascha L. Teichmann diff --git a/common/basetilecreator.go b/common/basetilecreator.go index 83fb8a2..84e2003 100644 --- a/common/basetilecreator.go +++ b/common/basetilecreator.go @@ -6,6 +6,7 @@ package common import ( "image/color" + "io/ioutil" "log" "path/filepath" "strconv" @@ -41,13 +42,16 @@ var tileDepths = [...][2]int16{ {-1024, -257}, {-1934, -1025}} +var BackgroundColor = color.RGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xff} + type BaseTileCreator struct { - client *RedisClient - colors *Colors - renderer *Renderer - yOrder *YOrder - baseDir string - update bool + client *RedisClient + colors *Colors + renderer *Renderer + yOrder *YOrder + baseDir string + update bool + emptyImage []byte } func NewBaseTileCreator( @@ -116,11 +120,25 @@ func (btc *BaseTileCreator) CreateTile(x, z int16, i, j int) error { oareas, nareas = nareas, oareas[0:0] } + path := filepath.Join(btc.baseDir, strconv.Itoa(i), strconv.Itoa(j)+".png") + + // Empty images are likely to be produced during seeding. + if !btc.update && btc.renderer.IsEmpty() { + // To avoid redundant encoding cache the resulting empty image. + if btc.emptyImage == nil { + var err error + m := BackgroundImage((tileWidth-2)*16, (tileHeight-2)*16, BackgroundColor) + if btc.emptyImage, err = EncodeToMem(m); err != nil { + return err + } + } + log.Printf("Writing empty (%d, %d) to file %s\n", x, z, path) + return ioutil.WriteFile(path, btc.emptyImage, 0666) + } + image := btc.renderer.CreateShadedImage( 16, 16, (tileWidth-2)*16, (tileHeight-2)*16, - btc.colors, color.RGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xff}) - - path := filepath.Join(btc.baseDir, strconv.Itoa(i), strconv.Itoa(j)+".png") + btc.colors, BackgroundColor) log.Printf("Writing (%d, %d) to file %s\n", x, z, path) diff --git a/common/image.go b/common/image.go index 1e2f81f..8ac0d98 100644 --- a/common/image.go +++ b/common/image.go @@ -6,6 +6,7 @@ package common import ( "bufio" + "bytes" "errors" "image" "image/png" @@ -35,6 +36,15 @@ func nextSuffix() string { return strconv.Itoa(int(1e9 + r%1e9))[1:] } +func EncodeToMem(img image.Image) ([]byte, error) { + var buf bytes.Buffer + enc := png.Encoder{png.BestCompression} + if err := enc.Encode(&buf, img); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + func SaveAsPNG(path string, img image.Image) (err error) { var file *os.File if file, err = os.Create(path); err != nil { diff --git a/common/renderer.go b/common/renderer.go index 1af1c8d..778de37 100644 --- a/common/renderer.go +++ b/common/renderer.go @@ -8,6 +8,7 @@ import ( "container/heap" "image" "image/color" + "image/draw" "math" ) @@ -181,6 +182,10 @@ func (r *Renderer) IsFilled() bool { return true } +func (r *Renderer) IsEmpty() bool { + return r.SolidBlocks == 0 && r.TransparentBlocks == 0 +} + func (r *Renderer) RenderBlock(block *Block, colors *Colors) (err error) { bx := block.Coord.X - r.xOfs @@ -392,6 +397,12 @@ func safeColor(x int32) uint8 { } } +func BackgroundImage(width, height int, bg color.RGBA) *image.RGBA { + m := image.NewRGBA(image.Rect(0, 0, width, height)) + draw.Draw(m, m.Bounds(), &image.Uniform{bg}, image.ZP, draw.Src) + return m +} + func (r *Renderer) CreateShadedImage( xOfs, zOfs, width, height int, cols *Colors, background color.RGBA) *image.RGBA { From 93d4aedffa354dfc64821d2f49d7adb2be5e392c Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Thu, 23 Jul 2015 16:20:42 +0200 Subject: [PATCH 4/9] Fixed non-interleaved sqlite support. --- cmd/mtredisalize/sqlite.go | 124 +++++++++---------------------------- common/basetilecreator.go | 2 +- 2 files changed, 29 insertions(+), 97 deletions(-) diff --git a/cmd/mtredisalize/sqlite.go b/cmd/mtredisalize/sqlite.go index 4929f29..e72b890 100644 --- a/cmd/mtredisalize/sqlite.go +++ b/cmd/mtredisalize/sqlite.go @@ -17,18 +17,13 @@ import ( var globalLock sync.RWMutex const ( - fetchSQL = "SELECT data FROM blocks WHERE pos = ?" - existsSQL = "SELECT 1 FROM blocks WHERE pos = ?" - updateSQL = "UPDATE blocks SET data = ? WHERE pos = ?" - insertSQL = "INSERT INTO blocks (pos, data) VALUES (?, ?)" - countSQL = "SELECT count(*) FROM blocks" - keysSQL = "SELECT pos FROM blocks" - rangeSQL = "SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ? ORDER BY pos" - rangeDuffSQL = "SELECT pos, data FROM blocks WHERE " + - "pos BETWEEN ? AND ? OR " + - "pos BETWEEN ? AND ? OR " + - "pos BETWEEN ? AND ? OR " + - "pos BETWEEN ? AND ?" + fetchSQL = "SELECT data FROM blocks WHERE pos = ?" + existsSQL = "SELECT 1 FROM blocks WHERE pos = ?" + updateSQL = "UPDATE blocks SET data = ? WHERE pos = ?" + insertSQL = "INSERT INTO blocks (pos, data) VALUES (?, ?)" + countSQL = "SELECT count(*) FROM blocks" + keysSQL = "SELECT pos FROM blocks" + rangeSQL = "SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ? ORDER BY pos" ) type SqliteBackend struct { @@ -105,14 +100,7 @@ func NewSqliteBackend( return } - var rS string - if interleaved { - rS = rangeSQL - } else { - rS = rangeDuffSQL - } - - if res.rangeStmt, err = res.db.Prepare(rS); err != nil { + if res.rangeStmt, err = res.db.Prepare(rangeSQL); err != nil { res.closeAll() return } @@ -421,52 +409,6 @@ func (ss *SqliteSession) interleavedSpatialQuery(first, second []byte, done chan return } -type duffStmt struct { - stmt *sql.Stmt - counter int - params [8]int64 -} - -func (ds *duffStmt) push(a, b int64) bool { - ds.params[ds.counter] = a - ds.counter++ - ds.params[ds.counter] = b - ds.counter++ - return ds.counter > 7 -} - -func (ds *duffStmt) Query() (*sql.Rows, error) { - c := ds.counter - ds.counter = 0 - switch c { - case 8: - return ds.stmt.Query( - ds.params[0], ds.params[1], - ds.params[2], ds.params[3], - ds.params[4], ds.params[5], - ds.params[0], ds.params[1]) - case 6: - return ds.stmt.Query( - ds.params[0], ds.params[1], - ds.params[2], ds.params[3], - ds.params[4], ds.params[5], - ds.params[0], ds.params[1]) - case 4: - return ds.stmt.Query( - ds.params[0], ds.params[1], - ds.params[2], ds.params[3], - ds.params[0], ds.params[1], - ds.params[0], ds.params[1]) - case 2: - return ds.stmt.Query( - ds.params[0], ds.params[1], - ds.params[0], ds.params[1], - ds.params[0], ds.params[1], - ds.params[0], ds.params[1]) - } - return nil, nil -} - func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struct{}) (blocks chan Block, err error) { var ( @@ -490,18 +432,17 @@ func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struc go func() { defer globalLock.RUnlock() defer close(blocks) - rangeStmt := duffStmt{stmt: ss.txStmt(ss.backend.rangeStmt)} + rangeStmt := ss.txStmt(ss.backend.rangeStmt) send := func(rows *sql.Rows, err error) bool { - if rows == nil { - return true - } if err != nil { log.Printf("Error in range query: %s\n", err) return false } defer rows.Close() + count := 0 + for rows.Next() { var key int64 var data []byte @@ -519,6 +460,7 @@ func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struc case <-done: return false } + count++ } if err = rows.Err(); err != nil { log.Printf("Error in range query: %s\n", err) @@ -527,37 +469,27 @@ func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struc return true } - if ss.backend.coverage == nil { - a, b := common.Coord{X: c1.X}, common.Coord{X: c2.X} - for a.Y = c2.Y; a.Y >= c1.Y; a.Y-- { - b.Y = a.Y - for a.Z = c1.Z; a.Z <= c2.Z; a.Z++ { - b.Z = a.Z - // Ordering should not be necessary. - from, to := order(common.CoordToPlain(a), common.CoordToPlain(b)) - if rangeStmt.push(from, to) && !send(rangeStmt.Query()) { - return - } - } - } - } else { - var a, b common.Coord + var a, b common.Coord - for _, r := range ss.backend.coverage.Query(c1, c2) { - a.Z, b.Z = int16(r.Z), int16(r.Z) - a.X, b.X = int16(r.X1), int16(r.X2) - // log.Printf("y1 y2 x1 x2 z: %d %d, %d %d, %d\n", r.Y1, r.Y2, r.X1, r.X2, r.Z) - for y := r.Y2; y >= r.Y1; y-- { - a.Y, b.Y = int16(y), int16(y) - from, to := order(common.CoordToPlain(a), common.CoordToPlain(b)) - if rangeStmt.push(from, to) && !send(rangeStmt.Query()) { - return - } + for _, r := range ss.backend.coverage.Query(c1, c2) { + a.Z, b.Z = int16(r.Z), int16(r.Z) + a.X, b.X = int16(r.X1), int16(r.X2) + // log.Printf("y1 y2 x1 x2 z: %d %d, %d %d, %d\n", r.Y1, r.Y2, r.X1, r.X2, r.Z) + for a.Y = r.Y2; a.Y >= r.Y1; a.Y-- { + b.Y = a.Y + from, to := common.CoordToPlain(a), common.CoordToPlain(b) + if !send(rangeStmt.Query(from, to)) { + return } } } - send(rangeStmt.Query()) }() return } + +func clone(a []byte) []byte { + b := make([]byte, len(a)) + copy(b, a) + return b +} diff --git a/common/basetilecreator.go b/common/basetilecreator.go index 84e2003..5517580 100644 --- a/common/basetilecreator.go +++ b/common/basetilecreator.go @@ -132,7 +132,7 @@ func (btc *BaseTileCreator) CreateTile(x, z int16, i, j int) error { return err } } - log.Printf("Writing empty (%d, %d) to file %s\n", x, z, path) + //log.Printf("Writing empty (%d, %d) to file %s\n", x, z, path) return ioutil.WriteFile(path, btc.emptyImage, 0666) } From 3fc89ad0f5b981bda8d2d4c25c3c3ef2c2199aac Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Thu, 23 Jul 2015 21:14:35 +0200 Subject: [PATCH 5/9] Removed dead code. --- cmd/mtredisalize/sqlite.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cmd/mtredisalize/sqlite.go b/cmd/mtredisalize/sqlite.go index e72b890..9f827e5 100644 --- a/cmd/mtredisalize/sqlite.go +++ b/cmd/mtredisalize/sqlite.go @@ -441,8 +441,6 @@ func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struc } defer rows.Close() - count := 0 - for rows.Next() { var key int64 var data []byte @@ -460,7 +458,6 @@ func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struc case <-done: return false } - count++ } if err = rows.Err(); err != nil { log.Printf("Error in range query: %s\n", err) @@ -487,9 +484,3 @@ func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struc return } - -func clone(a []byte) []byte { - b := make([]byte, len(a)) - copy(b, a) - return b -} From d53cea250ce99dd4a9780219b6831ca9f1c518af Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Fri, 24 Jul 2015 08:13:24 +0200 Subject: [PATCH 6/9] Started coverage index support for non-interleaved LevelDb backends. --- cmd/mtredisalize/leveldb.go | 52 +++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/cmd/mtredisalize/leveldb.go b/cmd/mtredisalize/leveldb.go index 23c8109..f8ede70 100644 --- a/cmd/mtredisalize/leveldb.go +++ b/cmd/mtredisalize/leveldb.go @@ -14,11 +14,13 @@ import ( ) type LevelDBBackend struct { - cache *leveldb.Cache - db *leveldb.DB - interleaved bool - encoder common.KeyTranscoder - decoder common.KeyTranscoder + cache *leveldb.Cache + db *leveldb.DB + interleaved bool + coverage *common.Coverage3D + encoder common.KeyTranscoder + decoder common.KeyTranscoder + changeTracker *ChangeTracker mutex sync.RWMutex } @@ -71,9 +73,42 @@ func NewLeveDBBackend( encoder: encoder, decoder: decoder, changeTracker: changeTracker} + + if !interleaved { + if err = ldb.buildCoverage(); err != nil { + ldb.Shutdown() + ldb = nil + return + } + } return } +func (ldb *LevelDBBackend) buildCoverage() error { + log.Println("INFO: Start building coverage index (this may take some time)...") + + coverage := common.NewCoverage3D() + + ro := leveldb.NewReadOptions() + defer ro.Close() + ro.SetFillCache(false) + it := ldb.db.NewIterator(ro) + it.SeekToFirst() + for ; it.Valid(); it.Next() { + c, err := common.DecodeStringBytesToCoord(it.Key()) + if err != nil { + return err + } + coverage.Insert(c) + } + if err := it.GetError(); err != nil { + return err + } + ldb.coverage = coverage + log.Println("INFO: Finished building coverage index.") + return nil +} + func (ldb *LevelDBBackend) doRead(f func(db *leveldb.DB)) { ldb.mutex.RLock() f(ldb.db) @@ -156,6 +191,13 @@ func (ldbs *LevelDBSession) Store(hash, key, value []byte) (exists bool, err err }) // This technically too early because this done in transactions // which are commited (and possible fail) later. + if ldbs.backend.coverage != nil { + c, err := common.DecodeStringBytesToCoord(origKey) + if err != nil { + return false, err + } + ldbs.backend.coverage.Insert(c) + } if ldbs.backend.changeTracker != nil { ldbs.backend.changeTracker.BlockChanged(origKey) } From 29eeb5e3016e12da5bd3867e6ba582503f74404e Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Fri, 24 Jul 2015 08:30:52 +0200 Subject: [PATCH 7/9] Implemented usage of coverage index in non-interleaved LevelDB backend. Untested, yet! --- cmd/mtredisalize/leveldb.go | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/cmd/mtredisalize/leveldb.go b/cmd/mtredisalize/leveldb.go index f8ede70..b1307ca 100644 --- a/cmd/mtredisalize/leveldb.go +++ b/cmd/mtredisalize/leveldb.go @@ -274,14 +274,19 @@ func (ldbs *LevelDBSession) AllKeys(hash []byte, done chan struct{}) (keys chan return } -func (ldbs *LevelDBSession) SpatialQuery(hash, first, second []byte, done chan struct{}) (chan Block, error) { +func (ldbs *LevelDBSession) SpatialQuery( + hash, first, second []byte, + done chan struct{}) (chan Block, error) { + if ldbs.backend.interleaved { return ldbs.interleavedSpatialQuery(first, second, done) } return ldbs.plainSpatialQuery(first, second, done) } -func (ldbs *LevelDBSession) plainSpatialQuery(first, second []byte, done chan struct{}) (blocks chan Block, err error) { +func (ldbs *LevelDBSession) plainSpatialQuery( + first, second []byte, + done chan struct{}) (blocks chan Block, err error) { var ( firstKey int64 @@ -311,21 +316,20 @@ func (ldbs *LevelDBSession) plainSpatialQuery(first, second []byte, done chan st it := ldbs.backend.db.NewIterator(ro) defer it.Close() - a, b := common.Coord{X: c1.X}, common.Coord{X: c2.X} - + var a, b common.Coord var err error - for a.Z = c1.Z; a.Z <= c2.Z; a.Z++ { - b.Z = a.Z - for a.Y = c1.Y; a.Y <= c2.Y; a.Y++ { + for _, r := range ldbs.backend.coverage.Query(c1, c2) { + a.Z, b.Z = int16(r.Z), int16(r.Z) + a.X, b.X = int16(r.X1), int16(r.X2) + // log.Printf("y1 y2 x1 x2 z: %d %d, %d %d, %d\n", r.Y1, r.Y2, r.X1, r.X2, r.Z) + for a.Y = r.Y2; a.Y >= r.Y1; a.Y-- { b.Y = a.Y - from, to := order(common.CoordToPlain(a), common.CoordToPlain(b)) + from, to := common.CoordToPlain(a), common.CoordToPlain(b) it.Seek(common.StringToBytes(from)) for ; it.Valid(); it.Next() { - var ( - key = it.Key() - pos int64 - ) + key := it.Key() + var pos int64 if pos, err = common.DecodeStringFromBytes(key); err != nil { log.Printf("decoding key failed: %s\n", err) return @@ -349,7 +353,10 @@ func (ldbs *LevelDBSession) plainSpatialQuery(first, second []byte, done chan st return } -func (ldbs *LevelDBSession) interleavedSpatialQuery(first, second []byte, done chan struct{}) (blocks chan Block, err error) { +func (ldbs *LevelDBSession) interleavedSpatialQuery( + first, second []byte, + done chan struct{}) (blocks chan Block, err error) { + var ( firstKey int64 secondKey int64 From fe6a551f06d511674fafb5766877320712fe0ed8 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Fri, 24 Jul 2015 10:01:39 +0200 Subject: [PATCH 8/9] Break some over-long lines. --- cmd/mtredisalize/leveldb.go | 5 ++++- cmd/mtredisalize/sqlite.go | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/cmd/mtredisalize/leveldb.go b/cmd/mtredisalize/leveldb.go index b1307ca..caf0977 100644 --- a/cmd/mtredisalize/leveldb.go +++ b/cmd/mtredisalize/leveldb.go @@ -226,7 +226,10 @@ func (ldbs *LevelDBSession) CommitTransaction() (err error) { return } -func (ldbs *LevelDBSession) AllKeys(hash []byte, done chan struct{}) (keys chan []byte, n int, err error) { +func (ldbs *LevelDBSession) AllKeys( + hash []byte, + done chan struct{}) (keys chan []byte, n int, err error) { + ldbs.backend.mutex.RLock() ro := leveldb.NewReadOptions() diff --git a/cmd/mtredisalize/sqlite.go b/cmd/mtredisalize/sqlite.go index 9f827e5..b10abf8 100644 --- a/cmd/mtredisalize/sqlite.go +++ b/cmd/mtredisalize/sqlite.go @@ -277,7 +277,9 @@ func (ss *SqliteSession) CommitTransaction() error { return tx.Commit() } -func (ss *SqliteSession) AllKeys(hash []byte, done chan struct{}) (keys chan []byte, n int, err error) { +func (ss *SqliteSession) AllKeys( + hash []byte, + done chan struct{}) (keys chan []byte, n int, err error) { globalLock.RLock() countStmt := ss.txStmt(ss.backend.countStmt) @@ -324,7 +326,9 @@ func (ss *SqliteSession) AllKeys(hash []byte, done chan struct{}) (keys chan []b return } -func (ss *SqliteSession) SpatialQuery(hash, first, second []byte, done chan struct{}) (chan Block, error) { +func (ss *SqliteSession) SpatialQuery( + hash, first, second []byte, + done chan struct{}) (chan Block, error) { if ss.backend.interleaved { return ss.interleavedSpatialQuery(first, second, done) @@ -333,7 +337,9 @@ func (ss *SqliteSession) SpatialQuery(hash, first, second []byte, done chan stru return ss.plainSpatialQuery(first, second, done) } -func (ss *SqliteSession) interleavedSpatialQuery(first, second []byte, done chan struct{}) (blocks chan Block, err error) { +func (ss *SqliteSession) interleavedSpatialQuery( + first, second []byte, + done chan struct{}) (blocks chan Block, err error) { var ( firstKey int64 secondKey int64 @@ -409,7 +415,9 @@ func (ss *SqliteSession) interleavedSpatialQuery(first, second []byte, done chan return } -func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struct{}) (blocks chan Block, err error) { +func (ss *SqliteSession) plainSpatialQuery( + first, second []byte, + done chan struct{}) (blocks chan Block, err error) { var ( firstKey int64 From c9686a7f24600c71528252444bdd8136676917ac Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Fri, 24 Jul 2015 13:15:05 +0200 Subject: [PATCH 9/9] Fixed non-interleaved LevelDB backend. --- cmd/mtredisalize/leveldb.go | 40 +++++++++++++++---------------------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/cmd/mtredisalize/leveldb.go b/cmd/mtredisalize/leveldb.go index caf0977..a03c6a9 100644 --- a/cmd/mtredisalize/leveldb.go +++ b/cmd/mtredisalize/leveldb.go @@ -314,42 +314,34 @@ func (ldbs *LevelDBSession) plainSpatialQuery( ro := leveldb.NewReadOptions() defer ro.Close() - ro.SetFillCache(false) - - it := ldbs.backend.db.NewIterator(ro) - defer it.Close() var a, b common.Coord - var err error for _, r := range ldbs.backend.coverage.Query(c1, c2) { a.Z, b.Z = int16(r.Z), int16(r.Z) a.X, b.X = int16(r.X1), int16(r.X2) - // log.Printf("y1 y2 x1 x2 z: %d %d, %d %d, %d\n", r.Y1, r.Y2, r.X1, r.X2, r.Z) for a.Y = r.Y2; a.Y >= r.Y1; a.Y-- { b.Y = a.Y - from, to := common.CoordToPlain(a), common.CoordToPlain(b) - it.Seek(common.StringToBytes(from)) - for ; it.Valid(); it.Next() { - key := it.Key() - var pos int64 - if pos, err = common.DecodeStringFromBytes(key); err != nil { - log.Printf("decoding key failed: %s\n", err) + // The keys in the database are stored and ordered as strings + // "1", "10", ..., "19", "2", "20", "21" so you cannot use + // an iterator and assume it is numerical ordered. + // Each block is fetched with a Get instead. + for f, t := common.CoordToPlain(a), common.CoordToPlain(b); f <= t; f++ { + key := common.StringToBytes(f) + value, err := ldbs.backend.db.Get(ro, key) + if err != nil { + log.Printf("get failed: %s\n", err) return } - if pos > to { - break - } - select { - case blocks <- Block{Key: key, Data: it.Value()}: - case <-done: - return + if value != nil { + select { + case blocks <- Block{Key: key, Data: value}: + case <-done: + return + } } } - if err = it.GetError(); err != nil { - log.Printf("iterating failed: %s\n", err) - return - } + } } }()