Some little experiment with a coverage index to speed up spatial queries if backend has no Z order support.

cmd/mtredisalize/sqlite.go

@@ -24,6 +24,11 @@ const (
 	countSQL     = "SELECT count(*) FROM blocks"
 	keysSQL      = "SELECT pos FROM blocks"
 	rangeSQL     = "SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ? ORDER BY pos"
+	rangeDuffSQL = "SELECT pos, data FROM blocks WHERE " +
+		"pos BETWEEN ? AND ? OR " +
+		"pos BETWEEN ? AND ? OR " +
+		"pos BETWEEN ? AND ? OR " +
+		"pos BETWEEN ? AND ?"
 )
 
 type SqliteBackend struct {
@@ -32,6 +37,7 @@ type SqliteBackend struct {
 	decoder       common.KeyDecoder
 	changeTracker *ChangeTracker
 	interleaved   bool
+	coverage      *common.Coverage3D
 	existsStmt    *sql.Stmt
 	fetchStmt     *sql.Stmt
 	insertStmt    *sql.Stmt
@@ -99,7 +105,14 @@ func NewSqliteBackend(
 		return
 	}
 
-	if res.rangeStmt, err = res.db.Prepare(rangeSQL); err != nil {
+	var rS string
+	if interleaved {
+		rS = rangeSQL
+	} else {
+		rS = rangeDuffSQL
+	}
+
+	if res.rangeStmt, err = res.db.Prepare(rS); err != nil {
 		res.closeAll()
 		return
 	}
@@ -112,10 +125,38 @@ func NewSqliteBackend(
 		res.decoder = common.DecodeStringFromBytes
 	}
 
+	if !interleaved {
+		if err = res.buildCoverage(); err != nil {
+			return
+		}
+	}
+
 	sqlb = &res
 	return
 }
 
+func (sb *SqliteBackend) buildCoverage() (err error) {
+	log.Println("INFO: Start building coverage index (this may take some time)...")
+	sb.coverage = common.NewCoverage3D()
+
+	var rows *sql.Rows
+	if rows, err = sb.keysStmt.Query(); err != nil {
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var key int64
+		if err = rows.Scan(&key); err != nil {
+			return
+		}
+		sb.coverage.Insert(common.PlainToCoord(key))
+	}
+	err = rows.Err()
+	log.Println("INFO: Finished building coverage index.")
+	return
+}
+
 func closeStmt(stmt **sql.Stmt) error {
 	s := *stmt
 	if s != nil {
@@ -212,6 +253,10 @@ func (ss *SqliteSession) Store(hash, key, value []byte) (exists bool, err error)
 	}
 	// This technically too early because this done in transactions
 	// which are commited (and possible fail) later.
+	if ss.backend.coverage != nil {
+		ss.backend.coverage.Insert(common.PlainToCoord(pos))
+	}
+
 	if ss.backend.changeTracker != nil {
 		ss.backend.changeTracker.BlockChanged(key)
 	}
@@ -376,6 +421,52 @@ func (ss *SqliteSession) interleavedSpatialQuery(first, second []byte, done chan
 	return
 }
 
+type duffStmt struct {
+	stmt    *sql.Stmt
+	counter int
+	params  [8]int64
+}
+
+func (ds *duffStmt) push(a, b int64) bool {
+	ds.params[ds.counter] = a
+	ds.counter++
+	ds.params[ds.counter] = b
+	ds.counter++
+	return ds.counter > 7
+}
+
+func (ds *duffStmt) Query() (*sql.Rows, error) {
+	c := ds.counter
+	ds.counter = 0
+	switch c {
+	case 8:
+		return ds.stmt.Query(
+			ds.params[0], ds.params[1],
+			ds.params[2], ds.params[3],
+			ds.params[4], ds.params[5],
+			ds.params[0], ds.params[1])
+	case 6:
+		return ds.stmt.Query(
+			ds.params[0], ds.params[1],
+			ds.params[2], ds.params[3],
+			ds.params[4], ds.params[5],
+			ds.params[0], ds.params[1])
+	case 4:
+		return ds.stmt.Query(
+			ds.params[0], ds.params[1],
+			ds.params[2], ds.params[3],
+			ds.params[0], ds.params[1],
+			ds.params[0], ds.params[1])
+	case 2:
+		return ds.stmt.Query(
+			ds.params[0], ds.params[1],
+			ds.params[0], ds.params[1],
+			ds.params[0], ds.params[1],
+			ds.params[0], ds.params[1])
+	}
+	return nil, nil
+}
+
 func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struct{}) (blocks chan Block, err error) {
 
 	var (
@@ -399,49 +490,73 @@ func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struc
 	go func() {
 		defer globalLock.RUnlock()
 		defer close(blocks)
-		rangeStmt := ss.txStmt(ss.backend.rangeStmt)
+		rangeStmt := duffStmt{stmt: ss.txStmt(ss.backend.rangeStmt)}
 
-		a, b := common.Coord{X: c1.X}, common.Coord{X: c2.X}
-
-		for a.Z = c1.Z; a.Z <= c2.Z; a.Z++ {
-			b.Z = a.Z
-			for a.Y = c1.Y; a.Y <= c2.Y; a.Y++ {
-				b.Y = a.Y
-				var (
-					err  error
-					rows *sql.Rows
-				)
-				// Ordering should not be necessary.
-				from, to := order(common.CoordToPlain(a), common.CoordToPlain(b))
-				if rows, err = rangeStmt.Query(from, to); err != nil {
-					log.Printf("Error in range query: %s\n", err)
-					return
+		send := func(rows *sql.Rows, err error) bool {
+			if rows == nil {
+				return true
 			}
+			if err != nil {
+				log.Printf("Error in range query: %s\n", err)
+				return false
+			}
+			defer rows.Close()
+
 			for rows.Next() {
 				var key int64
 				var data []byte
 				if err = rows.Scan(&key, &data); err != nil {
 					log.Printf("Error in scanning row: %s\n", err)
-						break
+					return false
 				}
 				var encodedKey []byte
 				if encodedKey, err = common.EncodeStringToBytes(key); err != nil {
 					log.Printf("Key encoding failed: %s\n", err)
-						break
+					return false
 				}
 				select {
 				case blocks <- Block{Key: encodedKey, Data: data}:
 				case <-done:
-						rows.Close()
-						return
+					return false
 				}
 			}
 			if err = rows.Err(); err != nil {
 				log.Printf("Error in range query: %s\n", err)
+				return false
 			}
-				rows.Close()
+			return true
+		}
+
+		if ss.backend.coverage == nil {
+			a, b := common.Coord{X: c1.X}, common.Coord{X: c2.X}
+			for a.Y = c2.Y; a.Y >= c1.Y; a.Y-- {
+				b.Y = a.Y
+				for a.Z = c1.Z; a.Z <= c2.Z; a.Z++ {
+					b.Z = a.Z
+					// Ordering should not be necessary.
+					from, to := order(common.CoordToPlain(a), common.CoordToPlain(b))
+					if rangeStmt.push(from, to) && !send(rangeStmt.Query()) {
+						return
 					}
 				}
+			}
+		} else {
+			var a, b common.Coord
+
+			for _, r := range ss.backend.coverage.Query(c1, c2) {
+				a.Z, b.Z = int16(r.Z), int16(r.Z)
+				a.X, b.X = int16(r.X1), int16(r.X2)
+				// log.Printf("y1 y2 x1 x2 z: %d %d, %d %d, %d\n", r.Y1, r.Y2, r.X1, r.X2, r.Z)
+				for y := r.Y2; y >= r.Y1; y-- {
+					a.Y, b.Y = int16(y), int16(y)
+					from, to := order(common.CoordToPlain(a), common.CoordToPlain(b))
+					if rangeStmt.push(from, to) && !send(rangeStmt.Query()) {
+						return
+					}
+				}
+			}
+		}
+		send(rangeStmt.Query())
 	}()
 
 	return

common/coords.go

@@ -45,20 +45,6 @@ func (c Coord) String() string {
 	return fmt.Sprintf("(%d, %d, %d)", c.X, c.Y, c.Z)
 }
 
-func minComponent(a, b int16) int16 {
-	if a < b {
-		return a
-	}
-	return b
-}
-
-func maxComponent(a, b int16) int16 {
-	if a > b {
-		return a
-	}
-	return b
-}
-
 func clipComponent(x int16) int16 {
 	if x < minValue {
 		return minValue
@@ -78,16 +64,16 @@ func ClipCoord(c Coord) Coord {
 
 func MinCoord(a, b Coord) Coord {
 	return Coord{
-		X: minComponent(a.X, b.X),
-		Y: minComponent(a.Y, b.Y),
-		Z: minComponent(a.Z, b.Z)}
+		X: min16(a.X, b.X),
+		Y: min16(a.Y, b.Y),
+		Z: min16(a.Z, b.Z)}
 }
 
 func MaxCoord(a, b Coord) Coord {
 	return Coord{
-		X: maxComponent(a.X, b.X),
-		Y: maxComponent(a.Y, b.Y),
-		Z: maxComponent(a.Z, b.Z)}
+		X: max16(a.X, b.X),
+		Y: max16(a.Y, b.Y),
+		Z: max16(a.Z, b.Z)}
 }
 
 // DecodeStringFromBytes constructs a database key out of byte slice.

common/coverage.go

@@ -0,0 +1,82 @@
+package common
+
+import "sync"
+
+type zRange struct {
+	y1     int16
+	y2     int16
+	xRange *Span
+}
+
+type Coverage3D struct {
+	pool    *SpanPool
+	zRanges map[int16]*zRange
+	mu      sync.RWMutex
+}
+
+type Range struct {
+	Z  int16
+	Y1 int16
+	Y2 int16
+	X1 int16
+	X2 int16
+}
+
+func NewCoverage3D() *Coverage3D {
+	return &Coverage3D{
+		pool:    NewSpanPool(),
+		zRanges: map[int16]*zRange{}}
+}
+
+func (s2d *Coverage3D) Insert(c Coord) {
+	s2d.mu.Lock()
+	defer s2d.mu.Unlock()
+	zr := s2d.zRanges[c.Z]
+	if zr == nil {
+		xr := s2d.pool.Alloc()
+		xr.From = int32(c.X)
+		xr.To = int32(c.X)
+		xr.Next = nil
+		s2d.zRanges[c.Z] = &zRange{
+			y1:     c.Y,
+			y2:     c.Y,
+			xRange: xr}
+		return
+	}
+	zr.xRange = s2d.pool.Insert(zr.xRange, int32(c.X), 0)
+	if c.Y < zr.y1 {
+		zr.y1 = c.Y
+	}
+	if c.Y > zr.y2 {
+		zr.y2 = c.Y
+	}
+}
+
+func (s2d *Coverage3D) Query(c1, c2 Coord) []Range {
+
+	c1, c2 = MinCoord(c1, c2), MaxCoord(c1, c2)
+
+	s2d.mu.RLock()
+	defer s2d.mu.RUnlock()
+
+	r := make([]Range, 0, 32)
+	for z := c1.Z; z <= c2.Z; z++ {
+		zr := s2d.zRanges[z]
+		if zr == nil || c1.Y > zr.y2 || c2.Y < zr.y1 {
+			continue
+		}
+		y1, y2 := max16(c1.Y, zr.y1), min16(c2.Y, zr.y2)
+		for xr := zr.xRange; xr != nil && xr.From <= int32(c2.X); xr = xr.Next {
+			if xr.To < int32(c1.X) {
+				continue
+			}
+			r = append(r, Range{
+				Z:  z,
+				Y1: y1,
+				Y2: y2,
+				X1: max16(c1.X, int16(xr.From)),
+				X2: min16(c2.X, int16(xr.To))})
+		}
+	}
+	return r
+}

common/math.go

@@ -0,0 +1,36 @@
+package common
+
+func max(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func max32(a, b int32) int32 {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func min32(a, b int32) int32 {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func max16(a, b int16) int16 {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func min16(a, b int16) int16 {
+	if a < b {
+		return a
+	}
+	return b
+}

common/renderer.go

@@ -48,20 +48,6 @@ func (yo *YOrder) Reset() {
 	yo.blocks = yo.blocks[0:0]
 }
 
-func max(a, b int) int {
-	if a > b {
-		return a
-	}
-	return b
-}
-
-func max32(a, b int32) int32 {
-	if a > b {
-		return a
-	}
-	return b
-}
-
 func copyData(data []byte) []byte {
 	l := len(data)
 	ndata := make([]byte, l, max(l, 8*1024))