mirror of
https://bitbucket.org/s_l_teichmann/mtsatellite
synced 2025-06-28 22:26:47 +02:00
Some little experiment with a coverage index to speed up spatial queries if backend has no Z order support.
This commit is contained in:
Sascha L. Teichmann
@ -17,13 +17,18 @@ import (
|
||||
var globalLock sync.RWMutex
|
||||
|
||||
const (
|
||||
fetchSQL = "SELECT data FROM blocks WHERE pos = ?"
|
||||
existsSQL = "SELECT 1 FROM blocks WHERE pos = ?"
|
||||
updateSQL = "UPDATE blocks SET data = ? WHERE pos = ?"
|
||||
insertSQL = "INSERT INTO blocks (pos, data) VALUES (?, ?)"
|
||||
countSQL = "SELECT count(*) FROM blocks"
|
||||
keysSQL = "SELECT pos FROM blocks"
|
||||
rangeSQL = "SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ? ORDER BY pos"
|
||||
fetchSQL = "SELECT data FROM blocks WHERE pos = ?"
|
||||
existsSQL = "SELECT 1 FROM blocks WHERE pos = ?"
|
||||
updateSQL = "UPDATE blocks SET data = ? WHERE pos = ?"
|
||||
insertSQL = "INSERT INTO blocks (pos, data) VALUES (?, ?)"
|
||||
countSQL = "SELECT count(*) FROM blocks"
|
||||
keysSQL = "SELECT pos FROM blocks"
|
||||
rangeSQL = "SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ? ORDER BY pos"
|
||||
rangeDuffSQL = "SELECT pos, data FROM blocks WHERE " +
|
||||
"pos BETWEEN ? AND ? OR " +
|
||||
"pos BETWEEN ? AND ? OR " +
|
||||
"pos BETWEEN ? AND ? OR " +
|
||||
"pos BETWEEN ? AND ?"
|
||||
)
|
||||
|
||||
type SqliteBackend struct {
|
||||
@ -32,6 +37,7 @@ type SqliteBackend struct {
|
||||
decoder common.KeyDecoder
|
||||
changeTracker *ChangeTracker
|
||||
interleaved bool
|
||||
coverage *common.Coverage3D
|
||||
existsStmt *sql.Stmt
|
||||
fetchStmt *sql.Stmt
|
||||
insertStmt *sql.Stmt
|
||||
@ -99,7 +105,14 @@ func NewSqliteBackend(
|
||||
return
|
||||
}
|
||||
|
||||
if res.rangeStmt, err = res.db.Prepare(rangeSQL); err != nil {
|
||||
var rS string
|
||||
if interleaved {
|
||||
rS = rangeSQL
|
||||
} else {
|
||||
rS = rangeDuffSQL
|
||||
}
|
||||
|
||||
if res.rangeStmt, err = res.db.Prepare(rS); err != nil {
|
||||
res.closeAll()
|
||||
return
|
||||
}
|
||||
@ -112,10 +125,38 @@ func NewSqliteBackend(
|
||||
res.decoder = common.DecodeStringFromBytes
|
||||
}
|
||||
|
||||
if !interleaved {
|
||||
if err = res.buildCoverage(); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
sqlb = &res
|
||||
return
|
||||
}
|
||||
|
||||
func (sb *SqliteBackend) buildCoverage() (err error) {
|
||||
log.Println("INFO: Start building coverage index (this may take some time)...")
|
||||
sb.coverage = common.NewCoverage3D()
|
||||
|
||||
var rows *sql.Rows
|
||||
if rows, err = sb.keysStmt.Query(); err != nil {
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var key int64
|
||||
if err = rows.Scan(&key); err != nil {
|
||||
return
|
||||
}
|
||||
sb.coverage.Insert(common.PlainToCoord(key))
|
||||
}
|
||||
err = rows.Err()
|
||||
log.Println("INFO: Finished building coverage index.")
|
||||
return
|
||||
}
|
||||
|
||||
func closeStmt(stmt **sql.Stmt) error {
|
||||
s := *stmt
|
||||
if s != nil {
|
||||
@ -212,6 +253,10 @@ func (ss *SqliteSession) Store(hash, key, value []byte) (exists bool, err error)
|
||||
}
|
||||
// This technically too early because this done in transactions
|
||||
// which are commited (and possible fail) later.
|
||||
if ss.backend.coverage != nil {
|
||||
ss.backend.coverage.Insert(common.PlainToCoord(pos))
|
||||
}
|
||||
|
||||
if ss.backend.changeTracker != nil {
|
||||
ss.backend.changeTracker.BlockChanged(key)
|
||||
}
|
||||
@ -376,6 +421,52 @@ func (ss *SqliteSession) interleavedSpatialQuery(first, second []byte, done chan
|
||||
return
|
||||
}
|
||||
|
||||
type duffStmt struct {
|
||||
stmt *sql.Stmt
|
||||
counter int
|
||||
params [8]int64
|
||||
}
|
||||
|
||||
func (ds *duffStmt) push(a, b int64) bool {
|
||||
ds.params[ds.counter] = a
|
||||
ds.counter++
|
||||
ds.params[ds.counter] = b
|
||||
ds.counter++
|
||||
return ds.counter > 7
|
||||
}
|
||||
|
||||
func (ds *duffStmt) Query() (*sql.Rows, error) {
|
||||
c := ds.counter
|
||||
ds.counter = 0
|
||||
switch c {
|
||||
case 8:
|
||||
return ds.stmt.Query(
|
||||
ds.params[0], ds.params[1],
|
||||
ds.params[2], ds.params[3],
|
||||
ds.params[4], ds.params[5],
|
||||
ds.params[0], ds.params[1])
|
||||
case 6:
|
||||
return ds.stmt.Query(
|
||||
ds.params[0], ds.params[1],
|
||||
ds.params[2], ds.params[3],
|
||||
ds.params[4], ds.params[5],
|
||||
ds.params[0], ds.params[1])
|
||||
case 4:
|
||||
return ds.stmt.Query(
|
||||
ds.params[0], ds.params[1],
|
||||
ds.params[2], ds.params[3],
|
||||
ds.params[0], ds.params[1],
|
||||
ds.params[0], ds.params[1])
|
||||
case 2:
|
||||
return ds.stmt.Query(
|
||||
ds.params[0], ds.params[1],
|
||||
ds.params[0], ds.params[1],
|
||||
ds.params[0], ds.params[1],
|
||||
ds.params[0], ds.params[1])
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struct{}) (blocks chan Block, err error) {
|
||||
|
||||
var (
|
||||
@ -399,49 +490,73 @@ func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struc
|
||||
go func() {
|
||||
defer globalLock.RUnlock()
|
||||
defer close(blocks)
|
||||
rangeStmt := ss.txStmt(ss.backend.rangeStmt)
|
||||
rangeStmt := duffStmt{stmt: ss.txStmt(ss.backend.rangeStmt)}
|
||||
|
||||
a, b := common.Coord{X: c1.X}, common.Coord{X: c2.X}
|
||||
send := func(rows *sql.Rows, err error) bool {
|
||||
if rows == nil {
|
||||
return true
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("Error in range query: %s\n", err)
|
||||
return false
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for a.Z = c1.Z; a.Z <= c2.Z; a.Z++ {
|
||||
b.Z = a.Z
|
||||
for a.Y = c1.Y; a.Y <= c2.Y; a.Y++ {
|
||||
b.Y = a.Y
|
||||
var (
|
||||
err error
|
||||
rows *sql.Rows
|
||||
)
|
||||
// Ordering should not be necessary.
|
||||
from, to := order(common.CoordToPlain(a), common.CoordToPlain(b))
|
||||
if rows, err = rangeStmt.Query(from, to); err != nil {
|
||||
log.Printf("Error in range query: %s\n", err)
|
||||
return
|
||||
for rows.Next() {
|
||||
var key int64
|
||||
var data []byte
|
||||
if err = rows.Scan(&key, &data); err != nil {
|
||||
log.Printf("Error in scanning row: %s\n", err)
|
||||
return false
|
||||
}
|
||||
for rows.Next() {
|
||||
var key int64
|
||||
var data []byte
|
||||
if err = rows.Scan(&key, &data); err != nil {
|
||||
log.Printf("Error in scanning row: %s\n", err)
|
||||
break
|
||||
}
|
||||
var encodedKey []byte
|
||||
if encodedKey, err = common.EncodeStringToBytes(key); err != nil {
|
||||
log.Printf("Key encoding failed: %s\n", err)
|
||||
break
|
||||
}
|
||||
select {
|
||||
case blocks <- Block{Key: encodedKey, Data: data}:
|
||||
case <-done:
|
||||
rows.Close()
|
||||
var encodedKey []byte
|
||||
if encodedKey, err = common.EncodeStringToBytes(key); err != nil {
|
||||
log.Printf("Key encoding failed: %s\n", err)
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case blocks <- Block{Key: encodedKey, Data: data}:
|
||||
case <-done:
|
||||
return false
|
||||
}
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
log.Printf("Error in range query: %s\n", err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
if ss.backend.coverage == nil {
|
||||
a, b := common.Coord{X: c1.X}, common.Coord{X: c2.X}
|
||||
for a.Y = c2.Y; a.Y >= c1.Y; a.Y-- {
|
||||
b.Y = a.Y
|
||||
for a.Z = c1.Z; a.Z <= c2.Z; a.Z++ {
|
||||
b.Z = a.Z
|
||||
// Ordering should not be necessary.
|
||||
from, to := order(common.CoordToPlain(a), common.CoordToPlain(b))
|
||||
if rangeStmt.push(from, to) && !send(rangeStmt.Query()) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
log.Printf("Error in range query: %s\n", err)
|
||||
}
|
||||
} else {
|
||||
var a, b common.Coord
|
||||
|
||||
for _, r := range ss.backend.coverage.Query(c1, c2) {
|
||||
a.Z, b.Z = int16(r.Z), int16(r.Z)
|
||||
a.X, b.X = int16(r.X1), int16(r.X2)
|
||||
// log.Printf("y1 y2 x1 x2 z: %d %d, %d %d, %d\n", r.Y1, r.Y2, r.X1, r.X2, r.Z)
|
||||
for y := r.Y2; y >= r.Y1; y-- {
|
||||
a.Y, b.Y = int16(y), int16(y)
|
||||
from, to := order(common.CoordToPlain(a), common.CoordToPlain(b))
|
||||
if rangeStmt.push(from, to) && !send(rangeStmt.Query()) {
|
||||
return
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
}
|
||||
send(rangeStmt.Query())
|
||||
}()
|
||||
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user