mtcontrib
/
mtsatellite
mirror of https://bitbucket.org/s_l_teichmann/mtsatellite
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Some little experiment with a coverage index to speed up spatial queries if backend has no Z order support.

This commit is contained in:
Sascha L. Teichmann 2015-07-21 22:01:10 +02:00
parent 470b47f70a
commit 1921b8211b
5 changed files with 281 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

199
cmd/mtredisalize/sqlite.go
View File

@ -17,13 +17,18 @@ import (
var globalLock sync.RWMutex
const (
fetchSQL = "SELECT data FROM blocks WHERE pos = ?"
existsSQL = "SELECT 1 FROM blocks WHERE pos = ?"
updateSQL = "UPDATE blocks SET data = ? WHERE pos = ?"
insertSQL = "INSERT INTO blocks (pos, data) VALUES (?, ?)"
countSQL = "SELECT count(*) FROM blocks"
keysSQL = "SELECT pos FROM blocks"
rangeSQL = "SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ? ORDER BY pos"
fetchSQL = "SELECT data FROM blocks WHERE pos = ?"
existsSQL = "SELECT 1 FROM blocks WHERE pos = ?"
updateSQL = "UPDATE blocks SET data = ? WHERE pos = ?"
insertSQL = "INSERT INTO blocks (pos, data) VALUES (?, ?)"
countSQL = "SELECT count(*) FROM blocks"
keysSQL = "SELECT pos FROM blocks"
rangeSQL = "SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ? ORDER BY pos"
rangeDuffSQL = "SELECT pos, data FROM blocks WHERE " +
"pos BETWEEN ? AND ? OR " +
"pos BETWEEN ? AND ? OR " +
"pos BETWEEN ? AND ? OR " +
"pos BETWEEN ? AND ?"
)
type SqliteBackend struct {
@ -32,6 +37,7 @@ type SqliteBackend struct {
decoder common.KeyDecoder
changeTracker *ChangeTracker
interleaved bool
coverage *common.Coverage3D
existsStmt *sql.Stmt
fetchStmt *sql.Stmt
insertStmt *sql.Stmt
@ -99,7 +105,14 @@ func NewSqliteBackend(
return
}
if res.rangeStmt, err = res.db.Prepare(rangeSQL); err != nil {
var rS string
if interleaved {
rS = rangeSQL
} else {
rS = rangeDuffSQL
}
if res.rangeStmt, err = res.db.Prepare(rS); err != nil {
res.closeAll()
return
}
@ -112,10 +125,38 @@ func NewSqliteBackend(
res.decoder = common.DecodeStringFromBytes
}
if !interleaved {
if err = res.buildCoverage(); err != nil {
return
}
}
sqlb = &res
return
}
func (sb *SqliteBackend) buildCoverage() (err error) {
log.Println("INFO: Start building coverage index (this may take some time)...")
sb.coverage = common.NewCoverage3D()
var rows *sql.Rows
if rows, err = sb.keysStmt.Query(); err != nil {
return
}
defer rows.Close()
for rows.Next() {
var key int64
if err = rows.Scan(&key); err != nil {
return
}
sb.coverage.Insert(common.PlainToCoord(key))
}
err = rows.Err()
log.Println("INFO: Finished building coverage index.")
return
}
func closeStmt(stmt **sql.Stmt) error {
s := *stmt
if s != nil {
@ -212,6 +253,10 @@ func (ss *SqliteSession) Store(hash, key, value []byte) (exists bool, err error)
}
// This technically too early because this done in transactions
// which are commited (and possible fail) later.
if ss.backend.coverage != nil {
ss.backend.coverage.Insert(common.PlainToCoord(pos))
}
if ss.backend.changeTracker != nil {
ss.backend.changeTracker.BlockChanged(key)
}
@ -376,6 +421,52 @@ func (ss *SqliteSession) interleavedSpatialQuery(first, second []byte, done chan
return
}
type duffStmt struct {
stmt *sql.Stmt
counter int
params [8]int64
}
func (ds *duffStmt) push(a, b int64) bool {
ds.params[ds.counter] = a
ds.counter++
ds.params[ds.counter] = b
ds.counter++
return ds.counter > 7
}
func (ds *duffStmt) Query() (*sql.Rows, error) {
c := ds.counter
ds.counter = 0
switch c {
case 8:
return ds.stmt.Query(
ds.params[0], ds.params[1],
ds.params[2], ds.params[3],
ds.params[4], ds.params[5],
ds.params[0], ds.params[1])
case 6:
return ds.stmt.Query(
ds.params[0], ds.params[1],
ds.params[2], ds.params[3],
ds.params[4], ds.params[5],
ds.params[0], ds.params[1])
case 4:
return ds.stmt.Query(
ds.params[0], ds.params[1],
ds.params[2], ds.params[3],
ds.params[0], ds.params[1],
ds.params[0], ds.params[1])
case 2:
return ds.stmt.Query(
ds.params[0], ds.params[1],
ds.params[0], ds.params[1],
ds.params[0], ds.params[1],
ds.params[0], ds.params[1])
}
return nil, nil
}
func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struct{}) (blocks chan Block, err error) {
var (
@ -399,49 +490,73 @@ func (ss *SqliteSession) plainSpatialQuery(first, second []byte, done chan struc
go func() {
defer globalLock.RUnlock()
defer close(blocks)
rangeStmt := ss.txStmt(ss.backend.rangeStmt)
rangeStmt := duffStmt{stmt: ss.txStmt(ss.backend.rangeStmt)}
a, b := common.Coord{X: c1.X}, common.Coord{X: c2.X}
send := func(rows *sql.Rows, err error) bool {
if rows == nil {
return true
}
if err != nil {
log.Printf("Error in range query: %s\n", err)
return false
}
defer rows.Close()
for a.Z = c1.Z; a.Z <= c2.Z; a.Z++ {
b.Z = a.Z
for a.Y = c1.Y; a.Y <= c2.Y; a.Y++ {
b.Y = a.Y
var (
err error
rows *sql.Rows
)
// Ordering should not be necessary.
from, to := order(common.CoordToPlain(a), common.CoordToPlain(b))
if rows, err = rangeStmt.Query(from, to); err != nil {
log.Printf("Error in range query: %s\n", err)
return
for rows.Next() {
var key int64
var data []byte
if err = rows.Scan(&key, &data); err != nil {
log.Printf("Error in scanning row: %s\n", err)
return false
}
for rows.Next() {
var key int64
var data []byte
if err = rows.Scan(&key, &data); err != nil {
log.Printf("Error in scanning row: %s\n", err)
break
}
var encodedKey []byte
if encodedKey, err = common.EncodeStringToBytes(key); err != nil {
log.Printf("Key encoding failed: %s\n", err)
break
}
select {
case blocks <- Block{Key: encodedKey, Data: data}:
case <-done:
rows.Close()
var encodedKey []byte
if encodedKey, err = common.EncodeStringToBytes(key); err != nil {
log.Printf("Key encoding failed: %s\n", err)
return false
}
select {
case blocks <- Block{Key: encodedKey, Data: data}:
case <-done:
return false
}
}
if err = rows.Err(); err != nil {
log.Printf("Error in range query: %s\n", err)
return false
}
return true
}
if ss.backend.coverage == nil {
a, b := common.Coord{X: c1.X}, common.Coord{X: c2.X}
for a.Y = c2.Y; a.Y >= c1.Y; a.Y-- {
b.Y = a.Y
for a.Z = c1.Z; a.Z <= c2.Z; a.Z++ {
b.Z = a.Z
// Ordering should not be necessary.
from, to := order(common.CoordToPlain(a), common.CoordToPlain(b))
if rangeStmt.push(from, to) && !send(rangeStmt.Query()) {
return
}
}
if err = rows.Err(); err != nil {
log.Printf("Error in range query: %s\n", err)
}
} else {
var a, b common.Coord
for _, r := range ss.backend.coverage.Query(c1, c2) {
a.Z, b.Z = int16(r.Z), int16(r.Z)
a.X, b.X = int16(r.X1), int16(r.X2)
// log.Printf("y1 y2 x1 x2 z: %d %d, %d %d, %d\n", r.Y1, r.Y2, r.X1, r.X2, r.Z)
for y := r.Y2; y >= r.Y1; y-- {
a.Y, b.Y = int16(y), int16(y)
from, to := order(common.CoordToPlain(a), common.CoordToPlain(b))
if rangeStmt.push(from, to) && !send(rangeStmt.Query()) {
return
}
}
rows.Close()
}
}
send(rangeStmt.Query())
}()
return

26
common/coords.go
View File

@ -45,20 +45,6 @@ func (c Coord) String() string {
return fmt.Sprintf("(%d, %d, %d)", c.X, c.Y, c.Z)
}
func minComponent(a, b int16) int16 {
if a < b {
return a
}
return b
}
func maxComponent(a, b int16) int16 {
if a > b {
return a
}
return b
}
func clipComponent(x int16) int16 {
if x < minValue {
return minValue
@ -78,16 +64,16 @@ func ClipCoord(c Coord) Coord {
func MinCoord(a, b Coord) Coord {
return Coord{
X: minComponent(a.X, b.X),
Y: minComponent(a.Y, b.Y),
Z: minComponent(a.Z, b.Z)}
X: min16(a.X, b.X),
Y: min16(a.Y, b.Y),
Z: min16(a.Z, b.Z)}
}
func MaxCoord(a, b Coord) Coord {
return Coord{
X: maxComponent(a.X, b.X),
Y: maxComponent(a.Y, b.Y),
Z: maxComponent(a.Z, b.Z)}
X: max16(a.X, b.X),
Y: max16(a.Y, b.Y),
Z: max16(a.Z, b.Z)}
}
// DecodeStringFromBytes constructs a database key out of byte slice.

82
common/coverage.go Normal file
View File

@ -0,0 +1,82 @@
package common
import "sync"
type zRange struct {
y1 int16
y2 int16
xRange *Span
}
type Coverage3D struct {
pool *SpanPool
zRanges map[int16]*zRange
mu sync.RWMutex
}
type Range struct {
Z int16
Y1 int16
Y2 int16
X1 int16
X2 int16
}
func NewCoverage3D() *Coverage3D {
return &Coverage3D{
pool: NewSpanPool(),
zRanges: map[int16]*zRange{}}
}
func (s2d *Coverage3D) Insert(c Coord) {
s2d.mu.Lock()
defer s2d.mu.Unlock()
zr := s2d.zRanges[c.Z]
if zr == nil {
xr := s2d.pool.Alloc()
xr.From = int32(c.X)
xr.To = int32(c.X)
xr.Next = nil
s2d.zRanges[c.Z] = &zRange{
y1: c.Y,
y2: c.Y,
xRange: xr}
return
}
zr.xRange = s2d.pool.Insert(zr.xRange, int32(c.X), 0)
if c.Y < zr.y1 {
zr.y1 = c.Y
}
if c.Y > zr.y2 {
zr.y2 = c.Y
}
}
func (s2d *Coverage3D) Query(c1, c2 Coord) []Range {
c1, c2 = MinCoord(c1, c2), MaxCoord(c1, c2)
s2d.mu.RLock()
defer s2d.mu.RUnlock()
r := make([]Range, 0, 32)
for z := c1.Z; z <= c2.Z; z++ {
zr := s2d.zRanges[z]
if zr == nil || c1.Y > zr.y2 || c2.Y < zr.y1 {
continue
}
y1, y2 := max16(c1.Y, zr.y1), min16(c2.Y, zr.y2)
for xr := zr.xRange; xr != nil && xr.From <= int32(c2.X); xr = xr.Next {
if xr.To < int32(c1.X) {
continue
}
r = append(r, Range{
Z: z,
Y1: y1,
Y2: y2,
X1: max16(c1.X, int16(xr.From)),
X2: min16(c2.X, int16(xr.To))})
}
}
return r
}

36
common/math.go Normal file
View File

@ -0,0 +1,36 @@
package common
func max(a, b int) int {
if a > b {
return a
}
return b
}
func max32(a, b int32) int32 {
if a > b {
return a
}
return b
}
func min32(a, b int32) int32 {
if a < b {
return a
}
return b
}
func max16(a, b int16) int16 {
if a > b {
return a
}
return b
}
func min16(a, b int16) int16 {
if a < b {
return a
}
return b
}

14
common/renderer.go
View File

@ -48,20 +48,6 @@ func (yo *YOrder) Reset() {
yo.blocks = yo.blocks[0:0]
}
func max(a, b int) int {
if a > b {
return a
}
return b
}
func max32(a, b int32) int32 {
if a > b {
return a
}
return b
}
func copyData(data []byte) []byte {
l := len(data)
ndata := make([]byte, l, max(l, 8*1024))