From 34d01762f079119d0d5275b6f2fd46d648af1bc1 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sun, 27 Feb 2022 21:02:16 +0100 Subject: [PATCH 01/26] Started with db abstraction in seeder. --- cmd/mtseeder/baselevel.go | 14 +++----------- cmd/mtseeder/main.go | 12 +++--------- common/basetilecreator.go | 4 ++-- common/dbclient.go | 10 ++++++++++ 4 files changed, 18 insertions(+), 22 deletions(-) create mode 100644 common/dbclient.go diff --git a/cmd/mtseeder/baselevel.go b/cmd/mtseeder/baselevel.go index 72b0c74..65250c6 100644 --- a/cmd/mtseeder/baselevel.go +++ b/cmd/mtseeder/baselevel.go @@ -10,7 +10,6 @@ import ( "os" "path/filepath" "strconv" - "strings" "sync" "bitbucket.org/s_l_teichmann/mtsatellite/common" @@ -57,7 +56,7 @@ func createTiles( } func createBaseLevel( - address string, + dbcc dbClientCreator, xMin, yMin, zMin, xMax, yMax, zMax int, transparent bool, transparentDim float32, colorsFile string, bg color.RGBA, outDir string, @@ -79,17 +78,10 @@ func createBaseLevel( jobs := make(chan blockPos) var done sync.WaitGroup - var proto string - if strings.ContainsRune(address, '/') { - proto = "unix" - } else { - proto = "tcp" - } - for i := 0; i < numWorkers; i++ { - var client *common.RedisClient + var client common.DBClient - if client, err = common.NewRedisClient(proto, address); err != nil { + if client, err = dbcc(); err != nil { return } done.Add(1) diff --git a/cmd/mtseeder/main.go b/cmd/mtseeder/main.go index 8af83df..4b5edc2 100644 --- a/cmd/mtseeder/main.go +++ b/cmd/mtseeder/main.go @@ -6,9 +6,7 @@ package main import ( "flag" - "fmt" "log" - "strings" "bitbucket.org/s_l_teichmann/mtsatellite/common" ) @@ -70,16 +68,12 @@ func main() { bg := common.ParseColorDefault(bgColor, common.BackgroundColor) + dbcc := createDBClientCreator(host, port) + if !skipBaseLevel { td := common.Clamp32f(float32(transparentDim/100.0), 0.0, 1.0) - var address string - if strings.ContainsRune(host, '/') { - address = host - } else { - address = fmt.Sprintf("%s:%d", host, port) - } if err := createBaseLevel( - address, + dbcc, xMin, yMin, zMin, xMax, yMax, zMax, transparent, td, colorsFile, bg, diff --git a/common/basetilecreator.go b/common/basetilecreator.go index 264ee6c..b6719a4 100644 --- a/common/basetilecreator.go +++ b/common/basetilecreator.go @@ -52,7 +52,7 @@ var BackgroundColor = color.RGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xff} type BaseTileUpdateFunc func(x, y int, hash []byte) bool type BaseTileCreator struct { - client *RedisClient + client DBClient colors *Colors renderer *Renderer yOrder *YOrder @@ -64,7 +64,7 @@ type BaseTileCreator struct { } func NewBaseTileCreator( - client *RedisClient, + client DBClient, colors *Colors, bg color.RGBA, yMin, yMax int16, diff --git a/common/dbclient.go b/common/dbclient.go new file mode 100644 index 0000000..7d05236 --- /dev/null +++ b/common/dbclient.go @@ -0,0 +1,10 @@ +package common + +// Copyright 2022 by Sascha L. Teichmann +// Use of this source code is governed by the MIT license +// that can be found in the LICENSE file. + +type DBClient interface { + QueryCuboid(cuboid Cuboid, fn func(*Block) *Block) (count int, err error) + Close() error +} From 4f7fedf0b9ee0fe3f2a29721669e8d0915dbaf72 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sun, 27 Feb 2022 21:17:43 +0100 Subject: [PATCH 02/26] Use db client factory in seeder. --- cmd/mtseeder/baselevel.go | 2 +- cmd/mtseeder/main.go | 2 +- common/clientfactory.go | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 common/clientfactory.go diff --git a/cmd/mtseeder/baselevel.go b/cmd/mtseeder/baselevel.go index 65250c6..96bf923 100644 --- a/cmd/mtseeder/baselevel.go +++ b/cmd/mtseeder/baselevel.go @@ -56,7 +56,7 @@ func createTiles( } func createBaseLevel( - dbcc dbClientCreator, + dbcc common.DBClientCreator, xMin, yMin, zMin, xMax, yMax, zMax int, transparent bool, transparentDim float32, colorsFile string, bg color.RGBA, outDir string, diff --git a/cmd/mtseeder/main.go b/cmd/mtseeder/main.go index 4b5edc2..7ba1fba 100644 --- a/cmd/mtseeder/main.go +++ b/cmd/mtseeder/main.go @@ -68,7 +68,7 @@ func main() { bg := common.ParseColorDefault(bgColor, common.BackgroundColor) - dbcc := createDBClientCreator(host, port) + dbcc := common.CreateDBClientCreator(host, port) if !skipBaseLevel { td := common.Clamp32f(float32(transparentDim/100.0), 0.0, 1.0) diff --git a/common/clientfactory.go b/common/clientfactory.go new file mode 100644 index 0000000..43a1e8f --- /dev/null +++ b/common/clientfactory.go @@ -0,0 +1,33 @@ +// Copyright 2022 by Sascha L. Teichmann +// Use of this source code is governed by the MIT license +// that can be found in the LICENSE file. + +package common + +import ( + "fmt" + "strings" +) + +type DBClientCreator func() (DBClient, error) + +func CreateDBClientCreator(host string, port int) DBClientCreator { + + var address string + if strings.ContainsRune(host, '/') { + address = host + } else { + address = fmt.Sprintf("%s:%d", host, port) + } + + var proto string + if strings.ContainsRune(address, '/') { + proto = "unix" + } else { + proto = "tcp" + } + + return func() (DBClient, error) { + return NewRedisClient(proto, address) + } +} From ae230d5abfc81861fc517411e877ed1dbddec61b Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sun, 27 Feb 2022 21:31:55 +0100 Subject: [PATCH 03/26] Use db abstraction in webmapper. --- cmd/mtwebmapper/main.go | 12 +++---- cmd/mtwebmapper/tilesupdater.go | 60 +++++++++++++++------------------ 2 files changed, 31 insertions(+), 41 deletions(-) diff --git a/cmd/mtwebmapper/main.go b/cmd/mtwebmapper/main.go index 690d2fb..0141943 100644 --- a/cmd/mtwebmapper/main.go +++ b/cmd/mtwebmapper/main.go @@ -10,7 +10,6 @@ import ( "log" "net" "net/http" - "strings" "bitbucket.org/s_l_teichmann/mtsatellite/common" @@ -107,6 +106,7 @@ func main() { } if redisHost != "" { + var colors *common.Colors var err error if colors, err = common.ParseColors(colorsFile); err != nil { @@ -114,12 +114,8 @@ func main() { } colors.TransparentDim = common.Clamp32f( float32(transparentDim/100.0), 0.0, 100.0) - var redisAddress string - if strings.ContainsRune(redisHost, '/') { - redisAddress = redisHost - } else { - redisAddress = fmt.Sprintf("%s:%d", redisHost, redisPort) - } + + dbcc := common.CreateDBClientCreator(redisHost, redisPort) var allowedUpdateIps []net.IP if allowedUpdateIps, err = ipsFromHosts(updateHosts); err != nil { @@ -128,7 +124,7 @@ func main() { tu := newTileUpdater( mapDir, - redisAddress, + dbcc, allowedUpdateIps, colors, bg, yMin, yMax, diff --git a/cmd/mtwebmapper/tilesupdater.go b/cmd/mtwebmapper/tilesupdater.go index 45bddbc..71045c2 100644 --- a/cmd/mtwebmapper/tilesupdater.go +++ b/cmd/mtwebmapper/tilesupdater.go @@ -32,18 +32,18 @@ type baseTilesUpdates interface { } type tileUpdater struct { - changes map[xz]struct{} - btu baseTilesUpdates - mapDir string - redisAddress string - ips []net.IP - colors *common.Colors - bg color.RGBA - yMin, yMax int16 - workers int - transparent bool - cond *sync.Cond - mu sync.Mutex + changes map[xz]struct{} + btu baseTilesUpdates + mapDir string + dbcc common.DBClientCreator + ips []net.IP + colors *common.Colors + bg color.RGBA + yMin, yMax int16 + workers int + transparent bool + cond *sync.Cond + mu sync.Mutex } type xz struct { @@ -78,7 +78,8 @@ func (c xz) parent() xzm { } func newTileUpdater( - mapDir, redisAddress string, + mapDir string, + dbcc common.DBClientCreator, ips []net.IP, colors *common.Colors, bg color.RGBA, @@ -88,17 +89,17 @@ func newTileUpdater( btu baseTilesUpdates) *tileUpdater { tu := tileUpdater{ - btu: btu, - mapDir: mapDir, - redisAddress: redisAddress, - ips: ips, - changes: map[xz]struct{}{}, - colors: colors, - bg: bg, - yMin: int16(yMin), - yMax: int16(yMax), - transparent: transparent, - workers: workers} + btu: btu, + mapDir: mapDir, + dbcc: dbcc, + ips: ips, + changes: map[xz]struct{}{}, + colors: colors, + bg: bg, + yMin: int16(yMin), + yMax: int16(yMax), + transparent: transparent, + workers: workers} tu.cond = sync.NewCond(&tu.mu) return &tu } @@ -196,17 +197,10 @@ func (tu *tileUpdater) doUpdates() { jobs := make(chan *xzc) var done sync.WaitGroup - var proto string - if strings.ContainsRune(tu.redisAddress, '/') { - proto = "unix" - } else { - proto = "tcp" - } - for i, n := 0, common.Min(tu.workers, len(changes)); i < n; i++ { - var client *common.RedisClient + var client common.DBClient var err error - if client, err = common.NewRedisClient(proto, tu.redisAddress); err != nil { + if client, err = tu.dbcc(); err != nil { log.Printf("WARN: Cannot connect to redis server: %s\n", err) continue } From fdbbde8855a48385489647d7a996fc23a2119827 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sun, 27 Feb 2022 21:39:46 +0100 Subject: [PATCH 04/26] Updated dependencies. --- go.mod | 9 +++++---- go.sum | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 008c27d..a1bc2e6 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,10 @@ go 1.13 require ( github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b - github.com/gorilla/mux v1.7.3 - github.com/gorilla/websocket v1.4.1 + github.com/gorilla/mux v1.8.0 + github.com/gorilla/websocket v1.5.0 github.com/jmhodges/levigo v1.0.0 - github.com/mattn/go-sqlite3 v1.12.0 - golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1 + github.com/mattn/go-sqlite3 v1.14.11 + golang.org/x/crypto v0.0.0-20220214200702-86341886e292 + golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7 // indirect ) diff --git a/go.sum b/go.sum index 9106b20..6fefa53 100644 --- a/go.sum +++ b/go.sum @@ -2,17 +2,34 @@ github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b h1:5Ci5wpOL75rYF6RQGRo github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b/go.mod h1:obBQGGIFbbv9KWg92Qu9UHeD94JXmHD1jovY/z6I3O8= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/mattn/go-sqlite3 v1.12.0 h1:u/x3mp++qUxvYfulZ4HKOvVO0JWhk7HtE8lWhbGz/Do= github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.11 h1:gt+cp9c0XGqe9S/wAHTL3n/7MqY+siPWgWJgqdsFrzQ= +github.com/mattn/go-sqlite3 v1.14.11/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1 h1:anGSYQpPhQwXlwsu5wmfq0nWkCNaMEMUwAv13Y92hd8= golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7 h1:BXxu8t6QN0G1uff4bzZzSkpsax8+ALqTGUtz08QrV00= +golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From d32c0d1a90a18314317aeaf15a0eaa5010468c36 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sun, 27 Feb 2022 22:48:20 +0100 Subject: [PATCH 05/26] build rudimentary postgresq block client --- common/clientfactory.go | 7 ++ common/dbclient.go | 4 +- common/pgclient.go | 98 +++++++++++++++++++++++++ go.mod | 2 + go.sum | 157 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 266 insertions(+), 2 deletions(-) create mode 100644 common/pgclient.go diff --git a/common/clientfactory.go b/common/clientfactory.go index 43a1e8f..ad12a16 100644 --- a/common/clientfactory.go +++ b/common/clientfactory.go @@ -13,6 +13,13 @@ type DBClientCreator func() (DBClient, error) func CreateDBClientCreator(host string, port int) DBClientCreator { + if strings.HasPrefix(host, "postgres:") { + host = host[len("postgres:"):] + return func() (DBClient, error) { + return NewPGClient(host) + } + } + var address string if strings.ContainsRune(host, '/') { address = host diff --git a/common/dbclient.go b/common/dbclient.go index 7d05236..29af14c 100644 --- a/common/dbclient.go +++ b/common/dbclient.go @@ -1,9 +1,9 @@ -package common - // Copyright 2022 by Sascha L. Teichmann // Use of this source code is governed by the MIT license // that can be found in the LICENSE file. +package common + type DBClient interface { QueryCuboid(cuboid Cuboid, fn func(*Block) *Block) (count int, err error) Close() error diff --git a/common/pgclient.go b/common/pgclient.go new file mode 100644 index 0000000..4b814ed --- /dev/null +++ b/common/pgclient.go @@ -0,0 +1,98 @@ +// Copyright 2022 by Sascha L. Teichmann +// Use of this source code is governed by the MIT license +// that can be found in the LICENSE file. + +package common + +import ( + "context" + "database/sql" + + _ "github.com/jackc/pgx/v4/stdlib" +) + +const queryCuboidSQL = ` +SELECT posx, posy, posz, data FROM blocks +WHERE + posx BETWEEN $1 AND $2 AND + posy BETWEEN $3 AND $4 AND + posz BETWEEN $5 AND $6` + +type PGClient struct { + db *sql.DB + queryCuboidStmt *sql.Stmt +} + +func NewPGClient(connS string) (*PGClient, error) { + db, err := sql.Open("pgx", connS) + if err != nil { + return nil, err + } + stmt, err := db.Prepare(queryCuboidSQL) + if err != nil { + return nil, err + } + client := PGClient{ + db: db, + queryCuboidStmt: stmt, + } + return &client, nil +} + +func (pgc *PGClient) QueryCuboid( + cuboid Cuboid, + fn func(*Block) *Block, +) (int, error) { + + rows, err := pgc.queryCuboidStmt.QueryContext( + context.Background(), + cuboid.P1.X, cuboid.P2.X, + cuboid.P1.Y, cuboid.P2.Y, + cuboid.P1.Z, cuboid.P2.Z) + + if err != nil { + return 0, err + } + defer rows.Close() + + var ( + posX, posY, posZ int + data []byte + count int + block *Block + ) + + for ; rows.Next(); count++ { + if err := rows.Scan( + &posX, &posY, posZ, + &data, + ); err != nil { + return count, err + } + c := Coord{ + X: int16(posX), + Y: int16(posY), + Z: int16(posZ), + } + + if block == nil { + block = &Block{Coord: c, Data: data} + } else { + *block = Block{Coord: c, Data: data} + } + if block = fn(block); block != nil { + data = block.Data[:0] + } else { + data = nil + } + } + + return count, rows.Err() +} + +func (pgc *PGClient) Close() error { + if pgc.queryCuboidStmt != nil { + pgc.queryCuboidStmt.Close() + } + return pgc.db.Close() +} diff --git a/go.mod b/go.mod index a1bc2e6..0b3df31 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,10 @@ require ( github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 + github.com/jackc/pgx/v4 v4.15.0 // indirect github.com/jmhodges/levigo v1.0.0 github.com/mattn/go-sqlite3 v1.14.11 golang.org/x/crypto v0.0.0-20220214200702-86341886e292 golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7 // indirect + golang.org/x/text v0.3.7 // indirect ) diff --git a/go.sum b/go.sum index 6fefa53..892fd50 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,18 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b h1:5Ci5wpOL75rYF6RQGRoqhEAU6xLJ6n/D4SckXX1yB74= github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b/go.mod h1:obBQGGIFbbv9KWg92Qu9UHeD94JXmHD1jovY/z6I3O8= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= @@ -8,28 +21,172 @@ github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvK github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= +github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= +github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= +github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.12.0 h1:u/x3mp++qUxvYfulZ4HKOvVO0JWhk7HtE8lWhbGz/Do= github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.11 h1:gt+cp9c0XGqe9S/wAHTL3n/7MqY+siPWgWJgqdsFrzQ= github.com/mattn/go-sqlite3 v1.14.11/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1 h1:anGSYQpPhQwXlwsu5wmfq0nWkCNaMEMUwAv13Y92hd8= golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7 h1:BXxu8t6QN0G1uff4bzZzSkpsax8+ALqTGUtz08QrV00= golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= From 4b654672e72e1b345b4e66e61b0a9dfd4b3fe46a Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 28 Feb 2022 00:55:03 +0100 Subject: [PATCH 06/26] Fixed handling of query parameter. --- common/pgclient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/pgclient.go b/common/pgclient.go index 4b814ed..a419a09 100644 --- a/common/pgclient.go +++ b/common/pgclient.go @@ -64,7 +64,7 @@ func (pgc *PGClient) QueryCuboid( for ; rows.Next(); count++ { if err := rows.Scan( - &posX, &posY, posZ, + &posX, &posY, &posZ, &data, ); err != nil { return count, err From d98df1c1cd9aba4ea31a5a13198f6e0606e02670 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 28 Feb 2022 11:07:50 +0100 Subject: [PATCH 07/26] WIP: add tile updating in in web mapper with pg listen/notify --- cmd/mtwebmapper/main.go | 10 ++++- cmd/mtwebmapper/tilesupdater.go | 78 +++++++++++++++++++++++++++++---- common/clientfactory.go | 10 ++++- go.mod | 2 +- go.sum | 23 ++++++---- 5 files changed, 102 insertions(+), 21 deletions(-) diff --git a/cmd/mtwebmapper/main.go b/cmd/mtwebmapper/main.go index 0141943..0465733 100644 --- a/cmd/mtwebmapper/main.go +++ b/cmd/mtwebmapper/main.go @@ -10,6 +10,7 @@ import ( "log" "net" "net/http" + "time" "bitbucket.org/s_l_teichmann/mtsatellite/common" @@ -35,6 +36,7 @@ func main() { version bool yMin int yMax int + changeDuration time.Duration ) defaultBgColor := common.ColorToHex(common.BackgroundColor) @@ -74,6 +76,7 @@ func main() { flag.StringVar(&playersFIFO, "ps", "", "Path to FIFO file to read active players from (shorthand).") flag.IntVar(&yMin, "ymin", common.MinHeight, "Minimum y in blocks.") flag.IntVar(&yMax, "ymax", common.MaxHeight, "Maximum y in blocks.") + flag.CommandLine.DurationVar(&changeDuration, "change-duration", time.Second, "Duration to aggregate changes. (PG only)") flag.BoolVar(&version, "version", false, "Print version and exit.") flag.Parse() @@ -132,7 +135,12 @@ func main() { workers, btu) go tu.doUpdates() - router.Path("/update").Methods("POST").Handler(tu) + + if pgHost, ok := common.IsPostgreSQL(redisHost); btu != nil && ok { + go tu.listen(pgHost, changeDuration) + } else { + router.Path("/update").Methods("POST").Handler(tu) + } } router.PathPrefix("/").Handler(http.FileServer(http.Dir(webDir))) diff --git a/cmd/mtwebmapper/tilesupdater.go b/cmd/mtwebmapper/tilesupdater.go index 71045c2..ebb8173 100644 --- a/cmd/mtwebmapper/tilesupdater.go +++ b/cmd/mtwebmapper/tilesupdater.go @@ -5,6 +5,7 @@ package main import ( + "context" "encoding/json" "image" "image/color" @@ -16,8 +17,10 @@ import ( "strconv" "strings" "sync" + "time" "github.com/bamiaux/rez" + "github.com/jackc/pgx/v4" "bytes" @@ -131,6 +134,60 @@ func (tu *tileUpdater) checkIP(r *http.Request) bool { return false } +func (tu *tileUpdater) listen(host string, changeDuration time.Duration) { + + xzCh := make(chan xz) + + go func() { + ctx := context.Background() + + conn, err := pgx.Connect(ctx, host) + if err != nil { + log.Printf("error: %v\n", err) + return + } + defer conn.Close(ctx) + + if _, err := conn.Exec(ctx, "listen blockchanges"); err != nil { + log.Printf("error: %v\n", err) + return + } + + for { + n, err := conn.WaitForNotification(ctx) + if err != nil { + log.Printf("error: %v\n", err) + continue + } + if n.Payload == "" { + continue + } + var c xz + dec := json.NewDecoder(strings.NewReader(n.Payload)) + if err := dec.Decode(&c); err != nil { + log.Printf("error: %v\n", err) + continue + } + xzCh <- c + } + }() + + ticker := time.NewTicker(changeDuration) + defer ticker.Stop() + + var changes []xz + + for { + select { + case c := <-xzCh: + changes = append(changes, c) + case <-ticker.C: + tu.sendChanges(changes) + changes = changes[:0] + } + } +} + func (tu *tileUpdater) ServeHTTP(rw http.ResponseWriter, r *http.Request) { if !tu.checkIP(r) { log.Printf("WARN: Unauthorized update request from '%s'\n", r.RemoteAddr) @@ -147,18 +204,23 @@ func (tu *tileUpdater) ServeHTTP(rw http.ResponseWriter, r *http.Request) { return } - if len(newChanges) > 0 { - tu.cond.L.Lock() - for _, c := range newChanges { - tu.changes[c.quantize()] = struct{}{} - } - tu.cond.L.Unlock() - tu.cond.Signal() - } + tu.sendChanges(newChanges) rw.WriteHeader(http.StatusOK) } +func (tu *tileUpdater) sendChanges(changes []xz) { + if len(changes) == 0 { + return + } + tu.cond.L.Lock() + for _, c := range changes { + tu.changes[c.quantize()] = struct{}{} + } + tu.cond.L.Unlock() + tu.cond.Signal() +} + func extractChanges(changes map[xz]struct{}) []xzc { chs := make([]xzc, len(changes)) var i int diff --git a/common/clientfactory.go b/common/clientfactory.go index ad12a16..fb87343 100644 --- a/common/clientfactory.go +++ b/common/clientfactory.go @@ -11,10 +11,16 @@ import ( type DBClientCreator func() (DBClient, error) +func IsPostgreSQL(host string) (string, bool) { + if strings.HasPrefix(host, "postgres:") { + return host[len("postgres:"):], true + } + return "", false +} + func CreateDBClientCreator(host string, port int) DBClientCreator { - if strings.HasPrefix(host, "postgres:") { - host = host[len("postgres:"):] + if host, ok := IsPostgreSQL(host); ok { return func() (DBClient, error) { return NewPGClient(host) } diff --git a/go.mod b/go.mod index 0b3df31..e1986c6 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 - github.com/jackc/pgx/v4 v4.15.0 // indirect + github.com/jackc/pgx/v4 v4.15.0 github.com/jmhodges/levigo v1.0.0 github.com/mattn/go-sqlite3 v1.14.11 golang.org/x/crypto v0.0.0-20220214200702-86341886e292 diff --git a/go.sum b/go.sum index 892fd50..947d427 100644 --- a/go.sum +++ b/go.sum @@ -1,24 +1,24 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b h1:5Ci5wpOL75rYF6RQGRoqhEAU6xLJ6n/D4SckXX1yB74= github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b/go.mod h1:obBQGGIFbbv9KWg92Qu9UHeD94JXmHD1jovY/z6I3O8= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= @@ -38,6 +38,7 @@ github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= @@ -74,24 +75,27 @@ github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-sqlite3 v1.12.0 h1:u/x3mp++qUxvYfulZ4HKOvVO0JWhk7HtE8lWhbGz/Do= -github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.11 h1:gt+cp9c0XGqe9S/wAHTL3n/7MqY+siPWgWJgqdsFrzQ= github.com/mattn/go-sqlite3 v1.14.11/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= @@ -99,6 +103,7 @@ github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OK github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -109,6 +114,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -127,8 +133,6 @@ golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaE golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1 h1:anGSYQpPhQwXlwsu5wmfq0nWkCNaMEMUwAv13Y92hd8= -golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -149,7 +153,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -184,9 +187,11 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= From 3c4b217364f2c17fac532e0a51793e2a23663c93 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 28 Feb 2022 11:38:16 +0100 Subject: [PATCH 08/26] pg listen/notify part --- pg/send_block_changes.sql | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 pg/send_block_changes.sql diff --git a/pg/send_block_changes.sql b/pg/send_block_changes.sql new file mode 100644 index 0000000..5017c0b --- /dev/null +++ b/pg/send_block_changes.sql @@ -0,0 +1,23 @@ +BEGIN; + + CREATE OR REPLACE FUNCTION send_block_changes() + RETURNS TRIGGER AS +$$ +BEGIN + PERFORM pg_notify('blockchanges', + json_build_object( + 'X', NEW.posx, + 'Z', NEW.posz + )::text + ); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER blocks_changed + AFTER INSERT OR UPDATE + ON blocks + FOR EACH ROW + EXECUTE PROCEDURE send_block_changes(); + +END; From 8c6ab34e6a891637cd5c43823e715c35436dd96c Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 28 Feb 2022 13:21:28 +0100 Subject: [PATCH 09/26] Fetch player pos from postgres --- cmd/mtwebmapper/players.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/cmd/mtwebmapper/players.go b/cmd/mtwebmapper/players.go index 72c0982..bc9485f 100644 --- a/cmd/mtwebmapper/players.go +++ b/cmd/mtwebmapper/players.go @@ -7,6 +7,8 @@ package main import ( "bufio" "bytes" + "context" + "database/sql" "encoding/json" "html/template" "log" @@ -16,6 +18,8 @@ import ( "sort" "sync" "time" + + "bitbucket.org/s_l_teichmann/mtsatellite/common" ) const sleepInterval = time.Second * 5 @@ -78,7 +82,38 @@ func (pls sortPlayersByName) Swap(i, j int) { pls[i], pls[j] = pls[j], pls[i] } +const selectPlayersSQL = ` +SELECT posx, posy, posz, name +FROM player +WHERE modification_date > now() - '2m'::interval` + +func playersFromPostgreSQL(connS string) ([]*player, error) { + db, err := sql.Open("pgx", connS) + if err != nil { + return nil, err + } + defer db.Close() + rows, err := db.QueryContext(context.Background(), selectPlayersSQL) + if err != nil { + return nil, err + } + defer rows.Close() + var pls []*player + for rows.Next() { + var p player + if err := rows.Scan(&p.Y, &p.Y, &p.Z, &p.Name); err != nil { + return nil, err + } + pls = append(pls, &p) + } + return pls, rows.Err() +} + func (ps *players) readFromFIFO() ([]*player, error) { + if host, ok := common.IsPostgreSQL(ps.fifo); ok { + return playersFromPostgreSQL(host) + } + file, err := os.Open(ps.fifo) if err != nil { return nil, err From 1ec9d8916b8887971e3f7688af1b2e017ffc1cac Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 28 Feb 2022 13:37:00 +0100 Subject: [PATCH 10/26] select players position from database. --- cmd/mtwebmapper/players.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/mtwebmapper/players.go b/cmd/mtwebmapper/players.go index bc9485f..f62c820 100644 --- a/cmd/mtwebmapper/players.go +++ b/cmd/mtwebmapper/players.go @@ -22,7 +22,10 @@ import ( "bitbucket.org/s_l_teichmann/mtsatellite/common" ) -const sleepInterval = time.Second * 5 +const ( + sleepInterval = time.Second * 5 + sleepPG = time.Second +) var geoJSONTmpl = template.Must(template.New("geojson").Parse( `{ "type": "Feature", @@ -83,11 +86,12 @@ func (pls sortPlayersByName) Swap(i, j int) { } const selectPlayersSQL = ` -SELECT posx, posy, posz, name +SELECT posx/10.0, posy/10.0, posz/10.0, name FROM player WHERE modification_date > now() - '2m'::interval` func playersFromPostgreSQL(connS string) ([]*player, error) { + time.Sleep(sleepPG) db, err := sql.Open("pgx", connS) if err != nil { return nil, err @@ -101,7 +105,7 @@ func playersFromPostgreSQL(connS string) ([]*player, error) { var pls []*player for rows.Next() { var p player - if err := rows.Scan(&p.Y, &p.Y, &p.Z, &p.Name); err != nil { + if err := rows.Scan(&p.X, &p.Y, &p.Z, &p.Name); err != nil { return nil, err } pls = append(pls, &p) From db45013f4e2404c2a1e2078189bf984dd09c38a3 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 28 Feb 2022 15:49:14 +0100 Subject: [PATCH 11/26] Simplified player forwarding. Forward empty player list --- cmd/mtwebmapper/forwardupdates.go | 19 +++++-------------- cmd/mtwebmapper/players.go | 18 +++--------------- 2 files changed, 8 insertions(+), 29 deletions(-) diff --git a/cmd/mtwebmapper/forwardupdates.go b/cmd/mtwebmapper/forwardupdates.go index d492d9d..0ede522 100644 --- a/cmd/mtwebmapper/forwardupdates.go +++ b/cmd/mtwebmapper/forwardupdates.go @@ -27,8 +27,8 @@ type connection struct { } type msg struct { - tiles []xz - pls []*player + Tiles []xz `json:"tiles,omitempty"` + Pls []*player `json:"pls,omitempty"` } func newWebsocketForwarder() *websocketForwarder { @@ -55,19 +55,10 @@ func (wsf *websocketForwarder) run() { if len(wsf.connections) == 0 { continue } - encMsg := map[string]interface{}{} - - if message.tiles != nil { - encMsg["tiles"] = message.tiles - } - - if message.pls != nil { - encMsg["players"] = message.pls - } var buf bytes.Buffer encoder := json.NewEncoder(&buf) - if err := encoder.Encode(encMsg); err != nil { + if err := encoder.Encode(message); err != nil { log.Printf("encoding changes failed: %s\n", err) continue } @@ -98,11 +89,11 @@ func (wsf *websocketForwarder) ServeHTTP(rw http.ResponseWriter, r *http.Request } func (wsf *websocketForwarder) BaseTilesUpdated(changes []xz) { - wsf.broadcast <- msg{tiles: changes} + wsf.broadcast <- msg{Tiles: changes} } func (wsf *websocketForwarder) BroadcastPlayers(pls []*player) { - wsf.broadcast <- msg{pls: pls} + wsf.broadcast <- msg{Pls: pls} } func (c *connection) writer() { diff --git a/cmd/mtwebmapper/players.go b/cmd/mtwebmapper/players.go index f62c820..dd8f2a9 100644 --- a/cmd/mtwebmapper/players.go +++ b/cmd/mtwebmapper/players.go @@ -71,20 +71,6 @@ func (p *player) same(o *player) bool { math.Abs(p.Z-o.Z) < 0.000001 } -type sortPlayersByName []*player - -func (pls sortPlayersByName) Len() int { - return len(pls) -} - -func (pls sortPlayersByName) Less(i, j int) bool { - return pls[i].Name < pls[j].Name -} - -func (pls sortPlayersByName) Swap(i, j int) { - pls[i], pls[j] = pls[j], pls[i] -} - const selectPlayersSQL = ` SELECT posx/10.0, posy/10.0, posz/10.0, name FROM player @@ -158,7 +144,9 @@ func (ps *players) run() { continue } //log.Printf("%+q\n", pls) - sort.Sort(sortPlayersByName(pls)) + sort.Slice(pls, func(i, j int) bool { + return pls[i].Name < pls[j].Name + }) var change bool ps.mu.Lock() //log.Printf("%+q\n", pls) From 6ffa0e1001508b12f881684ef1452d0477a2190f Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 28 Feb 2022 16:21:26 +0100 Subject: [PATCH 12/26] Name players correctly. --- cmd/mtwebmapper/forwardupdates.go | 2 +- common/basetilecreator.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/mtwebmapper/forwardupdates.go b/cmd/mtwebmapper/forwardupdates.go index 0ede522..a51c6a1 100644 --- a/cmd/mtwebmapper/forwardupdates.go +++ b/cmd/mtwebmapper/forwardupdates.go @@ -28,7 +28,7 @@ type connection struct { type msg struct { Tiles []xz `json:"tiles,omitempty"` - Pls []*player `json:"pls,omitempty"` + Pls []*player `json:"players,omitempty"` } func newWebsocketForwarder() *websocketForwarder { diff --git a/common/basetilecreator.go b/common/basetilecreator.go index b6719a4..c3fb367 100644 --- a/common/basetilecreator.go +++ b/common/basetilecreator.go @@ -200,7 +200,7 @@ func (btc *BaseTileCreator) WriteFunc(i, j int, update BaseTileUpdateFunc) func( return true, SaveAsPNGAtomic(path, image) } - log.Printf("(%d, %d) is unchanged.\n", x, z) + //log.Printf("(%d, %d) is unchanged.\n", x, z) return false, nil } } From 6fb338dbb802a5223ddc97a5e02ce87f988ada6f Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 28 Feb 2022 21:26:04 +0100 Subject: [PATCH 13/26] Another try to fix not removing last player. --- cmd/mtwebmapper/players.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/mtwebmapper/players.go b/cmd/mtwebmapper/players.go index dd8f2a9..54d43d4 100644 --- a/cmd/mtwebmapper/players.go +++ b/cmd/mtwebmapper/players.go @@ -133,13 +133,17 @@ func samePlayers(a, b []*player) bool { func (ps *players) run() { for { + ps.mu.Lock() + empty := len(ps.pls) == 0 + ps.mu.Unlock() + pls, err := ps.readFromFIFO() if err != nil { //log.Printf("err: %s\n", err) time.Sleep(sleepInterval) continue } - if pls == nil { + if empty && pls == nil { //log.Println("no players") continue } From 68ce57aa757de9d8b3082cc97bf9aa8bf410bf3a Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 1 Mar 2022 13:50:18 +0100 Subject: [PATCH 14/26] Fixed command line parameter. --- cmd/mtwebmapper/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/mtwebmapper/main.go b/cmd/mtwebmapper/main.go index 0465733..e80b5a8 100644 --- a/cmd/mtwebmapper/main.go +++ b/cmd/mtwebmapper/main.go @@ -76,7 +76,7 @@ func main() { flag.StringVar(&playersFIFO, "ps", "", "Path to FIFO file to read active players from (shorthand).") flag.IntVar(&yMin, "ymin", common.MinHeight, "Minimum y in blocks.") flag.IntVar(&yMax, "ymax", common.MaxHeight, "Maximum y in blocks.") - flag.CommandLine.DurationVar(&changeDuration, "change-duration", time.Second, "Duration to aggregate changes. (PG only)") + flag.DurationVar(&changeDuration, "change-duration", time.Second, "Duration to aggregate changes. (PG only)") flag.BoolVar(&version, "version", false, "Print version and exit.") flag.Parse() From 834c8a9bc6acf47bd46ff00550a37fc243c80d13 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 1 Mar 2022 13:59:41 +0100 Subject: [PATCH 15/26] Use database abstration in tilemapper, too. --- cmd/mttilemapper/main.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/cmd/mttilemapper/main.go b/cmd/mttilemapper/main.go index 55d6ffc..1801d7c 100644 --- a/cmd/mttilemapper/main.go +++ b/cmd/mttilemapper/main.go @@ -6,12 +6,10 @@ package main import ( "flag" - "fmt" "image" "log" "os" "runtime/pprof" - "strings" "bitbucket.org/s_l_teichmann/mtsatellite/common" ) @@ -85,17 +83,9 @@ func main() { colors.TransparentDim = common.Clamp32f( float32(transparentDim/100.0), 0.0, 100.0) - var proto, address string - if strings.ContainsRune(host, '/') { - proto, address = "unix", host - } else { - proto, address = "tcp", fmt.Sprintf("%s:%d", host, port) - } - - var client *common.RedisClient - - if client, err = common.NewRedisClient(proto, address); err != nil { - log.Fatalf("Cannot connect to '%s': %s", address, err) + client, err := common.CreateDBClientCreator(host, port)() + if err != nil { + log.Fatalf("Cannot connect to '%s:%d': %s", host, port, err) } defer client.Close() From c507663826a927d4eaa2e620b4c38ced8880f1b3 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 1 Mar 2022 14:47:14 +0100 Subject: [PATCH 16/26] Use a more general db client factory approach. --- cmd/mtseeder/baselevel.go | 4 ++-- cmd/mtseeder/main.go | 8 +++++-- cmd/mttilemapper/main.go | 7 ++++++- cmd/mtwebmapper/main.go | 8 +++++-- cmd/mtwebmapper/tilesupdater.go | 9 ++++---- common/clientfactory.go | 37 ++++++++++++--------------------- common/dbclient.go | 10 --------- common/pgclient.go | 35 ++++++++++++++++++++++++------- common/redisclient.go | 34 ++++++++++++++++++++++++++++++ 9 files changed, 99 insertions(+), 53 deletions(-) delete mode 100644 common/dbclient.go diff --git a/cmd/mtseeder/baselevel.go b/cmd/mtseeder/baselevel.go index 96bf923..d11484a 100644 --- a/cmd/mtseeder/baselevel.go +++ b/cmd/mtseeder/baselevel.go @@ -56,7 +56,7 @@ func createTiles( } func createBaseLevel( - dbcc common.DBClientCreator, + dbcf common.DBClientFactory, xMin, yMin, zMin, xMax, yMax, zMax int, transparent bool, transparentDim float32, colorsFile string, bg color.RGBA, outDir string, @@ -81,7 +81,7 @@ func createBaseLevel( for i := 0; i < numWorkers; i++ { var client common.DBClient - if client, err = dbcc(); err != nil { + if client, err = dbcf.Create(); err != nil { return } done.Add(1) diff --git a/cmd/mtseeder/main.go b/cmd/mtseeder/main.go index 7ba1fba..31d8664 100644 --- a/cmd/mtseeder/main.go +++ b/cmd/mtseeder/main.go @@ -68,12 +68,16 @@ func main() { bg := common.ParseColorDefault(bgColor, common.BackgroundColor) - dbcc := common.CreateDBClientCreator(host, port) + dbcf, err := common.CreateDBClientFactory(host, port) + if err != nil { + log.Fatalf("error: %s\n", err) + } + defer dbcf.Close() if !skipBaseLevel { td := common.Clamp32f(float32(transparentDim/100.0), 0.0, 1.0) if err := createBaseLevel( - dbcc, + dbcf, xMin, yMin, zMin, xMax, yMax, zMax, transparent, td, colorsFile, bg, diff --git a/cmd/mttilemapper/main.go b/cmd/mttilemapper/main.go index 1801d7c..d24ae3b 100644 --- a/cmd/mttilemapper/main.go +++ b/cmd/mttilemapper/main.go @@ -83,7 +83,12 @@ func main() { colors.TransparentDim = common.Clamp32f( float32(transparentDim/100.0), 0.0, 100.0) - client, err := common.CreateDBClientCreator(host, port)() + cf, err := common.CreateDBClientFactory(host, port) + if err != nil { + log.Fatalf("error: %v\n", err) + } + + client, err := cf.Create() if err != nil { log.Fatalf("Cannot connect to '%s:%d': %s", host, port, err) } diff --git a/cmd/mtwebmapper/main.go b/cmd/mtwebmapper/main.go index e80b5a8..711685d 100644 --- a/cmd/mtwebmapper/main.go +++ b/cmd/mtwebmapper/main.go @@ -118,7 +118,11 @@ func main() { colors.TransparentDim = common.Clamp32f( float32(transparentDim/100.0), 0.0, 100.0) - dbcc := common.CreateDBClientCreator(redisHost, redisPort) + dbcf, err := common.CreateDBClientFactory(redisHost, redisPort) + if err != nil { + log.Fatalf("error: %v\n", err) + } + defer dbcf.Close() var allowedUpdateIps []net.IP if allowedUpdateIps, err = ipsFromHosts(updateHosts); err != nil { @@ -127,7 +131,7 @@ func main() { tu := newTileUpdater( mapDir, - dbcc, + dbcf, allowedUpdateIps, colors, bg, yMin, yMax, diff --git a/cmd/mtwebmapper/tilesupdater.go b/cmd/mtwebmapper/tilesupdater.go index ebb8173..7b764d2 100644 --- a/cmd/mtwebmapper/tilesupdater.go +++ b/cmd/mtwebmapper/tilesupdater.go @@ -38,7 +38,7 @@ type tileUpdater struct { changes map[xz]struct{} btu baseTilesUpdates mapDir string - dbcc common.DBClientCreator + dbcf common.DBClientFactory ips []net.IP colors *common.Colors bg color.RGBA @@ -82,7 +82,7 @@ func (c xz) parent() xzm { func newTileUpdater( mapDir string, - dbcc common.DBClientCreator, + dbcf common.DBClientFactory, ips []net.IP, colors *common.Colors, bg color.RGBA, @@ -94,7 +94,7 @@ func newTileUpdater( tu := tileUpdater{ btu: btu, mapDir: mapDir, - dbcc: dbcc, + dbcf: dbcf, ips: ips, changes: map[xz]struct{}{}, colors: colors, @@ -262,10 +262,11 @@ func (tu *tileUpdater) doUpdates() { for i, n := 0, common.Min(tu.workers, len(changes)); i < n; i++ { var client common.DBClient var err error - if client, err = tu.dbcc(); err != nil { + if client, err = tu.dbcf.Create(); err != nil { log.Printf("WARN: Cannot connect to redis server: %s\n", err) continue } + btc := common.NewBaseTileCreator( client, tu.colors, tu.bg, tu.yMin, tu.yMax, diff --git a/common/clientfactory.go b/common/clientfactory.go index fb87343..35333bc 100644 --- a/common/clientfactory.go +++ b/common/clientfactory.go @@ -5,11 +5,18 @@ package common import ( - "fmt" "strings" ) -type DBClientCreator func() (DBClient, error) +type DBClient interface { + QueryCuboid(cuboid Cuboid, fn func(*Block) *Block) (count int, err error) + Close() error +} + +type DBClientFactory interface { + Create() (DBClient, error) + Close() error +} func IsPostgreSQL(host string) (string, bool) { if strings.HasPrefix(host, "postgres:") { @@ -18,29 +25,11 @@ func IsPostgreSQL(host string) (string, bool) { return "", false } -func CreateDBClientCreator(host string, port int) DBClientCreator { +func CreateDBClientFactory(host string, port int) (DBClientFactory, error) { - if host, ok := IsPostgreSQL(host); ok { - return func() (DBClient, error) { - return NewPGClient(host) - } + if connS, ok := IsPostgreSQL(host); ok { + return NewPGClientFactory(connS) } - var address string - if strings.ContainsRune(host, '/') { - address = host - } else { - address = fmt.Sprintf("%s:%d", host, port) - } - - var proto string - if strings.ContainsRune(address, '/') { - proto = "unix" - } else { - proto = "tcp" - } - - return func() (DBClient, error) { - return NewRedisClient(proto, address) - } + return NewRedisClientFactory(host, port) } diff --git a/common/dbclient.go b/common/dbclient.go deleted file mode 100644 index 29af14c..0000000 --- a/common/dbclient.go +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2022 by Sascha L. Teichmann -// Use of this source code is governed by the MIT license -// that can be found in the LICENSE file. - -package common - -type DBClient interface { - QueryCuboid(cuboid Cuboid, fn func(*Block) *Block) (count int, err error) - Close() error -} diff --git a/common/pgclient.go b/common/pgclient.go index a419a09..7c691b3 100644 --- a/common/pgclient.go +++ b/common/pgclient.go @@ -19,24 +19,43 @@ WHERE posz BETWEEN $5 AND $6` type PGClient struct { - db *sql.DB + conn *sql.Conn queryCuboidStmt *sql.Stmt } -func NewPGClient(connS string) (*PGClient, error) { +type PGClientFactory struct { + db *sql.DB +} + +func NewPGClientFactory(connS string) (*PGClientFactory, error) { db, err := sql.Open("pgx", connS) if err != nil { return nil, err } - stmt, err := db.Prepare(queryCuboidSQL) + return &PGClientFactory{db: db}, nil +} + +func (pgcf *PGClientFactory) Close() error { + return pgcf.db.Close() +} + +func (pgcf *PGClientFactory) Create() (DBClient, error) { + + ctx := context.Background() + + conn, err := pgcf.db.Conn(ctx) if err != nil { return nil, err } - client := PGClient{ - db: db, - queryCuboidStmt: stmt, + stmt, err := conn.PrepareContext(ctx, queryCuboidSQL) + if err != nil { + conn.Close() + return nil, err } - return &client, nil + return &PGClient{ + conn: conn, + queryCuboidStmt: stmt, + }, nil } func (pgc *PGClient) QueryCuboid( @@ -94,5 +113,5 @@ func (pgc *PGClient) Close() error { if pgc.queryCuboidStmt != nil { pgc.queryCuboidStmt.Close() } - return pgc.db.Close() + return pgc.conn.Close() } diff --git a/common/redisclient.go b/common/redisclient.go index 23c04bb..b4d36c6 100644 --- a/common/redisclient.go +++ b/common/redisclient.go @@ -11,9 +11,43 @@ import ( "fmt" "net" "strconv" + "strings" "unicode" ) +type RedisClientFactory struct { + proto string + address string +} + +func NewRedisClientFactory(host string, port int) (*RedisClientFactory, error) { + var address string + if strings.ContainsRune(host, '/') { + address = host + } else { + address = fmt.Sprintf("%s:%d", host, port) + } + + var proto string + if strings.ContainsRune(address, '/') { + proto = "unix" + } else { + proto = "tcp" + } + return &RedisClientFactory{ + proto: proto, + address: address, + }, nil +} + +func (rcf *RedisClientFactory) Close() error { + return nil +} + +func (rcf *RedisClientFactory) Create() (DBClient, error) { + return NewRedisClient(rcf.proto, rcf.address) +} + type RedisClient struct { conn net.Conn reader *bufio.Reader From da2b3279858eb37d37ace85e64d3615da64f61a9 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 1 Mar 2022 21:02:08 +0100 Subject: [PATCH 17/26] Wait until all tiles are written out or are cancelled. --- cmd/mtwebmapper/tilesupdater.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/cmd/mtwebmapper/tilesupdater.go b/cmd/mtwebmapper/tilesupdater.go index 7b764d2..a59b8b9 100644 --- a/cmd/mtwebmapper/tilesupdater.go +++ b/cmd/mtwebmapper/tilesupdater.go @@ -400,26 +400,30 @@ func (tu *tileUpdater) updateBaseTiles( update common.BaseTileUpdateFunc) { type jobWriter struct { - job *xzc - wFn func() (bool, error) + canceled *bool + wFn func() (bool, error) } jWs := make(chan jobWriter) + asyncWrite := make(chan struct{}) + go func() { + defer close(asyncWrite) + for jw := range jWs { updated, err := jw.wFn() if err != nil { + *jw.canceled = true log.Printf("WARN: writing tile failed: %v.\n", err) } if !updated { - jw.job.canceled = true + *jw.canceled = true } } }() defer func() { - close(jWs) btc.Close() done.Done() }() @@ -431,6 +435,13 @@ func (tu *tileUpdater) updateBaseTiles( job.canceled = true continue } - jWs <- jobWriter{job, btc.WriteFunc(int(job.X), int(job.Z), update)} + jWs <- jobWriter{ + &job.canceled, + btc.WriteFunc(int(job.X), int(job.Z), update), + } } + + close(jWs) + // Wait until all tiles are written. + <-asyncWrite } From 25e62b3a8b76e0fcdb0b94af7e6bb62f4fce1e27 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 1 Mar 2022 22:29:56 +0100 Subject: [PATCH 18/26] Use more modern function channel approach in websocket forwarder. --- cmd/mtwebmapper/forwardupdates.go | 105 +++++++++++++++++------------- 1 file changed, 60 insertions(+), 45 deletions(-) diff --git a/cmd/mtwebmapper/forwardupdates.go b/cmd/mtwebmapper/forwardupdates.go index a51c6a1..fe5ae20 100644 --- a/cmd/mtwebmapper/forwardupdates.go +++ b/cmd/mtwebmapper/forwardupdates.go @@ -15,10 +15,8 @@ import ( type websocketForwarder struct { upgrader *websocket.Upgrader - register chan *connection - unregister chan *connection - broadcast chan msg - connections map[*connection]bool + connections map[*connection]struct{} + funcs chan func(*websocketForwarder) } type connection struct { @@ -32,49 +30,74 @@ type msg struct { } func newWebsocketForwarder() *websocketForwarder { - upgrader := &websocket.Upgrader{ReadBufferSize: 512, WriteBufferSize: 2048} + upgrader := &websocket.Upgrader{ + ReadBufferSize: 512, + WriteBufferSize: 2048, + //CheckOrigin: func(*http.Request) bool { return true }, + } return &websocketForwarder{ upgrader: upgrader, - register: make(chan *connection), - unregister: make(chan *connection), - broadcast: make(chan msg), - connections: make(map[*connection]bool)} + connections: make(map[*connection]struct{}), + funcs: make(chan func(*websocketForwarder)), + } } func (wsf *websocketForwarder) run() { - for { - select { - case c := <-wsf.register: - wsf.connections[c] = true - case c := <-wsf.unregister: - if _, ok := wsf.connections[c]; ok { + for fn := range wsf.funcs { + fn(wsf) + } +} + +func (wsf *websocketForwarder) register(c *connection) { + wsf.funcs <- func(wsf *websocketForwarder) { + wsf.connections[c] = struct{}{} + } +} + +func (wsf *websocketForwarder) unregister(c *connection) { + wsf.funcs <- func(wsf *websocketForwarder) { + if _, ok := wsf.connections[c]; ok { + delete(wsf.connections, c) + close(c.send) + } + } +} + +func (wsf *websocketForwarder) send(m *msg) { + wsf.funcs <- func(wsf *websocketForwarder) { + if len(wsf.connections) == 0 { + return + } + + var buf bytes.Buffer + + encoder := json.NewEncoder(&buf) + if err := encoder.Encode(m); err != nil { + log.Printf("encoding changes failed: %s\n", err) + return + } + + data := buf.Bytes() + + for c := range wsf.connections { + select { + case c.send <- data: + default: delete(wsf.connections, c) close(c.send) } - case message := <-wsf.broadcast: - if len(wsf.connections) == 0 { - continue - } - - var buf bytes.Buffer - encoder := json.NewEncoder(&buf) - if err := encoder.Encode(message); err != nil { - log.Printf("encoding changes failed: %s\n", err) - continue - } - m := buf.Bytes() - for c := range wsf.connections { - select { - case c.send <- m: - default: - delete(wsf.connections, c) - close(c.send) - } - } } } } +func (wsf *websocketForwarder) BaseTilesUpdated(changes []xz) { + wsf.send(&msg{Tiles: changes}) +} + +func (wsf *websocketForwarder) BroadcastPlayers(pls []*player) { + wsf.send(&msg{Pls: pls}) +} + func (wsf *websocketForwarder) ServeHTTP(rw http.ResponseWriter, r *http.Request) { ws, err := wsf.upgrader.Upgrade(rw, r, nil) if err != nil { @@ -82,20 +105,12 @@ func (wsf *websocketForwarder) ServeHTTP(rw http.ResponseWriter, r *http.Request return } c := &connection{ws: ws, send: make(chan []byte, 8)} - wsf.register <- c - defer func() { wsf.unregister <- c }() + wsf.register(c) + defer wsf.unregister(c) go c.writer() c.reader() } -func (wsf *websocketForwarder) BaseTilesUpdated(changes []xz) { - wsf.broadcast <- msg{Tiles: changes} -} - -func (wsf *websocketForwarder) BroadcastPlayers(pls []*player) { - wsf.broadcast <- msg{Pls: pls} -} - func (c *connection) writer() { defer c.ws.Close() for msg := range c.send { From e6ea8fe3d138342390dd25e765d9b98cfabc8d86 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 1 Mar 2022 23:30:31 +0100 Subject: [PATCH 19/26] Support for configuration file in webmapper --- cmd/mtwebmapper/config.go | 81 +++++++++++++++++++++++++++++ cmd/mtwebmapper/main.go | 106 +++++++++++--------------------------- go.mod | 1 + go.sum | 2 + 4 files changed, 115 insertions(+), 75 deletions(-) create mode 100644 cmd/mtwebmapper/config.go diff --git a/cmd/mtwebmapper/config.go b/cmd/mtwebmapper/config.go new file mode 100644 index 0000000..63407c7 --- /dev/null +++ b/cmd/mtwebmapper/config.go @@ -0,0 +1,81 @@ +// Copyright 2014, 2015, 2022 by Sascha L. Teichmann +// Use of this source code is governed by the MIT license +// that can be found in the LICENSE file. + +package main + +import ( + "flag" + "time" + + "github.com/BurntSushi/toml" + + "bitbucket.org/s_l_teichmann/mtsatellite/common" +) + +type config struct { + WebPort int `toml:"web_port"` + WebHost string `toml:"web_host"` + WebDir string `toml:"web"` + MapDir string `toml:"map"` + RedisPort int `toml:"redis_port"` + RedisHost string `toml:"redis_host"` + ColorsFile string `toml:"update_hosts"` + BGColor string `toml:"background"` + Workers int `toml:"workers"` + Transparent bool `toml:"transparent"` + TransparentDim float64 `toml:"transparent_dim"` + UpdateHosts string `toml:"update_hosts"` + Websockets bool `toml:"websockets"` + PlayersFIFO string `toml:"players"` + YMin int `toml:"ymin"` + YMax int `toml:"ymax"` + ChangeDuration time.Duration `toml:"change_duration"` +} + +func (cfg *config) bindFlags() { + defaultBgColor := common.ColorToHex(common.BackgroundColor) + + flag.IntVar(&cfg.WebPort, "web-port", 8808, "port of the web server") + flag.IntVar(&cfg.WebPort, "p", 8808, "port of the web server (shorthand)") + flag.StringVar(&cfg.WebHost, "web-host", "localhost", "address to bind web server") + flag.StringVar(&cfg.WebHost, "h", "localhost", "address to bind web server(shorthand)") + flag.StringVar(&cfg.WebDir, "web", "web", "static served web files.") + flag.StringVar(&cfg.WebDir, "w", "web", "static served web files (shorthand)") + flag.StringVar(&cfg.MapDir, "map", "map", "directory of prerendered tiles") + flag.StringVar(&cfg.MapDir, "m", "map", "directory of prerendered tiles (shorthand)") + flag.StringVar(&cfg.UpdateHosts, "update-hosts", "localhost", + "';' separated list of hosts which are allowed to send map update requests") + flag.StringVar(&cfg.UpdateHosts, "u", "localhost", + "';' separated list of hosts which are allowed to send map update requests (shorthand)") + flag.StringVar(&cfg.RedisHost, "redis-host", "", "address of the backend Redis server") + flag.StringVar(&cfg.RedisHost, "rh", "", "address of the backend Redis server (shorthand)") + flag.IntVar(&cfg.RedisPort, "redis-port", 6379, "port of the backend Redis server") + flag.IntVar(&cfg.RedisPort, "rp", 6379, "port of the backend Redis server (shorthand)") + flag.IntVar(&cfg.Workers, "workers", 1, "number of workers to render tiles") + flag.StringVar(&cfg.ColorsFile, "colors", "colors.txt", "colors used to render map tiles.") + flag.StringVar(&cfg.ColorsFile, "c", "colors.txt", "colors used to render map tiles (shorthand).") + flag.StringVar(&cfg.BGColor, "background", defaultBgColor, "background color") + flag.StringVar(&cfg.BGColor, "bg", defaultBgColor, "background color (shorthand)") + flag.BoolVar(&cfg.Transparent, "transparent", false, "Render transparent blocks.") + flag.BoolVar(&cfg.Transparent, "t", false, "Render transparent blocks (shorthand).") + flag.Float64Var(&cfg.TransparentDim, + "transparent-dim", common.DefaultTransparentDim*100.0, + "Extra dimming of transparent nodes each depth meter in percent.") + flag.Float64Var(&cfg.TransparentDim, + "td", common.DefaultTransparentDim*100.0, + "Extra dimming of transparent nodes each depth meter in percent. (shorthand)") + flag.BoolVar(&cfg.Websockets, "websockets", false, "Forward tile changes to clients via websockets.") + flag.BoolVar(&cfg.Websockets, "ws", false, "Forward tile changes to clients via websockets (shorthand).") + flag.StringVar(&cfg.PlayersFIFO, "players", "", "Path to FIFO file to read active players from.") + flag.StringVar(&cfg.PlayersFIFO, "ps", "", "Path to FIFO file to read active players from (shorthand).") + flag.IntVar(&cfg.YMin, "ymin", common.MinHeight, "Minimum y in blocks.") + flag.IntVar(&cfg.YMax, "ymax", common.MaxHeight, "Maximum y in blocks.") + flag.DurationVar(&cfg.ChangeDuration, + "change-duration", time.Second, "Duration to aggregate changes. (PG only)") +} + +func (cfg *config) load(fname string) error { + _, err := toml.DecodeFile(fname, &cfg) + return err +} diff --git a/cmd/mtwebmapper/main.go b/cmd/mtwebmapper/main.go index 711685d..84d8f27 100644 --- a/cmd/mtwebmapper/main.go +++ b/cmd/mtwebmapper/main.go @@ -10,7 +10,6 @@ import ( "log" "net" "net/http" - "time" "bitbucket.org/s_l_teichmann/mtsatellite/common" @@ -18,66 +17,17 @@ import ( ) func main() { + var ( - webPort int - webHost string - webDir string - mapDir string - redisPort int - redisHost string - colorsFile string - bgColor string - workers int - transparent bool - transparentDim float64 - updateHosts string - websockets bool - playersFIFO string - version bool - yMin int - yMax int - changeDuration time.Duration + cfg config + cfgFile string + version bool ) - defaultBgColor := common.ColorToHex(common.BackgroundColor) - - flag.IntVar(&webPort, "web-port", 8808, "port of the web server") - flag.IntVar(&webPort, "p", 8808, "port of the web server (shorthand)") - flag.StringVar(&webHost, "web-host", "localhost", "address to bind web server") - flag.StringVar(&webHost, "h", "localhost", "address to bind web server(shorthand)") - flag.StringVar(&webDir, "web", "web", "static served web files.") - flag.StringVar(&webDir, "w", "web", "static served web files (shorthand)") - flag.StringVar(&mapDir, "map", "map", "directory of prerendered tiles") - flag.StringVar(&mapDir, "m", "map", "directory of prerendered tiles (shorthand)") - flag.StringVar(&updateHosts, "update-hosts", "localhost", - "';' separated list of hosts which are allowed to send map update requests") - flag.StringVar(&updateHosts, "u", "localhost", - "';' separated list of hosts which are allowed to send map update requests (shorthand)") - flag.StringVar(&redisHost, "redis-host", "", "address of the backend Redis server") - flag.StringVar(&redisHost, "rh", "", "address of the backend Redis server (shorthand)") - flag.IntVar(&redisPort, "redis-port", 6379, "port of the backend Redis server") - flag.IntVar(&redisPort, "rp", 6379, "port of the backend Redis server (shorthand)") - flag.IntVar(&workers, "workers", 1, "number of workers to render tiles") - flag.StringVar(&colorsFile, "colors", "colors.txt", "colors used to render map tiles.") - flag.StringVar(&colorsFile, "c", "colors.txt", "colors used to render map tiles (shorthand).") - flag.StringVar(&bgColor, "background", defaultBgColor, "background color") - flag.StringVar(&bgColor, "bg", defaultBgColor, "background color (shorthand)") - flag.BoolVar(&transparent, "transparent", false, "Render transparent blocks.") - flag.BoolVar(&transparent, "t", false, "Render transparent blocks (shorthand).") - flag.Float64Var(&transparentDim, - "transparent-dim", common.DefaultTransparentDim*100.0, - "Extra dimming of transparent nodes each depth meter in percent.") - flag.Float64Var(&transparentDim, - "td", common.DefaultTransparentDim*100.0, - "Extra fimming of transparent nodes each depth meter in percent. (shorthand)") - flag.BoolVar(&websockets, "websockets", false, "Forward tile changes to clients via websockets.") - flag.BoolVar(&websockets, "ws", false, "Forward tile changes to clients via websockets (shorthand).") - flag.StringVar(&playersFIFO, "players", "", "Path to FIFO file to read active players from.") - flag.StringVar(&playersFIFO, "ps", "", "Path to FIFO file to read active players from (shorthand).") - flag.IntVar(&yMin, "ymin", common.MinHeight, "Minimum y in blocks.") - flag.IntVar(&yMax, "ymax", common.MaxHeight, "Maximum y in blocks.") - flag.DurationVar(&changeDuration, "change-duration", time.Second, "Duration to aggregate changes. (PG only)") + flag.StringVar(&cfgFile, "config", "", "configuration file") + flag.StringVar(&cfgFile, "c", "", "configuration file (shorthand)") flag.BoolVar(&version, "version", false, "Print version and exit.") + cfg.bindFlags() flag.Parse() @@ -85,72 +35,78 @@ func main() { common.PrintVersionAndExit() } - bg := common.ParseColorDefault(bgColor, common.BackgroundColor) + if cfgFile != "" { + if err := cfg.load(cfgFile); err != nil { + log.Fatalf("error: %v\n", err) + } + } + + bg := common.ParseColorDefault(cfg.BGColor, common.BackgroundColor) router := mux.NewRouter() - subBaseLine := newSubBaseLine(mapDir, bg) + subBaseLine := newSubBaseLine(cfg.MapDir, bg) router.Path("/map/{z:[0-9]+}/{x:[0-9]+}/{y:[0-9]+}.png").Handler(subBaseLine) var btu baseTilesUpdates var wsf *websocketForwarder - if websockets { + if cfg.Websockets { wsf = newWebsocketForwarder() go wsf.run() router.Path("/ws").Methods("GET").Handler(wsf) btu = wsf } - if playersFIFO != "" { - plys := newPlayers(playersFIFO, wsf) + if cfg.PlayersFIFO != "" { + plys := newPlayers(cfg.PlayersFIFO, wsf) go plys.run() router.Path("/players").Methods("GET").Handler(plys) } - if redisHost != "" { + if cfg.RedisHost != "" { var colors *common.Colors var err error - if colors, err = common.ParseColors(colorsFile); err != nil { + if colors, err = common.ParseColors(cfg.ColorsFile); err != nil { log.Fatalf("ERROR: problem loading colors: %s", err) } colors.TransparentDim = common.Clamp32f( - float32(transparentDim/100.0), 0.0, 100.0) + float32(cfg.TransparentDim/100.0), 0.0, 100.0) - dbcf, err := common.CreateDBClientFactory(redisHost, redisPort) + dbcf, err := common.CreateDBClientFactory(cfg.RedisHost, cfg.RedisPort) if err != nil { log.Fatalf("error: %v\n", err) } defer dbcf.Close() var allowedUpdateIps []net.IP - if allowedUpdateIps, err = ipsFromHosts(updateHosts); err != nil { + if allowedUpdateIps, err = ipsFromHosts(cfg.UpdateHosts); err != nil { log.Fatalf("ERROR: name resolving problem: %s", err) } tu := newTileUpdater( - mapDir, + cfg.MapDir, dbcf, allowedUpdateIps, colors, bg, - yMin, yMax, - transparent, - workers, + cfg.YMin, cfg.YMax, + cfg.Transparent, + cfg.Workers, btu) go tu.doUpdates() - if pgHost, ok := common.IsPostgreSQL(redisHost); btu != nil && ok { - go tu.listen(pgHost, changeDuration) + if pgHost, ok := common.IsPostgreSQL(cfg.RedisHost); btu != nil && ok { + go tu.listen(pgHost, cfg.ChangeDuration) } else { router.Path("/update").Methods("POST").Handler(tu) } } - router.PathPrefix("/").Handler(http.FileServer(http.Dir(webDir))) + router.PathPrefix("/").Handler(http.FileServer(http.Dir(cfg.WebDir))) http.Handle("/", router) - addr := fmt.Sprintf("%s:%d", webHost, webPort) + addr := fmt.Sprintf("%s:%d", cfg.WebHost, cfg.WebPort) if err := http.ListenAndServe(addr, nil); err != nil { log.Fatalf("Starting server failed: %s\n", err) } diff --git a/go.mod b/go.mod index e1986c6..155d8f8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module bitbucket.org/s_l_teichmann/mtsatellite go 1.13 require ( + github.com/BurntSushi/toml v1.0.0 // indirect github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 diff --git a/go.sum b/go.sum index 947d427..83f3915 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= +github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b h1:5Ci5wpOL75rYF6RQGRoqhEAU6xLJ6n/D4SckXX1yB74= From 1155930bc9e86dc4f2a83e9fe9af5568b7e07522 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 1 Mar 2022 23:39:01 +0100 Subject: [PATCH 20/26] Dont define c flag twice --- cmd/mtwebmapper/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/mtwebmapper/main.go b/cmd/mtwebmapper/main.go index 84d8f27..1f4ba41 100644 --- a/cmd/mtwebmapper/main.go +++ b/cmd/mtwebmapper/main.go @@ -25,7 +25,6 @@ func main() { ) flag.StringVar(&cfgFile, "config", "", "configuration file") - flag.StringVar(&cfgFile, "c", "", "configuration file (shorthand)") flag.BoolVar(&version, "version", false, "Print version and exit.") cfg.bindFlags() From c17bca9c94d3eec139c18c79be242938c79cf695 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 1 Mar 2022 23:45:25 +0100 Subject: [PATCH 21/26] Deserialize duration from config toml --- cmd/mtwebmapper/config.go | 46 ++++++++++++++++++++++++--------------- cmd/mtwebmapper/main.go | 2 +- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/cmd/mtwebmapper/config.go b/cmd/mtwebmapper/config.go index 63407c7..f4c5f0a 100644 --- a/cmd/mtwebmapper/config.go +++ b/cmd/mtwebmapper/config.go @@ -13,24 +13,34 @@ import ( "bitbucket.org/s_l_teichmann/mtsatellite/common" ) +type duration struct { + time.Duration +} + type config struct { - WebPort int `toml:"web_port"` - WebHost string `toml:"web_host"` - WebDir string `toml:"web"` - MapDir string `toml:"map"` - RedisPort int `toml:"redis_port"` - RedisHost string `toml:"redis_host"` - ColorsFile string `toml:"update_hosts"` - BGColor string `toml:"background"` - Workers int `toml:"workers"` - Transparent bool `toml:"transparent"` - TransparentDim float64 `toml:"transparent_dim"` - UpdateHosts string `toml:"update_hosts"` - Websockets bool `toml:"websockets"` - PlayersFIFO string `toml:"players"` - YMin int `toml:"ymin"` - YMax int `toml:"ymax"` - ChangeDuration time.Duration `toml:"change_duration"` + WebPort int `toml:"web_port"` + WebHost string `toml:"web_host"` + WebDir string `toml:"web"` + MapDir string `toml:"map"` + RedisPort int `toml:"redis_port"` + RedisHost string `toml:"redis_host"` + ColorsFile string `toml:"update_hosts"` + BGColor string `toml:"background"` + Workers int `toml:"workers"` + Transparent bool `toml:"transparent"` + TransparentDim float64 `toml:"transparent_dim"` + UpdateHosts string `toml:"update_hosts"` + Websockets bool `toml:"websockets"` + PlayersFIFO string `toml:"players"` + YMin int `toml:"ymin"` + YMax int `toml:"ymax"` + ChangeDuration duration `toml:"change_duration"` +} + +func (d *duration) UnmarshalText(text []byte) error { + var err error + d.Duration, err = time.ParseDuration(string(text)) + return err } func (cfg *config) bindFlags() { @@ -71,7 +81,7 @@ func (cfg *config) bindFlags() { flag.StringVar(&cfg.PlayersFIFO, "ps", "", "Path to FIFO file to read active players from (shorthand).") flag.IntVar(&cfg.YMin, "ymin", common.MinHeight, "Minimum y in blocks.") flag.IntVar(&cfg.YMax, "ymax", common.MaxHeight, "Maximum y in blocks.") - flag.DurationVar(&cfg.ChangeDuration, + flag.DurationVar(&cfg.ChangeDuration.Duration, "change-duration", time.Second, "Duration to aggregate changes. (PG only)") } diff --git a/cmd/mtwebmapper/main.go b/cmd/mtwebmapper/main.go index 1f4ba41..64c465f 100644 --- a/cmd/mtwebmapper/main.go +++ b/cmd/mtwebmapper/main.go @@ -96,7 +96,7 @@ func main() { go tu.doUpdates() if pgHost, ok := common.IsPostgreSQL(cfg.RedisHost); btu != nil && ok { - go tu.listen(pgHost, cfg.ChangeDuration) + go tu.listen(pgHost, cfg.ChangeDuration.Duration) } else { router.Path("/update").Methods("POST").Handler(tu) } From f533ed71e7bf76af24e48f12ece30e3c44fc5bb7 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Wed, 2 Mar 2022 00:15:30 +0100 Subject: [PATCH 22/26] Fixed mapping of color in configuration. --- cmd/mtwebmapper/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/mtwebmapper/config.go b/cmd/mtwebmapper/config.go index f4c5f0a..bb2bfda 100644 --- a/cmd/mtwebmapper/config.go +++ b/cmd/mtwebmapper/config.go @@ -24,7 +24,7 @@ type config struct { MapDir string `toml:"map"` RedisPort int `toml:"redis_port"` RedisHost string `toml:"redis_host"` - ColorsFile string `toml:"update_hosts"` + ColorsFile string `toml:"colors"` BGColor string `toml:"background"` Workers int `toml:"workers"` Transparent bool `toml:"transparent"` From c87ce03dbe86c3cb9072c3f1f8f56af31050e908 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Thu, 3 Mar 2022 20:22:37 +0100 Subject: [PATCH 23/26] send empty players array if no player is online any more. --- cmd/mtwebmapper/forwardupdates.go | 20 +++++++++++++------- cmd/mtwebmapper/players.go | 3 +++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/cmd/mtwebmapper/forwardupdates.go b/cmd/mtwebmapper/forwardupdates.go index fe5ae20..4333974 100644 --- a/cmd/mtwebmapper/forwardupdates.go +++ b/cmd/mtwebmapper/forwardupdates.go @@ -24,10 +24,14 @@ type connection struct { send chan []byte } -type msg struct { - Tiles []xz `json:"tiles,omitempty"` - Pls []*player `json:"players,omitempty"` -} +type ( + tilesMsg struct { + Tiles []xz `json:"tiles,omitempty"` + } + plsMsg struct { + Pls []*player `json:"players,omitempty"` + } +) func newWebsocketForwarder() *websocketForwarder { upgrader := &websocket.Upgrader{ @@ -63,7 +67,7 @@ func (wsf *websocketForwarder) unregister(c *connection) { } } -func (wsf *websocketForwarder) send(m *msg) { +func (wsf *websocketForwarder) send(m interface{}) { wsf.funcs <- func(wsf *websocketForwarder) { if len(wsf.connections) == 0 { return @@ -79,6 +83,8 @@ func (wsf *websocketForwarder) send(m *msg) { data := buf.Bytes() + log.Printf("sending: %s\n", string(data)) + for c := range wsf.connections { select { case c.send <- data: @@ -91,11 +97,11 @@ func (wsf *websocketForwarder) send(m *msg) { } func (wsf *websocketForwarder) BaseTilesUpdated(changes []xz) { - wsf.send(&msg{Tiles: changes}) + wsf.send(&tilesMsg{Tiles: changes}) } func (wsf *websocketForwarder) BroadcastPlayers(pls []*player) { - wsf.send(&msg{Pls: pls}) + wsf.send(&plsMsg{Pls: pls}) } func (wsf *websocketForwarder) ServeHTTP(rw http.ResponseWriter, r *http.Request) { diff --git a/cmd/mtwebmapper/players.go b/cmd/mtwebmapper/players.go index 54d43d4..da6413c 100644 --- a/cmd/mtwebmapper/players.go +++ b/cmd/mtwebmapper/players.go @@ -160,6 +160,9 @@ func (ps *players) run() { } ps.mu.Unlock() if change && ps.wsf != nil { + if pls == nil { + pls = []*player{} + } // TODO: Throttle this! ps.wsf.BroadcastPlayers(pls) } From 0968c5c07c9781710f9d5094b10a43c0ace0a066 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Thu, 3 Mar 2022 22:30:06 +0100 Subject: [PATCH 24/26] Fixed websocket sending once again. --- cmd/mtwebmapper/forwardupdates.go | 6 ++---- cmd/mtwebmapper/players.go | 15 ++++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cmd/mtwebmapper/forwardupdates.go b/cmd/mtwebmapper/forwardupdates.go index 4333974..ec40ab3 100644 --- a/cmd/mtwebmapper/forwardupdates.go +++ b/cmd/mtwebmapper/forwardupdates.go @@ -26,10 +26,10 @@ type connection struct { type ( tilesMsg struct { - Tiles []xz `json:"tiles,omitempty"` + Tiles []xz `json:"tiles"` } plsMsg struct { - Pls []*player `json:"players,omitempty"` + Pls []*player `json:"players"` } ) @@ -83,8 +83,6 @@ func (wsf *websocketForwarder) send(m interface{}) { data := buf.Bytes() - log.Printf("sending: %s\n", string(data)) - for c := range wsf.connections { select { case c.send <- data: diff --git a/cmd/mtwebmapper/players.go b/cmd/mtwebmapper/players.go index da6413c..e48aa39 100644 --- a/cmd/mtwebmapper/players.go +++ b/cmd/mtwebmapper/players.go @@ -133,9 +133,7 @@ func samePlayers(a, b []*player) bool { func (ps *players) run() { for { - ps.mu.Lock() - empty := len(ps.pls) == 0 - ps.mu.Unlock() + empty := len(ps.current()) == 0 pls, err := ps.readFromFIFO() if err != nil { @@ -169,12 +167,15 @@ func (ps *players) run() { } } +func (ps *players) current() []*player { + ps.mu.RLock() + defer ps.mu.RUnlock() + return ps.pls +} + func (ps *players) ServeHTTP(rw http.ResponseWriter, r *http.Request) { rw.Header().Set("Content-Type", "application/json") - var pls []*player - ps.mu.RLock() - pls = ps.pls - ps.mu.RUnlock() + pls := ps.current() encoder := json.NewEncoder(rw) if err := encoder.Encode(pls); err != nil { log.Printf("error: sending JSON failed: %s\n", err) From 449741db4cb91081040dcc2b113d449b137e21d2 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Thu, 3 Mar 2022 22:57:02 +0100 Subject: [PATCH 25/26] Send players to websocket on connect. --- cmd/mtwebmapper/forwardupdates.go | 41 +++++++++++++++++++++++++------ cmd/mtwebmapper/main.go | 2 ++ cmd/mtwebmapper/players.go | 4 +++ 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/cmd/mtwebmapper/forwardupdates.go b/cmd/mtwebmapper/forwardupdates.go index ec40ab3..394a1c6 100644 --- a/cmd/mtwebmapper/forwardupdates.go +++ b/cmd/mtwebmapper/forwardupdates.go @@ -5,7 +5,6 @@ package main import ( - "bytes" "encoding/json" "log" "net/http" @@ -17,6 +16,7 @@ type websocketForwarder struct { upgrader *websocket.Upgrader connections map[*connection]struct{} funcs chan func(*websocketForwarder) + init func(*websocketForwarder, *connection) } type connection struct { @@ -67,22 +67,24 @@ func (wsf *websocketForwarder) unregister(c *connection) { } } +func (wsf *websocketForwarder) setInit(init func(*websocketForwarder, *connection)) { + wsf.funcs <- func(wsf *websocketForwarder) { + wsf.init = init + } +} + func (wsf *websocketForwarder) send(m interface{}) { wsf.funcs <- func(wsf *websocketForwarder) { if len(wsf.connections) == 0 { return } - var buf bytes.Buffer - - encoder := json.NewEncoder(&buf) - if err := encoder.Encode(m); err != nil { - log.Printf("encoding changes failed: %s\n", err) + data, err := json.Marshal(m) + if err != nil { + log.Printf("encoding failed. %v\n", err) return } - data := buf.Bytes() - for c := range wsf.connections { select { case c.send <- data: @@ -94,6 +96,26 @@ func (wsf *websocketForwarder) send(m interface{}) { } } +func (wsf *websocketForwarder) singleSend(c *connection, m interface{}) { + wsf.funcs <- func(wsf *websocketForwarder) { + _, ok := wsf.connections[c] + if !ok { + return + } + data, err := json.Marshal(m) + if err != nil { + log.Printf("encoding failed. %v\n", err) + return + } + select { + case c.send <- data: + default: + delete(wsf.connections, c) + close(c.send) + } + } +} + func (wsf *websocketForwarder) BaseTilesUpdated(changes []xz) { wsf.send(&tilesMsg{Tiles: changes}) } @@ -112,6 +134,9 @@ func (wsf *websocketForwarder) ServeHTTP(rw http.ResponseWriter, r *http.Request wsf.register(c) defer wsf.unregister(c) go c.writer() + if wsf.init != nil { + wsf.init(wsf, c) + } c.reader() } diff --git a/cmd/mtwebmapper/main.go b/cmd/mtwebmapper/main.go index 64c465f..901434f 100644 --- a/cmd/mtwebmapper/main.go +++ b/cmd/mtwebmapper/main.go @@ -59,7 +59,9 @@ func main() { if cfg.PlayersFIFO != "" { plys := newPlayers(cfg.PlayersFIFO, wsf) + wsf.setInit(plys.initConnection) go plys.run() + router.Path("/players").Methods("GET").Handler(plys) } diff --git a/cmd/mtwebmapper/players.go b/cmd/mtwebmapper/players.go index e48aa39..97ad56e 100644 --- a/cmd/mtwebmapper/players.go +++ b/cmd/mtwebmapper/players.go @@ -173,6 +173,10 @@ func (ps *players) current() []*player { return ps.pls } +func (ps *players) initConnection(wsf *websocketForwarder, c *connection) { + wsf.singleSend(c, &plsMsg{Pls: ps.current()}) +} + func (ps *players) ServeHTTP(rw http.ResponseWriter, r *http.Request) { rw.Header().Set("Content-Type", "application/json") pls := ps.current() From d9dabed221f8c291bbee53d27855b317bb7f7e6e Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 7 Mar 2022 10:38:19 +0100 Subject: [PATCH 26/26] Say something about the not up-to-date documentation. --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index be590a5..d08b9ff 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,18 @@ # MTSatellite +**Attention** MTSatellite is currently getting some improvements +namely PostgreSQL backend support and and a new Vue-JS based web client. + +The new features are in but the documentation is not updated, yet. +Hopefully this will change soon. + MTSatellite is a "realtime" web mapping system for [Minetest](http://minetest.net) worlds. With this system you can play your world and you instantly have an online map of it which can be shared on the web. To get a glimpse what it does watch [Realtime Webmapping for Minetest worlds](http://youtu.be/iYEROGPj7RI) on YouTube. -A live map of an online world can be viewed [here](http://maps.mt.sha-bang.de/). +A live map of an online world can be viewed [here](https://maps.mt.intevation.de/). See [COMPILE](https://bitbucket.org/s_l_teichmann/mtsatellite/src/default/COMPILE.md) how to compile MTSatellite. Essentially you need Go 1.4 (or higher) and a GNU/Linux system.