From 216f641fa3a755a66f99204d6b9d467669e5a1bf Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sat, 20 Sep 2014 15:21:01 +0200 Subject: [PATCH] mtwebmapper: Move BaseTileCreator to common. TODO: Make mtseeder use it, too. --- cmd/mtwebmapper/main.go | 4 +- cmd/mtwebmapper/tilesupdater.go | 202 ++++++-------------------------- common/basetilecreator.go | 145 +++++++++++++++++++++++ 3 files changed, 182 insertions(+), 169 deletions(-) create mode 100644 common/basetilecreator.go diff --git a/cmd/mtwebmapper/main.go b/cmd/mtwebmapper/main.go index 43c195e..fdbed1f 100644 --- a/cmd/mtwebmapper/main.go +++ b/cmd/mtwebmapper/main.go @@ -57,9 +57,7 @@ func main() { } redisAddress := fmt.Sprintf("%s:%d", redisHost, redisPort) tu := newTileUpdater(mapDir, redisAddress, colors, workers) - if err = tu.doUpdates(); err != nil { - log.Fatalf("ERROR: Cannot start tile generation: %s", err) - } + go tu.doUpdates() router.Path("/update").Methods("POST").Handler(tu) } diff --git a/cmd/mtwebmapper/tilesupdater.go b/cmd/mtwebmapper/tilesupdater.go index 220cb2d..d929466 100644 --- a/cmd/mtwebmapper/tilesupdater.go +++ b/cmd/mtwebmapper/tilesupdater.go @@ -6,48 +6,14 @@ package main import ( "encoding/json" - "fmt" - "image/color" "log" "net/http" - "os" "path/filepath" - "strconv" "sync" "bitbucket.org/s_l_teichmann/mtredisalize/common" ) -const ( - tileWidth = 18 - tileHeight = 18 - yOrderCapacity = 512 -) - -// To scan the whole height in terms of the y coordinate -// the database is queried in height units defined in the tileDepths table. -var tileDepths = [...][2]int16{ - {1024, 1934}, - {256, 1023}, - {128, 255}, - {64, 127}, - {32, 63}, - {16, 31}, - {8, 15}, - {4, 7}, - {2, 3}, - {0, 1}, - {-1, 0}, - {-4, -2}, - {-8, -5}, - {-16, -9}, - {-32, -17}, - {-64, -33}, - {-128, -65}, - {-256, -129}, - {-1024, -257}, - {-1934, -1025}} - type tileUpdater struct { changes map[xz]bool mapDir string @@ -100,149 +66,53 @@ func (tu *tileUpdater) ServeHTTP(rw http.ResponseWriter, r *http.Request) { rw.WriteHeader(http.StatusOK) } -func (tu *tileUpdater) doUpdates() (err error) { +func (tu *tileUpdater) doUpdates() { - jobs := make(chan xz) - - baseDir := filepath.Join(tu.mapDir, "8") - - for i := 0; i < tu.workers; i++ { - var client *common.RedisClient - if client, err = common.NewRedisClient("tcp", tu.redisAddress); err != nil { - return + for { + var changes map[xz]bool + tu.cond.L.Lock() + for len(tu.changes) == 0 { + tu.cond.Wait() } - btc := NewBaseTileCreator(client, tu.colors, baseDir, true) - go btc.run(jobs) + changes = tu.changes + tu.changes = map[xz]bool{} + tu.cond.L.Unlock() + + baseDir := filepath.Join(tu.mapDir, "8") + + jobs := make(chan xz) + var done sync.WaitGroup + + for i := 0; i < tu.workers; i++ { + var client *common.RedisClient + var err error + if client, err = common.NewRedisClient("tcp", tu.redisAddress); err != nil { + log.Printf("WARN: Cannot connect to redis server: %s", err) + continue + } + btc := common.NewBaseTileCreator(client, tu.colors, baseDir, true) + done.Add(1) + go updateBaseTiles(jobs, btc, &done) + } + + for c, _ := range changes { + log.Printf("job: %+v", c) + jobs <- c + } + done.Wait() } - - go func() { - for { - var changes map[xz]bool - tu.cond.L.Lock() - for len(tu.changes) == 0 { - tu.cond.Wait() - } - changes = tu.changes - tu.changes = map[xz]bool{} - tu.cond.L.Unlock() - - for c, _ := range changes { - log.Printf("job: %+v", c) - jobs <- c - } - } - }() return } -type BaseTileCreator struct { - client *common.RedisClient - colors *common.Colors - renderer *common.Renderer - yOrder *common.YOrder - baseDir string - update bool -} - -func NewBaseTileCreator( - client *common.RedisClient, - colors *common.Colors, - baseDir string, - update bool) *BaseTileCreator { - renderer := common.NewRenderer(tileWidth, tileHeight) - return &BaseTileCreator{ - client: client, - colors: colors, - renderer: renderer, - yOrder: common.NewYOrder(renderer, yOrderCapacity), - baseDir: baseDir, - update: update} -} - -func (btc *BaseTileCreator) run(jobs chan xz) { +func updateBaseTiles(jobs chan xz, btc *common.BaseTileCreator, done *sync.WaitGroup) { for job := range jobs { x := job.X*16 + -1936 - 1 z := job.Z*16 + -1936 - 1 log.Printf("%d/%d %d/%d", x, z, job.X, job.Z) - if err := btc.createTile(x, z, int(job.X), int(job.Z)); err != nil { + if err := btc.CreateTile(x, z, int(job.X), int(job.Z)); err != nil { log.Printf("WARN: create tile failed: %s", err) } } -} - -func (btc *BaseTileCreator) close() error { - return btc.client.Close() -} - -func (btc *BaseTileCreator) createTile(x, z int16, i, j int) error { - btc.renderer.Reset() - btc.renderer.SetPos(x, z) - btc.yOrder.Reset() - - drawBlock := func(block *common.Block) { - if err := btc.yOrder.RenderBlock(block, btc.colors.NameIndex); err != nil { - log.Printf("WARN: rendering block failed: %s", err) - } - } - - var c1, c2 common.Coord - - nareas := make([]common.Area, 0, tileWidth*tileHeight/2) - oareas := make([]common.Area, 1, tileWidth*tileHeight/2) - - oareas[0] = common.Area{ - X1: 0, Z1: 0, - X2: int16(tileWidth) - 1, Z2: int16(tileHeight) - 1} - - for _, yRange := range tileDepths { - c1.Y = yRange[0] - c2.Y = yRange[1] - - nareas = btc.renderer.UncoveredAreas(nareas, oareas) - - if len(nareas) == 0 { - break - } - - for _, area := range nareas { - c1.X = area.X1 + x - c1.Z = area.Z1 + z - c2.X = area.X2 + x - c2.Z = area.Z2 + z - query := common.Cuboid{P1: c1, P2: c2} - if err := btc.client.QueryCuboid(query, drawBlock); err != nil { - return err - } - if err := btc.yOrder.Drain(btc.colors.NameIndex); err != nil { - log.Printf("WARN: rendering block failed: %s", err) - } - } - oareas, nareas = nareas, oareas[0:0] - } - - image := btc.renderer.CreateShadedImage( - 16, 16, (tileWidth-2)*16, (tileHeight-2)*16, - btc.colors.Colors, color.RGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xff}) - - path := filepath.Join(btc.baseDir, strconv.Itoa(i), fmt.Sprintf("%d.png", j)) - - log.Printf("Writing (%d, %d) to file %s", x, z, path) - - if !btc.update { - return common.SaveAsPNG(path, image) - } - - // Try to make creation of update "atomic" by first writing to tmp file - // and rename the tmp file to the original one afterwards. - pathTmp := path + "tmp" - pathTmpPre := pathTmp - tc := 0 - for _, err := os.Stat(pathTmp); err == nil; _, err = os.Stat(pathTmp) { - pathTmp = fmt.Sprintf("%s%d", pathTmpPre, tc) - tc++ - } - if err := common.SaveAsPNG(pathTmp, image); err != nil { - return err - } - return os.Rename(pathTmp, path) + btc.Close() // TODO: Keep redis connection open to update the pyramid tiles. + done.Done() } diff --git a/common/basetilecreator.go b/common/basetilecreator.go new file mode 100644 index 0000000..52b9aac --- /dev/null +++ b/common/basetilecreator.go @@ -0,0 +1,145 @@ +// Copyright 2014 by Sascha L. Teichmann +// Use of this source code is governed by the MIT license +// that can be found in the LICENSE file. + +package common + +import ( + "fmt" + "image/color" + "log" + "os" + "path/filepath" + "strconv" +) + +const ( + tileWidth = 18 + tileHeight = 18 + yOrderCapacity = 512 +) + +// To scan the whole height in terms of the y coordinate +// the database is queried in height units defined in the tileDepths table. +var tileDepths = [...][2]int16{ + {1024, 1934}, + {256, 1023}, + {128, 255}, + {64, 127}, + {32, 63}, + {16, 31}, + {8, 15}, + {4, 7}, + {2, 3}, + {0, 1}, + {-1, 0}, + {-4, -2}, + {-8, -5}, + {-16, -9}, + {-32, -17}, + {-64, -33}, + {-128, -65}, + {-256, -129}, + {-1024, -257}, + {-1934, -1025}} + +type BaseTileCreator struct { + client *RedisClient + colors *Colors + renderer *Renderer + yOrder *YOrder + baseDir string + update bool +} + +func NewBaseTileCreator( + client *RedisClient, + colors *Colors, + baseDir string, + update bool) *BaseTileCreator { + renderer := NewRenderer(tileWidth, tileHeight) + return &BaseTileCreator{ + client: client, + colors: colors, + renderer: renderer, + yOrder: NewYOrder(renderer, yOrderCapacity), + baseDir: baseDir, + update: update} +} + +func (btc *BaseTileCreator) Close() error { + return btc.client.Close() +} + +func (btc *BaseTileCreator) CreateTile(x, z int16, i, j int) error { + btc.renderer.Reset() + btc.renderer.SetPos(x, z) + btc.yOrder.Reset() + + drawBlock := func(block *Block) { + if err := btc.yOrder.RenderBlock(block, btc.colors.NameIndex); err != nil { + log.Printf("WARN: rendering block failed: %s", err) + } + } + + var c1, c2 Coord + + nareas := make([]Area, 0, tileWidth*tileHeight/2) + oareas := make([]Area, 1, tileWidth*tileHeight/2) + + oareas[0] = Area{ + X1: 0, Z1: 0, + X2: int16(tileWidth) - 1, Z2: int16(tileHeight) - 1} + + for _, yRange := range tileDepths { + c1.Y = yRange[0] + c2.Y = yRange[1] + + nareas = btc.renderer.UncoveredAreas(nareas, oareas) + + if len(nareas) == 0 { + break + } + + for _, area := range nareas { + c1.X = area.X1 + x + c1.Z = area.Z1 + z + c2.X = area.X2 + x + c2.Z = area.Z2 + z + query := Cuboid{P1: c1, P2: c2} + if err := btc.client.QueryCuboid(query, drawBlock); err != nil { + return err + } + if err := btc.yOrder.Drain(btc.colors.NameIndex); err != nil { + log.Printf("WARN: rendering block failed: %s", err) + } + } + oareas, nareas = nareas, oareas[0:0] + } + + image := btc.renderer.CreateShadedImage( + 16, 16, (tileWidth-2)*16, (tileHeight-2)*16, + btc.colors.Colors, color.RGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xff}) + + path := filepath.Join(btc.baseDir, strconv.Itoa(i), fmt.Sprintf("%d.png", j)) + + log.Printf("Writing (%d, %d) to file %s", x, z, path) + + if !btc.update { + return SaveAsPNG(path, image) + } + + // Try to make creation of update "atomic" by first writing to tmp file + // and rename the tmp file to the original one afterwards. + pathTmp := path + "tmp" + pathTmpPre := pathTmp + tc := 0 + for _, err := os.Stat(pathTmp); err == nil; _, err = os.Stat(pathTmp) { + pathTmp = fmt.Sprintf("%s%d", pathTmpPre, tc) + tc++ + } + if err := SaveAsPNG(pathTmp, image); err != nil { + return err + } + return os.Rename(pathTmp, path) +}