From 6a1356f0eb08af854aaddb031f9c875a767ae04e Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 6 Mar 2017 15:43:30 +0100 Subject: [PATCH 1/2] When an tile is rendered the writing to disk can be done in background concurrently to the rendering of the next tile. --- cmd/mtseeder/baselevel.go | 27 +++++++++-- cmd/mtwebmapper/tilesupdater.go | 45 ++++++++++++++----- common/basetilecreator.go | 79 +++++++++++++++++++-------------- common/image.go | 14 ++++-- common/renderer.go | 11 ++--- 5 files changed, 118 insertions(+), 58 deletions(-) diff --git a/cmd/mtseeder/baselevel.go b/cmd/mtseeder/baselevel.go index 1e66535..72b0c74 100644 --- a/cmd/mtseeder/baselevel.go +++ b/cmd/mtseeder/baselevel.go @@ -30,10 +30,29 @@ func createTiles( jobs chan blockPos, done *sync.WaitGroup) { - defer done.Done() - defer btc.Close() + wFns := make(chan func() (bool, error)) + + // Writing already rendered tiles to disk can be done in background. + go func() { + for wfn := range wFns { + if _, err := wfn(); err != nil { + log.Printf("WARN: writing file failed: %v.\n", err) + } + } + }() + + defer func() { + close(wFns) + btc.Close() + done.Done() + }() + for job := range jobs { - btc.CreateTile(job.x-1, job.z-1, job.i, job.j) + if err := btc.RenderArea(job.x-1, job.z-1); err != nil { + log.Printf("WARN: rendering failed: %v.\n", err) + continue + } + wFns <- btc.WriteFunc(job.i, job.j, nil) } } @@ -77,7 +96,7 @@ func createBaseLevel( btc := common.NewBaseTileCreator( client, colors, bg, int16(yMin), int16(yMax), - transparent, baseDir, nil) + transparent, baseDir) go createTiles(btc, jobs, &done) } diff --git a/cmd/mtwebmapper/tilesupdater.go b/cmd/mtwebmapper/tilesupdater.go index 6982183..45bddbc 100644 --- a/cmd/mtwebmapper/tilesupdater.go +++ b/cmd/mtwebmapper/tilesupdater.go @@ -213,9 +213,9 @@ func (tu *tileUpdater) doUpdates() { btc := common.NewBaseTileCreator( client, tu.colors, tu.bg, tu.yMin, tu.yMax, - tu.transparent, baseDir, bth.Update) + tu.transparent, baseDir) done.Add(1) - go tu.updateBaseTiles(jobs, btc, &done) + go tu.updateBaseTiles(jobs, btc, &done, bth.Update) } for i := range changes { @@ -338,19 +338,42 @@ func (tu *tileUpdater) updatePyramidTile(scratch, resized *image.RGBA, level int func (tu *tileUpdater) updateBaseTiles( jobs chan *xzc, - btc *common.BaseTileCreator, done *sync.WaitGroup) { + btc *common.BaseTileCreator, + done *sync.WaitGroup, + update common.BaseTileUpdateFunc) { + + type jobWriter struct { + job *xzc + wFn func() (bool, error) + } + + jWs := make(chan jobWriter) + + go func() { + for jw := range jWs { + updated, err := jw.wFn() + if err != nil { + log.Printf("WARN: writing tile failed: %v.\n", err) + } + if !updated { + jw.job.canceled = true + } + } + }() + + defer func() { + close(jWs) + btc.Close() + done.Done() + }() - defer btc.Close() - defer done.Done() for job := range jobs { xz := job.dequantize() - //log.Printf("%d/%d %d/%d", x, z, job.X, job.Z) - updated, err := btc.CreateTile(xz.X-1, xz.Z-1, int(job.X), int(job.Z)) - if err != nil { - log.Printf("WARN: create tile failed: %s\n", err) - } - if !updated { + if err := btc.RenderArea(xz.X-1, xz.Z-1); err != nil { + log.Printf("WARN: rendering tile failed: %v.\n", err) job.canceled = true + continue } + jWs <- jobWriter{job, btc.WriteFunc(int(job.X), int(job.Z), update)} } } diff --git a/common/basetilecreator.go b/common/basetilecreator.go index 89f5b08..9abaeb2 100644 --- a/common/basetilecreator.go +++ b/common/basetilecreator.go @@ -52,14 +52,14 @@ var BackgroundColor = color.RGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xff} type BaseTileUpdateFunc func(x, y int, hash []byte) bool type BaseTileCreator struct { - client *RedisClient - colors *Colors - renderer *Renderer - yOrder *YOrder - yMin int16 - yMax int16 - baseDir string - update BaseTileUpdateFunc + client *RedisClient + colors *Colors + renderer *Renderer + yOrder *YOrder + yMin int16 + yMax int16 + baseDir string + //update BaseTileUpdateFunc emptyImage []byte bg color.RGBA } @@ -70,8 +70,8 @@ func NewBaseTileCreator( bg color.RGBA, yMin, yMax int16, transparent bool, - baseDir string, - update BaseTileUpdateFunc) *BaseTileCreator { + baseDir string) *BaseTileCreator { + //update BaseTileUpdateFunc) *BaseTileCreator { renderer := NewRenderer(tileWidth, tileHeight, transparent) yMin, yMax = Order16(yMin, yMax) btc := &BaseTileCreator{ @@ -82,7 +82,8 @@ func NewBaseTileCreator( yMin: yMin, yMax: yMax, baseDir: baseDir, - update: update} + } + //update: update} btc.yOrder = NewYOrder(btc.renderBlock, yOrderCapacity) return btc } @@ -105,7 +106,7 @@ func (btc *BaseTileCreator) blockLoaded(block *Block) *Block { return block } -func (btc *BaseTileCreator) CreateTile(x, z int16, i, j int) (bool, error) { +func (btc *BaseTileCreator) RenderArea(x, z int16) error { btc.renderer.Reset() btc.renderer.SetPos(x, z) btc.yOrder.Reset() @@ -136,7 +137,7 @@ func (btc *BaseTileCreator) CreateTile(x, z int16, i, j int) (bool, error) { var count int var err error if count, err = btc.client.QueryCuboid(query, btc.blockLoaded); err != nil { - return false, err + return err } if err = btc.yOrder.Drain(); err != nil { log.Printf("WARN: rendering block failed: %s\n", err) @@ -155,38 +156,50 @@ func (btc *BaseTileCreator) CreateTile(x, z int16, i, j int) (bool, error) { } areas, nareas = nareas, areas[:0] } + return nil +} + +func (btc *BaseTileCreator) blankImage() []byte { + // To avoid redundant encoding cache the resulting empty image. + if btc.emptyImage == nil { + m := BackgroundImage((tileWidth-2)*16, (tileHeight-2)*16, btc.bg) + btc.emptyImage = EncodeToMem(m) + } + return btc.emptyImage +} + +func (btc *BaseTileCreator) WriteFunc(i, j int, update BaseTileUpdateFunc) func() (bool, error) { path := filepath.Join(btc.baseDir, strconv.Itoa(i), strconv.Itoa(j)+".png") // Empty images are likely to be produced during seeding. - if btc.update == nil && btc.renderer.IsEmpty() { - // To avoid redundant encoding cache the resulting empty image. - if btc.emptyImage == nil { - var err error - m := BackgroundImage((tileWidth-2)*16, (tileHeight-2)*16, btc.bg) - if btc.emptyImage, err = EncodeToMem(m); err != nil { - return false, err - } + if update == nil && btc.renderer.IsEmpty() { + return func() (bool, error) { + //log.Printf("Writing empty (%d, %d) to file %s\n", x, z, path) + return true, ioutil.WriteFile(path, btc.blankImage(), 0666) } - //log.Printf("Writing empty (%d, %d) to file %s\n", x, z, path) - return true, ioutil.WriteFile(path, btc.emptyImage, 0666) } image := btc.renderer.CreateShadedImage( 16, 16, (tileWidth-2)*16, (tileHeight-2)*16, btc.colors, btc.bg) - if btc.update == nil { - log.Printf("Writing (%d, %d) to file %s.\n", x, z, path) - return true, SaveAsPNG(path, image) + x, z := btc.renderer.GetPos() + + if update == nil { + return func() (bool, error) { + log.Printf("Writing (%d, %d) to file %s.\n", x, z, path) + return true, SaveAsPNG(path, image) + } } - if btc.update(i, j, HashImage(image)) { - log.Printf("Writing (%d, %d) to file %s.\n", x, z, path) - return true, SaveAsPNGAtomic(path, image) + return func() (bool, error) { + if update(i, j, HashImage(image)) { + log.Printf("Writing (%d, %d) to file %s.\n", x, z, path) + return true, SaveAsPNGAtomic(path, image) + } + + log.Printf("(%d, %d) is unchanged.\n", x, z) + return false, nil } - - log.Printf("(%d, %d) is unchanged.\n", x, z) - - return false, nil } diff --git a/common/image.go b/common/image.go index 2fed583..04a7033 100644 --- a/common/image.go +++ b/common/image.go @@ -10,6 +10,7 @@ import ( "errors" "image" "image/color" + "image/draw" "image/png" "log" "os" @@ -44,13 +45,14 @@ func nextSuffix() string { return strconv.Itoa(int(1e9 + r%1e9))[1:] } -func EncodeToMem(img image.Image) ([]byte, error) { +func EncodeToMem(img image.Image) []byte { var buf bytes.Buffer enc := png.Encoder{CompressionLevel: png.BestCompression} if err := enc.Encode(&buf, img); err != nil { - return nil, err + // This really should not happen. + panic(err) } - return buf.Bytes(), nil + return buf.Bytes() } func SaveAsPNG(path string, img image.Image) (err error) { @@ -126,3 +128,9 @@ func HashImage(img *image.RGBA) []byte { return hash.Sum(nil) } + +func BackgroundImage(width, height int, bg color.RGBA) *image.RGBA { + m := image.NewRGBA(image.Rect(0, 0, width, height)) + draw.Draw(m, m.Bounds(), &image.Uniform{bg}, image.ZP, draw.Src) + return m +} diff --git a/common/renderer.go b/common/renderer.go index 70b0cf9..7dd834a 100644 --- a/common/renderer.go +++ b/common/renderer.go @@ -7,7 +7,6 @@ package common import ( "image" "image/color" - "image/draw" "math" ) @@ -59,6 +58,10 @@ func (r *Renderer) SetPos(xOfs, zOfs int16) { r.zOfs = zOfs } +func (r *Renderer) GetPos() (int16, int16) { + return r.xOfs, r.zOfs +} + func (r *Renderer) initBuffers() { yb := r.yBuffer yb = yb[:len(yb)] @@ -269,12 +272,6 @@ func safeColor(x int32) uint8 { } } -func BackgroundImage(width, height int, bg color.RGBA) *image.RGBA { - m := image.NewRGBA(image.Rect(0, 0, width, height)) - draw.Draw(m, m.Bounds(), &image.Uniform{bg}, image.ZP, draw.Src) - return m -} - func (r *Renderer) CreateShadedImage( xOfs, zOfs, width, height int, cols *Colors, background color.RGBA) *image.RGBA { From 8badf4aaca9d0ff3d58197ae7ed94c0e879f1ffe Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sat, 15 Apr 2017 13:08:19 +0200 Subject: [PATCH 2/2] Added a meaningful comment to the method of the base tile creator which creates the backgound task. --- common/basetilecreator.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/common/basetilecreator.go b/common/basetilecreator.go index 9abaeb2..264ee6c 100644 --- a/common/basetilecreator.go +++ b/common/basetilecreator.go @@ -52,14 +52,13 @@ var BackgroundColor = color.RGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xff} type BaseTileUpdateFunc func(x, y int, hash []byte) bool type BaseTileCreator struct { - client *RedisClient - colors *Colors - renderer *Renderer - yOrder *YOrder - yMin int16 - yMax int16 - baseDir string - //update BaseTileUpdateFunc + client *RedisClient + colors *Colors + renderer *Renderer + yOrder *YOrder + yMin int16 + yMax int16 + baseDir string emptyImage []byte bg color.RGBA } @@ -71,7 +70,7 @@ func NewBaseTileCreator( yMin, yMax int16, transparent bool, baseDir string) *BaseTileCreator { - //update BaseTileUpdateFunc) *BaseTileCreator { + renderer := NewRenderer(tileWidth, tileHeight, transparent) yMin, yMax = Order16(yMin, yMax) btc := &BaseTileCreator{ @@ -83,7 +82,6 @@ func NewBaseTileCreator( yMax: yMax, baseDir: baseDir, } - //update: update} btc.yOrder = NewYOrder(btc.renderBlock, yOrderCapacity) return btc } @@ -168,6 +166,9 @@ func (btc *BaseTileCreator) blankImage() []byte { return btc.emptyImage } +// WriteFunc returns a function intended to be run in background so +// the creation of the next tile with this creator can be done +// concurrently. func (btc *BaseTileCreator) WriteFunc(i, j int, update BaseTileUpdateFunc) func() (bool, error) { path := filepath.Join(btc.baseDir, strconv.Itoa(i), strconv.Itoa(j)+".png")