Merged write-tiless-in-background back into default.

cmd/mtseeder/baselevel.go

@@ -30,10 +30,29 @@ func createTiles(
 	jobs chan blockPos,
 	done *sync.WaitGroup) {
 
-	defer done.Done()
-	defer btc.Close()
+	wFns := make(chan func() (bool, error))
+
+	// Writing already rendered tiles to disk can be done in background.
+	go func() {
+		for wfn := range wFns {
+			if _, err := wfn(); err != nil {
+				log.Printf("WARN: writing file failed: %v.\n", err)
+			}
+		}
+	}()
+
+	defer func() {
+		close(wFns)
+		btc.Close()
+		done.Done()
+	}()
+
 	for job := range jobs {
-		btc.CreateTile(job.x-1, job.z-1, job.i, job.j)
+		if err := btc.RenderArea(job.x-1, job.z-1); err != nil {
+			log.Printf("WARN: rendering failed: %v.\n", err)
+			continue
+		}
+		wFns <- btc.WriteFunc(job.i, job.j, nil)
 	}
 }
 
@@ -77,7 +96,7 @@ func createBaseLevel(
 		btc := common.NewBaseTileCreator(
 			client, colors, bg,
 			int16(yMin), int16(yMax),
-			transparent, baseDir, nil)
+			transparent, baseDir)
 		go createTiles(btc, jobs, &done)
 	}
 

cmd/mtwebmapper/tilesupdater.go

@@ -213,9 +213,9 @@ func (tu *tileUpdater) doUpdates() {
 			btc := common.NewBaseTileCreator(
 				client, tu.colors, tu.bg,
 				tu.yMin, tu.yMax,
-				tu.transparent, baseDir, bth.Update)
+				tu.transparent, baseDir)
 			done.Add(1)
-			go tu.updateBaseTiles(jobs, btc, &done)
+			go tu.updateBaseTiles(jobs, btc, &done, bth.Update)
 		}
 
 		for i := range changes {
@@ -338,19 +338,42 @@ func (tu *tileUpdater) updatePyramidTile(scratch, resized *image.RGBA, level int
 
 func (tu *tileUpdater) updateBaseTiles(
 	jobs chan *xzc,
-	btc *common.BaseTileCreator, done *sync.WaitGroup) {
+	btc *common.BaseTileCreator,
+	done *sync.WaitGroup,
+	update common.BaseTileUpdateFunc) {
 
-	defer btc.Close()
-	defer done.Done()
-	for job := range jobs {
-		xz := job.dequantize()
-		//log.Printf("%d/%d %d/%d", x, z, job.X, job.Z)
-		updated, err := btc.CreateTile(xz.X-1, xz.Z-1, int(job.X), int(job.Z))
+	type jobWriter struct {
+		job *xzc
+		wFn func() (bool, error)
+	}
+
+	jWs := make(chan jobWriter)
+
+	go func() {
+		for jw := range jWs {
+			updated, err := jw.wFn()
 			if err != nil {
-			log.Printf("WARN: create tile failed: %s\n", err)
+				log.Printf("WARN: writing tile failed: %v.\n", err)
 			}
 			if !updated {
-			job.canceled = true
+				jw.job.canceled = true
 			}
 		}
+	}()
+
+	defer func() {
+		close(jWs)
+		btc.Close()
+		done.Done()
+	}()
+
+	for job := range jobs {
+		xz := job.dequantize()
+		if err := btc.RenderArea(xz.X-1, xz.Z-1); err != nil {
+			log.Printf("WARN: rendering tile failed: %v.\n", err)
+			job.canceled = true
+			continue
+		}
+		jWs <- jobWriter{job, btc.WriteFunc(int(job.X), int(job.Z), update)}
+	}
 }

common/basetilecreator.go

@@ -59,7 +59,6 @@ type BaseTileCreator struct {
 	yMin       int16
 	yMax       int16
 	baseDir    string
-	update     BaseTileUpdateFunc
 	emptyImage []byte
 	bg         color.RGBA
 }
@@ -70,8 +69,8 @@ func NewBaseTileCreator(
 	bg color.RGBA,
 	yMin, yMax int16,
 	transparent bool,
-	baseDir string,
-	update BaseTileUpdateFunc) *BaseTileCreator {
+	baseDir string) *BaseTileCreator {
+
 	renderer := NewRenderer(tileWidth, tileHeight, transparent)
 	yMin, yMax = Order16(yMin, yMax)
 	btc := &BaseTileCreator{
@@ -82,7 +81,7 @@ func NewBaseTileCreator(
 		yMin:     yMin,
 		yMax:     yMax,
 		baseDir:  baseDir,
-		update:   update}
+	}
 	btc.yOrder = NewYOrder(btc.renderBlock, yOrderCapacity)
 	return btc
 }
@@ -105,7 +104,7 @@ func (btc *BaseTileCreator) blockLoaded(block *Block) *Block {
 	return block
 }
 
-func (btc *BaseTileCreator) CreateTile(x, z int16, i, j int) (bool, error) {
+func (btc *BaseTileCreator) RenderArea(x, z int16) error {
 	btc.renderer.Reset()
 	btc.renderer.SetPos(x, z)
 	btc.yOrder.Reset()
@@ -136,7 +135,7 @@ func (btc *BaseTileCreator) CreateTile(x, z int16, i, j int) (bool, error) {
 			var count int
 			var err error
 			if count, err = btc.client.QueryCuboid(query, btc.blockLoaded); err != nil {
-				return false, err
+				return err
 			}
 			if err = btc.yOrder.Drain(); err != nil {
 				log.Printf("WARN: rendering block failed: %s\n", err)
@@ -155,38 +154,53 @@ func (btc *BaseTileCreator) CreateTile(x, z int16, i, j int) (bool, error) {
 		}
 		areas, nareas = nareas, areas[:0]
 	}
+	return nil
+}
+
+func (btc *BaseTileCreator) blankImage() []byte {
+	// To avoid redundant encoding cache the resulting empty image.
+	if btc.emptyImage == nil {
+		m := BackgroundImage((tileWidth-2)*16, (tileHeight-2)*16, btc.bg)
+		btc.emptyImage = EncodeToMem(m)
+	}
+	return btc.emptyImage
+}
+
+// WriteFunc returns a function intended to be run in background so
+// the creation of the next tile with this creator can be done
+// concurrently.
+func (btc *BaseTileCreator) WriteFunc(i, j int, update BaseTileUpdateFunc) func() (bool, error) {
 
 	path := filepath.Join(btc.baseDir, strconv.Itoa(i), strconv.Itoa(j)+".png")
 
 	// Empty images are likely to be produced during seeding.
-	if btc.update == nil && btc.renderer.IsEmpty() {
-		// To avoid redundant encoding cache the resulting empty image.
-		if btc.emptyImage == nil {
-			var err error
-			m := BackgroundImage((tileWidth-2)*16, (tileHeight-2)*16, btc.bg)
-			if btc.emptyImage, err = EncodeToMem(m); err != nil {
-				return false, err
-			}
-		}
+	if update == nil && btc.renderer.IsEmpty() {
+		return func() (bool, error) {
 			//log.Printf("Writing empty (%d, %d) to file %s\n", x, z, path)
-		return true, ioutil.WriteFile(path, btc.emptyImage, 0666)
+			return true, ioutil.WriteFile(path, btc.blankImage(), 0666)
+		}
 	}
 
 	image := btc.renderer.CreateShadedImage(
 		16, 16, (tileWidth-2)*16, (tileHeight-2)*16,
 		btc.colors, btc.bg)
 
-	if btc.update == nil {
+	x, z := btc.renderer.GetPos()
+
+	if update == nil {
+		return func() (bool, error) {
 			log.Printf("Writing (%d, %d) to file %s.\n", x, z, path)
 			return true, SaveAsPNG(path, image)
 		}
+	}
 
-	if btc.update(i, j, HashImage(image)) {
+	return func() (bool, error) {
+		if update(i, j, HashImage(image)) {
 			log.Printf("Writing (%d, %d) to file %s.\n", x, z, path)
 			return true, SaveAsPNGAtomic(path, image)
 		}
 
 		log.Printf("(%d, %d) is unchanged.\n", x, z)
-
 		return false, nil
+	}
 }

common/image.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"image"
 	"image/color"
+	"image/draw"
 	"image/png"
 	"log"
 	"os"
@@ -44,13 +45,14 @@ func nextSuffix() string {
 	return strconv.Itoa(int(1e9 + r%1e9))[1:]
 }
 
-func EncodeToMem(img image.Image) ([]byte, error) {
+func EncodeToMem(img image.Image) []byte {
 	var buf bytes.Buffer
 	enc := png.Encoder{CompressionLevel: png.BestCompression}
 	if err := enc.Encode(&buf, img); err != nil {
-		return nil, err
+		// This really should not happen.
+		panic(err)
 	}
-	return buf.Bytes(), nil
+	return buf.Bytes()
 }
 
 func SaveAsPNG(path string, img image.Image) (err error) {
@@ -126,3 +128,9 @@ func HashImage(img *image.RGBA) []byte {
 
 	return hash.Sum(nil)
 }
+
+func BackgroundImage(width, height int, bg color.RGBA) *image.RGBA {
+	m := image.NewRGBA(image.Rect(0, 0, width, height))
+	draw.Draw(m, m.Bounds(), &image.Uniform{bg}, image.ZP, draw.Src)
+	return m
+}

common/renderer.go

@@ -7,7 +7,6 @@ package common
 import (
 	"image"
 	"image/color"
-	"image/draw"
 	"math"
 )
 
@@ -59,6 +58,10 @@ func (r *Renderer) SetPos(xOfs, zOfs int16) {
 	r.zOfs = zOfs
 }
 
+func (r *Renderer) GetPos() (int16, int16) {
+	return r.xOfs, r.zOfs
+}
+
 func (r *Renderer) initBuffers() {
 	yb := r.yBuffer
 	yb = yb[:len(yb)]
@@ -269,12 +272,6 @@ func safeColor(x int32) uint8 {
 	}
 }
 
-func BackgroundImage(width, height int, bg color.RGBA) *image.RGBA {
-	m := image.NewRGBA(image.Rect(0, 0, width, height))
-	draw.Draw(m, m.Bounds(), &image.Uniform{bg}, image.ZP, draw.Src)
-	return m
-}
-
 func (r *Renderer) CreateShadedImage(
 	xOfs, zOfs, width, height int,
 	cols *Colors, background color.RGBA) *image.RGBA {