mtwebmapper: First version of pyramid tile updater. Needs testing.

cmd/mtseeder/pyramid.go

@@ -5,11 +5,9 @@
 package main
 
 import (
-	"bufio"
 	"fmt"
 	"image"
 	"image/draw"
-	"image/png"
 	"io/ioutil"
 	"log"
 	"math"
@@ -158,7 +156,7 @@ var dps = [4]image.Point{
 	image.Pt(256, 256),
 	image.Pt(256, 0)}
 
-func fuseTile(scratch *image.RGBA, job *pyramidJob) (err error) {
+func fuseTile(scratch *image.RGBA, job *pyramidJob) error {
 
 	for i, path := range job.src {
 
@@ -172,21 +170,7 @@ func fuseTile(scratch *image.RGBA, job *pyramidJob) (err error) {
 
 	resized := resize.Resize(256, 256, scratch, resize.Lanczos3)
 
-	var outFile *os.File
-	if outFile, err = os.Create(job.dst); err != nil {
-		return
-	}
-
-	out := bufio.NewWriter(outFile)
-
-	if err = png.Encode(out, resized); err != nil {
-		outFile.Close()
-		return
-	}
-
-	out.Flush()
-	err = outFile.Close()
-	return
+	return common.SaveAsPNG(job.dst, resized)
 }
 
 func fuseTiles(jobs chan pyramidJob, done *sync.WaitGroup) {

cmd/mtwebmapper/tilesupdater.go

@@ -6,11 +6,17 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
+	"image"
+	"image/draw"
 	"log"
 	"net/http"
 	"path/filepath"
+	"strconv"
 	"sync"
 
+	"github.com/nfnt/resize"
+
 	"bitbucket.org/s_l_teichmann/mtredisalize/common"
 )
 
@@ -47,7 +53,16 @@ func (c xz) parent() xzm {
 	zp, zr := c.Z>>1, uint16(c.Z&1)
 	return xzm{
 		P:    xz{X: xp, Z: zp},
-		Mask: ((1 << xr) << 2) | (1 << zr)}
+		Mask: 1 << (xr<<1 | zr)}
+}
+
+func (c xzm) numChanges() (n int) {
+	for i := uint16(0); i < 4; i++ {
+		if c.Mask&(1<<i) != 0 {
+			n++
+		}
+	}
+	return
 }
 
 func newTileUpdater(mapDir, redisAddress string, colors *common.Colors, workers int) *tileUpdater {
@@ -151,11 +166,75 @@ func min(a, b int) int {
 
 func updatePyramidTiles(level int, baseDir string, jobs chan xzm, done *sync.WaitGroup) {
 	defer done.Done()
+	scratch := image.NewRGBA(image.Rect(0, 0, 256, 256))
+
 	for job := range jobs {
-		_ = job
+		if err := updatePyramidTile(scratch, level, baseDir, job); err != nil {
+			log.Printf("Updating pyramid tile failed: %s", err)
+		}
 	}
 }
 
+var dps = [4]image.Point{
+	image.Pt(0, 128),
+	image.Pt(0, 0),
+	image.Pt(128, 128),
+	image.Pt(128, 0)}
+
+var ofs = [4][2]int{
+	{0, 0},
+	{1, 0},
+	{0, 1},
+	{1, 1}}
+
+func clip8(x int) int {
+	switch {
+	case x < 0:
+		return 0
+	case x > 256:
+		return 256
+	}
+	return x
+}
+
+func clipRect(r image.Rectangle) image.Rectangle {
+	return image.Rectangle{
+		Min: image.Point{X: clip8(r.Min.X), Y: clip8(r.Min.Y)},
+		Max: image.Point{X: clip8(r.Max.X), Y: clip8(r.Max.Y)}}
+}
+
+func updatePyramidTile(scratch *image.RGBA, level int, baseDir string, j xzm) error {
+
+	var orig image.Image
+
+	origPath := filepath.Join(
+		baseDir, strconv.Itoa(level), strconv.Itoa(int(j.P.X)), fmt.Sprintf("%d.png", j.P.Z))
+
+	if j.numChanges() < 4 {
+		orig = common.LoadPNG(origPath)
+	}
+
+	for i := uint16(0); i < 4; i++ {
+		if j.Mask&(1<<i) != 0 {
+			o := ofs[i]
+			bx, bz := int(2*j.P.X), int(2*j.P.Z)
+			path := filepath.Join(
+				baseDir, strconv.Itoa(level+1), strconv.Itoa(bx+o[0]), fmt.Sprintf("%d.png", bz+o[1]))
+			img := common.LoadPNG(path)
+			img = resize.Resize(128, 128, img, resize.Lanczos3)
+			sr := img.Bounds()
+			r := sr.Sub(sr.Min).Add(dps[i])
+			draw.Draw(scratch, r, img, sr.Min, draw.Src)
+		} else {
+			sr := clipRect(orig.Bounds())
+			r := sr.Sub(sr.Min).Add(dps[i])
+			draw.Draw(scratch, r, orig, sr.Min, draw.Src)
+		}
+	}
+
+	return common.SaveAsPNGAtomic(origPath, scratch)
+}
+
 func updateBaseTiles(jobs chan xz, btc *common.BaseTileCreator, done *sync.WaitGroup) {
 	defer btc.Close()
 	defer done.Done()

common/basetilecreator.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"image/color"
 	"log"
-	"os"
 	"path/filepath"
 	"strconv"
 )
@@ -129,17 +128,5 @@ func (btc *BaseTileCreator) CreateTile(x, z int16, i, j int) error {
 		return SaveAsPNG(path, image)
 	}
 
-	// Try to make creation of update "atomic" by first writing to tmp file
-	// and rename the tmp file to the original one afterwards.
-	pathTmp := path + "tmp"
-	pathTmpPre := pathTmp
-	tc := 0
-	for _, err := os.Stat(pathTmp); err == nil; _, err = os.Stat(pathTmp) {
-		pathTmp = fmt.Sprintf("%s%d", pathTmpPre, tc)
-		tc++
-	}
-	if err := SaveAsPNG(pathTmp, image); err != nil {
-		return err
-	}
-	return os.Rename(pathTmp, path)
+	return SaveAsPNGAtomic(path, image)
 }

common/image.go

@@ -6,12 +6,35 @@ package common
 
 import (
 	"bufio"
+	"errors"
 	"image"
 	"image/png"
 	"log"
 	"os"
+	"strconv"
+	"sync"
+	"time"
 )
 
+var rand uint32
+var randmu sync.Mutex
+
+func reseed() uint32 {
+	return uint32(time.Now().UnixNano() + int64(os.Getpid()))
+}
+
+func nextSuffix() string {
+	randmu.Lock()
+	r := rand
+	if r == 0 {
+		r = reseed()
+	}
+	r = r*1664525 + 1013904223 // constants from Numerical Recipes
+	rand = r
+	randmu.Unlock()
+	return strconv.Itoa(int(1e9 + r%1e9))[1:]
+}
+
 func SaveAsPNG(path string, img image.Image) (err error) {
 	var file *os.File
 	if file, err = os.Create(path); err != nil {
@@ -24,6 +47,38 @@ func SaveAsPNG(path string, img image.Image) (err error) {
 	return
 }
 
+func tmpName(tmpl string) (string, error) {
+	tmpPre := tmpl + ".tmp"
+	nconflict := 0
+	for i := 0; i < 10000; i++ {
+		tmp := tmpPre + nextSuffix()
+		if _, err := os.Stat(tmp); err != nil {
+			if os.IsNotExist(err) {
+				return tmp, nil
+			}
+			return "", err
+		}
+		if nconflict++; nconflict > 10 {
+			nconflict = 0
+			rand = reseed()
+		}
+	}
+	return "", errors.New("Cannot create temp name")
+}
+
+func SaveAsPNGAtomic(path string, img image.Image) (err error) {
+
+	var tmpPath string
+	if tmpPath, err = tmpName(path); err != nil {
+		return
+	}
+	// Still a bit racy
+	if err = SaveAsPNG(tmpPath, img); err != nil {
+		return
+	}
+	return os.Rename(tmpPath, path)
+}
+
 func LoadPNG(path string) image.Image {
 	var err error
 	var file *os.File