Regenerate base tiles when db signals changes. TODO remove duplicate code from mtseeder.

cmd/mtwebmapper/main.go

@@ -10,17 +10,21 @@ import (
 	"log"
 	"net/http"
 
+	"bitbucket.org/s_l_teichmann/mtredisalize/common"
+
 	"github.com/gorilla/mux"
 )
 
 func main() {
 	var (
-		webPort   int
-		webHost   string
-		webDir    string
-		mapDir    string
-		redisPort int
-		redisHost string
+		webPort    int
+		webHost    string
+		webDir     string
+		mapDir     string
+		redisPort  int
+		redisHost  string
+		colorsFile string
+		workers    int
 	)
 	flag.IntVar(&webPort, "web-port", 8808, "port of the web server")
 	flag.IntVar(&webPort, "p", 8808, "port of the web server (shorthand)")
@@ -34,6 +38,9 @@ func main() {
 	flag.StringVar(&redisHost, "rh", "", "address of the backend Redis server (shorthand)")
 	flag.IntVar(&redisPort, "redis-port", 6379, "port of the backend Redis server")
 	flag.IntVar(&redisPort, "rp", 6379, "port of the backend Redis server (shorthand)")
+	flag.IntVar(&workers, "workers", 1, "number of workers to render tiles")
+	flag.StringVar(&colorsFile, "colors", "colors.txt", "colors used to render map tiles.")
+	flag.StringVar(&colorsFile, "c", "colors.txt", "colors used to render map tiles (shorthand).")
 
 	flag.Parse()
 
@@ -43,8 +50,17 @@ func main() {
 	router.Path("/map/{z:[0-9]+}/{x:[0-9]+}/{y:[0-9]+}.png").Handler(subBaseLine)
 
 	if redisHost != "" {
-		tileUpdater := newTileUpdater(mapDir, redisHost, redisPort)
-		router.Path("/update").Methods("POST").Handler(tileUpdater)
+		var colors *common.Colors
+		var err error
+		if colors, err = common.ParseColors(colorsFile); err != nil {
+			log.Fatalf("ERROR: problem loading colors: %s", err)
+		}
+		redisAddress := fmt.Sprintf("%s:%d", redisHost, redisPort)
+		tu := newTileUpdater(mapDir, redisAddress, colors, workers)
+		if err = tu.doUpdates(); err != nil {
+			log.Fatalf("ERROR: Cannot start tile generation: %s", err)
+		}
+		router.Path("/update").Methods("POST").Handler(tu)
 	}
 
 	router.PathPrefix("/").Handler(http.FileServer(http.Dir(webDir)))

cmd/mtwebmapper/subbaseline.go

@@ -14,6 +14,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"time"
 
 	"bitbucket.org/s_l_teichmann/mtredisalize/common"
 	"github.com/gorilla/mux"
@@ -35,8 +36,10 @@ func (sb *subBaseLine) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 
 	x, y, z := toUint(xs), toUint(ys), toUint(zs)
 	if z < 9 {
-		filename := fmt.Sprintf("%d/%d/%d.png", z, x, y)
-		http.ServeFile(rw, r, filepath.Join(sb.mapDir, filename))
+		http.ServeFile(rw, r, filepath.Join(sb.mapDir,
+			strconv.Itoa(int(z)),
+			strconv.Itoa(int(x)),
+			fmt.Sprintf("%d.png", y)))
 		return
 	}
 
@@ -46,20 +49,24 @@ func (sb *subBaseLine) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 	tx := x >> (z - 8)
 	ty := y >> (z - 8)
 
-	baseTile := filepath.Join(sb.mapDir, fmt.Sprintf("8/%d/%d.png", tx, ty))
+	baseTile := filepath.Join(
+		sb.mapDir, "8", strconv.Itoa(int(tx)), fmt.Sprintf("%d.png", ty))
+
+	rw.Header().Set("Cache-Control", "private, max-age=0, no-cache")
 
-	var etag string
 	var err error
+	var fi os.FileInfo
+	if fi, err = os.Stat(baseTile); err != nil {
+		http.NotFound(rw, r)
+		return
+	}
 
-	if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch != "" {
-		if etag, err = createETag(baseTile); err != nil {
-			http.NotFound(rw, r)
-			return
-		}
-		if ifNoneMatch == etag {
-			http.Error(rw, http.StatusText(http.StatusNotModified), http.StatusNotModified)
-			return
-		}
+	if checkLastModified(rw, r, fi.ModTime()) {
+		return
+	}
+
+	if checkETag(rw, r, fi) {
+		return
 	}
 
 	rx := x & ^(^uint(0) << (z - 8))
@@ -82,23 +89,14 @@ func (sb *subBaseLine) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 	} else {
 		// Should not happen.
 		http.Error(rw,
-			http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+			http.StatusText(http.StatusInternalServerError),
+			http.StatusInternalServerError)
+		return
 	}
 
 	img = blowUp(img)
 
 	rw.Header().Set("Content-Type", "image/png")
-	if etag == "" {
-		if etag, err = createETag(baseTile); err != nil {
-			// Unlikely
-			log.Printf("Cannot create ETag: %s", baseTile)
-		} else {
-			rw.Header().Set("ETag", etag)
-		}
-	} else {
-		rw.Header().Set("ETag", etag)
-	}
-
 	if err = png.Encode(rw, img); err != nil {
 		log.Printf("WARN: encoding image failed: %s", err)
 		return
@@ -137,13 +135,30 @@ func blowUp(src image.Image) *image.RGBA {
 	return dst
 }
 
-func createETag(path string) (etag string, err error) {
-	var fi os.FileInfo
-	if fi, err = os.Stat(path); err != nil {
-		return
+func checkETag(w http.ResponseWriter, r *http.Request, fi os.FileInfo) bool {
+	etag := fmt.Sprintf("%x-%x", fi.ModTime().Unix(), fi.Size())
+	if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch == etag {
+		w.WriteHeader(http.StatusNotModified)
+		return true
 	}
-	etag = fmt.Sprintf("%x-%x", fi.ModTime().Unix(), fi.Size())
-	return
+	w.Header().Set("ETag", etag)
+	return false
+}
+
+func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool {
+
+	if modtime.IsZero() {
+		return false
+	}
+
+	// The Date-Modified header truncates sub-second precision, so
+	// use mtime < t+1s instead of mtime <= t to check for unmodified.
+	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) {
+		w.WriteHeader(http.StatusNotModified)
+		return true
+	}
+	w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
+	return false
 }
 
 func toUint(s string) uint {

cmd/mtwebmapper/tilesupdater.go

@@ -6,14 +6,54 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
+	"image/color"
 	"log"
 	"net/http"
+	"path/filepath"
+	"sync"
+
+	"bitbucket.org/s_l_teichmann/mtredisalize/common"
 )
 
+const (
+	tileWidth      = 18
+	tileHeight     = 18
+	yOrderCapacity = 512
+)
+
+// To scan the whole height in terms of the y coordinate
+// the database is queried in height units defined in the yRanges table.
+var yRanges = [][]int16{
+	{1024, 1934},
+	{256, 1023},
+	{128, 255},
+	{64, 127},
+	{32, 63},
+	{16, 31},
+	{8, 15},
+	{4, 7},
+	{2, 3},
+	{0, 1},
+	{-1, 0},
+	{-4, -2},
+	{-8, -5},
+	{-16, -9},
+	{-32, -17},
+	{-64, -33},
+	{-128, -65},
+	{-256, -129},
+	{-1024, -257},
+	{-1934, -1025}}
+
 type tileUpdater struct {
-	mapDir    string
-	redisHost string
-	redisPort int
+	changes      map[xz]bool
+	mapDir       string
+	redisAddress string
+	colors       *common.Colors
+	workers      int
+	cond         *sync.Cond
+	mu           sync.Mutex
 }
 
 type xz struct {
@@ -21,23 +61,164 @@ type xz struct {
 	Z int16
 }
 
-func newTileUpdater(mapDir, redisHost string, redisPort int) *tileUpdater {
-	return &tileUpdater{
-		mapDir:    mapDir,
-		redisHost: redisHost,
-		redisPort: redisPort}
+func (c xz) quantize() xz {
+	return xz{X: (c.X - -1936) / 16, Z: (c.Z - -1936) / 16}
+}
 
+func newTileUpdater(mapDir, redisAddress string, colors *common.Colors, workers int) *tileUpdater {
+	tu := tileUpdater{
+		mapDir:       mapDir,
+		redisAddress: redisAddress,
+		changes:      map[xz]bool{},
+		colors:       colors,
+		workers:      workers}
+	tu.cond = sync.NewCond(&tu.mu)
+	return &tu
 }
 
 func (tu *tileUpdater) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 	var err error
-	var data []xz
+	var newChanges []xz
 	decoder := json.NewDecoder(r.Body)
-	if err = decoder.Decode(&data); err != nil {
+	if err = decoder.Decode(&newChanges); err != nil {
 		log.Printf("WARN: JSON document broken: %s", err)
 		http.Error(rw, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
 		return
 	}
-	log.Printf("Changes: #%d %+v\n", len(data), data)
+
+	tu.cond.L.Lock()
+	for _, c := range newChanges {
+		tu.changes[c.quantize()] = true
+	}
+	tu.cond.L.Unlock()
+	tu.cond.Signal()
+
 	rw.WriteHeader(http.StatusOK)
 }
+
+func (tu *tileUpdater) doUpdates() (err error) {
+
+	jobs := make(chan xz)
+
+	baseDir := filepath.Join(tu.mapDir, "8")
+
+	for i := 0; i < tu.workers; i++ {
+		var client *common.RedisClient
+		if client, err = common.NewRedisClient("tcp", tu.redisAddress); err != nil {
+			return
+		}
+		btc := NewBaseTileCreator(client, tu.colors, baseDir)
+		go btc.run(jobs)
+	}
+
+	go func() {
+		for {
+			var changes map[xz]bool
+			tu.cond.L.Lock()
+			for len(tu.changes) == 0 {
+				tu.cond.Wait()
+			}
+			changes = tu.changes
+			tu.changes = map[xz]bool{}
+			tu.cond.L.Unlock()
+
+			for c, _ := range changes {
+				log.Printf("job: %+v", c)
+				jobs <- c
+			}
+		}
+	}()
+	return
+}
+
+type BaseTileCreator struct {
+	client   *common.RedisClient
+	colors   *common.Colors
+	renderer *common.Renderer
+	yOrder   *common.YOrder
+	baseDir  string
+}
+
+func NewBaseTileCreator(client *common.RedisClient,
+	colors *common.Colors, baseDir string) *BaseTileCreator {
+	renderer := common.NewRenderer(tileWidth, tileHeight)
+	return &BaseTileCreator{
+		client:   client,
+		colors:   colors,
+		baseDir:  baseDir,
+		renderer: renderer,
+		yOrder:   common.NewYOrder(renderer, yOrderCapacity)}
+}
+
+func (btc *BaseTileCreator) run(jobs chan xz) {
+	for job := range jobs {
+		x := job.X*16 + -1936 - 1
+		z := job.Z*16 + -1936 - 1
+		log.Printf("%d/%d %d/%d", x, z, job.X, job.Z)
+		if err := btc.createTile(x, z, int(job.X), int(job.Z)); err != nil {
+			log.Printf("WARN: create tile failed: %s", err)
+		}
+	}
+}
+
+func (btc *BaseTileCreator) close() error {
+	return btc.client.Close()
+}
+
+func (btc *BaseTileCreator) createTile(x, z int16, i, j int) error {
+	btc.renderer.Reset()
+	btc.renderer.SetPos(x, z)
+	btc.yOrder.Reset()
+
+	drawBlock := func(block *common.Block) {
+		if err := btc.yOrder.RenderBlock(block, btc.colors.NameIndex); err != nil {
+			log.Printf("WARN: rendering block failed: %s", err)
+		}
+	}
+
+	var c1, c2 common.Coord
+
+	nareas := make([]common.Area, 0, tileWidth*tileHeight/2)
+	oareas := make([]common.Area, 1, tileWidth*tileHeight/2)
+
+	oareas[0] = common.Area{
+		X1: 0, Z1: 0,
+		X2: int16(tileWidth) - 1, Z2: int16(tileHeight) - 1}
+
+	for _, yRange := range yRanges {
+		c1.Y = yRange[0]
+		c2.Y = yRange[1]
+
+		nareas = btc.renderer.UncoveredAreas(nareas, oareas)
+
+		if len(nareas) == 0 {
+			break
+		}
+
+		for _, area := range nareas {
+			c1.X = area.X1 + x
+			c1.Z = area.Z1 + z
+			c2.X = area.X2 + x
+			c2.Z = area.Z2 + z
+			query := common.Cuboid{P1: c1, P2: c2}
+			if err := btc.client.QueryCuboid(query, drawBlock); err != nil {
+				return err
+			}
+			if err := btc.yOrder.Drain(btc.colors.NameIndex); err != nil {
+				log.Printf("WARN: rendering block failed: %s", err)
+			}
+		}
+		oareas, nareas = nareas, oareas[0:0]
+	}
+
+	image := btc.renderer.CreateShadedImage(
+		16, 16, (tileWidth-2)*16, (tileHeight-2)*16,
+		btc.colors.Colors, color.RGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xff})
+
+	path := filepath.Join(btc.baseDir, fmt.Sprintf("%d", i), fmt.Sprintf("%d.png", j))
+	log.Printf("file path: %s", path)
+
+	log.Printf("Writing (%d, %d) (%d, %d)", i, j, x, z)
+
+	return common.SaveAsPNG(path, image)
+}