diff --git a/cmd/mtwebmapper/main.go b/cmd/mtwebmapper/main.go index f4bacd5..43c195e 100644 --- a/cmd/mtwebmapper/main.go +++ b/cmd/mtwebmapper/main.go @@ -10,17 +10,21 @@ import ( "log" "net/http" + "bitbucket.org/s_l_teichmann/mtredisalize/common" + "github.com/gorilla/mux" ) func main() { var ( - webPort int - webHost string - webDir string - mapDir string - redisPort int - redisHost string + webPort int + webHost string + webDir string + mapDir string + redisPort int + redisHost string + colorsFile string + workers int ) flag.IntVar(&webPort, "web-port", 8808, "port of the web server") flag.IntVar(&webPort, "p", 8808, "port of the web server (shorthand)") @@ -34,6 +38,9 @@ func main() { flag.StringVar(&redisHost, "rh", "", "address of the backend Redis server (shorthand)") flag.IntVar(&redisPort, "redis-port", 6379, "port of the backend Redis server") flag.IntVar(&redisPort, "rp", 6379, "port of the backend Redis server (shorthand)") + flag.IntVar(&workers, "workers", 1, "number of workers to render tiles") + flag.StringVar(&colorsFile, "colors", "colors.txt", "colors used to render map tiles.") + flag.StringVar(&colorsFile, "c", "colors.txt", "colors used to render map tiles (shorthand).") flag.Parse() @@ -43,8 +50,17 @@ func main() { router.Path("/map/{z:[0-9]+}/{x:[0-9]+}/{y:[0-9]+}.png").Handler(subBaseLine) if redisHost != "" { - tileUpdater := newTileUpdater(mapDir, redisHost, redisPort) - router.Path("/update").Methods("POST").Handler(tileUpdater) + var colors *common.Colors + var err error + if colors, err = common.ParseColors(colorsFile); err != nil { + log.Fatalf("ERROR: problem loading colors: %s", err) + } + redisAddress := fmt.Sprintf("%s:%d", redisHost, redisPort) + tu := newTileUpdater(mapDir, redisAddress, colors, workers) + if err = tu.doUpdates(); err != nil { + log.Fatalf("ERROR: Cannot start tile generation: %s", err) + } + router.Path("/update").Methods("POST").Handler(tu) } router.PathPrefix("/").Handler(http.FileServer(http.Dir(webDir))) diff --git a/cmd/mtwebmapper/subbaseline.go b/cmd/mtwebmapper/subbaseline.go index b218d00..f28e033 100644 --- a/cmd/mtwebmapper/subbaseline.go +++ b/cmd/mtwebmapper/subbaseline.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "strconv" + "time" "bitbucket.org/s_l_teichmann/mtredisalize/common" "github.com/gorilla/mux" @@ -35,8 +36,10 @@ func (sb *subBaseLine) ServeHTTP(rw http.ResponseWriter, r *http.Request) { x, y, z := toUint(xs), toUint(ys), toUint(zs) if z < 9 { - filename := fmt.Sprintf("%d/%d/%d.png", z, x, y) - http.ServeFile(rw, r, filepath.Join(sb.mapDir, filename)) + http.ServeFile(rw, r, filepath.Join(sb.mapDir, + strconv.Itoa(int(z)), + strconv.Itoa(int(x)), + fmt.Sprintf("%d.png", y))) return } @@ -46,20 +49,24 @@ func (sb *subBaseLine) ServeHTTP(rw http.ResponseWriter, r *http.Request) { tx := x >> (z - 8) ty := y >> (z - 8) - baseTile := filepath.Join(sb.mapDir, fmt.Sprintf("8/%d/%d.png", tx, ty)) + baseTile := filepath.Join( + sb.mapDir, "8", strconv.Itoa(int(tx)), fmt.Sprintf("%d.png", ty)) + + rw.Header().Set("Cache-Control", "private, max-age=0, no-cache") - var etag string var err error + var fi os.FileInfo + if fi, err = os.Stat(baseTile); err != nil { + http.NotFound(rw, r) + return + } - if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch != "" { - if etag, err = createETag(baseTile); err != nil { - http.NotFound(rw, r) - return - } - if ifNoneMatch == etag { - http.Error(rw, http.StatusText(http.StatusNotModified), http.StatusNotModified) - return - } + if checkLastModified(rw, r, fi.ModTime()) { + return + } + + if checkETag(rw, r, fi) { + return } rx := x & ^(^uint(0) << (z - 8)) @@ -82,23 +89,14 @@ func (sb *subBaseLine) ServeHTTP(rw http.ResponseWriter, r *http.Request) { } else { // Should not happen. http.Error(rw, - http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + http.StatusText(http.StatusInternalServerError), + http.StatusInternalServerError) + return } img = blowUp(img) rw.Header().Set("Content-Type", "image/png") - if etag == "" { - if etag, err = createETag(baseTile); err != nil { - // Unlikely - log.Printf("Cannot create ETag: %s", baseTile) - } else { - rw.Header().Set("ETag", etag) - } - } else { - rw.Header().Set("ETag", etag) - } - if err = png.Encode(rw, img); err != nil { log.Printf("WARN: encoding image failed: %s", err) return @@ -137,13 +135,30 @@ func blowUp(src image.Image) *image.RGBA { return dst } -func createETag(path string) (etag string, err error) { - var fi os.FileInfo - if fi, err = os.Stat(path); err != nil { - return +func checkETag(w http.ResponseWriter, r *http.Request, fi os.FileInfo) bool { + etag := fmt.Sprintf("%x-%x", fi.ModTime().Unix(), fi.Size()) + if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch == etag { + w.WriteHeader(http.StatusNotModified) + return true } - etag = fmt.Sprintf("%x-%x", fi.ModTime().Unix(), fi.Size()) - return + w.Header().Set("ETag", etag) + return false +} + +func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool { + + if modtime.IsZero() { + return false + } + + // The Date-Modified header truncates sub-second precision, so + // use mtime < t+1s instead of mtime <= t to check for unmodified. + if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) { + w.WriteHeader(http.StatusNotModified) + return true + } + w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) + return false } func toUint(s string) uint { diff --git a/cmd/mtwebmapper/tilesupdater.go b/cmd/mtwebmapper/tilesupdater.go index 1a8e712..a77745e 100644 --- a/cmd/mtwebmapper/tilesupdater.go +++ b/cmd/mtwebmapper/tilesupdater.go @@ -6,14 +6,54 @@ package main import ( "encoding/json" + "fmt" + "image/color" "log" "net/http" + "path/filepath" + "sync" + + "bitbucket.org/s_l_teichmann/mtredisalize/common" ) +const ( + tileWidth = 18 + tileHeight = 18 + yOrderCapacity = 512 +) + +// To scan the whole height in terms of the y coordinate +// the database is queried in height units defined in the yRanges table. +var yRanges = [][]int16{ + {1024, 1934}, + {256, 1023}, + {128, 255}, + {64, 127}, + {32, 63}, + {16, 31}, + {8, 15}, + {4, 7}, + {2, 3}, + {0, 1}, + {-1, 0}, + {-4, -2}, + {-8, -5}, + {-16, -9}, + {-32, -17}, + {-64, -33}, + {-128, -65}, + {-256, -129}, + {-1024, -257}, + {-1934, -1025}} + type tileUpdater struct { - mapDir string - redisHost string - redisPort int + changes map[xz]bool + mapDir string + redisAddress string + colors *common.Colors + workers int + cond *sync.Cond + mu sync.Mutex } type xz struct { @@ -21,23 +61,164 @@ type xz struct { Z int16 } -func newTileUpdater(mapDir, redisHost string, redisPort int) *tileUpdater { - return &tileUpdater{ - mapDir: mapDir, - redisHost: redisHost, - redisPort: redisPort} +func (c xz) quantize() xz { + return xz{X: (c.X - -1936) / 16, Z: (c.Z - -1936) / 16} +} +func newTileUpdater(mapDir, redisAddress string, colors *common.Colors, workers int) *tileUpdater { + tu := tileUpdater{ + mapDir: mapDir, + redisAddress: redisAddress, + changes: map[xz]bool{}, + colors: colors, + workers: workers} + tu.cond = sync.NewCond(&tu.mu) + return &tu } func (tu *tileUpdater) ServeHTTP(rw http.ResponseWriter, r *http.Request) { var err error - var data []xz + var newChanges []xz decoder := json.NewDecoder(r.Body) - if err = decoder.Decode(&data); err != nil { + if err = decoder.Decode(&newChanges); err != nil { log.Printf("WARN: JSON document broken: %s", err) http.Error(rw, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) return } - log.Printf("Changes: #%d %+v\n", len(data), data) + + tu.cond.L.Lock() + for _, c := range newChanges { + tu.changes[c.quantize()] = true + } + tu.cond.L.Unlock() + tu.cond.Signal() + rw.WriteHeader(http.StatusOK) } + +func (tu *tileUpdater) doUpdates() (err error) { + + jobs := make(chan xz) + + baseDir := filepath.Join(tu.mapDir, "8") + + for i := 0; i < tu.workers; i++ { + var client *common.RedisClient + if client, err = common.NewRedisClient("tcp", tu.redisAddress); err != nil { + return + } + btc := NewBaseTileCreator(client, tu.colors, baseDir) + go btc.run(jobs) + } + + go func() { + for { + var changes map[xz]bool + tu.cond.L.Lock() + for len(tu.changes) == 0 { + tu.cond.Wait() + } + changes = tu.changes + tu.changes = map[xz]bool{} + tu.cond.L.Unlock() + + for c, _ := range changes { + log.Printf("job: %+v", c) + jobs <- c + } + } + }() + return +} + +type BaseTileCreator struct { + client *common.RedisClient + colors *common.Colors + renderer *common.Renderer + yOrder *common.YOrder + baseDir string +} + +func NewBaseTileCreator(client *common.RedisClient, + colors *common.Colors, baseDir string) *BaseTileCreator { + renderer := common.NewRenderer(tileWidth, tileHeight) + return &BaseTileCreator{ + client: client, + colors: colors, + baseDir: baseDir, + renderer: renderer, + yOrder: common.NewYOrder(renderer, yOrderCapacity)} +} + +func (btc *BaseTileCreator) run(jobs chan xz) { + for job := range jobs { + x := job.X*16 + -1936 - 1 + z := job.Z*16 + -1936 - 1 + log.Printf("%d/%d %d/%d", x, z, job.X, job.Z) + if err := btc.createTile(x, z, int(job.X), int(job.Z)); err != nil { + log.Printf("WARN: create tile failed: %s", err) + } + } +} + +func (btc *BaseTileCreator) close() error { + return btc.client.Close() +} + +func (btc *BaseTileCreator) createTile(x, z int16, i, j int) error { + btc.renderer.Reset() + btc.renderer.SetPos(x, z) + btc.yOrder.Reset() + + drawBlock := func(block *common.Block) { + if err := btc.yOrder.RenderBlock(block, btc.colors.NameIndex); err != nil { + log.Printf("WARN: rendering block failed: %s", err) + } + } + + var c1, c2 common.Coord + + nareas := make([]common.Area, 0, tileWidth*tileHeight/2) + oareas := make([]common.Area, 1, tileWidth*tileHeight/2) + + oareas[0] = common.Area{ + X1: 0, Z1: 0, + X2: int16(tileWidth) - 1, Z2: int16(tileHeight) - 1} + + for _, yRange := range yRanges { + c1.Y = yRange[0] + c2.Y = yRange[1] + + nareas = btc.renderer.UncoveredAreas(nareas, oareas) + + if len(nareas) == 0 { + break + } + + for _, area := range nareas { + c1.X = area.X1 + x + c1.Z = area.Z1 + z + c2.X = area.X2 + x + c2.Z = area.Z2 + z + query := common.Cuboid{P1: c1, P2: c2} + if err := btc.client.QueryCuboid(query, drawBlock); err != nil { + return err + } + if err := btc.yOrder.Drain(btc.colors.NameIndex); err != nil { + log.Printf("WARN: rendering block failed: %s", err) + } + } + oareas, nareas = nareas, oareas[0:0] + } + + image := btc.renderer.CreateShadedImage( + 16, 16, (tileWidth-2)*16, (tileHeight-2)*16, + btc.colors.Colors, color.RGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xff}) + + path := filepath.Join(btc.baseDir, fmt.Sprintf("%d", i), fmt.Sprintf("%d.png", j)) + log.Printf("file path: %s", path) + + log.Printf("Writing (%d, %d) (%d, %d)", i, j, x, z) + + return common.SaveAsPNG(path, image) +}