mtcontrib/mtsatellite
1
0
Fork 0
mirror of https://bitbucket.org/s_l_teichmann/mtsatellite synced 2024-11-17 15:48:17 +01:00
Code Issues Projects Releases Wiki Activity

Regenerate base tiles when db signals changes. TODO remove duplicate code from mtseeder.

Browse Source
This commit is contained in:
Sascha L. Teichmann 2014-09-19 13:06:04 +02:00
parent 3f13738d41
commit e899b13889
3 changed files with 262 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
cmd/mtwebmapper/main.go
View File

@ -10,6 +10,8 @@ import (
"log"
"net/http"
"bitbucket.org/s_l_teichmann/mtredisalize/common"
"github.com/gorilla/mux"
)
@ -21,6 +23,8 @@ func main() {
mapDir string
redisPort int
redisHost string
colorsFile string
workers int
)
flag.IntVar(&webPort, "web-port", 8808, "port of the web server")
flag.IntVar(&webPort, "p", 8808, "port of the web server (shorthand)")
@ -34,6 +38,9 @@ func main() {
flag.StringVar(&redisHost, "rh", "", "address of the backend Redis server (shorthand)")
flag.IntVar(&redisPort, "redis-port", 6379, "port of the backend Redis server")
flag.IntVar(&redisPort, "rp", 6379, "port of the backend Redis server (shorthand)")
flag.IntVar(&workers, "workers", 1, "number of workers to render tiles")
flag.StringVar(&colorsFile, "colors", "colors.txt", "colors used to render map tiles.")
flag.StringVar(&colorsFile, "c", "colors.txt", "colors used to render map tiles (shorthand).")
flag.Parse()
@ -43,8 +50,17 @@ func main() {
router.Path("/map/{z:[0-9]+}/{x:[0-9]+}/{y:[0-9]+}.png").Handler(subBaseLine)
if redisHost != "" {
tileUpdater := newTileUpdater(mapDir, redisHost, redisPort)
router.Path("/update").Methods("POST").Handler(tileUpdater)
var colors *common.Colors
var err error
if colors, err = common.ParseColors(colorsFile); err != nil {
log.Fatalf("ERROR: problem loading colors: %s", err)
}
redisAddress := fmt.Sprintf("%s:%d", redisHost, redisPort)
tu := newTileUpdater(mapDir, redisAddress, colors, workers)
if err = tu.doUpdates(); err != nil {
log.Fatalf("ERROR: Cannot start tile generation: %s", err)
}
router.Path("/update").Methods("POST").Handler(tu)
}
router.PathPrefix("/").Handler(http.FileServer(http.Dir(webDir)))

69
cmd/mtwebmapper/subbaseline.go
View File

@ -14,6 +14,7 @@ import (
"os"
"path/filepath"
"strconv"
"time"
"bitbucket.org/s_l_teichmann/mtredisalize/common"
"github.com/gorilla/mux"
@ -35,8 +36,10 @@ func (sb *subBaseLine) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
x, y, z := toUint(xs), toUint(ys), toUint(zs)
if z < 9 {
filename := fmt.Sprintf("%d/%d/%d.png", z, x, y)
http.ServeFile(rw, r, filepath.Join(sb.mapDir, filename))
http.ServeFile(rw, r, filepath.Join(sb.mapDir,
strconv.Itoa(int(z)),
strconv.Itoa(int(x)),
fmt.Sprintf("%d.png", y)))
return
}
@ -46,20 +49,24 @@ func (sb *subBaseLine) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
tx := x >> (z - 8)
ty := y >> (z - 8)
baseTile := filepath.Join(sb.mapDir, fmt.Sprintf("8/%d/%d.png", tx, ty))
baseTile := filepath.Join(
sb.mapDir, "8", strconv.Itoa(int(tx)), fmt.Sprintf("%d.png", ty))
rw.Header().Set("Cache-Control", "private, max-age=0, no-cache")
var etag string
var err error
if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch != "" {
if etag, err = createETag(baseTile); err != nil {
var fi os.FileInfo
if fi, err = os.Stat(baseTile); err != nil {
http.NotFound(rw, r)
return
}
if ifNoneMatch == etag {
http.Error(rw, http.StatusText(http.StatusNotModified), http.StatusNotModified)
if checkLastModified(rw, r, fi.ModTime()) {
return
}
if checkETag(rw, r, fi) {
return
}
rx := x & ^(^uint(0) << (z - 8))
@ -82,23 +89,14 @@ func (sb *subBaseLine) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
} else {
// Should not happen.
http.Error(rw,
http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
http.StatusText(http.StatusInternalServerError),
http.StatusInternalServerError)
return
}
img = blowUp(img)
rw.Header().Set("Content-Type", "image/png")
if etag == "" {
if etag, err = createETag(baseTile); err != nil {
// Unlikely
log.Printf("Cannot create ETag: %s", baseTile)
} else {
rw.Header().Set("ETag", etag)
}
} else {
rw.Header().Set("ETag", etag)
}
if err = png.Encode(rw, img); err != nil {
log.Printf("WARN: encoding image failed: %s", err)
return
@ -137,13 +135,30 @@ func blowUp(src image.Image) *image.RGBA {
return dst
}
func createETag(path string) (etag string, err error) {
var fi os.FileInfo
if fi, err = os.Stat(path); err != nil {
return
func checkETag(w http.ResponseWriter, r *http.Request, fi os.FileInfo) bool {
etag := fmt.Sprintf("%x-%x", fi.ModTime().Unix(), fi.Size())
if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch == etag {
w.WriteHeader(http.StatusNotModified)
return true
}
etag = fmt.Sprintf("%x-%x", fi.ModTime().Unix(), fi.Size())
return
w.Header().Set("ETag", etag)
return false
}
func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool {
if modtime.IsZero() {
return false
}
// The Date-Modified header truncates sub-second precision, so
// use mtime < t+1s instead of mtime <= t to check for unmodified.
if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) {
w.WriteHeader(http.StatusNotModified)
return true
}
w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
return false
}
func toUint(s string) uint {

201
cmd/mtwebmapper/tilesupdater.go
View File

@ -6,14 +6,54 @@ package main
import (
"encoding/json"
"fmt"
"image/color"
"log"
"net/http"
"path/filepath"
"sync"
"bitbucket.org/s_l_teichmann/mtredisalize/common"
)
const (
tileWidth = 18
tileHeight = 18
yOrderCapacity = 512
)
// To scan the whole height in terms of the y coordinate
// the database is queried in height units defined in the yRanges table.
var yRanges = [][]int16{
{1024, 1934},
{256, 1023},
{128, 255},
{64, 127},
{32, 63},
{16, 31},
{8, 15},
{4, 7},
{2, 3},
{0, 1},
{-1, 0},
{-4, -2},
{-8, -5},
{-16, -9},
{-32, -17},
{-64, -33},
{-128, -65},
{-256, -129},
{-1024, -257},
{-1934, -1025}}
type tileUpdater struct {
changes map[xz]bool
mapDir string
redisHost string
redisPort int
redisAddress string
colors *common.Colors
workers int
cond *sync.Cond
mu sync.Mutex
}
type xz struct {
@ -21,23 +61,164 @@ type xz struct {
Z int16
}
func newTileUpdater(mapDir, redisHost string, redisPort int) *tileUpdater {
return &tileUpdater{
mapDir: mapDir,
redisHost: redisHost,
redisPort: redisPort}
func (c xz) quantize() xz {
return xz{X: (c.X - -1936) / 16, Z: (c.Z - -1936) / 16}
}
func newTileUpdater(mapDir, redisAddress string, colors *common.Colors, workers int) *tileUpdater {
tu := tileUpdater{
mapDir: mapDir,
redisAddress: redisAddress,
changes: map[xz]bool{},
colors: colors,
workers: workers}
tu.cond = sync.NewCond(&tu.mu)
return &tu
}
func (tu *tileUpdater) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
var err error
var data []xz
var newChanges []xz
decoder := json.NewDecoder(r.Body)
if err = decoder.Decode(&data); err != nil {
if err = decoder.Decode(&newChanges); err != nil {
log.Printf("WARN: JSON document broken: %s", err)
http.Error(rw, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
log.Printf("Changes: #%d %+v\n", len(data), data)
tu.cond.L.Lock()
for _, c := range newChanges {
tu.changes[c.quantize()] = true
}
tu.cond.L.Unlock()
tu.cond.Signal()
rw.WriteHeader(http.StatusOK)
}
func (tu *tileUpdater) doUpdates() (err error) {
jobs := make(chan xz)
baseDir := filepath.Join(tu.mapDir, "8")
for i := 0; i < tu.workers; i++ {
var client *common.RedisClient
if client, err = common.NewRedisClient("tcp", tu.redisAddress); err != nil {
return
}
btc := NewBaseTileCreator(client, tu.colors, baseDir)
go btc.run(jobs)
}
go func() {
for {
var changes map[xz]bool
tu.cond.L.Lock()
for len(tu.changes) == 0 {
tu.cond.Wait()
}
changes = tu.changes
tu.changes = map[xz]bool{}
tu.cond.L.Unlock()
for c, _ := range changes {
log.Printf("job: %+v", c)
jobs <- c
}
}
}()
return
}
type BaseTileCreator struct {
client *common.RedisClient
colors *common.Colors
renderer *common.Renderer
yOrder *common.YOrder
baseDir string
}
func NewBaseTileCreator(client *common.RedisClient,
colors *common.Colors, baseDir string) *BaseTileCreator {
renderer := common.NewRenderer(tileWidth, tileHeight)
return &BaseTileCreator{
client: client,
colors: colors,
baseDir: baseDir,
renderer: renderer,
yOrder: common.NewYOrder(renderer, yOrderCapacity)}
}
func (btc *BaseTileCreator) run(jobs chan xz) {
for job := range jobs {
x := job.X*16 + -1936 - 1
z := job.Z*16 + -1936 - 1
log.Printf("%d/%d %d/%d", x, z, job.X, job.Z)
if err := btc.createTile(x, z, int(job.X), int(job.Z)); err != nil {
log.Printf("WARN: create tile failed: %s", err)
}
}
}
func (btc *BaseTileCreator) close() error {
return btc.client.Close()
}
func (btc *BaseTileCreator) createTile(x, z int16, i, j int) error {
btc.renderer.Reset()
btc.renderer.SetPos(x, z)
btc.yOrder.Reset()
drawBlock := func(block *common.Block) {
if err := btc.yOrder.RenderBlock(block, btc.colors.NameIndex); err != nil {
log.Printf("WARN: rendering block failed: %s", err)
}
}
var c1, c2 common.Coord
nareas := make([]common.Area, 0, tileWidth*tileHeight/2)
oareas := make([]common.Area, 1, tileWidth*tileHeight/2)
oareas[0] = common.Area{
X1: 0, Z1: 0,
X2: int16(tileWidth) - 1, Z2: int16(tileHeight) - 1}
for _, yRange := range yRanges {
c1.Y = yRange[0]
c2.Y = yRange[1]
nareas = btc.renderer.UncoveredAreas(nareas, oareas)
if len(nareas) == 0 {
break
}
for _, area := range nareas {
c1.X = area.X1 + x
c1.Z = area.Z1 + z
c2.X = area.X2 + x
c2.Z = area.Z2 + z
query := common.Cuboid{P1: c1, P2: c2}
if err := btc.client.QueryCuboid(query, drawBlock); err != nil {
return err
}
if err := btc.yOrder.Drain(btc.colors.NameIndex); err != nil {
log.Printf("WARN: rendering block failed: %s", err)
}
}
oareas, nareas = nareas, oareas[0:0]
}
image := btc.renderer.CreateShadedImage(
16, 16, (tileWidth-2)*16, (tileHeight-2)*16,
btc.colors.Colors, color.RGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xff})
path := filepath.Join(btc.baseDir, fmt.Sprintf("%d", i), fmt.Sprintf("%d.png", j))
log.Printf("file path: %s", path)
log.Printf("Writing (%d, %d) (%d, %d)", i, j, x, z)
return common.SaveAsPNG(path, image)
}