From 91959685f51e43a68a5036dc12554c58e2884f99 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Fri, 3 Mar 2017 23:00:29 +0100 Subject: [PATCH] Re-worked memory management of blocks loaded from redis client to avoid unnecessary copying them around. --- cmd/mttilemapper/main.go | 15 +++++++++---- common/basetilecreator.go | 6 ++++-- common/redisclient.go | 45 ++++++++++++++++++++++++--------------- common/yorder.go | 33 +++++++--------------------- 4 files changed, 51 insertions(+), 48 deletions(-) diff --git a/cmd/mttilemapper/main.go b/cmd/mttilemapper/main.go index e129524..e394df5 100644 --- a/cmd/mttilemapper/main.go +++ b/cmd/mttilemapper/main.go @@ -105,14 +105,21 @@ func main() { renderer := common.NewRenderer(width, height, transparent) renderer.SetPos(q1x, q1z) - yOrder := common.NewYOrder(renderer, 512) + + renderFn := func(block *common.Block) error { + return renderer.RenderBlock(block, colors) + } + + yOrder := common.NewYOrder(renderFn, 512) numBlocks := 0 - drawBlock := func(block *common.Block) { - if err := yOrder.RenderBlock(block, colors); err != nil { + drawBlock := func(block *common.Block) *common.Block { + block, err := yOrder.RenderBlock(block) + if err != nil { log.Printf("WARN: rendering block failed: %s\n", err) } numBlocks++ + return block } c1 := common.Coord{X: q1x, Z: q1z} @@ -126,7 +133,7 @@ func main() { if _, err = client.QueryCuboid(cuboid, drawBlock); err != nil { log.Fatalf("query failed: %s", err) } - if err = yOrder.Drain(colors); err != nil { + if err = yOrder.Drain(); err != nil { log.Printf("WARN: rendering block failed: %s\n", err) } if renderer.IsFilled() { diff --git a/common/basetilecreator.go b/common/basetilecreator.go index 5f62160..89f5b08 100644 --- a/common/basetilecreator.go +++ b/common/basetilecreator.go @@ -97,10 +97,12 @@ func (btc *BaseTileCreator) renderBlock(block *Block) error { } // blockLoaded is a callback for RedisClient.QueryCuboid. -func (btc *BaseTileCreator) blockLoaded(block *Block) { - if err := btc.yOrder.RenderBlock(block); err != nil { +func (btc *BaseTileCreator) blockLoaded(block *Block) *Block { + block, err := btc.yOrder.RenderBlock(block) + if err != nil { log.Printf("WARN: rendering block failed: %s\n", err) } + return block } func (btc *BaseTileCreator) CreateTile(x, z int16, i, j int) (bool, error) { diff --git a/common/redisclient.go b/common/redisclient.go index 1e4126d..7a8e48e 100644 --- a/common/redisclient.go +++ b/common/redisclient.go @@ -11,13 +11,13 @@ import ( "fmt" "net" "strconv" - "sync" "unicode" ) type RedisClient struct { conn net.Conn reader *bufio.Reader + arena []byte scratch [80]byte } @@ -89,6 +89,16 @@ func parseSize(line []byte) (int, error) { return int(v), err } +func (client *RedisClient) alloc(size int) []byte { + a := client.arena + if len(a) < size { + a = make([]byte, Max(size, 16*1024)) + } + x := a[:size:size] + client.arena = a[size:] + return x +} + func (client *RedisClient) readBulkString(data *[]byte) (size int, err error) { var line []byte if line, err = client.reader.ReadBytes('\n'); err != nil { @@ -101,7 +111,9 @@ func (client *RedisClient) readBulkString(data *[]byte) (size int, err error) { return } if cap(*data) < size { - *data = make([]byte, size) + *data = client.alloc(size) + } else { + *data = (*data)[:size] } for rest := size; rest > 0; { var n int @@ -114,26 +126,18 @@ func (client *RedisClient) readBulkString(data *[]byte) (size int, err error) { return } -var dataPool = sync.Pool{ - New: func() interface{} { - return make([]byte, 8*1024) - }, -} - -func (client *RedisClient) QueryCuboid(cuboid Cuboid, fn func(*Block)) (count int, err error) { +func (client *RedisClient) QueryCuboid(cuboid Cuboid, fn func(*Block) *Block) (count int, err error) { p1 := CoordToPlain(cuboid.P1) p2 := CoordToPlain(cuboid.P2) if err = client.writeHSpatial(p1, p2); err != nil { return } - data := dataPool.Get().([]byte) - defer dataPool.Put(data[:0]) - var ( - block = Block{} + block *Block size int key int64 + data []byte ) for s := client.scratch[:]; ; count++ { @@ -144,15 +148,22 @@ func (client *RedisClient) QueryCuboid(cuboid Cuboid, fn func(*Block)) (count in if size <= 0 { break } - if key, err = DecodeStringFromBytes((*p)[:size]); err != nil { + if key, err = DecodeStringFromBytes(*p); err != nil { return } - block.Coord = PlainToCoord(key) if size, err = client.readBulkString(&data); err != nil || size < 0 { return } - block.Data = data[:size] - fn(&block) + if block == nil { + block = &Block{Coord: PlainToCoord(key), Data: data} + } else { + *block = Block{Coord: PlainToCoord(key), Data: data} + } + if block = fn(block); block != nil { + data = block.Data[:0] + } else { + data = nil + } } return diff --git a/common/yorder.go b/common/yorder.go index 4aa523b..155ffca 100644 --- a/common/yorder.go +++ b/common/yorder.go @@ -43,40 +43,23 @@ func (yo *YOrder) Reset() { yo.blocks = blocks[:0] } -func copyData(data []byte) []byte { - l := len(data) - ndata := make([]byte, l, Max(l, 8*1024)) - copy(ndata, data) - return ndata -} - -func (yo *YOrder) RenderBlock(block *Block) (err error) { - var nblock *Block +func (yo *YOrder) RenderBlock(block *Block) (*Block, error) { if len(yo.blocks) == yo.capacity { oblock := yo.blocks[0] if oblock.Coord.Y < block.Coord.Y { // New one is above highest old. Directly render new. - err = yo.RenderFn(block) - return + err := yo.RenderFn(block) + return block, err } // Render old one. Store copy of new in heap. heap.Pop(yo) - err = yo.RenderFn(oblock) - l := len(block.Data) - if cap(oblock.Data) < l { - oblock.Data = make([]byte, l, Max(l, 8*1024)) - } else { - oblock.Data = oblock.Data[:l] - } - copy(oblock.Data, block.Data) - oblock.Coord = block.Coord - nblock = oblock - } else { - nblock = &Block{Coord: block.Coord, Data: copyData(block.Data)} + heap.Push(yo, block) + err := yo.RenderFn(oblock) + return oblock, err } - heap.Push(yo, nblock) - return + heap.Push(yo, block) + return nil, nil } func (yo *YOrder) Drain() error {