mtsatellite/common/block.go

388 lines
8.4 KiB
Go
Raw Normal View History

2015-07-26 16:44:51 +02:00
// Copyright 2014, 2015 by Sascha L. Teichmann
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.
package common
import (
"bytes"
"compress/zlib"
"encoding/binary"
"errors"
"io"
"log"
"sync"
2024-01-05 14:41:06 +01:00
"github.com/klauspost/compress/zstd"
)
// Error returned if a Producer has run to its end.
var (
2024-01-06 10:17:25 +01:00
ErrNoMoreBlocks = errors.New("no more blocks")
ErrMapContentSizeMismatch = errors.New("content size does not match")
ErrBlockTruncated = errors.New("block is truncated")
)
const (
mapBlockSize = 16
nodeCount = mapBlockSize * mapBlockSize * mapBlockSize
)
type (
// Block data from Minetest database.
Block struct {
Coord Coord
Data []byte
}
// BlockProducer is used to over a existing Minetest database
// and return its content block by block.
BlockProducer interface {
// error is ErrNoMoreBlocks if it run out of blocks.
Next(*Block) error
// Closes the open database connections.
Close() error
}
// BlockConsumer is used to store blocks in a new Minetest database.
BlockConsumer interface {
Consume(*Block) error
// Closes the open database connections.
Close() error
}
DecodedBlock struct {
Version byte
Transparent bool
MapContent []byte
AirID int32
IgnoreID int32
IndexMap map[int32]int32
}
)
// zlibEmpty is a minimal zlib stream with zero length.
// zlib.NewReader needs a valid zlib stream to start with
// even if Reset is called directly afterwards.
var zlibEmpty = []byte{
0x78, 0x9c, 0x00, 0x00,
0x00, 0xff, 0xff, 0x01,
0x00, 0x00, 0xff, 0xff,
0x00, 0x00, 0x00, 0x01}
// zlibReaderPool is a pool of zlib Readers to be reused
// for decoding the compressed parts of database blocks.
// Decoding blocks relies heavly on zlib decompression.
// Reusing the internal structures of already allocated
// zlib readers speeds up the decoding significantly.
var zlibReaderPool = sync.Pool{
New: func() interface{} {
reader, _ := zlib.NewReader(bytes.NewBuffer(zlibEmpty))
return reader
},
}
// The content of the map and the meta data are compressed with zlib.
// Unfortunately the byte length of this two structures are not stored
// explicitly in the block data. To access the informations behind
// them (e.g. the node id mappings) we have to count the bytes consumed
// by the zlib reader and continue our extraction process behind this
// offset. posBuf implements such a counting reader source.
type posBuf struct {
Data []byte
Pos int
}
2024-01-06 01:40:03 +01:00
func decode29(data []byte, colors *Colors) (*DecodedBlock, error) {
dec, err := zstd.NewReader(bytes.NewReader(data[1:]))
2024-01-06 01:40:03 +01:00
if err != nil {
return nil, err
}
2024-01-07 16:02:00 +01:00
defer dec.Close()
2024-01-06 01:40:03 +01:00
2024-01-07 15:31:42 +01:00
be := bigEndianReader{parent: dec}
2024-01-06 01:40:03 +01:00
2024-01-07 15:31:42 +01:00
be.u8() // flags
be.u16() // lightning_complete
be.u32() // timestamp
be.u8() // name_id_mapping_version
2024-01-06 01:40:03 +01:00
airID, ignoreID := int32(-1), int32(-1)
indexMap := make(map[int32]int32)
transparent := false
2024-01-07 15:31:42 +01:00
numNameIDMappings, _ := be.u16()
2024-01-06 01:40:03 +01:00
for i := uint16(0); i < numNameIDMappings; i++ {
2024-01-07 15:31:42 +01:00
var (
id, _ = be.u16()
nlen, _ = be.u16()
name, _ = be.str(int(nlen))
)
2024-01-06 01:40:03 +01:00
switch name {
case "air":
2024-01-07 15:31:42 +01:00
airID = int32(id)
2024-01-06 01:40:03 +01:00
case "ignore":
2024-01-07 15:31:42 +01:00
ignoreID = int32(id)
2024-01-06 01:40:03 +01:00
default:
if index, found := colors.NameIndex[name]; found {
2024-01-07 15:31:42 +01:00
indexMap[int32(id)] = index
2024-01-06 01:40:03 +01:00
if !transparent && colors.IsTransparent(index) {
transparent = true
}
} else {
logMissing(name)
}
}
}
2024-01-07 15:31:42 +01:00
be.u8() // content_width
be.u8() // params_width
2024-01-06 01:40:03 +01:00
2024-01-07 15:31:42 +01:00
mapContent, err := be.full(2 * 4096)
if err != nil {
2024-01-06 01:40:03 +01:00
return nil, ErrBlockTruncated
}
return &DecodedBlock{
Version: data[0],
Transparent: transparent,
MapContent: mapContent,
AirID: airID,
IgnoreID: ignoreID,
IndexMap: indexMap,
}, nil
}
func NewDecodedBlock(data []byte, colors *Colors) (db *DecodedBlock, err error) {
dataLen := len(data)
if dataLen < 4 {
return nil, ErrBlockTruncated
}
version := data[0]
2024-01-06 01:40:03 +01:00
if version >= 29 {
return decode29(data, colors)
}
var offset int
2024-01-06 01:40:03 +01:00
contentWidth := Min(int(data[2]), 2)
paramsWidth := Min(int(data[3]), 2)
2024-01-05 14:41:06 +01:00
2024-01-06 01:40:03 +01:00
uncompressedLen := nodeCount * (contentWidth + paramsWidth)
2024-01-05 14:41:06 +01:00
2024-01-06 01:40:03 +01:00
switch {
case version >= 27:
offset = 6
case version >= 22:
offset = 4
default:
offset = 2
}
2024-01-06 01:40:03 +01:00
zr := zlibReaderPool.Get().(interface {
io.ReadCloser
zlib.Resetter
})
defer func() {
zr.Close() // This sould not be necessary.
zlibReaderPool.Put(zr)
}()
2024-01-06 01:40:03 +01:00
buf := posBuf{Data: data[offset:]}
if err = zr.Reset(&buf, nil); err != nil {
return
}
2024-01-06 01:40:03 +01:00
mapContent := make([]byte, uncompressedLen)
2024-01-06 01:40:03 +01:00
var k int
k, err = io.ReadFull(zr, mapContent)
if err != nil {
return
}
2024-01-06 01:40:03 +01:00
if k != uncompressedLen {
err = ErrMapContentSizeMismatch
return
}
2024-01-05 14:41:06 +01:00
2024-01-06 01:40:03 +01:00
// There is a bug before Go 1.7 that enforces
// to add 4 as an offset after the compressed
// geometry data. As we now require at least 1.20
// this is no longer an issue.
const afterCompressOfs = 0
2024-01-06 01:40:03 +01:00
offset += buf.Pos + afterCompressOfs
buf.Pos = 0
if offset >= dataLen {
return nil, ErrBlockTruncated
}
buf.Data = data[offset:]
2024-01-05 14:41:06 +01:00
2024-01-06 01:40:03 +01:00
if err = zr.(zlib.Resetter).Reset(&buf, nil); err != nil {
return
}
// Discard the meta data.
if _, err = io.Copy(io.Discard, zr); err != nil {
return
}
2024-01-05 14:41:06 +01:00
2024-01-06 01:40:03 +01:00
offset += buf.Pos
2024-01-05 14:41:06 +01:00
2024-01-06 01:40:03 +01:00
switch {
case version <= 21:
offset += 2
case version == 23:
offset++
case version == 24:
if offset >= dataLen {
return nil, ErrBlockTruncated
}
ver := data[offset]
offset++
if ver == 1 {
if offset+1 >= dataLen {
return nil, ErrBlockTruncated
}
2024-01-06 01:40:03 +01:00
num := int(binary.BigEndian.Uint16(data[offset:]))
offset += 2 + 10*num
}
}
offset++
if offset+1 >= dataLen {
return nil, ErrBlockTruncated
}
numStaticObjects := int(binary.BigEndian.Uint16(data[offset:]))
offset += 2
for i := 0; i < numStaticObjects; i++ {
offset += 13
if offset+1 >= dataLen {
return nil, ErrBlockTruncated
}
dataSize := int(binary.BigEndian.Uint16(data[offset:]))
offset += dataSize + 2
}
offset += 4
airID, ignoreID := int32(-1), int32(-1)
indexMap := make(map[int32]int32)
var transparent bool
if version >= 22 {
offset++
if offset+1 >= dataLen {
return nil, ErrBlockTruncated
}
numMappings := int(binary.BigEndian.Uint16(data[offset:]))
offset += 2
// Be a bit more tolerant with truncated node name table.
// We should probably issue an error here, too!?
const outOfBounds = "Offset in node id table out of bounds. Ignored."
for i := 0; i < numMappings; i++ {
if offset+1 >= dataLen {
log.Println(outOfBounds)
break
}
nodeID := int32(binary.BigEndian.Uint16(data[offset:]))
offset += 2
if offset+1 >= dataLen {
log.Println(outOfBounds)
break
}
nameLen := int(binary.BigEndian.Uint16(data[offset:]))
offset += 2
if offset+nameLen-1 >= dataLen {
log.Println(outOfBounds)
break
}
name := string(data[offset : offset+nameLen])
offset += nameLen
switch name {
case "air":
airID = nodeID
case "ignore":
ignoreID = nodeID
default:
if index, found := colors.NameIndex[name]; found {
indexMap[nodeID] = index
if !transparent && colors.IsTransparent(index) {
transparent = true
}
} else {
logMissing(name)
}
}
}
}
db = &DecodedBlock{
Version: version,
Transparent: transparent,
MapContent: mapContent,
AirID: airID,
IgnoreID: ignoreID,
IndexMap: indexMap}
return
}
var missingColors = struct {
sync.Mutex
cols map[string]struct{}
}{cols: map[string]struct{}{}}
func logMissing(name string) {
missingColors.Lock()
defer missingColors.Unlock()
if _, found := missingColors.cols[name]; !found {
missingColors.cols[name] = struct{}{}
log.Printf("Missing color entry for %s.\n", name)
}
}
func (db *DecodedBlock) AirOnly() bool {
return db.AirID != -1 && len(db.IndexMap) == 0
}
func (db *DecodedBlock) Content(x, y, z int) (content int32, found bool) {
pos := z<<8 + y<<4 + x
switch {
case db.Version >= 24:
pos <<= 1
content = int32(db.MapContent[pos])<<8 | int32(db.MapContent[pos+1])
case db.Version >= 20:
if c := db.MapContent[pos]; c <= 0x80 {
content = int32(c)
} else {
content = int32(c)<<4 | int32(db.MapContent[pos+0x2000])>>4
}
default:
return
}
if content != db.AirID && content != db.IgnoreID {
content, found = db.IndexMap[content]
}
return
}
func (pb *posBuf) Read(p []byte) (int, error) {
pl := len(p)
ml := len(pb.Data)
if pb.Pos >= ml {
return 0, io.EOF
}
rest := ml - pb.Pos
if pl > rest {
copy(p, pb.Data[pb.Pos:])
pb.Pos = ml
return rest, io.EOF
}
copy(p, pb.Data[pb.Pos:pb.Pos+pl])
pb.Pos += pl
return pl, nil
}