// Copyright 2014, 2015 by Sascha L. Teichmann // Use of this source code is governed by the MIT license // that can be found in the LICENSE file. package common import ( "bytes" "compress/zlib" "encoding/binary" "errors" "io" "log" "sync" "github.com/klauspost/compress/zstd" ) // Error returned if a Producer has run to its end. var ( ErrNoMoreBlocks = errors.New("no more blocks") ErrMapContentSizeMismatch = errors.New("content size does not match") ErrBlockTruncated = errors.New("block is truncated") ) const ( mapBlockSize = 16 nodeCount = mapBlockSize * mapBlockSize * mapBlockSize ) type ( // Block data from Minetest database. Block struct { Coord Coord Data []byte } // BlockProducer is used to over a existing Minetest database // and return its content block by block. BlockProducer interface { // error is ErrNoMoreBlocks if it run out of blocks. Next(*Block) error // Closes the open database connections. Close() error } // BlockConsumer is used to store blocks in a new Minetest database. BlockConsumer interface { Consume(*Block) error // Closes the open database connections. Close() error } DecodedBlock struct { Version byte Transparent bool MapContent []byte AirID int32 IgnoreID int32 IndexMap map[int32]int32 } ) // zlibEmpty is a minimal zlib stream with zero length. // zlib.NewReader needs a valid zlib stream to start with // even if Reset is called directly afterwards. var zlibEmpty = []byte{ 0x78, 0x9c, 0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01} // zlibReaderPool is a pool of zlib Readers to be reused // for decoding the compressed parts of database blocks. // Decoding blocks relies heavly on zlib decompression. // Reusing the internal structures of already allocated // zlib readers speeds up the decoding significantly. var zlibReaderPool = sync.Pool{ New: func() interface{} { reader, _ := zlib.NewReader(bytes.NewBuffer(zlibEmpty)) return reader }, } // The content of the map and the meta data are compressed with zlib. // Unfortunately the byte length of this two structures are not stored // explicitly in the block data. To access the informations behind // them (e.g. the node id mappings) we have to count the bytes consumed // by the zlib reader and continue our extraction process behind this // offset. posBuf implements such a counting reader source. type posBuf struct { Data []byte Pos int } func decode29(data []byte, colors *Colors) (*DecodedBlock, error) { dec, err := zstd.NewReader(bytes.NewReader(data[1:])) if err != nil { return nil, err } defer dec.Close() be := bigEndianReader{parent: dec} be.u8() // flags be.u16() // lightning_complete be.u32() // timestamp be.u8() // name_id_mapping_version airID, ignoreID := int32(-1), int32(-1) indexMap := make(map[int32]int32) transparent := false numNameIDMappings, _ := be.u16() for i := uint16(0); i < numNameIDMappings; i++ { var ( id, _ = be.u16() nlen, _ = be.u16() name, _ = be.str(int(nlen)) ) switch name { case "air": airID = int32(id) case "ignore": ignoreID = int32(id) default: if index, found := colors.NameIndex[name]; found { indexMap[int32(id)] = index if !transparent && colors.IsTransparent(index) { transparent = true } } else { logMissing(name) } } } be.u8() // content_width be.u8() // params_width mapContent, err := be.full(2 * 4096) if err != nil { return nil, ErrBlockTruncated } return &DecodedBlock{ Version: data[0], Transparent: transparent, MapContent: mapContent, AirID: airID, IgnoreID: ignoreID, IndexMap: indexMap, }, nil } func NewDecodedBlock(data []byte, colors *Colors) (db *DecodedBlock, err error) { dataLen := len(data) if dataLen < 4 { return nil, ErrBlockTruncated } version := data[0] if version >= 29 { return decode29(data, colors) } var offset int contentWidth := Min(int(data[2]), 2) paramsWidth := Min(int(data[3]), 2) uncompressedLen := nodeCount * (contentWidth + paramsWidth) switch { case version >= 27: offset = 6 case version >= 22: offset = 4 default: offset = 2 } zr := zlibReaderPool.Get().(interface { io.ReadCloser zlib.Resetter }) defer func() { zr.Close() // This sould not be necessary. zlibReaderPool.Put(zr) }() buf := posBuf{Data: data[offset:]} if err = zr.Reset(&buf, nil); err != nil { return } mapContent := make([]byte, uncompressedLen) var k int k, err = io.ReadFull(zr, mapContent) if err != nil { return } if k != uncompressedLen { err = ErrMapContentSizeMismatch return } // There is a bug before Go 1.7 that enforces // to add 4 as an offset after the compressed // geometry data. As we now require at least 1.20 // this is no longer an issue. const afterCompressOfs = 0 offset += buf.Pos + afterCompressOfs buf.Pos = 0 if offset >= dataLen { return nil, ErrBlockTruncated } buf.Data = data[offset:] if err = zr.(zlib.Resetter).Reset(&buf, nil); err != nil { return } // Discard the meta data. if _, err = io.Copy(io.Discard, zr); err != nil { return } offset += buf.Pos switch { case version <= 21: offset += 2 case version == 23: offset++ case version == 24: if offset >= dataLen { return nil, ErrBlockTruncated } ver := data[offset] offset++ if ver == 1 { if offset+1 >= dataLen { return nil, ErrBlockTruncated } num := int(binary.BigEndian.Uint16(data[offset:])) offset += 2 + 10*num } } offset++ if offset+1 >= dataLen { return nil, ErrBlockTruncated } numStaticObjects := int(binary.BigEndian.Uint16(data[offset:])) offset += 2 for i := 0; i < numStaticObjects; i++ { offset += 13 if offset+1 >= dataLen { return nil, ErrBlockTruncated } dataSize := int(binary.BigEndian.Uint16(data[offset:])) offset += dataSize + 2 } offset += 4 airID, ignoreID := int32(-1), int32(-1) indexMap := make(map[int32]int32) var transparent bool if version >= 22 { offset++ if offset+1 >= dataLen { return nil, ErrBlockTruncated } numMappings := int(binary.BigEndian.Uint16(data[offset:])) offset += 2 // Be a bit more tolerant with truncated node name table. // We should probably issue an error here, too!? const outOfBounds = "Offset in node id table out of bounds. Ignored." for i := 0; i < numMappings; i++ { if offset+1 >= dataLen { log.Println(outOfBounds) break } nodeID := int32(binary.BigEndian.Uint16(data[offset:])) offset += 2 if offset+1 >= dataLen { log.Println(outOfBounds) break } nameLen := int(binary.BigEndian.Uint16(data[offset:])) offset += 2 if offset+nameLen-1 >= dataLen { log.Println(outOfBounds) break } name := string(data[offset : offset+nameLen]) offset += nameLen switch name { case "air": airID = nodeID case "ignore": ignoreID = nodeID default: if index, found := colors.NameIndex[name]; found { indexMap[nodeID] = index if !transparent && colors.IsTransparent(index) { transparent = true } } else { logMissing(name) } } } } db = &DecodedBlock{ Version: version, Transparent: transparent, MapContent: mapContent, AirID: airID, IgnoreID: ignoreID, IndexMap: indexMap} return } var missingColors = struct { sync.Mutex cols map[string]struct{} }{cols: map[string]struct{}{}} func logMissing(name string) { missingColors.Lock() defer missingColors.Unlock() if _, found := missingColors.cols[name]; !found { missingColors.cols[name] = struct{}{} log.Printf("Missing color entry for %s.\n", name) } } func (db *DecodedBlock) AirOnly() bool { return db.AirID != -1 && len(db.IndexMap) == 0 } func (db *DecodedBlock) Content(x, y, z int) (content int32, found bool) { pos := z<<8 + y<<4 + x switch { case db.Version >= 24: pos <<= 1 content = int32(db.MapContent[pos])<<8 | int32(db.MapContent[pos+1]) case db.Version >= 20: if c := db.MapContent[pos]; c <= 0x80 { content = int32(c) } else { content = int32(c)<<4 | int32(db.MapContent[pos+0x2000])>>4 } default: return } if content != db.AirID && content != db.IgnoreID { content, found = db.IndexMap[content] } return } func (pb *posBuf) Read(p []byte) (int, error) { pl := len(p) ml := len(pb.Data) if pb.Pos >= ml { return 0, io.EOF } rest := ml - pb.Pos if pl > rest { copy(p, pb.Data[pb.Pos:]) pb.Pos = ml return rest, io.EOF } copy(p, pb.Data[pb.Pos:pb.Pos+pl]) pb.Pos += pl return pl, nil } func (pb *posBuf) ReadByte() (byte, error) { if pb.Pos >= len(pb.Data) { return 0, io.EOF } c := pb.Data[pb.Pos] pb.Pos++ return c, nil }