From b3937128015e97f5125a311fb16a46b22b1438b5 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Thu, 21 Aug 2014 14:46:34 +0200 Subject: [PATCH] Completed implementation of interleaver. Needs testing! --- common/coords.go | 28 ++++++++++ interleaver/interfaces.go | 13 ++++- interleaver/leveldb.go | 6 +- interleaver/main.go | 115 +++++++++++++++++++++++++++++++++++++- interleaver/sqlite.go | 8 +-- 5 files changed, 158 insertions(+), 12 deletions(-) diff --git a/common/coords.go b/common/coords.go index bc1d913..ed162aa 100644 --- a/common/coords.go +++ b/common/coords.go @@ -143,3 +143,31 @@ func TransformPlainToInterleaved(pos int64) int64 { func TransformInterleavedPlain(pos int64) int64 { return CoordToPlain(InterleavedToCoord(pos)) } + +func SelectKeySplitter(interleaved bool) KeySplitter { + if interleaved { + return InterleavedToCoord + } + return PlainToCoord +} + +func SelectKeyJoiner(interleaved bool) KeyJoiner { + if interleaved { + return CoordToInterleaved + } + return CoordToPlain +} + +func SelectKeyDecoder(interleaved bool) KeyDecoder { + if interleaved { + return DecodeFromBigEndian + } + return DecodeStringFromBytes +} + +func SelectKeyEncoder(interleaved bool) KeyEncoder { + if interleaved { + return EncodeToBigEndian + } + return EncodeStringToBytes +} diff --git a/interleaver/interfaces.go b/interleaver/interfaces.go index bc4c404..444122c 100644 --- a/interleaver/interfaces.go +++ b/interleaver/interfaces.go @@ -10,22 +10,29 @@ import ( "bitbucket.org/s_l_teichmann/mtredisalize/common" ) -var NoMoreBlocksErr = errors.New("No more blocks.") +// Error returned if a Producer has run to its end. +var ErrNoMoreBlocks = errors.New("No more blocks.") type ( + // Block data from Minetest database. Block struct { Coord common.Coord Data []byte } - BlocKProducer interface { - // error is NoMoreBlocksErr if it run out of blocks. + // BlockProducer is used to over a existing Minetest database + // and return its content block by block. + BlockProducer interface { + // error is ErrNoMoreBlocks if it run out of blocks. Next(*Block) error + // Closes the open database connections. Close() error } + // BlockConsumer is used to store blocks in a new Minetest database. BlockConsumer interface { Consume(*Block) error + // Closes the open database connections. Close() error } ) diff --git a/interleaver/leveldb.go b/interleaver/leveldb.go index 6a854cf..36021e3 100644 --- a/interleaver/leveldb.go +++ b/interleaver/leveldb.go @@ -31,7 +31,7 @@ type ( } ) -func NewLevelDBlockProducer(path string, +func NewLevelDBBlockProducer(path string, splitter common.KeySplitter, decoder common.KeyDecoder) (ldbp *LevelDBBlockProducer, err error) { @@ -77,12 +77,12 @@ func (ldbp *LevelDBBlockProducer) Close() error { func (ldbp *LevelDBBlockProducer) Next(block *Block) (err error) { if ldbp.iterator == nil { - err = NoMoreBlocksErr + err = ErrNoMoreBlocks return } if !ldbp.iterator.Valid() { if err = ldbp.iterator.GetError(); err == nil { - err = NoMoreBlocksErr + err = ErrNoMoreBlocks } ldbp.iterator.Close() ldbp.iterator = nil diff --git a/interleaver/main.go b/interleaver/main.go index 61f84b9..1971e7d 100644 --- a/interleaver/main.go +++ b/interleaver/main.go @@ -4,6 +4,117 @@ package main -func main() { - // TODO: Implement me! +import ( + "flag" + "fmt" + "log" + "os" + + "bitbucket.org/s_l_teichmann/mtredisalize/common" +) + +func usage() { + fmt.Fprintf(os.Stderr, + "Usage: %s [] \n", os.Args[0]) + fmt.Fprintln(os.Stderr, "Options:") + flag.PrintDefaults() +} + +func copyProducerToConsumer(producer BlockProducer, consumer BlockConsumer) (err error) { + var block Block + i := 0 + for { + if err = producer.Next(&block); err != nil { + if err == ErrNoMoreBlocks { + err = nil + break + } + return + } + if err = consumer.Consume(&block); err != nil { + return + } + i++ + if i%1000 == 0 { + log.Printf("%i blocks transferred.\n", i) + } + } + log.Printf("%i blocks transferred in total.\n", i) + + return +} + +func main() { + var ( + srcBackend string + dstBackend string + srcInterleaved bool + dstInterleaved bool + ) + + flag.Usage = usage + + flag.StringVar(&srcBackend, "source-backend", "sqlite", + "type of source database (leveldb, sqlite)") + flag.StringVar(&srcBackend, "sb", "sqlite", + "type of source database (leveldb, sqlite). Shorthand") + flag.StringVar(&dstBackend, "dest-backend", "leveldb", + "type of destination database (leveldb, sqlite)") + flag.StringVar(&dstBackend, "db", "leveldb", + "type of destination database (leveldb, sqlite). Shorthand") + flag.BoolVar(&srcInterleaved, "source-interleaved", false, + "Is source database interleaved?") + flag.BoolVar(&srcInterleaved, "si", false, + "Is source database interleaved? Shorthand") + flag.BoolVar(&dstInterleaved, "dest-interleaved", true, + "Should dest database be interleaved?") + flag.BoolVar(&dstInterleaved, "di", true, + "Should source database be interleaved? Shorthand") + + flag.Parse() + + if flag.NArg() < 2 { + log.Fatal("Missing source and/or destination database.") + } + + var ( + producer BlockProducer + consumer BlockConsumer + err error + ) + + if srcBackend == "sqlite" { + if producer, err = NewSQLiteBlockProducer( + flag.Arg(0), + common.SelectKeySplitter(srcInterleaved)); err != nil { + log.Fatalf("Cannot open '%s': %s", flag.Arg(0), err) + } + } else { // LevelDB + if producer, err = NewLevelDBBlockProducer( + flag.Arg(0), + common.SelectKeySplitter(srcInterleaved), + common.SelectKeyDecoder(srcInterleaved)); err != nil { + log.Fatalf("Cannot open '%s': %s", flag.Arg(0), err) + } + } + defer producer.Close() + + if dstBackend == "sqlite" { + if consumer, err = NewSQLiteBlockConsumer( + flag.Arg(1), common.SelectKeyJoiner(dstInterleaved)); err != nil { + log.Fatalf("Cannot open '%s': %s", flag.Arg(1), err) + } + } else { // LevelDB + if consumer, err = NewLevelDBBlockConsumer( + flag.Arg(1), + common.SelectKeyJoiner(dstInterleaved), + common.SelectKeyEncoder(dstInterleaved)); err != nil { + log.Fatalf("Cannot open '%s': %s", flag.Arg(1), err) + } + } + defer consumer.Close() + + if err = copyProducerToConsumer(producer, consumer); err != nil { + log.Fatalf("Database transfer failed: %s\n", err) + } } diff --git a/interleaver/sqlite.go b/interleaver/sqlite.go index a202455..52f1813 100644 --- a/interleaver/sqlite.go +++ b/interleaver/sqlite.go @@ -21,7 +21,7 @@ const ( selectSql = "SELECT pos, data FROM blocks" ) -var DatabaseNotExistsErr = errors.New("Database does not exists.") +var ErrDatabaseNotExists = errors.New("Database does not exists.") type ( SQLiteBlockProducer struct { @@ -89,7 +89,7 @@ func NewSQLiteBlockProducer( splitter common.KeySplitter) (sbp *SQLiteBlockProducer, err error) { if !fileExists(path) { - err = DatabaseNotExistsErr + err = ErrDatabaseNotExists return } @@ -114,7 +114,7 @@ func NewSQLiteBlockProducer( func (sbp *SQLiteBlockProducer) Next(block *Block) (err error) { if sbp.rows == nil { - err = NoMoreBlocksErr + err = ErrNoMoreBlocks return } if sbp.rows.Next() { @@ -125,7 +125,7 @@ func (sbp *SQLiteBlockProducer) Next(block *Block) (err error) { } else { sbp.rows.Close() sbp.rows = nil - err = NoMoreBlocksErr + err = ErrNoMoreBlocks } return }