Completed implementation of interleaver. Needs testing!

common/coords.go

@@ -143,3 +143,31 @@ func TransformPlainToInterleaved(pos int64) int64 {
 func TransformInterleavedPlain(pos int64) int64 {
 	return CoordToPlain(InterleavedToCoord(pos))
 }
+
+func SelectKeySplitter(interleaved bool) KeySplitter {
+	if interleaved {
+		return InterleavedToCoord
+	}
+	return PlainToCoord
+}
+
+func SelectKeyJoiner(interleaved bool) KeyJoiner {
+	if interleaved {
+		return CoordToInterleaved
+	}
+	return CoordToPlain
+}
+
+func SelectKeyDecoder(interleaved bool) KeyDecoder {
+	if interleaved {
+		return DecodeFromBigEndian
+	}
+	return DecodeStringFromBytes
+}
+
+func SelectKeyEncoder(interleaved bool) KeyEncoder {
+	if interleaved {
+		return EncodeToBigEndian
+	}
+	return EncodeStringToBytes
+}

interleaver/interfaces.go

@@ -10,22 +10,29 @@ import (
 	"bitbucket.org/s_l_teichmann/mtredisalize/common"
 )
 
-var NoMoreBlocksErr = errors.New("No more blocks.")
+// Error returned if a Producer has run to its end.
+var ErrNoMoreBlocks = errors.New("No more blocks.")
 
 type (
+	// Block data from Minetest database.
 	Block struct {
 		Coord common.Coord
 		Data  []byte
 	}
 
-	BlocKProducer interface {
-		// error is NoMoreBlocksErr if it run out of blocks.
+	// BlockProducer is used to over a existing Minetest database
+	// and return its content block by block.
+	BlockProducer interface {
+		// error is ErrNoMoreBlocks if it run out of blocks.
 		Next(*Block) error
+		// Closes the open database connections.
 		Close() error
 	}
 
+	// BlockConsumer is used to store blocks in a new Minetest database.
 	BlockConsumer interface {
 		Consume(*Block) error
+		// Closes the open database connections.
 		Close() error
 	}
 )

interleaver/leveldb.go

@@ -31,7 +31,7 @@ type (
 	}
 )
 
-func NewLevelDBlockProducer(path string,
+func NewLevelDBBlockProducer(path string,
 	splitter common.KeySplitter,
 	decoder common.KeyDecoder) (ldbp *LevelDBBlockProducer, err error) {
 
@@ -77,12 +77,12 @@ func (ldbp *LevelDBBlockProducer) Close() error {
 
 func (ldbp *LevelDBBlockProducer) Next(block *Block) (err error) {
 	if ldbp.iterator == nil {
-		err = NoMoreBlocksErr
+		err = ErrNoMoreBlocks
 		return
 	}
 	if !ldbp.iterator.Valid() {
 		if err = ldbp.iterator.GetError(); err == nil {
-			err = NoMoreBlocksErr
+			err = ErrNoMoreBlocks
 		}
 		ldbp.iterator.Close()
 		ldbp.iterator = nil

interleaver/main.go

@@ -4,6 +4,117 @@
 
 package main
 
-func main() {
-	// TODO: Implement me!
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	"bitbucket.org/s_l_teichmann/mtredisalize/common"
+)
+
+func usage() {
+	fmt.Fprintf(os.Stderr,
+		"Usage: %s [<options>] <source database> <dest database>\n", os.Args[0])
+	fmt.Fprintln(os.Stderr, "Options:")
+	flag.PrintDefaults()
+}
+
+func copyProducerToConsumer(producer BlockProducer, consumer BlockConsumer) (err error) {
+	var block Block
+	i := 0
+	for {
+		if err = producer.Next(&block); err != nil {
+			if err == ErrNoMoreBlocks {
+				err = nil
+				break
+			}
+			return
+		}
+		if err = consumer.Consume(&block); err != nil {
+			return
+		}
+		i++
+		if i%1000 == 0 {
+			log.Printf("%i blocks transferred.\n", i)
+		}
+	}
+	log.Printf("%i blocks transferred in total.\n", i)
+
+	return
+}
+
+func main() {
+	var (
+		srcBackend     string
+		dstBackend     string
+		srcInterleaved bool
+		dstInterleaved bool
+	)
+
+	flag.Usage = usage
+
+	flag.StringVar(&srcBackend, "source-backend", "sqlite",
+		"type of source database (leveldb, sqlite)")
+	flag.StringVar(&srcBackend, "sb", "sqlite",
+		"type of source database (leveldb, sqlite). Shorthand")
+	flag.StringVar(&dstBackend, "dest-backend", "leveldb",
+		"type of destination database (leveldb, sqlite)")
+	flag.StringVar(&dstBackend, "db", "leveldb",
+		"type of destination database (leveldb, sqlite). Shorthand")
+	flag.BoolVar(&srcInterleaved, "source-interleaved", false,
+		"Is source database interleaved?")
+	flag.BoolVar(&srcInterleaved, "si", false,
+		"Is source database interleaved? Shorthand")
+	flag.BoolVar(&dstInterleaved, "dest-interleaved", true,
+		"Should dest database be interleaved?")
+	flag.BoolVar(&dstInterleaved, "di", true,
+		"Should source database be interleaved? Shorthand")
+
+	flag.Parse()
+
+	if flag.NArg() < 2 {
+		log.Fatal("Missing source and/or destination database.")
+	}
+
+	var (
+		producer BlockProducer
+		consumer BlockConsumer
+		err      error
+	)
+
+	if srcBackend == "sqlite" {
+		if producer, err = NewSQLiteBlockProducer(
+			flag.Arg(0),
+			common.SelectKeySplitter(srcInterleaved)); err != nil {
+			log.Fatalf("Cannot open '%s': %s", flag.Arg(0), err)
+		}
+	} else { // LevelDB
+		if producer, err = NewLevelDBBlockProducer(
+			flag.Arg(0),
+			common.SelectKeySplitter(srcInterleaved),
+			common.SelectKeyDecoder(srcInterleaved)); err != nil {
+			log.Fatalf("Cannot open '%s': %s", flag.Arg(0), err)
+		}
+	}
+	defer producer.Close()
+
+	if dstBackend == "sqlite" {
+		if consumer, err = NewSQLiteBlockConsumer(
+			flag.Arg(1), common.SelectKeyJoiner(dstInterleaved)); err != nil {
+			log.Fatalf("Cannot open '%s': %s", flag.Arg(1), err)
+		}
+	} else { // LevelDB
+		if consumer, err = NewLevelDBBlockConsumer(
+			flag.Arg(1),
+			common.SelectKeyJoiner(dstInterleaved),
+			common.SelectKeyEncoder(dstInterleaved)); err != nil {
+			log.Fatalf("Cannot open '%s': %s", flag.Arg(1), err)
+		}
+	}
+	defer consumer.Close()
+
+	if err = copyProducerToConsumer(producer, consumer); err != nil {
+		log.Fatalf("Database transfer failed: %s\n", err)
+	}
 }

interleaver/sqlite.go

@@ -21,7 +21,7 @@ const (
 	selectSql = "SELECT pos, data FROM blocks"
 )
 
-var DatabaseNotExistsErr = errors.New("Database does not exists.")
+var ErrDatabaseNotExists = errors.New("Database does not exists.")
 
 type (
 	SQLiteBlockProducer struct {
@@ -89,7 +89,7 @@ func NewSQLiteBlockProducer(
 	splitter common.KeySplitter) (sbp *SQLiteBlockProducer, err error) {
 
 	if !fileExists(path) {
-		err = DatabaseNotExistsErr
+		err = ErrDatabaseNotExists
 		return
 	}
 
@@ -114,7 +114,7 @@ func NewSQLiteBlockProducer(
 
 func (sbp *SQLiteBlockProducer) Next(block *Block) (err error) {
 	if sbp.rows == nil {
-		err = NoMoreBlocksErr
+		err = ErrNoMoreBlocks
 		return
 	}
 	if sbp.rows.Next() {
@@ -125,7 +125,7 @@ func (sbp *SQLiteBlockProducer) Next(block *Block) (err error) {
 	} else {
 		sbp.rows.Close()
 		sbp.rows = nil
-		err = NoMoreBlocksErr
+		err = ErrNoMoreBlocks
 	}
 	return
 }