Merged.

COMPILE.md

@@ -12,10 +12,10 @@ A quick and dirty way to produce the binaries of `mtdbconverter`,
 
     # Assuming you have a 64bit GNU/Linux system. For other systems take
     # the corresponding version from https://golang.org/dl/
-    $ wget https://storage.googleapis.com/golang/go1.4.1.linux-amd64.tar.gz
+    $ wget https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz
 
-    $ sha1sum go1.4.1.linux-amd64.tar.gz
-    3e871200e13c0b059b14866d428910de0a4c51ed go1.4.1.linux-amd64.tar.gz
+    $ sha1sum go1.4.2.linux-amd64.tar.gz
+    5020af94b52b65cc9b6f11d50a67e4bae07b0aff go1.4.2.linux-amd64.tar.gz
 
     $ tar xf go1.3.3.linux-amd64.tar.gz
 

cmd/mtredisalize/changetracker.go

@@ -55,7 +55,7 @@ func (ct *ChangeTracker) FlushChanges(url string) {
 	go func() {
 		changes := make([]quantizedXZ, len(oldChanges))
 		i := 0
-		for change, _ := range oldChanges {
+		for change := range oldChanges {
 			changes[i] = change
 			i++
 		}

cmd/mtredisalize/connection.go

@@ -22,16 +22,18 @@ var (
 )
 
 type Connection struct {
-	conn      net.Conn
-	session   Session
-	boolArray []bool
+	conn              net.Conn
+	session           Session
+	maxBulkStringSize int64
+	boolArray         []bool
 }
 
-func NewConnection(conn net.Conn, session Session) *Connection {
+func NewConnection(conn net.Conn, session Session, maxBulkStringSize int64) *Connection {
 	return &Connection{
-		conn:      conn,
-		session:   session,
-		boolArray: []bool{}}
+		conn:              conn,
+		session:           session,
+		maxBulkStringSize: maxBulkStringSize,
+		boolArray:         []bool{}}
 }
 
 func (c *Connection) Run() {
@@ -41,7 +43,7 @@ func (c *Connection) Run() {
 	}()
 	rce := NewRedisCommandExecutor(c)
 	r := bufio.NewReaderSize(c.conn, 8*1024)
-	parser := NewRedisParser(r, rce)
+	parser := NewRedisParser(r, rce, c.maxBulkStringSize)
 	parser.Parse()
 	log.Println("client disconnected")
 }

cmd/mtredisalize/main.go

@@ -16,9 +16,10 @@ import (
 )
 
 const (
-	Version        = "0.3"
-	GCDuration     = "24h"
-	ChangeDuration = "30s"
+	defaultMaxBulkStringSize = 32 * 1024 * 1024
+	Version                  = "0.3"
+	GCDuration               = "24h"
+	ChangeDuration           = "30s"
 )
 
 func usage() {
@@ -31,15 +32,16 @@ func usage() {
 func main() {
 
 	var (
-		port           int
-		host           string
-		driver         string
-		cacheSize      int
-		version        bool
-		interleaved    bool
-		changeUrl      string
-		gcDuration     string
-		changeDuration string
+		port              int
+		host              string
+		driver            string
+		cacheSize         int
+		version           bool
+		interleaved       bool
+		changeUrl         string
+		gcDuration        string
+		changeDuration    string
+		maxBulkStringSize int64
 	)
 
 	flag.Usage = usage
@@ -56,6 +58,8 @@ func main() {
 	flag.StringVar(&changeDuration,
 		"change-duration", ChangeDuration, "Duration to aggregate changes.")
 	flag.StringVar(&changeUrl, "change-url", "", "URL to send changes to.")
+	flag.Int64Var(&maxBulkStringSize, "max-bulk-string-size", defaultMaxBulkStringSize,
+		"max size of a bulk string to be accepted as input (in bytes).")
 	flag.Parse()
 
 	if version {
@@ -147,7 +151,7 @@ func main() {
 				log.Printf("Cannot create session: %s", err)
 				conn.Close()
 			} else {
-				go NewConnection(conn, session).Run()
+				go NewConnection(conn, session, maxBulkStringSize).Run()
 			}
 		case <-sigChan:
 			log.Println("Shutting down")

cmd/mtredisalize/parser.go

@@ -14,8 +14,6 @@ import (
 	"strings"
 )
 
-const maxBulkStringSize = 8 * 1024 * 1024
-
 type RedisConsumer interface {
 	ConsumeInteger(int64) bool
 	ConsumeArray(int64) bool
@@ -25,14 +23,17 @@ type RedisConsumer interface {
 }
 
 type RedisParser struct {
-	reader   *bufio.Reader
-	consumer RedisConsumer
+	reader            *bufio.Reader
+	consumer          RedisConsumer
+	maxBulkStringSize int64
 }
 
-func NewRedisParser(reader *bufio.Reader, consumer RedisConsumer) *RedisParser {
+func NewRedisParser(reader *bufio.Reader, consumer RedisConsumer,
+	maxBulkStringSize int64) *RedisParser {
 	return &RedisParser{
-		reader:   reader,
-		consumer: consumer}
+		reader:            reader,
+		consumer:          consumer,
+		maxBulkStringSize: maxBulkStringSize}
 }
 
 func (rp *RedisParser) Parse() {
@@ -95,7 +96,7 @@ func (rp *RedisParser) bulkString(line []byte) bool {
 	case i == 0:
 		return rp.consumer.ConsumeBulkString([]byte{})
 	default:
-		if i > maxBulkStringSize { // prevent denial of service.
+		if i > rp.maxBulkStringSize { // prevent denial of service.
 			return rp.consumer.ConsumeError(
 				fmt.Errorf("Bulk string too large (%d bytes).\n", i))
 		}

cmd/mtseeder/baselevel.go

@@ -15,10 +15,7 @@ import (
 )
 
 const (
-	width          = 18
-	height         = 18
-	baseLevelDir   = "8"
-	yOrderCapacity = 512
+	baseLevelDir = "8"
 )
 
 type blockPos struct {

common/coords.go

@@ -145,10 +145,6 @@ func CoordToInterleaved(c Coord) (result int64) {
 	return
 }
 
-func invert16(x int16) int16 {
-	return -x - 1
-}
-
 func InterleavedToCoord(pos int64) Coord {
 	const end = 1 << (numBitsPerComponent + 1)
 	var x, y, z int16

common/coords_test.go

@@ -5,7 +5,6 @@
 package common
 
 import (
-	"math"
 	"math/rand"
 	"testing"
 )
@@ -181,15 +180,6 @@ func outsiders(zmin, zmax int64, fn func(int64)) {
 	}
 }
 
-func TestInvert16(t *testing.T) {
-	if invert16(math.MaxInt16) != math.MinInt16 {
-		t.Errorf("invert16(max) != min\n")
-	}
-	if invert16(math.MinInt16) != math.MaxInt16 {
-		t.Errorf("invert16(max) != min\n")
-	}
-}
-
 func TestBigMin(t *testing.T) {
 	const tries = 20
 	for i := 0; i < tries; i++ {