diff --git a/COMPILE.md b/COMPILE.md index 6514959..13ad1e3 100644 --- a/COMPILE.md +++ b/COMPILE.md @@ -12,10 +12,10 @@ A quick and dirty way to produce the binaries of `mtdbconverter`, # Assuming you have a 64bit GNU/Linux system. For other systems take # the corresponding version from https://golang.org/dl/ - $ wget https://storage.googleapis.com/golang/go1.4.1.linux-amd64.tar.gz + $ wget https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz - $ sha1sum go1.4.1.linux-amd64.tar.gz - 3e871200e13c0b059b14866d428910de0a4c51ed go1.4.1.linux-amd64.tar.gz + $ sha1sum go1.4.2.linux-amd64.tar.gz + 5020af94b52b65cc9b6f11d50a67e4bae07b0aff go1.4.2.linux-amd64.tar.gz $ tar xf go1.3.3.linux-amd64.tar.gz diff --git a/cmd/mtredisalize/changetracker.go b/cmd/mtredisalize/changetracker.go index 413d4d0..78da29e 100644 --- a/cmd/mtredisalize/changetracker.go +++ b/cmd/mtredisalize/changetracker.go @@ -55,7 +55,7 @@ func (ct *ChangeTracker) FlushChanges(url string) { go func() { changes := make([]quantizedXZ, len(oldChanges)) i := 0 - for change, _ := range oldChanges { + for change := range oldChanges { changes[i] = change i++ } diff --git a/cmd/mtredisalize/connection.go b/cmd/mtredisalize/connection.go index 64cea82..252134b 100644 --- a/cmd/mtredisalize/connection.go +++ b/cmd/mtredisalize/connection.go @@ -22,16 +22,18 @@ var ( ) type Connection struct { - conn net.Conn - session Session - boolArray []bool + conn net.Conn + session Session + maxBulkStringSize int64 + boolArray []bool } -func NewConnection(conn net.Conn, session Session) *Connection { +func NewConnection(conn net.Conn, session Session, maxBulkStringSize int64) *Connection { return &Connection{ - conn: conn, - session: session, - boolArray: []bool{}} + conn: conn, + session: session, + maxBulkStringSize: maxBulkStringSize, + boolArray: []bool{}} } func (c *Connection) Run() { @@ -41,7 +43,7 @@ func (c *Connection) Run() { }() rce := NewRedisCommandExecutor(c) r := bufio.NewReaderSize(c.conn, 8*1024) - parser := NewRedisParser(r, rce) + parser := NewRedisParser(r, rce, c.maxBulkStringSize) parser.Parse() log.Println("client disconnected") } diff --git a/cmd/mtredisalize/main.go b/cmd/mtredisalize/main.go index 7a3c2f6..3a612ac 100644 --- a/cmd/mtredisalize/main.go +++ b/cmd/mtredisalize/main.go @@ -16,9 +16,10 @@ import ( ) const ( - Version = "0.3" - GCDuration = "24h" - ChangeDuration = "30s" + defaultMaxBulkStringSize = 32 * 1024 * 1024 + Version = "0.3" + GCDuration = "24h" + ChangeDuration = "30s" ) func usage() { @@ -31,15 +32,16 @@ func usage() { func main() { var ( - port int - host string - driver string - cacheSize int - version bool - interleaved bool - changeUrl string - gcDuration string - changeDuration string + port int + host string + driver string + cacheSize int + version bool + interleaved bool + changeUrl string + gcDuration string + changeDuration string + maxBulkStringSize int64 ) flag.Usage = usage @@ -56,6 +58,8 @@ func main() { flag.StringVar(&changeDuration, "change-duration", ChangeDuration, "Duration to aggregate changes.") flag.StringVar(&changeUrl, "change-url", "", "URL to send changes to.") + flag.Int64Var(&maxBulkStringSize, "max-bulk-string-size", defaultMaxBulkStringSize, + "max size of a bulk string to be accepted as input (in bytes).") flag.Parse() if version { @@ -147,7 +151,7 @@ func main() { log.Printf("Cannot create session: %s", err) conn.Close() } else { - go NewConnection(conn, session).Run() + go NewConnection(conn, session, maxBulkStringSize).Run() } case <-sigChan: log.Println("Shutting down") diff --git a/cmd/mtredisalize/parser.go b/cmd/mtredisalize/parser.go index daaae23..9bce2fd 100644 --- a/cmd/mtredisalize/parser.go +++ b/cmd/mtredisalize/parser.go @@ -14,8 +14,6 @@ import ( "strings" ) -const maxBulkStringSize = 8 * 1024 * 1024 - type RedisConsumer interface { ConsumeInteger(int64) bool ConsumeArray(int64) bool @@ -25,14 +23,17 @@ type RedisConsumer interface { } type RedisParser struct { - reader *bufio.Reader - consumer RedisConsumer + reader *bufio.Reader + consumer RedisConsumer + maxBulkStringSize int64 } -func NewRedisParser(reader *bufio.Reader, consumer RedisConsumer) *RedisParser { +func NewRedisParser(reader *bufio.Reader, consumer RedisConsumer, + maxBulkStringSize int64) *RedisParser { return &RedisParser{ - reader: reader, - consumer: consumer} + reader: reader, + consumer: consumer, + maxBulkStringSize: maxBulkStringSize} } func (rp *RedisParser) Parse() { @@ -95,7 +96,7 @@ func (rp *RedisParser) bulkString(line []byte) bool { case i == 0: return rp.consumer.ConsumeBulkString([]byte{}) default: - if i > maxBulkStringSize { // prevent denial of service. + if i > rp.maxBulkStringSize { // prevent denial of service. return rp.consumer.ConsumeError( fmt.Errorf("Bulk string too large (%d bytes).\n", i)) } diff --git a/cmd/mtseeder/baselevel.go b/cmd/mtseeder/baselevel.go index 8b13cdf..f9503bb 100644 --- a/cmd/mtseeder/baselevel.go +++ b/cmd/mtseeder/baselevel.go @@ -15,10 +15,7 @@ import ( ) const ( - width = 18 - height = 18 - baseLevelDir = "8" - yOrderCapacity = 512 + baseLevelDir = "8" ) type blockPos struct { diff --git a/common/coords.go b/common/coords.go index c2816e9..c4b20b6 100644 --- a/common/coords.go +++ b/common/coords.go @@ -145,10 +145,6 @@ func CoordToInterleaved(c Coord) (result int64) { return } -func invert16(x int16) int16 { - return -x - 1 -} - func InterleavedToCoord(pos int64) Coord { const end = 1 << (numBitsPerComponent + 1) var x, y, z int16 diff --git a/common/coords_test.go b/common/coords_test.go index 011b05f..519ac4a 100644 --- a/common/coords_test.go +++ b/common/coords_test.go @@ -5,7 +5,6 @@ package common import ( - "math" "math/rand" "testing" ) @@ -181,15 +180,6 @@ func outsiders(zmin, zmax int64, fn func(int64)) { } } -func TestInvert16(t *testing.T) { - if invert16(math.MaxInt16) != math.MinInt16 { - t.Errorf("invert16(max) != min\n") - } - if invert16(math.MinInt16) != math.MaxInt16 { - t.Errorf("invert16(max) != min\n") - } -} - func TestBigMin(t *testing.T) { const tries = 20 for i := 0; i < tries; i++ {