This commit is contained in:
Sascha L. Teichmann 2015-05-26 18:14:48 +02:00
commit af469a3173
8 changed files with 41 additions and 51 deletions

View File

@ -12,10 +12,10 @@ A quick and dirty way to produce the binaries of `mtdbconverter`,
# Assuming you have a 64bit GNU/Linux system. For other systems take # Assuming you have a 64bit GNU/Linux system. For other systems take
# the corresponding version from https://golang.org/dl/ # the corresponding version from https://golang.org/dl/
$ wget https://storage.googleapis.com/golang/go1.4.1.linux-amd64.tar.gz $ wget https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz
$ sha1sum go1.4.1.linux-amd64.tar.gz $ sha1sum go1.4.2.linux-amd64.tar.gz
3e871200e13c0b059b14866d428910de0a4c51ed go1.4.1.linux-amd64.tar.gz 5020af94b52b65cc9b6f11d50a67e4bae07b0aff go1.4.2.linux-amd64.tar.gz
$ tar xf go1.3.3.linux-amd64.tar.gz $ tar xf go1.3.3.linux-amd64.tar.gz

View File

@ -55,7 +55,7 @@ func (ct *ChangeTracker) FlushChanges(url string) {
go func() { go func() {
changes := make([]quantizedXZ, len(oldChanges)) changes := make([]quantizedXZ, len(oldChanges))
i := 0 i := 0
for change, _ := range oldChanges { for change := range oldChanges {
changes[i] = change changes[i] = change
i++ i++
} }

View File

@ -22,16 +22,18 @@ var (
) )
type Connection struct { type Connection struct {
conn net.Conn conn net.Conn
session Session session Session
boolArray []bool maxBulkStringSize int64
boolArray []bool
} }
func NewConnection(conn net.Conn, session Session) *Connection { func NewConnection(conn net.Conn, session Session, maxBulkStringSize int64) *Connection {
return &Connection{ return &Connection{
conn: conn, conn: conn,
session: session, session: session,
boolArray: []bool{}} maxBulkStringSize: maxBulkStringSize,
boolArray: []bool{}}
} }
func (c *Connection) Run() { func (c *Connection) Run() {
@ -41,7 +43,7 @@ func (c *Connection) Run() {
}() }()
rce := NewRedisCommandExecutor(c) rce := NewRedisCommandExecutor(c)
r := bufio.NewReaderSize(c.conn, 8*1024) r := bufio.NewReaderSize(c.conn, 8*1024)
parser := NewRedisParser(r, rce) parser := NewRedisParser(r, rce, c.maxBulkStringSize)
parser.Parse() parser.Parse()
log.Println("client disconnected") log.Println("client disconnected")
} }

View File

@ -16,9 +16,10 @@ import (
) )
const ( const (
Version = "0.3" defaultMaxBulkStringSize = 32 * 1024 * 1024
GCDuration = "24h" Version = "0.3"
ChangeDuration = "30s" GCDuration = "24h"
ChangeDuration = "30s"
) )
func usage() { func usage() {
@ -31,15 +32,16 @@ func usage() {
func main() { func main() {
var ( var (
port int port int
host string host string
driver string driver string
cacheSize int cacheSize int
version bool version bool
interleaved bool interleaved bool
changeUrl string changeUrl string
gcDuration string gcDuration string
changeDuration string changeDuration string
maxBulkStringSize int64
) )
flag.Usage = usage flag.Usage = usage
@ -56,6 +58,8 @@ func main() {
flag.StringVar(&changeDuration, flag.StringVar(&changeDuration,
"change-duration", ChangeDuration, "Duration to aggregate changes.") "change-duration", ChangeDuration, "Duration to aggregate changes.")
flag.StringVar(&changeUrl, "change-url", "", "URL to send changes to.") flag.StringVar(&changeUrl, "change-url", "", "URL to send changes to.")
flag.Int64Var(&maxBulkStringSize, "max-bulk-string-size", defaultMaxBulkStringSize,
"max size of a bulk string to be accepted as input (in bytes).")
flag.Parse() flag.Parse()
if version { if version {
@ -147,7 +151,7 @@ func main() {
log.Printf("Cannot create session: %s", err) log.Printf("Cannot create session: %s", err)
conn.Close() conn.Close()
} else { } else {
go NewConnection(conn, session).Run() go NewConnection(conn, session, maxBulkStringSize).Run()
} }
case <-sigChan: case <-sigChan:
log.Println("Shutting down") log.Println("Shutting down")

View File

@ -14,8 +14,6 @@ import (
"strings" "strings"
) )
const maxBulkStringSize = 8 * 1024 * 1024
type RedisConsumer interface { type RedisConsumer interface {
ConsumeInteger(int64) bool ConsumeInteger(int64) bool
ConsumeArray(int64) bool ConsumeArray(int64) bool
@ -25,14 +23,17 @@ type RedisConsumer interface {
} }
type RedisParser struct { type RedisParser struct {
reader *bufio.Reader reader *bufio.Reader
consumer RedisConsumer consumer RedisConsumer
maxBulkStringSize int64
} }
func NewRedisParser(reader *bufio.Reader, consumer RedisConsumer) *RedisParser { func NewRedisParser(reader *bufio.Reader, consumer RedisConsumer,
maxBulkStringSize int64) *RedisParser {
return &RedisParser{ return &RedisParser{
reader: reader, reader: reader,
consumer: consumer} consumer: consumer,
maxBulkStringSize: maxBulkStringSize}
} }
func (rp *RedisParser) Parse() { func (rp *RedisParser) Parse() {
@ -95,7 +96,7 @@ func (rp *RedisParser) bulkString(line []byte) bool {
case i == 0: case i == 0:
return rp.consumer.ConsumeBulkString([]byte{}) return rp.consumer.ConsumeBulkString([]byte{})
default: default:
if i > maxBulkStringSize { // prevent denial of service. if i > rp.maxBulkStringSize { // prevent denial of service.
return rp.consumer.ConsumeError( return rp.consumer.ConsumeError(
fmt.Errorf("Bulk string too large (%d bytes).\n", i)) fmt.Errorf("Bulk string too large (%d bytes).\n", i))
} }

View File

@ -15,10 +15,7 @@ import (
) )
const ( const (
width = 18 baseLevelDir = "8"
height = 18
baseLevelDir = "8"
yOrderCapacity = 512
) )
type blockPos struct { type blockPos struct {

View File

@ -145,10 +145,6 @@ func CoordToInterleaved(c Coord) (result int64) {
return return
} }
func invert16(x int16) int16 {
return -x - 1
}
func InterleavedToCoord(pos int64) Coord { func InterleavedToCoord(pos int64) Coord {
const end = 1 << (numBitsPerComponent + 1) const end = 1 << (numBitsPerComponent + 1)
var x, y, z int16 var x, y, z int16

View File

@ -5,7 +5,6 @@
package common package common
import ( import (
"math"
"math/rand" "math/rand"
"testing" "testing"
) )
@ -181,15 +180,6 @@ func outsiders(zmin, zmax int64, fn func(int64)) {
} }
} }
func TestInvert16(t *testing.T) {
if invert16(math.MaxInt16) != math.MinInt16 {
t.Errorf("invert16(max) != min\n")
}
if invert16(math.MinInt16) != math.MaxInt16 {
t.Errorf("invert16(max) != min\n")
}
}
func TestBigMin(t *testing.T) { func TestBigMin(t *testing.T) {
const tries = 20 const tries = 20
for i := 0; i < tries; i++ { for i := 0; i < tries; i++ {