Merge pull request #4374 from benbjohnson/tsm1-quick

Add tsm1 quickcheck tests

CHANGELOG.md

@@ -46,6 +46,7 @@
 - [#4357](https://github.com/influxdb/influxdb/issues/4357): Fix similar float values encoding overflow Thanks @dgryski!
 - [#4344](https://github.com/influxdb/influxdb/issues/4344): Make client.Write default to client.precision if none is given.
 - [#3429](https://github.com/influxdb/influxdb/issues/3429)): Incorrect parsing of regex containing '/'
+- [#4374](https://github.com/influxdb/influxdb/issues/4374)): Add tsm1 quickcheck tests
 
 ## v0.9.4 [2015-09-14]
 

tsdb/engine/tsm1/bool_test.go

@@ -1,7 +1,9 @@
 package tsm1_test
 
 import (
+	"reflect"
 	"testing"
+	"testing/quick"
 
 	"github.com/influxdb/influxdb/tsdb/engine/tsm1"
 )
@@ -71,3 +73,33 @@ func Test_BoolEncoder_Multi_Compressed(t *testing.T) {
 		t.Fatalf("unexpected next value: got true, exp false")
 	}
 }
+
+func Test_BoolEncoder_Quick(t *testing.T) {
+	quick.Check(func(values []bool) bool {
+		// Write values to encoder.
+		enc := tsm1.NewBoolEncoder()
+		for _, v := range values {
+			enc.Write(v)
+		}
+
+		// Retrieve compressed bytes.
+		buf, err := enc.Bytes()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Read values out of decoder.
+		got := make([]bool, 0, len(values))
+		dec := tsm1.NewBoolDecoder(buf)
+		for dec.Next() {
+			got = append(got, dec.Read())
+		}
+
+		// Verify that input and output values match.
+		if !reflect.DeepEqual(values, got) {
+			t.Fatalf("mismatch:\n\nexp=%+v\n\ngot=%+v\n\n", values, got)
+		}
+
+		return true
+	}, nil)
+}

tsdb/engine/tsm1/cursor_test.go

@@ -0,0 +1,175 @@
+package tsm1_test
+
+import (
+	"github.com/influxdb/influxdb/tsdb"
+	"github.com/influxdb/influxdb/tsdb/engine/tsm1"
+	"math/rand"
+	"reflect"
+	"sort"
+	"testing"
+	"testing/quick"
+)
+
+func TestCombinedEngineCursor_Quick(t *testing.T) {
+	const tmin = 0
+	quick.Check(func(wc, ec *Cursor, ascending bool, seek int64) bool {
+		c := tsm1.NewCombinedEngineCursor(wc, ec, ascending)
+		// Read from cursor.
+		got := make([]int64, 0)
+		for k, _ := c.SeekTo(seek); k != tsdb.EOF; k, _ = c.Next() {
+			got = append(got, k)
+		}
+		// Merge cursors items.
+		merged := MergeCursorItems(wc.items, ec.items)
+		if !ascending {
+			sort.Sort(sort.Reverse(CursorItems(merged)))
+		}
+		// Filter out items outside of seek range.
+		exp := make([]int64, 0)
+		for _, item := range merged {
+			if (ascending && item.Key < seek) || (!ascending && item.Key > seek) {
+				continue
+			}
+			exp = append(exp, item.Key)
+		}
+		if !reflect.DeepEqual(got, exp) {
+			t.Fatalf("mismatch: seek=%v, ascending=%v\n\ngot=%#v\n\nexp=%#v\n\n", seek, ascending, got, exp)
+		}
+		return true
+	}, &quick.Config{Values: func(values []reflect.Value, rand *rand.Rand) {
+		ascending := rand.Intn(1) == 1
+		values[0] = reflect.ValueOf(GenerateCursor(tmin, 10, ascending, rand))
+		values[1] = reflect.ValueOf(GenerateCursor(tmin, 10, ascending, rand))
+		values[2] = reflect.ValueOf(ascending)
+		values[3] = reflect.ValueOf(rand.Int63n(100))
+	}})
+}
+
+// Cursor represents a simple test cursor that implements tsdb.Cursor.
+type Cursor struct {
+	i         int
+	items     []CursorItem
+	ascending bool
+}
+
+// NewCursor returns a new instance of Cursor.
+func NewCursor(items []CursorItem, ascending bool) *Cursor {
+	c := &Cursor{
+		items:     items,
+		ascending: ascending,
+	}
+	// Set initial position depending on cursor direction.
+	if ascending {
+		c.i = -1
+	} else {
+		c.i = len(c.items)
+	}
+	return c
+}
+
+// CursorItem represents an item in a test cursor.
+type CursorItem struct {
+	Key   int64
+	Value interface{}
+}
+
+// SeekTo moves the cursor to the first key greater than or equal to seek.
+func (c *Cursor) SeekTo(seek int64) (key int64, value interface{}) {
+	if c.ascending {
+		for i, item := range c.items {
+			if item.Key >= seek {
+				c.i = i
+				return item.Key, item.Value
+			}
+		}
+	} else {
+		for i := len(c.items) - 1; i >= 0; i-- {
+			if item := c.items[i]; item.Key <= seek {
+				c.i = i
+				return item.Key, item.Value
+			}
+		}
+	}
+	c.i = len(c.items)
+	return tsdb.EOF, nil
+}
+
+// Next returns the next key/value from the cursor.
+func (c *Cursor) Next() (key int64, value interface{}) {
+	if c.ascending {
+		c.i++
+		if c.i >= len(c.items) {
+			return tsdb.EOF, nil
+		}
+	} else if !c.ascending {
+		c.i--
+		if c.i < 0 {
+			return tsdb.EOF, nil
+		}
+	}
+	return c.items[c.i].Key, c.items[c.i].Value
+}
+
+// Ascending returns true if the cursor moves in ascending order.
+func (c *Cursor) Ascending() bool { return c.ascending }
+
+// CursorItems represents a list of CursorItem objects.
+type CursorItems []CursorItem
+
+func (a CursorItems) Len() int           { return len(a) }
+func (a CursorItems) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a CursorItems) Less(i, j int) bool { return a[i].Key < a[j].Key }
+
+// Keys returns a list of keys.
+func (a CursorItems) Keys() []int64 {
+	keys := make([]int64, len(a))
+	for i := range a {
+		keys[i] = a[i].Key
+	}
+	return keys
+}
+
+// GenerateCursor generates a cursor with a random data.
+func GenerateCursor(tmin, step int64, ascending bool, rand *rand.Rand) *Cursor {
+	key := tmin + rand.Int63n(10)
+	items := make([]CursorItem, 0)
+	for i, n := 0, rand.Intn(100); i < n; i++ {
+		items = append(items, CursorItem{
+			Key:   key,
+			Value: int64(0),
+		})
+		key += rand.Int63n(10)
+	}
+	return NewCursor(items, ascending)
+}
+
+// MergeCursorItems merges items in a & b together.
+// If two items share a timestamp then a takes precendence.
+func MergeCursorItems(a, b []CursorItem) []CursorItem {
+	items := make([]CursorItem, 0)
+	var ai, bi int
+	for {
+		if ai < len(a) && bi < len(b) {
+			if ak, bk := a[ai].Key, b[bi].Key; ak == bk {
+				items = append(items, a[ai])
+				ai++
+				bi++
+			} else if ak < bk {
+				items = append(items, a[ai])
+				ai++
+			} else {
+				items = append(items, b[bi])
+				bi++
+			}
+		} else if ai < len(a) {
+			items = append(items, a[ai])
+			ai++
+		} else if bi < len(b) {
+			items = append(items, b[bi])
+			bi++
+		} else {
+			break
+		}
+	}
+	return items
+}

tsdb/engine/tsm1/float.go

@@ -154,6 +154,13 @@ func (it *FloatDecoder) Next() bool {
 
 	if it.first {
 		it.first = false
+
+		// mark as finished if there were no values.
+		if math.IsNaN(it.val) {
+			it.finished = true
+			return false
+		}
+
 		return true
 	}
 

tsdb/engine/tsm1/float_test.go

@@ -1,7 +1,9 @@
 package tsm1_test
 
 import (
+	"reflect"
 	"testing"
+	"testing/quick"
 
 	"github.com/influxdb/influxdb/tsdb/engine/tsm1"
 )
@@ -174,6 +176,34 @@ func TestFloatEncoder_Roundtrip(t *testing.T) {
 	}
 }
 
+func Test_FloatEncoder_Quick(t *testing.T) {
+	quick.Check(func(values []float64) bool {
+		// Write values to encoder.
+		enc := tsm1.NewFloatEncoder()
+		for _, v := range values {
+			enc.Push(v)
+		}
+		enc.Finish()
+
+		// Read values out of decoder.
+		got := make([]float64, 0, len(values))
+		dec, err := tsm1.NewFloatDecoder(enc.Bytes())
+		if err != nil {
+			t.Fatal(err)
+		}
+		for dec.Next() {
+			got = append(got, dec.Values())
+		}
+
+		// Verify that input and output values match.
+		if !reflect.DeepEqual(values, got) {
+			t.Fatalf("mismatch:\n\nexp=%+v\n\ngot=%+v\n\n", values, got)
+		}
+
+		return true
+	}, nil)
+}
+
 func BenchmarkFloatEncoder(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		s := tsm1.NewFloatEncoder()

tsdb/engine/tsm1/int.go

@@ -2,7 +2,7 @@ package tsm1
 
 // Int64 encoding uses two different strategies depending on the range of values in
 // the uncompressed data.  Encoded values are first encoding used zig zag encoding.
-// This interleaves postiive and negative integers across a range of positive integers.
+// This interleaves positive and negative integers across a range of positive integers.
 //
 // For example, [-2,-1,0,1] becomes [3,1,0,2]. See
 // https://developers.google.com/protocol-buffers/docs/encoding?hl=en#signed-integers

tsdb/engine/tsm1/int_test.go

@@ -2,7 +2,9 @@ package tsm1_test
 
 import (
 	"math"
+	"reflect"
 	"testing"
+	"testing/quick"
 
 	"github.com/influxdb/influxdb/tsdb/engine/tsm1"
 )
@@ -255,6 +257,39 @@ func Test_Int64Encoder_AllNegative(t *testing.T) {
 
 }
 
+func Test_Int64Encoder_Quick(t *testing.T) {
+	quick.Check(func(values []int64) bool {
+		// Write values to encoder.
+		enc := tsm1.NewInt64Encoder()
+		for _, v := range values {
+			enc.Write(v)
+		}
+
+		// Retrieve encoded bytes from encoder.
+		buf, err := enc.Bytes()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Read values out of decoder.
+		got := make([]int64, 0, len(values))
+		dec := tsm1.NewInt64Decoder(buf)
+		for dec.Next() {
+			if err := dec.Error(); err != nil {
+				t.Fatal(err)
+			}
+			got = append(got, dec.Read())
+		}
+
+		// Verify that input and output values match.
+		if !reflect.DeepEqual(values, got) {
+			t.Fatalf("mismatch:\n\nexp=%+v\n\ngot=%+v\n\n", values, got)
+		}
+
+		return true
+	}, nil)
+}
+
 func BenchmarkInt64Encoder(b *testing.B) {
 	enc := tsm1.NewInt64Encoder()
 	x := make([]int64, 1024)

tsdb/engine/tsm1/string_test.go

@@ -2,7 +2,9 @@ package tsm1
 
 import (
 	"fmt"
+	"reflect"
 	"testing"
+	"testing/quick"
 )
 
 func Test_StringEncoder_NoValues(t *testing.T) {
@@ -83,3 +85,39 @@ func Test_StringEncoder_Multi_Compressed(t *testing.T) {
 		t.Fatalf("unexpected next value: got true, exp false")
 	}
 }
+
+func Test_StringEncoder_Quick(t *testing.T) {
+	quick.Check(func(values []string) bool {
+		// Write values to encoder.
+		enc := NewStringEncoder()
+		for _, v := range values {
+			enc.Write(v)
+		}
+
+		// Retrieve encoded bytes from encoder.
+		buf, err := enc.Bytes()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Read values out of decoder.
+		got := make([]string, 0, len(values))
+		dec, err := NewStringDecoder(buf)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for dec.Next() {
+			if err := dec.Error(); err != nil {
+				t.Fatal(err)
+			}
+			got = append(got, dec.Read())
+		}
+
+		// Verify that input and output values match.
+		if !reflect.DeepEqual(values, got) {
+			t.Fatalf("mismatch:\n\nexp=%+v\n\ngot=%+v\n\n", values, got)
+		}
+
+		return true
+	}, nil)
+}