Skip to content

Commit

Permalink
more encoding
Browse files Browse the repository at this point in the history
  • Loading branch information
@bbengfort
bbengfort committed Dec 10, 2024
1 parent 7be1349 commit 35e1f05
Show file tree
Hide file tree
Showing 6 changed files with 142 additions and 85 deletions.
94 changes: 64 additions & 30 deletions pkg/store/lamport/scalar.go
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
package lamport

import (
"bytes"
"encoding"
"encoding/binary"
"errors"
"fmt"
"regexp"
"strconv"

"github.com/rotationalio/honu/pkg/store/lani"
)
Expand All @@ -23,13 +28,51 @@ type Scalar struct {
}

var (
zero = &Scalar{0, 0}
_ lani.Encodable = &Scalar{}
_ lani.Decodable = &Scalar{}
zero = &Scalar{0, 0}
scre = regexp.MustCompile(`^(\d+)\.(\d+)$`)
_ lani.Encodable = &Scalar{}
_ lani.Decodable = &Scalar{}
_ encoding.BinaryMarshaler = &Scalar{}
_ encoding.BinaryUnmarshaler = &Scalar{}
_ encoding.TextMarshaler = &Scalar{}
_ encoding.TextUnmarshaler = &Scalar{}
)

const scalarSize = binary.MaxVarintLen32 + binary.MaxVarintLen64

// Compare returns an integer comparing two scalars using a happens before relationship.
// The result will be 0 if a == b, -1 if a < b (e.g. a happens before b), and
// +1 if a > b (e.g. b happens before a). A nil argument is equivalent to a zero scalar.
func Compare(a, b *Scalar) int {
if a == nil && b == nil {
return 0
}

if a == nil {
a = zero
}

if b == nil {
b = zero
}

if a.VID == b.VID {
switch {
case a.PID < b.PID:
return -1
case a.PID > b.PID:
return 1
default:
return 0
}
}

if a.VID > b.VID {
return 1
}
return -1
}

// Returns true if the scalar is the zero-valued scalar (0.0)
func (s *Scalar) IsZero() bool {
return s.PID == 0 && s.VID == 0
Expand Down Expand Up @@ -99,40 +142,31 @@ func (s *Scalar) UnmarshalBinary(data []byte) (err error) {
return s.Decode(d)
}

// Returns a scalar version representation in the form PID.VID using decimal notation.
func (s *Scalar) String() string {
return fmt.Sprintf("%d.%d", s.PID, s.VID)
func (s *Scalar) MarshalText() (_ []byte, err error) {
return []byte(s.String()), nil
}

// Compare returns an integer comparing two scalars using a happens before relationship.
// The result will be 0 if a == b, -1 if a < b (e.g. a happens before b), and
// +1 if a > b (e.g. b happens before a). A nil argument is equivalent to a zero scalar.
func Compare(a, b *Scalar) int {
if a == nil && b == nil {
return 0
func (s *Scalar) UnmarshalText(text []byte) (err error) {
if !scre.Match(text) {
return errors.New("could not parse text representation of a scalar")
}

if a == nil {
a = zero
}
parts := bytes.Split(text, []byte{'.'})

if b == nil {
b = zero
var pid uint64
if pid, err = strconv.ParseUint(string(parts[0]), 10, 32); err != nil {
panic("pid is not parseable even though regular expression matched.")
}
s.PID = uint32(pid)

if a.VID == b.VID {
switch {
case a.PID < b.PID:
return -1
case a.PID > b.PID:
return 1
default:
return 0
}
if s.VID, err = strconv.ParseUint(string(parts[1]), 10, 32); err != nil {
panic("pid is not parseable even though regular expression matched.")
}

if a.VID > b.VID {
return 1
}
return -1
return nil
}

// Returns a scalar version representation in the form PID.VID using decimal notation.
func (s *Scalar) String() string {
return fmt.Sprintf("%d.%d", s.PID, s.VID)
}
58 changes: 54 additions & 4 deletions pkg/store/lamport/scalar_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package lamport_test

import (
"encoding/json"
"math/rand/v2"
"testing"

Expand All @@ -19,10 +20,6 @@ func TestScalar(t *testing.T) {
require.False(t, (&Scalar{0, 1}).IsZero(), "0.1 should not be zero")
})

t.Run("String", func(t *testing.T) {
require.Equal(t, "1.1", one.String())
})

t.Run("Serialize", func(t *testing.T) {
current := &Scalar{}
for i := 0; i < 128; i++ {
Expand All @@ -38,6 +35,59 @@ func TestScalar(t *testing.T) {
current = randNextScalar(current)
}
})

t.Run("Text", func(t *testing.T) {
current := &Scalar{}
for i := 0; i < 128; i++ {
data, err := current.MarshalText()
require.NoError(t, err, "could not marshal %s", current)

cmpr := &Scalar{}
err = cmpr.UnmarshalText(data)
require.NoError(t, err, "could not unmarshal %d bytes", len(data))

require.Equal(t, current, cmpr, "unmarshaled scalar does not match marshaled one")

current = randNextScalar(current)
}
})

t.Run("BadText", func(t *testing.T) {
testCases := []string{
"123",
"a.b",
"a.123",
"1.abc",
"",
"1.1.1",
"1.",
".1",
}

for i, tc := range testCases {
err := (&Scalar{}).UnmarshalText([]byte(tc))
require.Error(t, err, "expected errror on test case %d", i)
}
})

t.Run("Binary", func(t *testing.T) {
vers := &Scalar{42, 198}
data, err := vers.MarshalBinary()
require.NoError(t, err, "could not marshal scalar as a binary value")
require.Equal(t, []byte{0x2a, 0xc6, 0x1}, data)
})

t.Run("JSON", func(t *testing.T) {
s := &Scalar{}
data := []byte(`"8.16"`)

require.NoError(t, json.Unmarshal(data, s), "could not unmarshal s")
require.Equal(t, &Scalar{8, 16}, s, "incorrect unmarshal")

cmpt, err := json.Marshal(s)
require.NoError(t, err, "could not marshal s")
require.Equal(t, data, cmpt, "unexpected marshaled data")
})
}

func TestCompare(t *testing.T) {
Expand Down
10 changes: 2 additions & 8 deletions pkg/store/metadata/testdata/metadata.json
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,8 @@
{
"version": {
"pid": 8,
"version": 12,
"scalar": "8.12",
"region": "us-central1",
"parent": {
"pid": 3,
"version": 11,
"region": "us-central1",
"created": "2024-11-30T10:29:59Z"
},
"parent": "3.11",
"created": "2024-11-30T10:29:59Z"
},
"schema": {
Expand Down
35 changes: 12 additions & 23 deletions pkg/store/metadata/version.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"encoding/binary"
"time"

"github.com/rotationalio/honu/pkg/store/lamport"
"github.com/rotationalio/honu/pkg/store/lani"
)

Expand All @@ -12,25 +13,22 @@ import (
//===========================================================================

type Version struct {
PID uint64 `json:"pid" msg:"pid"`
Version uint64 `json:"version" msg:"version"`
Region string `json:"region" msg:"region"`
Parent *Version `json:"parent,omitempty" msg:"parent,omitempty"`
Tombstone bool `json:"tombstone,omitempty" msg:"tombstone,omitempty"`
Created time.Time `json:"created" msg:"created"`
Scalar lamport.Scalar `json:"scalar" msg:"scalar"`
Region string `json:"region" msg:"region"`
Parent *lamport.Scalar `json:"parent,omitempty" msg:"parent,omitempty"`
Tombstone bool `json:"tombstone,omitempty" msg:"tombstone,omitempty"`
Created time.Time `json:"created" msg:"created"`
}

var _ lani.Encodable = &Version{}
var _ lani.Decodable = &Version{}

func (o *Version) Size() (s int) {
s += 2 * binary.MaxVarintLen64
s += len([]byte(o.Region)) + binary.MaxVarintLen64
s += o.Scalar.Size() // Scalar uint32 + uint64
s += 1 // Add 1 for the parent nil bool

if o.Parent != nil {
s += o.Parent.Size() + 1 // Add 1 for the not nil bool
} else {
s += 1 // Add 1 for the nil bool
s += o.Parent.Size()
}

s += 1 // Tombstone bool
Expand All @@ -41,12 +39,7 @@ func (o *Version) Size() (s int) {

func (o *Version) Encode(e *lani.Encoder) (n int, err error) {
var m int
if m, err = e.EncodeUint64(o.PID); err != nil {
return n + m, err
}
n += m

if m, err = e.EncodeUint64(o.Version); err != nil {
if m, err = o.Scalar.Encode(e); err != nil {
return n + m, err
}
n += m
Expand Down Expand Up @@ -75,11 +68,7 @@ func (o *Version) Encode(e *lani.Encoder) (n int, err error) {
}

func (o *Version) Decode(d *lani.Decoder) (err error) {
if o.PID, err = d.DecodeUint64(); err != nil {
return err
}

if o.Version, err = d.DecodeUint64(); err != nil {
if err = o.Scalar.Decode(d); err != nil {
return err
}

Expand All @@ -88,7 +77,7 @@ func (o *Version) Decode(d *lani.Decoder) (err error) {
}

var isNil bool
o.Parent = &Version{}
o.Parent = &lamport.Scalar{}

if isNil, err = d.DecodeStruct(o.Parent); err != nil {
return err
Expand Down
20 changes: 8 additions & 12 deletions pkg/store/object/object_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"time"

"github.com/oklog/ulid/v2"
"github.com/rotationalio/honu/pkg/store/lamport"
"github.com/rotationalio/honu/pkg/store/metadata"
"github.com/rotationalio/honu/pkg/store/object"
"github.com/stretchr/testify/require"
Expand All @@ -22,7 +23,7 @@ func TestObject(t *testing.T) {
obj, err := object.Marshal(meta, data)
require.NoError(t, err, "could not marshal object")

require.Len(t, obj, 1271, "unexpected length of encoded object")
require.Len(t, obj, 1248, "unexpected length of encoded object")
require.Equal(t, object.StorageVersion, obj.StorageVersion())

ometa, err := obj.Metadata()
Expand Down Expand Up @@ -141,7 +142,7 @@ func BenchmarkSerialization(b *testing.B) {

func generateRandomObject(size Size) (*metadata.Metadata, []byte) {
obj := &metadata.Metadata{
Version: randVersion(false),
Version: randVersion(),
Schema: randSchema(),
MIME: "application/random",
Owner: ulid.MustNew(ulid.Now(), rand.Reader),
Expand All @@ -165,22 +166,17 @@ func generateRandomObject(size Size) (*metadata.Metadata, []byte) {
return obj, data
}

func randVersion(isParent bool) *metadata.Version {
// 10% chance of nil
if mrand.Float32() < 0.1 {
return nil
}

func randVersion() *metadata.Version {
vers := &metadata.Version{
PID: mrand.Uint64(),
Version: mrand.Uint64(),
Scalar: lamport.Scalar{PID: mrand.Uint32(), VID: mrand.Uint64()},
Region: randRegion(),
Tombstone: mrand.Float32() < 0.25,
Created: randTime(),
}

if !isParent {
vers.Parent = randVersion(true)
// 10% chance of nil parent
if mrand.Float32() < 0.9 {
vers.Parent = &lamport.Scalar{PID: mrand.Uint32(), VID: mrand.Uint64()}
}

return vers
Expand Down
10 changes: 2 additions & 8 deletions pkg/store/object/testdata/metadata.json
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,8 @@
{
"version": {
"pid": 8,
"version": 12,
"scalar": "8.12",
"region": "us-central1",
"parent": {
"pid": 3,
"version": 11,
"region": "us-central1",
"created": "2024-11-30T10:29:59Z"
},
"parent": "3.11",
"created": "2024-11-30T10:29:59Z"
},
"schema": {
Expand Down

0 comments on commit 35e1f05

Please sign in to comment.