Skip to content

Commit

Permalink
encoding/mvt: use protoscan for unmarshalling
Browse files Browse the repository at this point in the history 
benchmark                     old ns/op     new ns/op     delta
BenchmarkUnmarshal-12         367590        246198        -33.02%

benchmark                     old allocs     new allocs     delta
BenchmarkUnmarshal-12         6629           2476           -62.65%

benchmark                     old bytes     new bytes     delta
BenchmarkUnmarshal-12         372954        211681        -43.24%
  • Loading branch information
@paulmach
paulmach committed Jan 16, 2021
1 parent 1a8c09c commit d655888
Show file tree
Hide file tree
Showing 7 changed files with 547 additions and 284 deletions.
164 changes: 0 additions & 164 deletions encoding/mvt/geometry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,170 +140,6 @@ type keyValueEncoder struct {
valueMap map[interface{}]uint32
}

// A geomDecoder holds state for geometry decoding.
type geomDecoder struct {
geom []uint32
i int

prev orb.Point
}

func decodeGeometry(geomType vectortile.Tile_GeomType, geom []uint32) (orb.Geometry, error) {
if len(geom) < 2 {
return nil, errors.Errorf("geom is not long enough: %v", geom)
}

gd := &geomDecoder{geom: geom}

switch geomType {
case vectortile.Tile_POINT:
return gd.decodePoint()
case vectortile.Tile_LINESTRING:
return gd.decodeLineString()
case vectortile.Tile_POLYGON:
return gd.decodePolygon()
}

return nil, errors.Errorf("unknown geometry type: %v", geomType)
}

func (gd *geomDecoder) decodePoint() (orb.Geometry, error) {
_, count, err := gd.cmdAndCount()
if err != nil {
return nil, err
}

if count == 1 {
return gd.NextPoint(), nil
}

mp := make(orb.MultiPoint, 0, count)
for i := uint32(0); i < count; i++ {
mp = append(mp, gd.NextPoint())
}

return mp, nil
}

func (gd *geomDecoder) decodeLine() (orb.LineString, error) {
cmd, count, err := gd.cmdAndCount()
if err != nil {
return nil, err
}

if cmd != moveTo || count != 1 {
return nil, errors.New("first command not one moveTo")
}

first := gd.NextPoint()
cmd, count, err = gd.cmdAndCount()
if err != nil {
return nil, err
}

if cmd != lineTo {
return nil, errors.New("second command not a lineTo")
}

ls := make(orb.LineString, 0, count+1)
ls = append(ls, first)

for i := uint32(0); i < count; i++ {
ls = append(ls, gd.NextPoint())
}

return ls, nil
}

func (gd *geomDecoder) decodeLineString() (orb.Geometry, error) {
var mls orb.MultiLineString
for !gd.done() {
ls, err := gd.decodeLine()
if err != nil {
return nil, err
}

if gd.done() && len(mls) == 0 {
return ls, nil
}

mls = append(mls, ls)
}

return mls, nil
}

func (gd *geomDecoder) decodePolygon() (orb.Geometry, error) {
var mp orb.MultiPolygon
var p orb.Polygon
for !gd.done() {
ls, err := gd.decodeLine()
if err != nil {
return nil, err
}

r := orb.Ring(ls)

cmd, _, err := gd.cmdAndCount()
if err != nil {
return nil, err
}

if cmd == closePath && !r.Closed() {
r = append(r, r[0])
}

// figure out if new polygon
if len(mp) == 0 && len(p) == 0 {
p = append(p, r)
} else {
if r.Orientation() == orb.CCW {
mp = append(mp, p)
p = orb.Polygon{r}
} else {
p = append(p, r)
}
}
}

if len(mp) == 0 {
return p, nil
}

return append(mp, p), nil
}

func (gd *geomDecoder) cmdAndCount() (uint32, uint32, error) {
if gd.i >= len(gd.geom) {
return 0, 0, errors.New("no more data")
}

v := gd.geom[gd.i]

cmd := v & 0x07
count := v >> 3
gd.i++

if cmd != closePath {
if v := gd.i + int(2*count); len(gd.geom) < v {
return 0, 0, errors.Errorf("data cut short: needed %d, have %d", v, len(gd.geom))
}
}

return cmd, count, nil
}

func (gd *geomDecoder) NextPoint() orb.Point {
gd.i += 2
gd.prev[0] += unzigzag(gd.geom[gd.i-2])
gd.prev[1] += unzigzag(gd.geom[gd.i-1])
return gd.prev
}

func (gd *geomDecoder) done() bool {
return gd.i >= len(gd.geom)
}

func newKeyValueEncoder() *keyValueEncoder {
return &keyValueEncoder{
keyMap: make(map[string]uint32),
Expand Down
32 changes: 31 additions & 1 deletion encoding/mvt/geometry_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/paulmach/orb"
"github.com/paulmach/orb/encoding/mvt/vectortile"
"github.com/paulmach/protoscan"
)

func TestGeometry_Point(t *testing.T) {
Expand Down Expand Up @@ -322,7 +323,8 @@ func compareGeometry(
t.Errorf("different encoding")
}

result, err := decodeGeometry(geomType, input)
d := &decoder{geom: sliceToIterator(input)}
result, err := d.Geometry(geomType)
if err != nil {
t.Fatalf("decode error: %v", err)
}
Expand All @@ -337,3 +339,31 @@ func compareGeometry(
t.Errorf("geometry not equal")
}
}

func sliceToIterator(vals []uint32) *protoscan.Iterator {
feature := &vectortile.Tile_Feature{
Geometry: vals,
}

data, err := feature.Marshal()
if err != nil {
panic(err)
}

msg := protoscan.New(data)
for msg.Next() {
switch msg.FieldNumber() {
case 4:
iter, err := msg.Iterator(nil)
if err != nil {
panic(err)
}

return iter
default:
msg.Skip()
}
}

panic("unreachable")
}
97 changes: 0 additions & 97 deletions encoding/mvt/marshal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"bytes"
"compress/gzip"
"fmt"
"io/ioutil"
"strconv"

"github.com/paulmach/orb/encoding/mvt/vectortile"
Expand Down Expand Up @@ -84,72 +83,6 @@ func Marshal(layers Layers) ([]byte, error) {
return proto.Marshal(vt)
}

// UnmarshalGzipped takes gzipped Mapbox Vector Tile (MVT) data and unzips it
// before decoding it into a set of layers, It does not project the coordinates.
func UnmarshalGzipped(data []byte) (Layers, error) {
gzreader, err := gzip.NewReader(bytes.NewBuffer(data))
if err != nil {
return nil, errors.WithMessage(err, "failed to create gzreader")
}

decoded, err := ioutil.ReadAll(gzreader)
if err != nil {
return nil, errors.WithMessage(err, "failed to unzip")
}

return Unmarshal(decoded)
}

// Unmarshal takes Mapbox Vector Tile (MVT) data and converts into a
// set of layers, It does not project the coordinates.
func Unmarshal(data []byte) (Layers, error) {
vt := &vectortile.Tile{}
err := vt.Unmarshal(data)
if err != nil {
return nil, err
}

return decode(vt)
}

func decode(vt *vectortile.Tile) (Layers, error) {
result := make(Layers, 0, len(vt.Layers))
for i, l := range vt.Layers {
layer := &Layer{
Name: l.GetName(),
Version: l.GetVersion(),
Extent: l.GetExtent(),
Features: make([]*geojson.Feature, 0, len(l.Features)),
}

for j, f := range l.Features {
geom, err := decodeGeometry(f.GetType(), f.Geometry)
if err != nil {
return nil, errors.WithMessage(err, fmt.Sprintf("layer %d: feature %d", i, j))
}

properties := decodeFeatureProperties(l.Keys, l.Values, f.Tags)

if geom != nil {
gjf := &geojson.Feature{
Geometry: geom,
Properties: properties,
}

if f.Id != nil {
gjf.ID = float64(*f.Id)
}

layer.Features = append(layer.Features, gjf)
}
}

result = append(result, layer)
}

return result, nil
}

func encodeProperties(kve *keyValueEncoder, properties geojson.Properties) ([]uint32, error) {
tags := make([]uint32, 0, 2*len(properties))
for k, v := range properties {
Expand All @@ -165,36 +98,6 @@ func encodeProperties(kve *keyValueEncoder, properties geojson.Properties) ([]ui
return tags, nil
}

func decodeFeatureProperties(
keys []string,
values []*vectortile.Tile_Value,
tags []uint32,
) geojson.Properties {
result := make(geojson.Properties, len(tags)/2)
if len(tags) == 0 {
return result
}

for i := 2; i <= len(tags); i += 2 {
vi := tags[i-1]
if int(vi) >= len(values) {
continue
}

v := decodeValue(values[vi])
if v != nil {
ti := tags[i-2]
if int(ti) >= len(keys) {
continue
}

result[keys[ti]] = v
}
}

return result
}

func convertID(id interface{}) *uint64 {
if id == nil {
return nil
Expand Down
22 changes: 0 additions & 22 deletions encoding/mvt/marshal_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"testing"

"github.com/paulmach/orb"
"github.com/paulmach/orb/encoding/mvt/vectortile"
"github.com/paulmach/orb/geojson"
"github.com/paulmach/orb/maptile"
)
Expand Down Expand Up @@ -484,27 +483,6 @@ func BenchmarkUnmarshal(b *testing.B) {
}
}

func BenchmarkDecode(b *testing.B) {
layers := NewLayers(loadGeoJSON(b, maptile.New(17896, 24449, 16)))
data, err := Marshal(layers)
if err != nil {
b.Fatalf("marshal error: %v", err)
}

vt := &vectortile.Tile{}
err = vt.Unmarshal(data)
if err != nil {
b.Fatalf("unmarshal error: %v", err)
}

b.ReportAllocs()
b.ResetTimer()

for i := 0; i < b.N; i++ {
decode(vt)
}
}

func BenchmarkProjectToTile(b *testing.B) {
tile := maptile.New(17896, 24449, 16)
layers := NewLayers(loadGeoJSON(b, maptile.New(17896, 24449, 16)))
Expand Down
Loading

0 comments on commit d655888

Please sign in to comment.