diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2b1678b --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.idea +*.test +*.out +*.png +*.exe \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..63b4b68 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) [year] [fullname] + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..c85b82c --- /dev/null +++ b/TODO.md @@ -0,0 +1,18 @@ +# Feature + +- [x] Skip object +- [ ] Support other tag than `json` + +# Improvement + +- [x] Nested skip (by using a stack) +- [x] Decode with stack (proven worse than use recursive directly) + +# Benchmark + +- [x] Reset +- [x] Skip switch/slice + +# Benchmark with other library + +- [ ] jsoniter diff --git a/decoder.go b/decoder.go new file mode 100644 index 0000000..09505c6 --- /dev/null +++ b/decoder.go @@ -0,0 +1,202 @@ +package jzon + +import ( + "reflect" + "sync" + "sync/atomic" +) + +var ( + DefaultDecoder = NewDecoder(&DecoderOption{ + // + }) +) + +type DecoderOption struct { + // custom value decoders + ValDecoders map[reflect.Type]ValDecoder + + // if the object key is case sensitive + // `false` by default + CaseSensitive bool + + // the tag name for structures + // `json` by default + Tag string +} + +type decoderCache = map[rtype]ValDecoder + +type Decoder struct { + cacheMu sync.Mutex + decoderCache atomic.Value + + caseSensitive bool + tag string +} + +func NewDecoder(opt *DecoderOption) *Decoder { + var dec Decoder + // add decoders to cache + m := decoderCache{} + if opt != nil { + for elemTyp, valDec := range opt.ValDecoders { + m[rtypeOfType(reflect.PtrTo(elemTyp))] = valDec + } + dec.caseSensitive = opt.CaseSensitive + dec.tag = opt.Tag + } + if dec.tag == "" { + dec.tag = "json" + } + dec.decoderCache.Store(m) + return &dec +} + +func (dec *Decoder) Unmarshal(data []byte, obj interface{}) error { + it := dec.NewIterator() + err := it.Unmarshal(data, obj) + dec.ReturnIterator(it) + return err +} + +func (dec *Decoder) getDecoderFromCache(rType rtype) ValDecoder { + return dec.decoderCache.Load().(decoderCache)[rType] +} + +// the typ must be a pointer type +func (dec *Decoder) createDecoder(rType rtype, ptrType reflect.Type) ValDecoder { + dec.cacheMu.Lock() + defer dec.cacheMu.Unlock() + cache := dec.decoderCache.Load().(decoderCache) + // double check + if vd := cache[rType]; vd != nil { + return vd + } + // make copy + newCache := decoderCache{} + for k, v := range cache { + newCache[k] = v + } + typesToCreate := []reflect.Type{ptrType} + dec.createDecoderInternal(newCache, typesToCreate) + dec.decoderCache.Store(newCache) + return newCache[rType] +} + +func (dec *Decoder) createDecoderInternal(cache decoderCache, typesToCreate []reflect.Type) { + rebuildMap := decoderCache{} + idx := len(typesToCreate) - 1 + for idx >= 0 { + // pop one + ptrType := typesToCreate[idx] + typesToCreate = typesToCreate[:idx] + idx -= 1 + + rType := rtypeOfType(ptrType) + if _, ok := cache[rType]; ok { // double check + continue + } + // check global decoders + if v, ok := globalValDecoders[rType]; ok { + cache[rType] = v + continue + } + // check json.Unmarshaler interface + if ptrType.Implements(jsonUnmarshalerType) { + cache[rType] = jsonUnmarshalerDecoder(rType) + continue + } + if ptrType.Implements(textUnmarshalerType) { + cache[rType] = textUnmarshalerDecoder(rType) + continue + } + elem := ptrType.Elem() + elemKind := elem.Kind() + if elemNativeRType := kindMap[elemKind]; elemNativeRType != 0 { + // TODO: shall we make this an option? + // TODO: so that only the native type is affected? + // check if the native type has a custom decoder + if v, ok := cache[elemNativeRType]; ok { + cache[rType] = v + continue + } + // otherwise check default native type decoder + if v := kindDecoders[elemKind]; v != nil { + cache[rType] = v + continue + } + } + switch elemKind { + case reflect.Interface: + if elem.NumMethod() == 0 { + cache[rType] = (*efaceDecoder)(nil) + } else { + cache[rType] = (*ifaceDecoder)(nil) + } + case reflect.Struct: + vd := dec.newStructDecoder(elem) + if vd == nil { + // no field to unmarshal + cache[rType] = (*skipDecoder)(nil) + } else { + for _, fi := range vd.fields { + typesToCreate = append(typesToCreate, fi.ptrType) + idx += 1 + } + cache[rType] = vd + rebuildMap[rType] = vd + } + case reflect.Ptr: + typesToCreate = append(typesToCreate, elem) + idx += 1 + vd := newPointerDecoder(elem) + cache[rType] = vd + rebuildMap[rType] = vd + case reflect.Array: + elemPtrType := reflect.PtrTo(elem.Elem()) + typesToCreate = append(typesToCreate, elemPtrType) + idx += 1 + vd := newArrayDecoder(elem) + cache[rType] = vd + rebuildMap[rType] = vd + case reflect.Slice: + elemPtrType := reflect.PtrTo(elem.Elem()) + typesToCreate = append(typesToCreate, elemPtrType) + idx += 1 + vd := newSliceDecoder(elem) + cache[rType] = vd + rebuildMap[rType] = vd + case reflect.Map: + vd := newMapDecoder(elem) + if vd == nil { + cache[rType] = notSupportedDecoder(ptrType.String()) + } else { + valuePtrType := reflect.PtrTo(elem.Elem()) + typesToCreate = append(typesToCreate, valuePtrType) + idx += 1 + cache[rType] = vd + rebuildMap[rType] = vd + } + default: + cache[rType] = notSupportedDecoder(ptrType.String()) + } + } + // rebuild some decoders + for _, vd := range rebuildMap { + switch x := vd.(type) { + case *pointerDecoder: + x.elemDec = cache[x.ptrRType] + case *structDecoder: + for _, field := range x.fields { + field.decoder = cache[field.rtype] + } + case *arrayDecoder: + x.elemDec = cache[x.elemPtrRType] + case *sliceDecoder: + x.elemDec = cache[x.elemPtrRType] + case *mapDecoder: + x.valDec = cache[x.valPtrRType] + } + } +} diff --git a/decoder_iterator.go b/decoder_iterator.go new file mode 100644 index 0000000..e2b8fe4 --- /dev/null +++ b/decoder_iterator.go @@ -0,0 +1,12 @@ +package jzon + +func (dec *Decoder) NewIterator() *Iterator { + it := defaultIteratorPool.BorrowIterator() + it.decoder = dec + return it +} + +func (dec *Decoder) ReturnIterator(it *Iterator) { + it.decoder = nil + defaultIteratorPool.ReturnIterator(it) +} diff --git a/decoder_test.go b/decoder_test.go new file mode 100644 index 0000000..20b8bc0 --- /dev/null +++ b/decoder_test.go @@ -0,0 +1 @@ +package jzon diff --git a/eface.go b/eface.go new file mode 100644 index 0000000..4907fbe --- /dev/null +++ b/eface.go @@ -0,0 +1,26 @@ +package jzon + +import ( + "reflect" + "unsafe" +) + +type rtype = uintptr + +type eface struct { + rtype rtype + data unsafe.Pointer +} + +func packEFace(rtype rtype, data unsafe.Pointer) interface{} { + var i interface{} + e := (*eface)(unsafe.Pointer(&i)) + e.rtype = rtype + e.data = data + return i +} + +func rtypeOfType(typ reflect.Type) rtype { + ef := (*eface)(unsafe.Pointer(&typ)) + return rtype(ef.data) +} diff --git a/eface_test.go b/eface_test.go new file mode 100644 index 0000000..bd57273 --- /dev/null +++ b/eface_test.go @@ -0,0 +1,37 @@ +package jzon + +import ( + "reflect" + "testing" + "unsafe" + + "github.com/stretchr/testify/require" +) + +func TestEface(t *testing.T) { + f := func(o interface{}) *eface { + return (*eface)(unsafe.Pointer(&o)) + } + a := 1 + b := 2 + ef1 := f(&a) + ef2 := f(&b) + require.Equal(t, ef1.rtype, ef2.rtype) + // require.Equal(t, uintptr(strconv.IntSize/8), uintptr(ef2.data)-uintptr(ef1.data)) + t.Logf("%x %x", ef1.rtype, ef1.data) + t.Logf("%x %x", ef2.rtype, ef2.data) + + // with reflect + r := reflect.TypeOf(&a) + ef3 := (*eface)(unsafe.Pointer(&r)) + require.Equal(t, ef1.rtype, uintptr(ef3.data)) + require.Equal(t, ef1.rtype, rtypeOfType(r)) + + // pack + packed := packEFace(rtype(ef3.data), ef1.data) + t.Logf("%+v", packed) + v, ok := packed.(*int) + t.Logf("%+v %+v", v, ok) + require.True(t, ok) + require.Equal(t, &a, v) +} diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..ac71171 --- /dev/null +++ b/errors.go @@ -0,0 +1,97 @@ +package jzon + +import ( + "errors" + "fmt" +) + +// DataRemainedError +var DataRemainedError = errors.New("expecting EOF, but there is still data") + +// PointerReceiverError +var PointerReceiverError = errors.New("the receiver is not a pointer") + +// NilPointerReceiverError +var NilPointerReceiverError = errors.New("the receiver is nil") + +// IFaceError +var IFaceError = errors.New("cannot unmarshal on empty iface") + +// InvalidStringCharError +type InvalidStringCharError struct { + c byte +} + +func (e InvalidStringCharError) Error() string { + return fmt.Sprintf("invalid character %x found", e.c) +} + +// InvalidEscapeCharError +type InvalidEscapeCharError struct { + c byte +} + +func (e InvalidEscapeCharError) Error() string { + return fmt.Sprintf("invalid escape character \\%x found", e.c) +} + +// InvalidUnicodeCharError +type InvalidUnicodeCharError struct { + c byte +} + +func (e InvalidUnicodeCharError) Error() string { + return fmt.Sprintf("invalid unicode character %x found", e.c) +} + +// UnexpectedByteError +type UnexpectedByteError struct { + got byte + exp byte + exp2 byte +} + +func (e UnexpectedByteError) Error() string { + if e.exp == 0 { + return fmt.Sprintf("unexpected character %q", e.got) + } + if e.exp2 == 0 { + return fmt.Sprintf("expecting %q but got %q", e.exp, e.got) + } + return fmt.Sprintf("expecting %q or %q but got %q", e.exp, e.exp2, e.got) +} + +// IntOverflow +type IntOverflowError struct { + typ string + value string +} + +func (e IntOverflowError) Error() string { + return fmt.Sprintf("overflow %s: %s", e.typ, e.value) +} + +// InvalidDigit +type InvalidDigitError struct { + c byte +} + +func (e InvalidDigitError) Error() string { + return fmt.Sprintf("invalid digit character: %q", e.c) +} + +// InvalidFloat +type InvalidFloatError struct { + c byte +} + +func (e InvalidFloatError) Error() string { + return fmt.Sprintf("invalid float character: %q", e.c) +} + +// TypeNotSupported +type TypeNotSupportedError string + +func (e TypeNotSupportedError) Error() string { + return fmt.Sprintf("%q is not supported", string(e)) +} diff --git a/helper.go b/helper.go new file mode 100644 index 0000000..c0a8122 --- /dev/null +++ b/helper.go @@ -0,0 +1,3 @@ +package jzon + +const charNum = 256 diff --git a/helper_test.go b/helper_test.go new file mode 100644 index 0000000..eaaa81d --- /dev/null +++ b/helper_test.go @@ -0,0 +1,42 @@ +package jzon + +import "io" + +type oneByteReader struct { + b []byte + err error +} + +var _ io.Reader = &oneByteReader{} + +func (o *oneByteReader) Read(p []byte) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if len(o.b) == 0 { + if o.err != nil { + return 0, o.err + } + return 0, io.EOF + } + p[0] = o.b[0] + o.b = o.b[1:] + return 1, nil +} + +type repeatByteReader struct { + b byte + count int +} + +var _ io.Reader = &repeatByteReader{} + +func (r *repeatByteReader) Read(p []byte) (n int, err error) { + l := len(p) + for r.count > 0 && n < l { + p[n] = r.b + r.count-- + n++ + } + return +} diff --git a/iface.go b/iface.go new file mode 100644 index 0000000..a685cc8 --- /dev/null +++ b/iface.go @@ -0,0 +1,23 @@ +package jzon + +import ( + "unsafe" +) + +type iface struct { + itab *itab + data unsafe.Pointer +} + +type itab struct { + ignore uintptr + rtype rtype +} + +func packIFace(ptr unsafe.Pointer) interface{} { + iface := (*iface)(ptr) + if iface.itab == nil { + return nil + } + return packEFace(iface.itab.rtype, iface.data) +} diff --git a/iterator.go b/iterator.go new file mode 100644 index 0000000..85a13df --- /dev/null +++ b/iterator.go @@ -0,0 +1,216 @@ +package jzon + +import ( + "bytes" + "fmt" + "io" +) + +type Iterator struct { + decoder *Decoder + + reader io.Reader + buffer []byte + + // a temp buffer is needed for string reading + // which include utf8 conversion + tmpBuffer []byte + + capture bool + offset int + + // the current index position + head int + tail int + + // path string + + // Error error +} + +func NewIterator() *Iterator { + return DefaultDecoder.NewIterator() +} + +func ReturnIterator(it *Iterator) { + DefaultDecoder.ReturnIterator(it) +} + +func (it *Iterator) reset() { + if it.reader == nil { + it.buffer = nil + } else { // it.reader != nil + it.reader = nil + releaseByteSlice(it.buffer) + it.buffer = nil + } +} + +/* + * In reset methods, explicit assignment is faster than then following + * *it = Iterator{ ... } + * When the above code is used, runtime.duffcopy and runtime.duffzero will be used + * which will slow down our code (correct me if I am wrong) + */ +func (it *Iterator) Reset(r io.Reader) { + switch v := r.(type) { + case nil: + it.reset() + return + case *bytes.Buffer: + it.ResetBytes(v.Bytes()) + return + } + var b []byte + if it.reader == nil { + b = getFullByteSlice() + } else { + b = it.buffer + } + it.reader = r + it.buffer = b + it.offset = 0 + it.head = 0 + it.tail = 0 +} + +func (it *Iterator) ResetBytes(data []byte) { + if it.reader != nil && it.buffer != nil { + releaseByteSlice(it.buffer) + } + it.reader = nil + it.buffer = data + it.offset = 0 + it.head = 0 + it.tail = len(data) +} + +func (it *Iterator) Buffer() []byte { + return it.buffer[:it.tail] +} + +// make sure that it.head == it.tail before call +func (it *Iterator) readMore() error { + if it.reader == nil { + return io.EOF + } + var ( + n int + err error + ) + // TODO: risk of infinite loop? + for { + if it.capture { + var buf [bufferSize]byte + n, err = it.reader.Read(buf[:]) + it.buffer = append(it.buffer[:it.tail], buf[:n]...) + it.tail += n + } else { + if it.head != it.tail { // debug, to be removed + panic(fmt.Errorf("head %d, tail %d", it.head, it.tail)) + } + n, err = it.reader.Read(it.buffer) + it.offset += it.tail + it.head = 0 + it.tail = n + } + if err != nil { + if err == io.EOF && n > 0 { + return nil + } + return err + } + if n > 0 { + return nil + } + } +} + +// will NOT skip whitespaces +// will NOT consume the character +// will report error on EOF +func (it *Iterator) nextByte() (ret byte, err error) { + if it.head == it.tail { + if err = it.readMore(); err != nil { + return + } + } + return it.buffer[it.head], nil +} + +// will consume the characters +func (it *Iterator) expectBytes(s string) error { + last := len(s) - 1 + j := 0 + for { + i := it.head + for ; i < it.tail; i++ { + c := it.buffer[i] + if c != s[j] { + return UnexpectedByteError{exp: s[j], got: c} + } + if j == last { + it.head = i + 1 + return nil + } + j++ + } + it.head = i + if err := it.readMore(); err != nil { + return err + } + } +} + +// Read until the first valid token is found, only the whitespaces are consumed +func (it *Iterator) nextToken() (ret byte, vt ValueType, err error) { + for { + i := it.head + for ; i < it.tail; i++ { + c := it.buffer[i] + vt := valueTypeMap[c] + if vt == WhiteSpaceValue { + continue + } + it.head = i + return c, vt, nil + } + // the head and tail will be reset by readMore + it.head = i + if err := it.readMore(); err != nil { + return 0, InvalidValue, err + } + } +} + +// Read until the first valid token is found, only the whitespaces are consumed +func (it *Iterator) NextValueType() (vt ValueType, err error) { + _, vt, err = it.nextToken() + return +} + +func (it *Iterator) Unmarshal(data []byte, obj interface{}) error { + it.ResetBytes(data) + err := it.ReadVal(obj) + if err != nil { + return err + } + _, _, err = it.nextToken() + if err == nil { + return DataRemainedError + } + if err != io.EOF { + return err + } + return nil +} + +func (it *Iterator) Valid(data []byte) bool { + it.ResetBytes(data) + err := it.Skip() + if err != nil { + return false + } + _, _, err = it.nextToken() + return err == io.EOF +} diff --git a/iterator_array.go b/iterator_array.go new file mode 100644 index 0000000..da14f31 --- /dev/null +++ b/iterator_array.go @@ -0,0 +1,88 @@ +package jzon + +/* + * var ( + * more bool + * err error + * ) + * for more, err = it.ReadArray(); + * more; + * more, err = it.ReadArrayMore() { + * } + * if err != nil { + * // error handling + * } + */ +func (it *Iterator) ReadArrayBegin() (ret bool, err error) { + c, _, err := it.nextToken() + if err != nil { + return false, err + } + if c != '[' { + return false, UnexpectedByteError{got: c, exp: '['} + } + it.head += 1 + c, _, err = it.nextToken() + if err != nil { + return false, err + } + if c == ']' { + it.head += 1 + return false, nil + } + return true, nil +} + +func (it *Iterator) ReadArrayMore() (ret bool, err error) { + c, _, err := it.nextToken() + if err != nil { + return false, err + } + switch c { + case ',': + it.head += 1 + return true, nil + case ']': + it.head += 1 + return false, nil + default: + return false, UnexpectedByteError{got: c, exp: ',', exp2: ']'} + } +} + +func (it *Iterator) ReadArrayCB(cb func(*Iterator) error) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c != '[' { + return UnexpectedByteError{got: c, exp: '['} + } + it.head += 1 + c, _, err = it.nextToken() + if err != nil { + return err + } + if c == ']' { + it.head += 1 + return nil + } + for { + if err := cb(it); err != nil { + return err + } + c, _, err = it.nextToken() + if err != nil { + return err + } + switch c { + case ',': + it.head += 1 + case ']': + it.head += 1 + return nil + default: + return UnexpectedByteError{got: c, exp: ',', exp2: ']'} + } + } +} diff --git a/iterator_array_test.go b/iterator_array_test.go new file mode 100644 index 0000000..13e18c9 --- /dev/null +++ b/iterator_array_test.go @@ -0,0 +1,189 @@ +package jzon + +import ( + "errors" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Array_ReadArrayBegin(t *testing.T) { + t.Run("leading space", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" [ ]")) + more, err := it.ReadArrayBegin() + require.NoError(t, err) + require.False(t, more) + }) + t.Run("eof", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadArrayBegin() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid first byte", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("1")) + _, err := it.ReadArrayBegin() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after bracket", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("[")) + _, err := it.ReadArrayBegin() + require.Equal(t, io.EOF, err) + }) + t.Run("more", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`["`)) + more, err := it.ReadArrayBegin() + require.NoError(t, err) + require.True(t, more) + }) +} + +func TestIterator_Array_ReadArrayMore(t *testing.T) { + t.Run("leading space", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" ,")) + more, err := it.ReadArrayMore() + require.NoError(t, err) + require.True(t, more) + }) + t.Run("eof", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadArrayMore() + require.Equal(t, io.EOF, err) + }) + t.Run("more", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(",")) + more, err := it.ReadArrayMore() + require.NoError(t, err) + require.True(t, more) + }) + t.Run("no more", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("]")) + more, err := it.ReadArrayMore() + require.NoError(t, err) + require.False(t, more) + }) + t.Run("error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("a")) + _, err := it.ReadArrayMore() + require.IsType(t, UnexpectedByteError{}, err) + }) +} + +func TestIterator_Array_ReadArray_Example(t *testing.T) { + must := require.New(t) + + it := NewIterator() + it.ResetBytes([]byte(" [ 0 , 1 , 2 ] ")) + more, err := it.ReadArrayBegin() + must.NoError(err) + i := 0 + for ; more; more, err = it.ReadArrayMore() { + ri, err := it.ReadInt() + must.NoError(err) + must.Equal(i, ri) + i += 1 + } + must.NoError(err) + _, err = it.NextValueType() + must.Equal(io.EOF, err) +} + +func TestIterator_Array_ReadArrayCB(t *testing.T) { + t.Run("leading space", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" []")) + err := it.ReadArrayCB(nil) + require.NoError(t, err) + }) + t.Run("eof", func(t *testing.T) { + it := NewIterator() + err := it.ReadArrayCB(nil) + require.Equal(t, io.EOF, err) + }) + t.Run("invalid bracket", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("a")) + err := it.ReadArrayCB(nil) + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after bracket", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("[")) + err := it.ReadArrayCB(nil) + require.Equal(t, io.EOF, err) + }) + t.Run("no element", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("[]")) + err := it.ReadArrayCB(nil) + require.NoError(t, err) + }) + t.Run("no element", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("[]")) + err := it.ReadArrayCB(nil) + require.NoError(t, err) + }) + t.Run("callback error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("[1]")) + e := errors.New("test") + err := it.ReadArrayCB(func(*Iterator) error { + return e + }) + require.Equal(t, e, err) + }) + t.Run("error on more", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("[1")) + err := it.ReadArrayCB(func(it *Iterator) (err error) { + _, err = it.ReadInt() + return + }) + require.Equal(t, io.EOF, err) + }) + t.Run("error on more 2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("[1a")) + err := it.ReadArrayCB(func(it *Iterator) (err error) { + _, err = it.ReadInt() + return + }) + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("2 items", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("[1,2]")) + err := it.ReadArrayCB(func(it *Iterator) (err error) { + _, err = it.ReadInt() + return + }) + require.NoError(t, err) + }) +} + +func TestIterator_Array_ReadArrayCB_Example(t *testing.T) { + must := require.New(t) + + it := NewIterator() + it.ResetBytes([]byte(" [ 0 , 1 , 2 ] ")) + i := 0 + err := it.ReadArrayCB(func(it *Iterator) (err error) { + j, err := it.ReadInt() + must.NoError(err) + must.Equal(i, j) + i += 1 + return nil + }) + must.NoError(err) + _, err = it.NextValueType() + must.Equal(io.EOF, err) +} diff --git a/iterator_bool.go b/iterator_bool.go new file mode 100644 index 0000000..d956eb3 --- /dev/null +++ b/iterator_bool.go @@ -0,0 +1,18 @@ +package jzon + +func (it *Iterator) ReadBool() (bool, error) { + c, _, err := it.nextToken() + if err != nil { + return false, err + } + switch c { + case 't': + it.head += 1 + return true, it.expectBytes("rue") + case 'f': + it.head += 1 + return false, it.expectBytes("alse") + default: + return false, UnexpectedByteError{got: c, exp: 't', exp2: 'f'} + } +} diff --git a/iterator_bool_test.go b/iterator_bool_test.go new file mode 100644 index 0000000..99f6841 --- /dev/null +++ b/iterator_bool_test.go @@ -0,0 +1,40 @@ +package jzon + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Bool_ReadBool(t *testing.T) { + t.Run("eof", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadBool() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid fist byte", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" a")) + _, err := it.ReadBool() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("true", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" true ")) + b, err := it.ReadBool() + require.NoError(t, err) + require.True(t, b) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("true", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" false ")) + b, err := it.ReadBool() + require.NoError(t, err) + require.False(t, b) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) +} diff --git a/iterator_float.go b/iterator_float.go new file mode 100644 index 0000000..62ddca0 --- /dev/null +++ b/iterator_float.go @@ -0,0 +1,23 @@ +package jzon + +const ( + invalidFloatDigit = -1 + dotInNumber = -2 + expInNumber = -3 +) + +var ( + floatDigits [charNum]int8 +) + +func init() { + for i := 0; i < charNum; i++ { + floatDigits[i] = invalidFloatDigit + } + for i := '0'; i <= '9'; i++ { + floatDigits[i] = int8(i - '0') + } + floatDigits['.'] = dotInNumber + floatDigits['e'] = expInNumber + floatDigits['E'] = expInNumber +} diff --git a/iterator_float32.go b/iterator_float32.go new file mode 100644 index 0000000..1593163 --- /dev/null +++ b/iterator_float32.go @@ -0,0 +1,184 @@ +package jzon + +import ( + "io" + "strconv" + "unsafe" +) + +func (it *Iterator) ReadFloat32() (float32, error) { + c, _, err := it.nextToken() + if err != nil { + return 0, err + } + it.head += 1 + if c == '-' { + c, err = it.nextByte() + if err != nil { + return 0, err + } + it.head += 1 + f, buf, err := it.readPositiveFloat32(c, it.tmpBuffer[:0]) + it.tmpBuffer = buf + return -f, err + } + f, buf, err := it.readPositiveFloat32(c, it.tmpBuffer[:0]) + it.tmpBuffer = buf + return f, err +} + +func (it *Iterator) parseFloat32(buf []byte) (ret float32, err error) { + f, err := strconv.ParseFloat(*(*string)(unsafe.Pointer(&buf)), 32) + return float32(f), err +} + +func (it *Iterator) readFloat32ExponentPart(buf []byte) (ret float32, _ []byte, err error) { + c, err := it.nextByte() + if err != nil { + return 0, buf, err + } + it.head += 1 + if c == '+' || c == '-' { + buf = append(buf, c) + c, err = it.nextByte() + if err != nil { + return 0, buf, err + } + it.head += 1 + } + if intDigits[c] == invalidDigit { + return 0, buf, InvalidFloatError{c: c} + } + buf = append(buf, c) + for { + i := it.head + for ; i < it.tail; i++ { + c = it.buffer[i] + digit := intDigits[c] + if digit == invalidDigit { + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + ret, err = it.parseFloat32(buf) + return ret, buf, err + } + } + // i == it.tail + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + if err = it.readMore(); err != nil { + if err == io.EOF { + ret, err = it.parseFloat32(buf) + return ret, buf, err + } + return 0, buf, err + } + } +} + +func (it *Iterator) readFloat32FractionPart(buf []byte) (ret float32, _ []byte, err error) { + c, err := it.nextByte() + if err != nil { + return 0, buf, err + } + if intDigits[c] == invalidDigit { + return 0, buf, InvalidFloatError{c: c} + } + it.head += 1 + buf = append(buf, c) + for { + i := it.head + for ; i < it.tail; i++ { + c = it.buffer[i] + digit := floatDigits[c] + if digit < 0 { + switch digit { + case expInNumber: + buf = append(buf, it.buffer[it.head:i+1]...) + it.head = i + 1 + return it.readFloat32ExponentPart(buf) + default: + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + ret, err = it.parseFloat32(buf) + return ret, buf, err + } + } + } + // i == it.tail + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + if err = it.readMore(); err != nil { + if err == io.EOF { + ret, err = it.parseFloat32(buf) + return ret, buf, err + } + return 0, buf, err + } + } +} + +func (it *Iterator) readPositiveFloat32(c byte, buf []byte) (ret float32, _ []byte, err error) { + u := intDigits[c] + if u == invalidDigit { + return 0, buf, InvalidFloatError{c: c} + } + + buf = append(buf, c) + if u == 0 { + if it.head == it.tail { + if err = it.readMore(); err != nil { + if err == io.EOF { + return 0, buf, nil + } + return 0, buf, err + } + } + switch floatDigits[it.buffer[it.head]] { + case dotInNumber: + it.head += 1 + buf = append(buf, '.') + return it.readFloat32FractionPart(buf) + case expInNumber: + it.head += 1 + buf = append(buf, 'e') + return it.readFloat32ExponentPart(buf) + default: + return 0, buf, nil + } + } else { + for { + i := it.head + for ; i < it.tail; i++ { + c = it.buffer[i] + digit := floatDigits[c] + if digit < 0 { + switch digit { + case dotInNumber: + buf = append(buf, it.buffer[it.head:i+1]...) + it.head = i + 1 + return it.readFloat32FractionPart(buf) + case expInNumber: + buf = append(buf, it.buffer[it.head:i+1]...) + it.head = i + 1 + return it.readFloat32ExponentPart(buf) + default: + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + ret, err = it.parseFloat32(buf) + return ret, buf, err + } + } + } + // i == it.tail + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + if err = it.readMore(); err != nil { + if err == io.EOF { + ret, err = it.parseFloat32(buf) + return ret, buf, err + } + return 0, buf, err + } + } + } +} diff --git a/iterator_float32_test.go b/iterator_float32_test.go new file mode 100644 index 0000000..902d2fe --- /dev/null +++ b/iterator_float32_test.go @@ -0,0 +1,220 @@ +package jzon + +import ( + "errors" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Float_ReadFloat32(t *testing.T) { + t.Run("leading space", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.Equal(t, float32(1), f) + }) + t.Run("first byte error", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadFloat32() + require.Equal(t, io.EOF, err) + }) + t.Run("negative error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte{'-'}) + _, err := it.ReadFloat32() + require.Equal(t, io.EOF, err) + }) + t.Run("negative", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-0")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.Equal(t, float32(0), f) + }) + t.Run("negative 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-3.1415926535")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.InDelta(t, float32(-3.1415926535), f, 1e-10) + }) + t.Run("positive", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.Equal(t, float32(0), f) + }) + t.Run("positive 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("3.1415926535")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.InDelta(t, float32(3.1415926535), f, 1e-10) + }) + t.Run("invalid first char", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte{'a'}) + _, err := it.ReadFloat32() + require.IsType(t, InvalidFloatError{}, err) + }) +} + +func TestIterator_Float_ReadFloat32_LeadingZero(t *testing.T) { + t.Run("reader error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte{'0'}, + err: e, + }) + _, err := it.ReadFloat32() + require.Equal(t, e, err) + }) + t.Run("fraction eof after dot", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.")) + _, err := it.ReadFloat32() + require.Equal(t, io.EOF, err) + }) + t.Run("fraction invalid byte after dot", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.a")) + _, err := it.ReadFloat32() + require.IsType(t, InvalidFloatError{}, err) + }) + t.Run("fraction 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.1a")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.InDelta(t, float32(0.1), f, 1e-10) + }) + t.Run("fraction 2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.1")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.InDelta(t, float32(0.1), f, 1e-10) + }) + t.Run("fraction reader error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte("0."), + err: e, + }) + _, err := it.ReadFloat32() + require.Equal(t, e, err) + }) + t.Run("fraction reader error 2", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte("0.1"), + err: e, + }) + _, err := it.ReadFloat32() + require.Equal(t, e, err) + }) + t.Run("exponent eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.1e")) + _, err := it.ReadFloat32() + require.Equal(t, io.EOF, err) + }) + t.Run("exponent eof after sign", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.1e+")) + _, err := it.ReadFloat32() + require.Equal(t, io.EOF, err) + }) + t.Run("exponent invalid byte after sign", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.1e++")) + _, err := it.ReadFloat32() + require.IsType(t, InvalidFloatError{}, err) + }) + t.Run("exponent 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.1e+1")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.InDelta(t, float32(1), f, 1e-10) + }) + t.Run("exponent 2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.1e+1a")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.InDelta(t, float32(1), f, 1e-10) + }) + t.Run("exponent reader error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte("0.1e+1"), + err: e, + }) + _, err := it.ReadFloat32() + require.Equal(t, e, err) + }) + t.Run("no fraction part", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0e+1")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.Equal(t, float32(0), f) + }) + t.Run("only zero", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0a")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.Equal(t, float32(0), f) + }) + t.Run("double zero", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("00")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.Equal(t, float32(0), f) + }) +} + +func TestIterator_Float_ReadFloat32_NonLeadingZero(t *testing.T) { + t.Run("exponent", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("1e1")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.InDelta(t, float32(10), f, 1e-10) + }) + t.Run("no fraction no exponent 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("10")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.InDelta(t, float32(10), f, 1e-10) + }) + t.Run("no fraction no exponent 2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("10a")) + f, err := it.ReadFloat32() + require.NoError(t, err) + require.InDelta(t, float32(10), f, 1e-10) + }) + t.Run("reader error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte("10"), + err: e, + }) + _, err := it.ReadFloat32() + require.Equal(t, e, err) + }) +} diff --git a/iterator_float64.go b/iterator_float64.go new file mode 100644 index 0000000..1b782dc --- /dev/null +++ b/iterator_float64.go @@ -0,0 +1,187 @@ +package jzon + +import ( + "io" + "strconv" + "unsafe" +) + +func (it *Iterator) readFloat64(c byte) (float64, error) { + if c == '-' { + c, err := it.nextByte() + if err != nil { + return 0, err + } + it.head += 1 + f, buf, err := it.readPositiveFloat64(c, it.tmpBuffer[:0]) + it.tmpBuffer = buf + return -f, err + } + f, buf, err := it.readPositiveFloat64(c, it.tmpBuffer[:0]) + it.tmpBuffer = buf + return f, err +} + +func (it *Iterator) ReadFloat64() (float64, error) { + c, _, err := it.nextToken() + if err != nil { + return 0, err + } + it.head += 1 + return it.readFloat64(c) +} + +func (it *Iterator) parseFloat64(buf []byte) (ret float64, err error) { + return strconv.ParseFloat(*(*string)(unsafe.Pointer(&buf)), 64) +} + +func (it *Iterator) readFloat64ExponentPart(buf []byte) (ret float64, _ []byte, err error) { + c, err := it.nextByte() + if err != nil { + return 0, buf, err + } + it.head += 1 + if c == '+' || c == '-' { + buf = append(buf, c) + c, err = it.nextByte() + if err != nil { + return 0, buf, err + } + it.head += 1 + } + if intDigits[c] == invalidDigit { + return 0, buf, InvalidFloatError{c: c} + } + buf = append(buf, c) + for { + i := it.head + for ; i < it.tail; i++ { + c = it.buffer[i] + digit := intDigits[c] + if digit == invalidDigit { + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + f, err := it.parseFloat64(buf) + return f, buf, err + } + } + // i == it.tail + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + if err = it.readMore(); err != nil { + if err == io.EOF { + f, err := it.parseFloat64(buf) + return f, buf, err + } + return 0, buf, err + } + } +} + +func (it *Iterator) readFloat64FractionPart(buf []byte) (ret float64, _ []byte, err error) { + c, err := it.nextByte() + if err != nil { + return 0, buf, err + } + if intDigits[c] == invalidDigit { + return 0, buf, InvalidFloatError{c: c} + } + it.head += 1 + buf = append(buf, c) + for { + i := it.head + for ; i < it.tail; i++ { + c = it.buffer[i] + digit := floatDigits[c] + if digit < 0 { + switch digit { + case expInNumber: + buf = append(buf, it.buffer[it.head:i+1]...) + it.head = i + 1 + return it.readFloat64ExponentPart(buf) + default: + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + f, err := it.parseFloat64(buf) + return f, buf, err + } + } + } + // i == it.tail + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + if err = it.readMore(); err != nil { + if err == io.EOF { + f, err := it.parseFloat64(buf) + return f, buf, err + } + return 0, buf, err + } + } +} + +func (it *Iterator) readPositiveFloat64(c byte, buf []byte) (ret float64, _ []byte, err error) { + u := intDigits[c] + if u == invalidDigit { + return 0, buf, InvalidFloatError{c: c} + } + + buf = append(buf, c) + if u == 0 { + if it.head == it.tail { + if err = it.readMore(); err != nil { + if err == io.EOF { + return 0, buf, nil + } + return 0, buf, err + } + } + switch floatDigits[it.buffer[it.head]] { + case dotInNumber: + it.head += 1 + buf = append(buf, '.') + return it.readFloat64FractionPart(buf) + case expInNumber: + it.head += 1 + buf = append(buf, 'e') + return it.readFloat64ExponentPart(buf) + default: + return 0, buf, nil + } + } else { + for { + i := it.head + for ; i < it.tail; i++ { + c = it.buffer[i] + digit := floatDigits[c] + if digit < 0 { + switch digit { + case dotInNumber: + buf = append(buf, it.buffer[it.head:i+1]...) + it.head = i + 1 + return it.readFloat64FractionPart(buf) + case expInNumber: + buf = append(buf, it.buffer[it.head:i+1]...) + it.head = i + 1 + return it.readFloat64ExponentPart(buf) + default: + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + f, err := it.parseFloat64(buf) + return f, buf, err + } + } + } + // i == it.tail + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + if err = it.readMore(); err != nil { + if err == io.EOF { + f, err := it.parseFloat64(buf) + return f, buf, err + } + return 0, buf, err + } + } + } +} diff --git a/iterator_float64_test.go b/iterator_float64_test.go new file mode 100644 index 0000000..013d2e2 --- /dev/null +++ b/iterator_float64_test.go @@ -0,0 +1,220 @@ +package jzon + +import ( + "errors" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Float_ReadFloat64(t *testing.T) { + t.Run("leading space", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.Equal(t, float64(1), f) + }) + t.Run("first byte error", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadFloat64() + require.Equal(t, io.EOF, err) + }) + t.Run("negative error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte{'-'}) + _, err := it.ReadFloat64() + require.Equal(t, io.EOF, err) + }) + t.Run("negative", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-0")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.Equal(t, float64(0), f) + }) + t.Run("negative 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-3.1415926535")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.InDelta(t, float64(-3.1415926535), f, 1e-10) + }) + t.Run("positive", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.Equal(t, float64(0), f) + }) + t.Run("positive 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("3.1415926535")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.InDelta(t, float64(3.1415926535), f, 1e-10) + }) + t.Run("invalid first char", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte{'a'}) + _, err := it.ReadFloat64() + require.IsType(t, InvalidFloatError{}, err) + }) +} + +func TestIterator_Float_ReadFloat64_LeadingZero(t *testing.T) { + t.Run("reader error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte{'0'}, + err: e, + }) + _, err := it.ReadFloat64() + require.Equal(t, e, err) + }) + t.Run("fraction eof after dot", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.")) + _, err := it.ReadFloat64() + require.Equal(t, io.EOF, err) + }) + t.Run("fraction invalid byte after dot", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.a")) + _, err := it.ReadFloat64() + require.IsType(t, InvalidFloatError{}, err) + }) + t.Run("fraction 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.1a")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.InDelta(t, float64(0.1), f, 1e-10) + }) + t.Run("fraction 2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.1")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.InDelta(t, float64(0.1), f, 1e-10) + }) + t.Run("fraction reader error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte("0."), + err: e, + }) + _, err := it.ReadFloat64() + require.Equal(t, e, err) + }) + t.Run("fraction reader error 2", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte("0.1"), + err: e, + }) + _, err := it.ReadFloat64() + require.Equal(t, e, err) + }) + t.Run("exponent eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.1e")) + _, err := it.ReadFloat64() + require.Equal(t, io.EOF, err) + }) + t.Run("exponent eof after sign", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.1e+")) + _, err := it.ReadFloat64() + require.Equal(t, io.EOF, err) + }) + t.Run("exponent invalid byte after sign", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.1e++")) + _, err := it.ReadFloat64() + require.IsType(t, InvalidFloatError{}, err) + }) + t.Run("exponent 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.1e+1")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.InDelta(t, float64(1), f, 1e-10) + }) + t.Run("exponent 2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0.1e+1a")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.InDelta(t, float64(1), f, 1e-10) + }) + t.Run("exponent reader error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte("0.1e+1"), + err: e, + }) + _, err := it.ReadFloat64() + require.Equal(t, e, err) + }) + t.Run("no fraction part", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0e+1")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.Equal(t, float64(0), f) + }) + t.Run("only zero", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0a")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.Equal(t, float64(0), f) + }) + t.Run("double zero", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("00")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.Equal(t, float64(0), f) + }) +} + +func TestIterator_Float_ReadFloat64_NonLeadingZero(t *testing.T) { + t.Run("exponent", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("1e1")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.InDelta(t, float64(10), f, 1e-10) + }) + t.Run("no fraction no exponent 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("10")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.InDelta(t, float64(10), f, 1e-10) + }) + t.Run("no fraction no exponent 2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("10a")) + f, err := it.ReadFloat64() + require.NoError(t, err) + require.InDelta(t, float64(10), f, 1e-10) + }) + t.Run("reader error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte("10"), + err: e, + }) + _, err := it.ReadFloat64() + require.Equal(t, e, err) + }) +} diff --git a/iterator_int.go b/iterator_int.go new file mode 100644 index 0000000..1978da0 --- /dev/null +++ b/iterator_int.go @@ -0,0 +1,51 @@ +package jzon + +import ( + "strconv" +) + +const ( + invalidDigit = -1 +) + +var ( + readInt func(it *Iterator) (int, error) + readUint func(it *Iterator) (uint, error) + intDigits [charNum]int8 +) + +func init() { + if strconv.IntSize == 32 { + readInt = func(it *Iterator) (int, error) { + i, err := it.ReadInt32() + return int(i), err + } + readUint = func(it *Iterator) (uint, error) { + u, err := it.ReadUint32() + return uint(u), err + } + } else { + readInt = func(it *Iterator) (int, error) { + i, err := it.ReadInt64() + return int(i), err + } + readUint = func(it *Iterator) (uint, error) { + u, err := it.ReadUint64() + return uint(u), err + } + } + for i := 0; i < charNum; i++ { + intDigits[i] = invalidDigit + } + for i := '0'; i <= '9'; i++ { + intDigits[i] = int8(i - '0') + } +} + +func (it *Iterator) ReadInt() (int, error) { + return readInt(it) +} + +func (it *Iterator) ReadUint() (uint, error) { + return readUint(it) +} diff --git a/iterator_int16.go b/iterator_int16.go new file mode 100644 index 0000000..7d4006c --- /dev/null +++ b/iterator_int16.go @@ -0,0 +1,99 @@ +package jzon + +import ( + "io" + "math" + "strconv" +) + +const ( + maxUint16Div10NumDigits = 4 + maxUint16Div10 = uint16(math.MaxUint16) / 10 + maxUint16Mod10 = int8(math.MaxUint16 - maxUint16Div10*10) +) + +func (it *Iterator) ReadUint16() (uint16, error) { + c, _, err := it.nextToken() + if err != nil { + return 0, err + } + it.head += 1 + return it.readUint16(c) +} + +func (it *Iterator) readUint16(c byte) (ret uint16, err error) { + u := intDigits[c] + if u == 0 { + return 0, nil + } + if u == invalidDigit { + return 0, InvalidDigitError{c: c} + } + ret = uint16(u) + numDigit := 1 + for { + i := it.head + for ; i < it.tail; i++ { + digit := intDigits[it.buffer[i]] + if digit == invalidDigit { + it.head = i + return ret, nil + } + if numDigit >= maxUint16Div10NumDigits { + if ret > maxUint16Div10 || + (ret == maxUint16Div10 && digit > maxUint16Mod10) { + it.head = i + err = IntOverflowError{} + return + } + } + ret = (ret << 3) + (ret << 1) + uint16(digit) + numDigit++ + } + it.head = i + if err = it.readMore(); err != nil { + if err == io.EOF { + err = nil + } + return + } + } +} + +func (it *Iterator) ReadInt16() (int16, error) { + c, _, err := it.nextToken() + if err != nil { + return 0, err + } + it.head += 1 + if c == '-' { + c, err = it.nextByte() + if err != nil { + return 0, err + } + it.head += 1 + v, err := it.readUint16(c) + if err != nil { + return 0, err + } + if v > math.MaxInt16+1 { + return 0, IntOverflowError{ + typ: "int16", + value: "-" + strconv.FormatUint(uint64(v), 10), + } + } + return -int16(v), nil + } else { + v, err := it.readUint16(c) + if err != nil { + return 0, err + } + if v > math.MaxInt16 { + return 0, IntOverflowError{ + typ: "int16", + value: strconv.FormatUint(uint64(v), 10), + } + } + return int16(v), nil + } +} diff --git a/iterator_int16_test.go b/iterator_int16_test.go new file mode 100644 index 0000000..b1cba76 --- /dev/null +++ b/iterator_int16_test.go @@ -0,0 +1,146 @@ +package jzon + +import ( + "fmt" + "io" + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Int_ReadUint16(t *testing.T) { + t.Run("leading space", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1")) + u, err := it.ReadUint16() + require.NoError(t, err) + require.Equal(t, uint16(1), u) + }) + t.Run("invalid first byte", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadUint16() + require.Equal(t, io.EOF, err) + }) + t.Run("zero", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0")) + i, err := it.ReadUint16() + require.NoError(t, err) + require.Equal(t, uint16(0), i) + }) + t.Run("invalid first digit", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-")) + _, err := it.ReadUint16() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("early return", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("1a")) + u, err := it.ReadUint16() + require.NoError(t, err) + require.Equal(t, uint16(1), u) + }) + t.Run("overflow 1", func(t *testing.T) { + it := NewIterator() + s := fmt.Sprintf("%d0", math.MaxUint16/10+1) + it.ResetBytes([]byte(s)) + _, err := it.ReadUint16() + require.IsType(t, IntOverflowError{}, err) + }) + t.Run("overflow 2", func(t *testing.T) { + it := NewIterator() + d := math.MaxUint16 / 10 + m := math.MaxUint16 - d*10 + s := fmt.Sprintf("%d%d", d, m+1) + it.ResetBytes([]byte(s)) + _, err := it.ReadUint16() + require.IsType(t, IntOverflowError{}, err) + }) + t.Run("max uint16", func(t *testing.T) { + it := NewIterator() + var m uint16 = math.MaxUint16 + it.ResetBytes([]byte(fmt.Sprint(m))) + u, err := it.ReadUint16() + require.NoError(t, err) + require.Equal(t, m, u) + }) + t.Run("reader", func(t *testing.T) { + it := NewIterator() + var m uint16 = math.MaxUint16 + it.Reset(&oneByteReader{ + b: []byte(fmt.Sprint(m)), + }) + u, err := it.ReadUint16() + require.NoError(t, err) + require.Equal(t, m, u) + }) +} + +func TestIterator_Int_ReadInt16(t *testing.T) { + t.Run("leading space", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" -1")) + i, err := it.ReadInt16() + require.NoError(t, err) + require.Equal(t, int16(-1), i) + }) + t.Run("first byte error", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadInt16() + require.Equal(t, io.EOF, err) + }) + t.Run("negative error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-")) + _, err := it.ReadInt16() + require.Equal(t, io.EOF, err) + }) + t.Run("negative readUint16 error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-a")) + _, err := it.ReadInt16() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("negative overflow", func(t *testing.T) { + it := NewIterator() + s := fmt.Sprint(-math.MaxInt16 - 2) + it.ResetBytes([]byte(s)) + _, err := it.ReadInt16() + e, ok := err.(IntOverflowError) + require.True(t, ok) + require.Equal(t, "int16", e.typ) + require.Equal(t, s, e.value) + }) + t.Run("negative max", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(fmt.Sprint(-math.MaxInt16 - 1))) + i, err := it.ReadInt16() + require.NoError(t, err) + require.Equal(t, int16(-math.MaxInt16-1), i) + }) + t.Run("positive readUint16 error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("a")) + _, err := it.ReadInt16() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("positive overflow", func(t *testing.T) { + it := NewIterator() + s := fmt.Sprint(math.MaxInt16 + 1) + it.ResetBytes([]byte(s)) + _, err := it.ReadInt16() + e, ok := err.(IntOverflowError) + require.True(t, ok) + require.Equal(t, "int16", e.typ) + require.Equal(t, s, e.value) + }) + t.Run("positive max", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(fmt.Sprint(math.MaxInt16))) + i, err := it.ReadInt16() + require.NoError(t, err) + require.Equal(t, int16(math.MaxInt16), i) + }) +} diff --git a/iterator_int32.go b/iterator_int32.go new file mode 100644 index 0000000..2e011f8 --- /dev/null +++ b/iterator_int32.go @@ -0,0 +1,99 @@ +package jzon + +import ( + "io" + "math" + "strconv" +) + +const ( + maxUint32Div10NumDigits = 9 + maxUint32Div10 = uint32(math.MaxUint32) / 10 + maxUint32Mod10 = int8(math.MaxUint32 - maxUint32Div10*10) +) + +func (it *Iterator) ReadUint32() (uint32, error) { + c, _, err := it.nextToken() + if err != nil { + return 0, err + } + it.head += 1 + return it.readUint32(c) +} + +func (it *Iterator) readUint32(c byte) (ret uint32, err error) { + u := intDigits[c] + if u == 0 { + return 0, nil + } + if u == invalidDigit { + return 0, InvalidDigitError{c: c} + } + ret = uint32(u) + numDigit := 1 + for { + i := it.head + for ; i < it.tail; i++ { + digit := intDigits[it.buffer[i]] + if digit == invalidDigit { + it.head = i + return + } + if numDigit >= maxUint32Div10NumDigits { + if ret > maxUint32Div10 || + (ret == maxUint32Div10 && digit > maxUint32Mod10) { + it.head = i + err = IntOverflowError{} + return + } + } + ret = (ret << 3) + (ret << 1) + uint32(digit) + numDigit++ + } + it.head = i + if err = it.readMore(); err != nil { + if err == io.EOF { + err = nil + } + return + } + } +} + +func (it *Iterator) ReadInt32() (int32, error) { + c, _, err := it.nextToken() + if err != nil { + return 0, err + } + it.head += 1 + if c == '-' { + c, err = it.nextByte() + if err != nil { + return 0, err + } + it.head += 1 + v, err := it.readUint32(c) + if err != nil { + return 0, err + } + if v > math.MaxInt32+1 { + return 0, IntOverflowError{ + typ: "int32", + value: "-" + strconv.FormatUint(uint64(v), 10), + } + } + return -int32(v), nil + } else { + v, err := it.readUint32(c) + if err != nil { + return 0, err + } + if v > math.MaxInt32 { + return 0, IntOverflowError{ + typ: "int32", + value: strconv.FormatUint(uint64(v), 10), + } + } + return int32(v), nil + } +} diff --git a/iterator_int32_test.go b/iterator_int32_test.go new file mode 100644 index 0000000..8b84906 --- /dev/null +++ b/iterator_int32_test.go @@ -0,0 +1,146 @@ +package jzon + +import ( + "fmt" + "io" + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Int_ReadUint32(t *testing.T) { + t.Run("leading space", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1")) + u, err := it.ReadUint32() + require.NoError(t, err) + require.Equal(t, uint32(1), u) + }) + t.Run("invalid first byte", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadUint32() + require.Equal(t, io.EOF, err) + }) + t.Run("zero", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0")) + i, err := it.ReadUint32() + require.NoError(t, err) + require.Equal(t, uint32(0), i) + }) + t.Run("invalid first digit", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-")) + _, err := it.ReadUint32() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("early return", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("1a")) + u, err := it.ReadUint32() + require.NoError(t, err) + require.Equal(t, uint32(1), u) + }) + t.Run("overflow 1", func(t *testing.T) { + it := NewIterator() + s := fmt.Sprintf("%d0", math.MaxUint32/10+1) + it.ResetBytes([]byte(s)) + _, err := it.ReadUint32() + require.IsType(t, IntOverflowError{}, err) + }) + t.Run("overflow 2", func(t *testing.T) { + it := NewIterator() + d := math.MaxUint32 / 10 + m := math.MaxUint32 - d*10 + s := fmt.Sprintf("%d%d", d, m+1) + it.ResetBytes([]byte(s)) + _, err := it.ReadUint32() + require.IsType(t, IntOverflowError{}, err) + }) + t.Run("max uint32", func(t *testing.T) { + it := NewIterator() + var m uint32 = math.MaxUint32 + it.ResetBytes([]byte(fmt.Sprint(m))) + u, err := it.ReadUint32() + require.NoError(t, err) + require.Equal(t, m, u) + }) + t.Run("reader", func(t *testing.T) { + it := NewIterator() + var m uint32 = math.MaxUint32 + it.Reset(&oneByteReader{ + b: []byte(fmt.Sprint(m)), + }) + u, err := it.ReadUint32() + require.NoError(t, err) + require.Equal(t, m, u) + }) +} + +func TestIterator_Int_ReadInt32(t *testing.T) { + t.Run("leading space", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" -1")) + i, err := it.ReadInt32() + require.NoError(t, err) + require.Equal(t, int32(-1), i) + }) + t.Run("first byte error", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadInt32() + require.Equal(t, io.EOF, err) + }) + t.Run("negative error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-")) + _, err := it.ReadInt32() + require.Equal(t, io.EOF, err) + }) + t.Run("negative readUint32 error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-a")) + _, err := it.ReadInt32() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("negative overflow", func(t *testing.T) { + it := NewIterator() + s := fmt.Sprint(-math.MaxInt32 - 2) + it.ResetBytes([]byte(s)) + _, err := it.ReadInt32() + e, ok := err.(IntOverflowError) + require.True(t, ok) + require.Equal(t, "int32", e.typ) + require.Equal(t, s, e.value) + }) + t.Run("negative max", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(fmt.Sprint(-math.MaxInt32 - 1))) + i, err := it.ReadInt32() + require.NoError(t, err) + require.Equal(t, int32(-math.MaxInt32-1), i) + }) + t.Run("positive readUint32 error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("a")) + _, err := it.ReadInt32() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("positive overflow", func(t *testing.T) { + it := NewIterator() + s := fmt.Sprint(math.MaxInt32 + 1) + it.ResetBytes([]byte(s)) + _, err := it.ReadInt32() + e, ok := err.(IntOverflowError) + require.True(t, ok) + require.Equal(t, "int32", e.typ) + require.Equal(t, s, e.value) + }) + t.Run("positive max", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(fmt.Sprint(math.MaxInt32))) + i, err := it.ReadInt32() + require.NoError(t, err) + require.Equal(t, int32(math.MaxInt32), i) + }) +} diff --git a/iterator_int64.go b/iterator_int64.go new file mode 100644 index 0000000..9a57578 --- /dev/null +++ b/iterator_int64.go @@ -0,0 +1,100 @@ +package jzon + +import ( + "io" + "math" + "strconv" +) + +const ( + maxUint64Div10NumDigits = 19 + maxUint64Div10 = uint64(math.MaxUint64) / 10 + maxUint64Mod10 = int8(math.MaxUint64 - maxUint64Div10*10) +) + +func (it *Iterator) ReadUint64() (uint64, error) { + c, _, err := it.nextToken() + if err != nil { + return 0, err + } + it.head += 1 + return it.readUint64(c) +} + +func (it *Iterator) readUint64(c byte) (ret uint64, err error) { + u := intDigits[c] + if u == 0 { + return 0, nil + } + if u == invalidDigit { + return 0, InvalidDigitError{c: c} + } + ret = uint64(u) + numDigit := 1 + // TODO: inline expansion + for { + i := it.head + for ; i < it.tail; i++ { + digit := intDigits[it.buffer[i]] + if digit == invalidDigit { + it.head = i + return ret, nil + } + if numDigit >= maxUint64Div10NumDigits { + if ret > maxUint64Div10 || + (ret == maxUint64Div10 && digit > maxUint64Mod10) { + it.head = i + err = IntOverflowError{} + return + } + } + ret = (ret << 3) + (ret << 1) + uint64(digit) + numDigit++ + } + it.head = i + if err = it.readMore(); err != nil { + if err == io.EOF { + err = nil + } + return + } + } +} + +func (it *Iterator) ReadInt64() (int64, error) { + c, _, err := it.nextToken() + if err != nil { + return 0, err + } + it.head += 1 + if c == '-' { + c, err = it.nextByte() + if err != nil { + return 0, err + } + it.head += 1 + v, err := it.readUint64(c) + if err != nil { + return 0, err + } + if v > math.MaxInt64+1 { + return 0, IntOverflowError{ + typ: "int64", + value: "-" + strconv.FormatUint(uint64(v), 10), + } + } + return -int64(v), nil + } else { + v, err := it.readUint64(c) + if err != nil { + return 0, err + } + if v > math.MaxInt64 { + return 0, IntOverflowError{ + typ: "int64", + value: strconv.FormatUint(uint64(v), 10), + } + } + return int64(v), nil + } +} diff --git a/iterator_int64_test.go b/iterator_int64_test.go new file mode 100644 index 0000000..b5a7ac7 --- /dev/null +++ b/iterator_int64_test.go @@ -0,0 +1,154 @@ +package jzon + +import ( + "fmt" + "io" + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Int_ReadUint64(t *testing.T) { + t.Run("leading space", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1")) + u, err := it.ReadUint64() + require.NoError(t, err) + require.Equal(t, uint64(1), u) + }) + t.Run("invalid first byte", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadUint64() + require.Equal(t, io.EOF, err) + }) + t.Run("zero", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0")) + i, err := it.ReadUint64() + require.NoError(t, err) + require.Equal(t, uint64(0), i) + }) + t.Run("invalid first digit", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-")) + _, err := it.ReadUint64() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("early return", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("1a")) + u, err := it.ReadUint64() + require.NoError(t, err) + require.Equal(t, uint64(1), u) + }) + t.Run("overflow 1", func(t *testing.T) { + it := NewIterator() + s := fmt.Sprintf("%d0", math.MaxUint64/10+1) + it.ResetBytes([]byte(s)) + _, err := it.ReadUint64() + require.IsType(t, IntOverflowError{}, err) + }) + t.Run("overflow 2", func(t *testing.T) { + it := NewIterator() + d := uint64(math.MaxUint64) / 10 + m := uint64(math.MaxUint64) - d*10 + 1 + require.Less(t, m, uint64(10)) + s := fmt.Sprintf("%d%d", d, m) + it.ResetBytes([]byte(s)) + _, err := it.ReadUint64() + require.IsType(t, IntOverflowError{}, err) + }) + t.Run("max uint64", func(t *testing.T) { + it := NewIterator() + var m uint64 = math.MaxUint64 + it.ResetBytes([]byte(fmt.Sprint(m))) + u, err := it.ReadUint64() + require.NoError(t, err) + require.Equal(t, m, u) + }) + t.Run("reader", func(t *testing.T) { + it := NewIterator() + var m uint64 = math.MaxUint64 + it.Reset(&oneByteReader{ + b: []byte(fmt.Sprint(m)), + }) + u, err := it.ReadUint64() + require.NoError(t, err) + require.Equal(t, m, u) + }) +} + +func TestIterator_Int_ReadInt64(t *testing.T) { + t.Run("leading space", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" -1")) + i, err := it.ReadInt64() + require.NoError(t, err) + require.Equal(t, int64(-1), i) + }) + t.Run("first byte error", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadInt64() + require.Equal(t, io.EOF, err) + }) + t.Run("negative error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-")) + _, err := it.ReadInt64() + require.Equal(t, io.EOF, err) + }) + t.Run("negative readUint64 error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-a")) + _, err := it.ReadInt64() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("negative overflow", func(t *testing.T) { + it := NewIterator() + d := int64(math.MaxInt64) / 10 + m := int64(math.MaxInt64) - d*10 + 2 + require.Less(t, m, int64(10)) + s := fmt.Sprintf("-%d%d", d, m) + it.ResetBytes([]byte(s)) + _, err := it.ReadInt64() + e, ok := err.(IntOverflowError) + require.True(t, ok) + require.Equal(t, "int64", e.typ) + require.Equal(t, s, e.value) + }) + t.Run("negative max", func(t *testing.T) { + it := NewIterator() + v := int64(-math.MaxInt64 - 1) + it.ResetBytes([]byte(fmt.Sprint(v))) + i, err := it.ReadInt64() + require.NoError(t, err) + require.Equal(t, v, i) + }) + t.Run("positive readUint64 error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("a")) + _, err := it.ReadInt64() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("positive overflow", func(t *testing.T) { + it := NewIterator() + d := int64(math.MaxInt64 / 10) + m := int64(math.MaxInt64-d*10) + 1 + require.Less(t, m, int64(10)) + s := fmt.Sprintf("%d%d", d, m) + it.ResetBytes([]byte(s)) + _, err := it.ReadInt64() + e, ok := err.(IntOverflowError) + require.True(t, ok) + require.Equal(t, "int64", e.typ) + require.Equal(t, s, e.value) + }) + t.Run("positive max", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(fmt.Sprint(math.MaxInt64))) + i, err := it.ReadInt64() + require.NoError(t, err) + require.Equal(t, int64(math.MaxInt64), i) + }) +} diff --git a/iterator_int8.go b/iterator_int8.go new file mode 100644 index 0000000..ff1ed37 --- /dev/null +++ b/iterator_int8.go @@ -0,0 +1,104 @@ +package jzon + +import ( + "io" + "math" + "strconv" +) + +const ( + maxUint8Div10 = uint8(math.MaxUint8) / 10 + maxUint8Mod10 = int8(math.MaxUint8 - maxUint8Div10*10) +) + +func (it *Iterator) ReadUint8() (uint8, error) { + c, _, err := it.nextToken() + if err != nil { + return 0, err + } + it.head += 1 + return it.readUint8(c) +} + +func (it *Iterator) readUint8(c byte) (ret uint8, err error) { + u := intDigits[c] + if u == 0 { + return 0, nil + } + if u == invalidDigit { + return 0, InvalidDigitError{c: c} + } + ret = uint8(u) + if it.head == it.tail { + if err = it.readMore(); err != nil { + if err == io.EOF { + err = nil + } + return + } + } + u = intDigits[it.buffer[it.head]] // second digit + if u == invalidDigit { + return + } + ret = (ret << 3) + (ret << 1) + uint8(u) + it.head += 1 + if it.head == it.tail { + if err = it.readMore(); err != nil { + if err == io.EOF { + err = nil + } + return + } + } + u = intDigits[it.buffer[it.head]] // third digit + if u == invalidDigit { + return + } + it.head += 1 + if ret > maxUint8Div10 || + (ret == maxUint8Div10 && u > maxUint8Mod10) { + err = IntOverflowError{} + return + } + ret = (ret << 3) + (ret << 1) + uint8(u) + return +} + +func (it *Iterator) ReadInt8() (int8, error) { + c, _, err := it.nextToken() + if err != nil { + return 0, err + } + it.head += 1 + if c == '-' { + c, err = it.nextByte() + if err != nil { + return 0, err + } + it.head += 1 + v, err := it.readUint8(c) + if err != nil { + return 0, err + } + if v > math.MaxInt8+1 { + return 0, IntOverflowError{ + typ: "int8", + value: "-" + strconv.FormatUint(uint64(v), 10), + } + } + return -int8(v), nil + } else { + v, err := it.readUint8(c) + if err != nil { + return 0, err + } + if v > math.MaxInt8 { + return 0, IntOverflowError{ + typ: "int8", + value: strconv.FormatUint(uint64(v), 10), + } + } + return int8(v), nil + } +} diff --git a/iterator_int8_test.go b/iterator_int8_test.go new file mode 100644 index 0000000..c67f642 --- /dev/null +++ b/iterator_int8_test.go @@ -0,0 +1,162 @@ +package jzon + +import ( + "fmt" + "io" + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Int_ReadUint8(t *testing.T) { + t.Run("leading space", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1")) + u, err := it.ReadUint8() + require.NoError(t, err) + require.Equal(t, uint8(1), u) + }) + t.Run("invalid first byte", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadUint8() + require.Equal(t, io.EOF, err) + }) + t.Run("zero", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("0")) + i, err := it.ReadUint8() + require.NoError(t, err) + require.Equal(t, uint8(0), i) + }) + t.Run("invalid first digit", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-")) + _, err := it.ReadUint8() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("early return", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("1a")) + u, err := it.ReadUint8() + require.NoError(t, err) + require.Equal(t, uint8(1), u) + }) + t.Run("early return2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("12a")) + u, err := it.ReadUint8() + require.NoError(t, err) + require.Equal(t, uint8(12), u) + }) + t.Run("overflow 1", func(t *testing.T) { + it := NewIterator() + s := fmt.Sprintf("%d0", math.MaxUint8/10+1) + it.ResetBytes([]byte(s)) + _, err := it.ReadUint8() + require.IsType(t, IntOverflowError{}, err) + }) + t.Run("overflow 2", func(t *testing.T) { + it := NewIterator() + d := math.MaxUint8 / 10 + m := math.MaxUint8 - d*10 + s := fmt.Sprintf("%d%d", d, m+1) + it.ResetBytes([]byte(s)) + _, err := it.ReadUint8() + require.IsType(t, IntOverflowError{}, err) + }) + t.Run("max uint8", func(t *testing.T) { + it := NewIterator() + var m uint8 = math.MaxUint8 + it.ResetBytes([]byte(fmt.Sprint(m))) + u, err := it.ReadUint8() + require.NoError(t, err) + require.Equal(t, m, u) + }) + t.Run("reader", func(t *testing.T) { + it := NewIterator() + var m uint8 = math.MaxUint8 + it.Reset(&oneByteReader{ + b: []byte(fmt.Sprint(m)), + }) + u, err := it.ReadUint8() + require.NoError(t, err) + require.Equal(t, m, u) + }) + t.Run("all values", func(t *testing.T) { + it := NewIterator() + for i := 0; i <= math.MaxUint8; i++ { + it.ResetBytes([]byte(fmt.Sprint(i))) + j, err := it.ReadUint8() + require.NoError(t, err, "value %d", i) + require.Equal(t, uint8(i), j) + } + }) +} + +func TestIterator_Int_ReadInt8(t *testing.T) { + t.Run("leading space", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" -1")) + i, err := it.ReadInt8() + require.NoError(t, err) + require.Equal(t, int8(-1), i) + }) + t.Run("first byte error", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadInt8() + require.Equal(t, io.EOF, err) + }) + t.Run("negative error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-")) + _, err := it.ReadInt8() + require.Equal(t, io.EOF, err) + }) + t.Run("negative readUint8 error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("-a")) + _, err := it.ReadInt8() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("negative overflow", func(t *testing.T) { + it := NewIterator() + s := fmt.Sprint(-math.MaxInt8 - 2) + it.ResetBytes([]byte(s)) + _, err := it.ReadInt8() + e, ok := err.(IntOverflowError) + require.True(t, ok) + require.Equal(t, "int8", e.typ) + require.Equal(t, s, e.value) + }) + t.Run("negative max", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(fmt.Sprint(-math.MaxInt8 - 1))) + i, err := it.ReadInt8() + require.NoError(t, err) + require.Equal(t, int8(-math.MaxInt8-1), i) + }) + t.Run("positive readUint8 error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("a")) + _, err := it.ReadInt8() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("positive overflow", func(t *testing.T) { + it := NewIterator() + s := fmt.Sprint(math.MaxInt8 + 1) + it.ResetBytes([]byte(s)) + _, err := it.ReadInt8() + e, ok := err.(IntOverflowError) + require.True(t, ok) + require.Equal(t, "int8", e.typ) + require.Equal(t, s, e.value) + }) + t.Run("positive max", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(fmt.Sprint(math.MaxInt8))) + i, err := it.ReadInt8() + require.NoError(t, err) + require.Equal(t, int8(math.MaxInt8), i) + }) +} diff --git a/iterator_int_test.go b/iterator_int_test.go new file mode 100644 index 0000000..20b8bc0 --- /dev/null +++ b/iterator_int_test.go @@ -0,0 +1 @@ +package jzon diff --git a/iterator_null.go b/iterator_null.go new file mode 100644 index 0000000..96add6d --- /dev/null +++ b/iterator_null.go @@ -0,0 +1,13 @@ +package jzon + +func (it *Iterator) ReadNull() error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c != 'n' { + return UnexpectedByteError{got: c, exp: 'n'} + } + it.head += 1 + return it.expectBytes("ull") +} diff --git a/iterator_null_test.go b/iterator_null_test.go new file mode 100644 index 0000000..130b4d4 --- /dev/null +++ b/iterator_null_test.go @@ -0,0 +1,42 @@ +package jzon + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Null_ReadNull(t *testing.T) { + t.Run("eof", func(t *testing.T) { + it := NewIterator() + err := it.ReadNull() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid first byte", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" a")) + err := it.ReadNull() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("error 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" n")) + err := it.ReadNull() + require.Equal(t, io.EOF, err) + }) + t.Run("error 2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" na")) + err := it.ReadNull() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("valid", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" null ")) + err := it.ReadNull() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) +} diff --git a/iterator_number.go b/iterator_number.go new file mode 100644 index 0000000..e57fa8c --- /dev/null +++ b/iterator_number.go @@ -0,0 +1,31 @@ +package jzon + +// do not step forward it.head before calling this +func (it *Iterator) readNumberAsString(c byte) (n string, err error) { + // start capture + oldCapture := it.capture + it.capture = true + + begin := it.head // save current location + it.head += 1 + err = skipNumber(it, c) + if err == nil { + n = string(it.buffer[begin:it.head]) + } + // end capture + it.capture = oldCapture + return +} + +func (it *Iterator) ReadNumber() (n Number, err error) { + c, vt, err := it.nextToken() + if err != nil { + return + } + if vt != NumberValue { + err = UnexpectedByteError{got: c} + return + } + s, err := it.readNumberAsString(c) + return Number(s), err +} diff --git a/iterator_number_test.go b/iterator_number_test.go new file mode 100644 index 0000000..800b51b --- /dev/null +++ b/iterator_number_test.go @@ -0,0 +1,278 @@ +package jzon + +import ( + "errors" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Number_ReadNumber(t *testing.T) { + t.Run("eof", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadNumber() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid type", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" +")) + _, err := it.ReadNumber() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("negative eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" -")) + _, err := it.ReadNumber() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid first byte after dash", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" --")) + _, err := it.ReadNumber() + require.IsType(t, InvalidDigitError{}, err) + }) +} + +func TestIterator_Number_ReadNumber_LeadingZero(t *testing.T) { + t.Run("one zero", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "0", string(n)) + }) + t.Run("reader error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte(" 0"), + err: e, + }) + _, err := it.ReadNumber() + require.Equal(t, e, err) + }) + t.Run("double zero", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 00")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "0", string(n)) + }) + t.Run("fraction eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0.")) + _, err := it.ReadNumber() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid fraction", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0.a")) + _, err := it.ReadNumber() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("fraction end with eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0.12")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "0.12", string(n)) + }) + t.Run("fraction end with other char", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0.12 ")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "0.12", string(n)) + }) + t.Run("fraction error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte(" 0.12"), + err: e, + }) + _, err := it.ReadNumber() + require.Equal(t, e, err) + }) + t.Run("exponent eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0.1e")) + _, err := it.ReadNumber() + require.Equal(t, io.EOF, err) + }) + t.Run("exponent eof after sign", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0.1e+")) + _, err := it.ReadNumber() + require.Equal(t, io.EOF, err) + }) + t.Run("exponent invalid byte after sign", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0.1e++")) + _, err := it.ReadNumber() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("exponent end with eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0.1e+2")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "0.1e+2", string(n)) + }) + t.Run("exponent end with another char", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0.1e+2 ")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "0.1e+2", string(n)) + }) + t.Run("exponent end error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte(" 0.1e+2"), + err: e, + }) + _, err := it.ReadNumber() + require.Equal(t, e, err) + }) + t.Run("exponent only", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0e+1 ")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "0e+1", string(n)) + }) +} + +func TestIterator_Number_ReadNumber_NonLeadingZero(t *testing.T) { + t.Run("integer", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "1", string(n)) + }) + t.Run("reader error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte(" 1"), + err: e, + }) + _, err := it.ReadNumber() + require.Equal(t, e, err) + }) + t.Run("double digit", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 12")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "12", string(n)) + }) + t.Run("double digit end with other char", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 12 ")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "12", string(n)) + }) + t.Run("fraction eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1.")) + _, err := it.ReadNumber() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid fraction", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1.+")) + _, err := it.ReadNumber() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("fraction end with eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1.23")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "1.23", string(n)) + }) + t.Run("fraction end with other char", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1.23 ")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "1.23", string(n)) + }) + t.Run("fraction error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte(" 1.23"), + err: e, + }) + _, err := it.ReadNumber() + require.Equal(t, e, err) + }) + t.Run("exponent eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1.2e")) + _, err := it.ReadNumber() + require.Equal(t, io.EOF, err) + }) + t.Run("exponent eof after sign", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1.2e+")) + _, err := it.ReadNumber() + require.Equal(t, io.EOF, err) + }) + t.Run("exponent invalid byte after sign", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1.2e++")) + _, err := it.ReadNumber() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("exponent end with eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1.2e+3")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "1.2e+3", string(n)) + }) + t.Run("exponent end with another char", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1.2e+3 ")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "1.2e+3", string(n)) + }) + t.Run("exponent end error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte(" 1.2e+3"), + err: e, + }) + _, err := it.ReadNumber() + require.Equal(t, e, err) + }) + t.Run("exponent only", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1e+2 ")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, "1e+2", string(n)) + }) +} + +func TestIterator_Number_ReadNumber_LargeNumber(t *testing.T) { + it := NewIterator() + s := "-" + strings.Repeat("123", 20) + "." + + strings.Repeat("0456", 20) + "e+" + + strings.Repeat("789", 20) + it.ResetBytes([]byte(" " + s + " ")) + n, err := it.ReadNumber() + require.NoError(t, err) + require.Equal(t, s, string(n)) +} diff --git a/iterator_object.go b/iterator_object.go new file mode 100644 index 0000000..372f4d6 --- /dev/null +++ b/iterator_object.go @@ -0,0 +1,143 @@ +package jzon + +// use after the first `"` is consumed +// will read the object field as well as the colon +// more must be true when err is nil +func (it *Iterator) readObjectFieldAsSlice(buf []byte, caseSensitive bool) ( + more bool, field []byte, err error) { + field, err = it.readStringAsSlice(buf, caseSensitive) + if err != nil { + return + } + c, _, err := it.nextToken() + if err != nil { + return + } + if c != ':' { + err = UnexpectedByteError{got: c, exp: ':'} + return + } + it.head += 1 + more = true + return +} + +func (it *Iterator) readObjectField() (bool, string, error) { + more, field, err := it.readObjectFieldAsSlice(it.tmpBuffer[:0], true) + it.tmpBuffer = field + return more, string(field), err +} + +func (it *Iterator) skipObjectField() (bool, error) { + more, field, err := it.readObjectFieldAsSlice(it.tmpBuffer[:0], true) + it.tmpBuffer = field + return more, err +} + +func (it *Iterator) ReadObjectBegin() (more bool, field string, err error) { + c, _, err := it.nextToken() + if err != nil { + return + } + if c != '{' { + err = UnexpectedByteError{got: c, exp: '{'} + return + } + it.head += 1 + c, _, err = it.nextToken() + if err != nil { + return + } + switch c { + case '}': + // no more items + it.head += 1 + return + case '"': + it.head += 1 + return it.readObjectField() + default: + err = UnexpectedByteError{got: c, exp: '}', exp2: '"'} + return + } +} + +func (it *Iterator) ReadObjectMore() (more bool, field string, err error) { + c, _, err := it.nextToken() + if err != nil { + return + } + switch c { + case '}': + it.head += 1 + return + case ',': + it.head += 1 + c, _, err = it.nextToken() + if err != nil { + return + } + if c != '"' { + err = UnexpectedByteError{got: c, exp: '"'} + return + } + it.head += 1 + return it.readObjectField() + default: + err = UnexpectedByteError{got: c, exp: '}', exp2: ','} + return + } +} + +func (it *Iterator) ReadObjectCB(cb func(it *Iterator, field string) error) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c != '{' { + return UnexpectedByteError{got: c, exp: '{'} + } + it.head += 1 + c, _, err = it.nextToken() + if err != nil { + return err + } + if c == '}' { + it.head += 1 + return nil + } + if c != '"' { + return UnexpectedByteError{got: c, exp: '"', exp2: '}'} + } + it.head += 1 + for { + _, field, err := it.readObjectField() + if err != nil { + return err + } + if err := cb(it, field); err != nil { + return err + } + c, _, err = it.nextToken() + if err != nil { + return err + } + switch c { + case '}': + it.head += 1 + return nil + case ',': + it.head += 1 + c, _, err = it.nextToken() + if err != nil { + return err + } + if c != '"' { + return UnexpectedByteError{got: c, exp: '"'} + } + it.head += 1 + default: + return UnexpectedByteError{got: c, exp: '}', exp2: ','} + } + } +} diff --git a/iterator_object_test.go b/iterator_object_test.go new file mode 100644 index 0000000..4dcd2fd --- /dev/null +++ b/iterator_object_test.go @@ -0,0 +1,293 @@ +package jzon + +import ( + "errors" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Object_ReadObjectBegin(t *testing.T) { + t.Run("eof", func(t *testing.T) { + it := NewIterator() + _, _, err := it.ReadObjectBegin() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid first byte", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte{'"'}) + _, _, err := it.ReadObjectBegin() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after first byte", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte{'{'}) + _, _, err := it.ReadObjectBegin() + require.Equal(t, io.EOF, err) + }) + t.Run("empty object", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" { } ")) + more, _, err := it.ReadObjectBegin() + require.NoError(t, err) + require.False(t, more) + }) + t.Run("invalid", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { "" `)) + _, _, err := it.ReadObjectBegin() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid colon", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { "" a `)) + _, _, err := it.ReadObjectBegin() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("valid", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { "" : `)) + more, field, err := it.ReadObjectBegin() + require.NoError(t, err) + require.True(t, more) + require.Equal(t, "", field) + }) + t.Run("invalid second token", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { 1 `)) + _, _, err := it.ReadObjectBegin() + require.IsType(t, UnexpectedByteError{}, err) + }) +} + +func TestIterator_Object_ReadObjectMore(t *testing.T) { + init := func(t *testing.T, s string) *Iterator { + it := NewIterator() + buf := append([]byte(` { "k" : 2 `), s...) + it.ResetBytes(buf) + more, field, err := it.ReadObjectBegin() + require.NoError(t, err) + require.True(t, more) + require.Equal(t, "k", field) + + i, err := it.ReadInt() + require.NoError(t, err) + require.Equal(t, 2, i) + return it + } + t.Run("eof", func(t *testing.T) { + it := init(t, "") + _, _, err := it.ReadObjectMore() + require.Equal(t, io.EOF, err) + }) + t.Run("valid ending", func(t *testing.T) { + it := init(t, "}") + more, _, err := it.ReadObjectMore() + require.NoError(t, err) + require.False(t, more) + }) + t.Run("eof after comma", func(t *testing.T) { + it := init(t, `, `) + _, _, err := it.ReadObjectMore() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid byte after comma", func(t *testing.T) { + it := init(t, `, a`) + _, _, err := it.ReadObjectMore() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("valid", func(t *testing.T) { + it := init(t, `, "k2" : `) + more, field, err := it.ReadObjectMore() + require.NoError(t, err) + require.True(t, more) + require.Equal(t, "k2", field) + }) + t.Run("invalid byte", func(t *testing.T) { + it := init(t, ` a`) + _, _, err := it.ReadObjectMore() + require.IsType(t, UnexpectedByteError{}, err) + }) +} + +func TestIterator_Object_ReadObject_Example(t *testing.T) { + must := require.New(t) + + it := NewIterator() + it.ResetBytes([]byte(` { "key" : "value" , "key2" : 1 } `)) + more, field, err := it.ReadObjectBegin() + must.NoError(err) + must.True(more) + must.Equal("key", field) + + s, err := it.ReadString() + must.NoError(err) + must.Equal("value", s) + + more, field, err = it.ReadObjectMore() + must.NoError(err) + must.True(more) + must.Equal("key2", field) + + i, err := it.ReadInt() + must.NoError(err) + must.Equal(1, i) + + more, _, err = it.ReadObjectMore() + must.NoError(err) + must.False(more) + + _, err = it.NextValueType() + must.Equal(io.EOF, err) +} + +func TestIterator_Object_ReadObjectCB(t *testing.T) { + t.Run("eof", func(t *testing.T) { + it := NewIterator() + err := it.ReadObjectCB(nil) + require.Equal(t, io.EOF, err) + }) + t.Run("invalid first byte", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" a")) + err := it.ReadObjectCB(nil) + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after first byte", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" { ")) + err := it.ReadObjectCB(nil) + require.Equal(t, io.EOF, err) + }) + t.Run("empty object", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" { } ")) + err := it.ReadObjectCB(nil) + require.NoError(t, err) + }) + t.Run("invalid field", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { a`)) + err := it.ReadObjectCB(nil) + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("invalid field 2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " `)) + err := it.ReadObjectCB(nil) + require.Equal(t, io.EOF, err) + }) + t.Run("error during callback", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : `)) + e := errors.New("test") + err := it.ReadObjectCB(func(it *Iterator, field string) error { + return e + }) + require.Equal(t, e, err) + }) + t.Run("eof after first item", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : 1`)) + err := it.ReadObjectCB(func(it *Iterator, field string) error { + i, err := it.ReadInt() + require.NoError(t, err) + require.Equal(t, 1, i) + return nil + }) + require.Equal(t, io.EOF, err) + }) + t.Run("end after first item", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : 1 } `)) + err := it.ReadObjectCB(func(it *Iterator, field string) error { + i, err := it.ReadInt() + require.NoError(t, err) + require.Equal(t, 1, i) + return nil + }) + require.NoError(t, err) + + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("eof after comma", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : 1 , `)) + err := it.ReadObjectCB(func(it *Iterator, field string) error { + i, err := it.ReadInt() + require.NoError(t, err) + require.Equal(t, 1, i) + return nil + }) + require.Equal(t, io.EOF, err) + }) + t.Run("invalid byte after comma", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : 1 , a `)) + err := it.ReadObjectCB(func(it *Iterator, field string) error { + i, err := it.ReadInt() + require.NoError(t, err) + require.Equal(t, 1, i) + return nil + }) + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after second field", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : 1 , " `)) + err := it.ReadObjectCB(func(it *Iterator, field string) error { + i, err := it.ReadInt() + require.NoError(t, err) + require.Equal(t, 1, i) + return nil + }) + require.Equal(t, io.EOF, err) + }) + t.Run("invalid comma", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : 1 a " `)) + err := it.ReadObjectCB(func(it *Iterator, field string) error { + i, err := it.ReadInt() + require.NoError(t, err) + require.Equal(t, 1, i) + return nil + }) + require.IsType(t, UnexpectedByteError{}, err) + }) +} + +func TestIterator_Object_ReadObjectCB_Example(t *testing.T) { + must := require.New(t) + + it := NewIterator() + it.ResetBytes([]byte(` { "key" : "value" , "key2" : "value2" } `)) + + m := map[string]string{} + + err := it.ReadObjectCB(func(it *Iterator, field string) (err error) { + value, err := it.ReadString() + if err == nil { + m[field] = value + } + return + }) + must.NoError(err) + must.Len(m, 2) + must.Equal("value", m["key"]) + must.Equal("value2", m["key2"]) + + _, err = it.NextValueType() + must.Equal(io.EOF, err) +} + +func TestIterator_Object_skipObjectField(t *testing.T) { + must := require.New(t) + + it := NewIterator() + it.ResetBytes([]byte(` key" : `)) + more, err := it.skipObjectField() + must.NoError(err) + must.True(more) +} diff --git a/iterator_pool.go b/iterator_pool.go new file mode 100644 index 0000000..872a039 --- /dev/null +++ b/iterator_pool.go @@ -0,0 +1,34 @@ +package jzon + +import ( + "sync" +) + +var ( + defaultIteratorPool = NewIteratorPool() +) + +type IteratorPool struct { + pool sync.Pool +} + +func NewIteratorPool() *IteratorPool { + return &IteratorPool{ + pool: sync.Pool{ + New: func() interface{} { + return &Iterator{ + tmpBuffer: getByteSlice(), + } + }, + }, + } +} + +func (p *IteratorPool) BorrowIterator() *Iterator { + return p.pool.Get().(*Iterator) +} + +func (p *IteratorPool) ReturnIterator(it *Iterator) { + it.reset() + p.pool.Put(it) +} diff --git a/iterator_pool_test.go b/iterator_pool_test.go new file mode 100644 index 0000000..48092a1 --- /dev/null +++ b/iterator_pool_test.go @@ -0,0 +1,65 @@ +package jzon + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIteratorPool(t *testing.T) { + must := require.New(t) + + pool := NewIteratorPool() + + // borrow 1 + it := pool.BorrowIterator() + must.Nil(it.reader) + must.Nil(it.buffer) + must.Equal(0, it.offset) + must.Equal(0, it.head) + must.Equal(0, it.tail) + + // reset reader + it.Reset(&oneByteReader{}) + + // return 1 + pool.ReturnIterator(it) + must.Nil(it.reader) + must.Nil(it.buffer) + + // borrow 2 + it = pool.BorrowIterator() + must.Nil(it.reader) + must.Nil(it.buffer) + must.Equal(0, it.offset) + must.Equal(0, it.head) + must.Equal(0, it.tail) + + // reset bytes buffer + data := []byte("test") + it.Reset(bytes.NewBuffer(data)) + must.Nil(it.reader) + must.Equal(data, it.buffer) + must.Equal(0, it.offset) + must.Equal(0, it.head) + must.Equal(len(data), it.tail) + + // return 2 + pool.ReturnIterator(it) + must.Nil(it.reader) + must.Nil(it.buffer) + + // reset bytes + it.ResetBytes(data) + must.Nil(it.reader) + must.Equal(data, it.buffer) + must.Equal(0, it.offset) + must.Equal(0, it.head) + must.Equal(len(data), it.tail) + + // return 3 + pool.ReturnIterator(it) + must.Nil(it.reader) + must.Nil(it.buffer) +} diff --git a/iterator_raw.go b/iterator_raw.go new file mode 100644 index 0000000..021a0ac --- /dev/null +++ b/iterator_raw.go @@ -0,0 +1,34 @@ +package jzon + +// No copy version +func (it *Iterator) SkipRaw() ([]byte, error) { + c, _, err := it.nextToken() + if err != nil { + return nil, err + } + oldCapture := it.capture + it.capture = true + begin := it.head + it.head += 1 + err = skipFunctions[c](it, c) + it.capture = oldCapture + if err != nil { + return nil, err + } + return it.buffer[begin:it.head], nil +} + +// copy version +func (it *Iterator) AppendRaw(in []byte) ([]byte, error) { + b, err := it.SkipRaw() + if err != nil { + return in, err + } + // https://github.com/go101/go101/wiki + return append(in, b...), nil +} + +// copy version +func (it *Iterator) ReadRaw() ([]byte, error) { + return it.AppendRaw(nil) +} diff --git a/iterator_raw_test.go b/iterator_raw_test.go new file mode 100644 index 0000000..4c04ecf --- /dev/null +++ b/iterator_raw_test.go @@ -0,0 +1,48 @@ +package jzon + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Raw_ReadRaw(t *testing.T) { + t.Run("eof", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadRaw() + require.Equal(t, io.EOF, err) + }) + t.Run("error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` + `)) + _, err := it.ReadRaw() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("no error", func(t *testing.T) { + it := NewIterator() + exp := `{ " " : null }` + + data := []byte(" " + exp + " ") + it.ResetBytes(data) + raw, err := it.ReadRaw() + require.NoError(t, err) + require.Equal(t, exp, string(raw)) + + copy(data, []byte(exp+" ")) // modify content + require.Equal(t, exp, string(raw)) + + _, err = it.ReadRaw() + require.Equal(t, io.EOF, err) + }) +} + +func TestIterator_Raw_AppendRaw(t *testing.T) { + it := NewIterator() + data := []byte(`{}`) + it.ResetBytes(data) + prefix := []byte(`test`) + b, err := it.AppendRaw(prefix) + require.NoError(t, err) + require.Equal(t, append(prefix, data...), b) +} diff --git a/iterator_read.go b/iterator_read.go new file mode 100644 index 0000000..bf26459 --- /dev/null +++ b/iterator_read.go @@ -0,0 +1,143 @@ +package jzon + +var ( + readFunctions [charNum]func(it *Iterator, c byte) (interface{}, error) +) + +func init() { + readFunctions['"'] = func(it *Iterator, c byte) (interface{}, error) { + return it.readString() + } + readFunctions['n'] = func(it *Iterator, c byte) (interface{}, error) { + return nil, it.expectBytes("ull") + } + readFunctions['t'] = func(it *Iterator, c byte) (interface{}, error) { + return true, it.expectBytes("rue") + } + readFunctions['f'] = func(it *Iterator, c byte) (interface{}, error) { + return false, it.expectBytes("alse") + } + readFunctions['['] = readArrayWithStack + readFunctions['{'] = readObjectWithStack + for _, c := range []byte("-0123456789") { + readFunctions[c] = func(it *Iterator, c byte) (interface{}, error) { + return it.readFloat64(c) + } + } + errFunc := func(it *Iterator, c byte) (interface{}, error) { + return nil, UnexpectedByteError{got: c} + } + for i := 0; i < charNum; i++ { + if readFunctions[i] == nil { + readFunctions[i] = errFunc + } + } +} + +func (it *Iterator) Read() (interface{}, error) { + c, _, err := it.nextToken() + if err != nil { + return nil, err + } + it.head += 1 + return readFunctions[c](it, c) +} + +func readWithStack(it *Iterator, top stackElement, s *stack, ns *nodeStack) ( + _ interface{}, err error) { + var c byte + for { + c, _, err = it.nextToken() + if err != nil { + return nil, err + } + it.head += 1 + if top&1 == 0 { + // stackElementObjectBegin + // stackElementObject + if c == '}' { + if top = s.pop(); top == stackElementNone { + return ns.topObject(), nil + } + ns.popObject() + continue + } + if top == stackElementObject { + if c != ',' { + return nil, UnexpectedByteError{got: c, exp: ','} + } + c, _, err = it.nextToken() + if err != nil { + return nil, err + } + it.head += 1 + } + if c != '"' { + return nil, UnexpectedByteError{got: c, exp: '"'} + } + _, field, err := it.readObjectField() + if err != nil { + return nil, err + } + c, _, err = it.nextToken() + if err != nil { + return nil, err + } + it.head += 1 + switch c { + case '[': + s.pushObject() + ns.pushArray(field) + top = stackElementArrayBegin + case '{': + s.pushObject() + ns.pushObject(field) + top = stackElementObjectBegin + default: + o, err := readFunctions[c](it, c) + if err != nil { + return nil, err + } + top = stackElementObject + ns.setTopObject(field, o) + } + } else { + // stackElementArrayBegin + // stackElementArray + if c == ']' { + if top = s.pop(); top == stackElementNone { + return ns.topArray(), nil + } + ns.popArray() + continue + } + if top == stackElementArray { + if c != ',' { + return nil, UnexpectedByteError{got: c, exp: ','} + } + c, _, err = it.nextToken() + if err != nil { + return nil, err + } + it.head += 1 + } + switch c { + case '[': + s.pushArray() + ns.pushArray("") + top = stackElementArrayBegin + case '{': + s.pushArray() + ns.pushObject("") + top = stackElementObjectBegin + default: + o, err := readFunctions[c](it, c) + if err != nil { + return nil, err + } + top = stackElementArray + ns.appendTopArray(o) + } + } + } +} diff --git a/iterator_read_array.go b/iterator_read_array.go new file mode 100644 index 0000000..022cb9b --- /dev/null +++ b/iterator_read_array.go @@ -0,0 +1,60 @@ +package jzon + +func readArrayWithStack(it *Iterator, _ byte) (interface{}, error) { + c, _, err := it.nextToken() + if err != nil { + return nil, err + } + it.head += 1 + topObj := make([]interface{}, 0) + if c == ']' { + return topObj, nil + } + for { + // We disabled the following switch to test benchmark + // comparing to using builtin stack + // the result is using our own stack will improve the + // performance by about 25% + switch c { + case '{': + s := stackPool.Get().(*stack).initArray() + ns := nodeStackPool.Get().(*nodeStack). + initArray(topObj). + pushObject("") + ret, err := readWithStack(it, stackElementObjectBegin, s, ns) + releaseNodeStack(ns) + stackPool.Put(s) + return ret, err + case '[': + s := stackPool.Get().(*stack).initArray() + ns := nodeStackPool.Get().(*nodeStack). + initArray(topObj). + pushArray("") + ret, err := readWithStack(it, stackElementArrayBegin, s, ns) + releaseNodeStack(ns) + stackPool.Put(s) + return ret, err + } + o, err := readFunctions[c](it, c) + if err != nil { + return nil, err + } + topObj = append(topObj, o) + c, _, err = it.nextToken() + if err != nil { + return nil, err + } + it.head += 1 + if c == ']' { + return topObj, nil + } + if c != ',' { + return nil, UnexpectedByteError{got: c, exp: ',', exp2: ']'} + } + c, _, err = it.nextToken() + if err != nil { + return nil, err + } + it.head += 1 + } +} diff --git a/iterator_read_object.go b/iterator_read_object.go new file mode 100644 index 0000000..653ff6d --- /dev/null +++ b/iterator_read_object.go @@ -0,0 +1,72 @@ +package jzon + +func readObjectWithStack(it *Iterator, _ byte) (interface{}, error) { + c, _, err := it.nextToken() + if err != nil { + return nil, err + } + topObj := map[string]interface{}{} + if c == '}' { + it.head += 1 + return topObj, nil + } + for { + if c != '"' { + return nil, UnexpectedByteError{got: c, exp: '"'} + } + it.head += 1 + _, field, err := it.readObjectField() + if err != nil { + return nil, err + } + c, _, err = it.nextToken() + if err != nil { + return nil, err + } + it.head += 1 + // We disabled the following switch to test benchmark + // comparing to using builtin stack + // the result is using ours own stack will improve the + // performance by about 25% + switch c { + case '{': + s := stackPool.Get().(*stack).initObject() + ns := nodeStackPool.Get().(*nodeStack). + initObject(topObj). + pushObject(field) + ret, err := readWithStack(it, stackElementObjectBegin, s, ns) + releaseNodeStack(ns) + stackPool.Put(s) + return ret, err + case '[': + s := stackPool.Get().(*stack).initObject() + ns := nodeStackPool.Get().(*nodeStack). + initObject(topObj). + pushArray(field) + ret, err := readWithStack(it, stackElementArrayBegin, s, ns) + releaseNodeStack(ns) + stackPool.Put(s) + return ret, err + } + o, err := readFunctions[c](it, c) + if err != nil { + return nil, err + } + topObj[field] = o + c, _, err = it.nextToken() + if err != nil { + return nil, err + } + it.head += 1 + if c == '}' { + return topObj, nil + } + if c != ',' { + return nil, UnexpectedByteError{got: c, exp: '}', exp2: ','} + } + c, _, err = it.nextToken() + if err != nil { + return nil, err + } + } +} diff --git a/iterator_read_test.go b/iterator_read_test.go new file mode 100644 index 0000000..4cff3cc --- /dev/null +++ b/iterator_read_test.go @@ -0,0 +1,334 @@ +package jzon + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Read_Read(t *testing.T) { + t.Run("eof", func(t *testing.T) { + it := NewIterator() + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("string", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"test"`)) + o, err := it.Read() + require.NoError(t, err) + require.Equal(t, "test", o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("null", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`null`)) + o, err := it.Read() + require.NoError(t, err) + require.Equal(t, nil, o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("true", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`true`)) + o, err := it.Read() + require.NoError(t, err) + require.Equal(t, true, o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("false", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`false`)) + o, err := it.Read() + require.NoError(t, err) + require.Equal(t, false, o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("number", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`-123.456e7`)) + o, err := it.Read() + require.NoError(t, err) + require.Equal(t, -123.456e7, o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`+`)) + _, err := it.Read() + require.IsType(t, UnexpectedByteError{}, err) + }) +} + +func TestIterator_Read_ReadArray(t *testing.T) { + t.Run("eof after bracket", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`[`)) + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("empty array", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`[]`)) + o, err := it.Read() + require.NoError(t, err) + require.Equal(t, []interface{}{}, o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("bad element", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ + `)) + _, err := it.Read() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after first element", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ 1 `)) + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("not nested", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ null ] `)) + o, err := it.Read() + require.NoError(t, err) + require.Equal(t, []interface{}{nil}, o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("bad char after element", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ 1 + `)) + _, err := it.Read() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after comma", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ 1 , `)) + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("mix elements", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ null , 1 , "a" , true , false ] `)) + o, err := it.Read() + require.NoError(t, err) + require.Equal(t, []interface{}{nil, float64(1), "a", true, false}, o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("nested eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ [ `)) + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("nested bad comma", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ [ ] [ `)) + _, err := it.Read() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("nested eof after comma", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ [ ] , `)) + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("nested bad item", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ [ ] , + `)) + _, err := it.Read() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("nested 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ [ ] , null ] `)) + o, err := it.Read() + require.NoError(t, err) + t.Log(o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("nested 2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(nestedArray1(10))) + o, err := it.Read() + require.NoError(t, err) + t.Log(o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("nested 3", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(nestedArray2(10))) + o, err := it.Read() + require.NoError(t, err) + t.Log(o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("nested with object", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(nestedArrayWithObject(10))) + o, err := it.Read() + require.NoError(t, err) + t.Log(o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) +} + +func TestIterator_Read_ReadObject(t *testing.T) { + t.Run("eof after bracket", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`{`)) + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("empty", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { } `)) + o, err := it.Read() + require.NoError(t, err) + require.Equal(t, map[string]interface{}{}, o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid char after bracket", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`{ ,`)) + _, err := it.Read() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("field eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`{ " `)) + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("eof after colon", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`{ " " : `)) + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("bad field value", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`{ " " : + `)) + _, err := it.Read() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after field value", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`{ " " : 1 `)) + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("one field", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { "k" : "v" } `)) + o, err := it.Read() + require.NoError(t, err) + require.Equal(t, map[string]interface{}{"k": "v"}, o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("bad char after field value", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`{ " " : 1 { `)) + _, err := it.Read() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after comma", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`{ " " : 1 , `)) + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("two fields", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { "k1" : "v1", "k2" : "v2" } `)) + o, err := it.Read() + require.NoError(t, err) + require.Equal(t, map[string]interface{}{"k1": "v1", "k2": "v2"}, o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("nested eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { "a" : { `)) + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("nested bad comma", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { "a" : { } + `)) + _, err := it.Read() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("nested eof after comma", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { "a" : { } , `)) + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("nested bad char after comma", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { "a" : { } , { `)) + _, err := it.Read() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("nested bad field", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { "a" : { } , " `)) + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("nested eof after field", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { "a" : { } , "b" : `)) + _, err := it.Read() + require.Equal(t, io.EOF, err) + }) + t.Run("nested bad value", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { "a" : { } , "b" : } `)) + _, err := it.Read() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("nested 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { "a" : { } , "b" : null } `)) + o, err := it.Read() + require.NoError(t, err) + t.Log(o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("nested 2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(nestedObject(10))) + o, err := it.Read() + require.NoError(t, err) + t.Log(o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("nested with array", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(nestedObjectWithArray(10))) + o, err := it.Read() + require.NoError(t, err) + t.Log(o) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) +} diff --git a/iterator_skip.go b/iterator_skip.go new file mode 100644 index 0000000..5e04b44 --- /dev/null +++ b/iterator_skip.go @@ -0,0 +1,128 @@ +package jzon + +var ( + skipFunctions [charNum]func(it *Iterator, c byte) error +) + +func init() { + skipFunctions['"'] = skipString + skipFunctions['n'] = func(it *Iterator, _ byte) error { + return it.expectBytes("ull") + } + skipFunctions['t'] = func(it *Iterator, _ byte) error { + return it.expectBytes("rue") + } + skipFunctions['f'] = func(it *Iterator, _ byte) error { + return it.expectBytes("alse") + } + skipFunctions['['] = skipArrayWithStack + skipFunctions['{'] = skipObjectWithStack + for _, c := range []byte("-0123456789") { + skipFunctions[c] = skipNumber + } + errFunc := func(it *Iterator, c byte) error { + return UnexpectedByteError{got: c} + } + for i := 0; i < charNum; i++ { + if skipFunctions[i] == nil { + skipFunctions[i] = errFunc + } + } +} + +func skipWithStack(it *Iterator, top stackElement, s *stack) (err error) { + var c byte + for { + c, _, err = it.nextToken() + if err != nil { + return err + } + it.head += 1 + if top&1 == 0 { + // stackElementObjectBegin + // stackElementObject + if c == '}' { + if top = s.pop(); top == stackElementNone { + return nil + } + continue + } + if top == stackElementObject { + if c != ',' { + return UnexpectedByteError{got: c, exp: ','} + } + c, _, err = it.nextToken() + if err != nil { + return err + } + it.head += 1 + } + if c != '"' { + return UnexpectedByteError{got: c, exp: '"'} + } + _, err := it.skipObjectField() + if err != nil { + return err + } + c, _, err = it.nextToken() + if err != nil { + return err + } + it.head += 1 + switch c { + case '[': + s.pushObject() + top = stackElementArrayBegin + case '{': + s.pushObject() + top = stackElementObjectBegin + default: + if err = skipFunctions[c](it, c); err != nil { + return err + } + top = stackElementObject + } + } else { + // stackElementArrayBegin + // stackElementArray + if c == ']' { + if top = s.pop(); top == stackElementNone { + return nil + } + continue + } + if top == stackElementArray { + if c != ',' { + return UnexpectedByteError{got: c, exp: ','} + } + c, _, err = it.nextToken() + if err != nil { + return err + } + it.head += 1 + } + switch c { + case '[': + s.pushArray() + top = stackElementArrayBegin + case '{': + s.pushArray() + top = stackElementObjectBegin + default: + if err = skipFunctions[c](it, c); err != nil { + return err + } + top = stackElementArray + } + } + } +} + +func (it *Iterator) Skip() error { + c, _, err := it.nextToken() + if err != nil { + return err + } + it.head += 1 + return skipFunctions[c](it, c) +} diff --git a/iterator_skip_array.go b/iterator_skip_array.go new file mode 100644 index 0000000..60d78f5 --- /dev/null +++ b/iterator_skip_array.go @@ -0,0 +1,57 @@ +package jzon + +func skipArrayWithStack(it *Iterator, _ byte) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + it.head += 1 + if c == ']' { + return nil + } + for { + switch c { + case '{': + s := stackPool.Get().(*stack).initArray() + err = skipWithStack(it, stackElementObjectBegin, s) + stackPool.Put(s) + return err + case '[': + s := stackPool.Get().(*stack).initArray() + err = skipWithStack(it, stackElementArrayBegin, s) + stackPool.Put(s) + return err + } + if err = skipFunctions[c](it, c); err != nil { + return err + } + c, _, err = it.nextToken() + if err != nil { + return err + } + it.head += 1 + if c == ']' { + return nil + } + if c != ',' { + return UnexpectedByteError{got: c, exp: ']', exp2: ','} + } + c, _, err = it.nextToken() + if err != nil { + return err + } + it.head += 1 + } +} + +func (it *Iterator) SkipArray() error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c != '[' { + return UnexpectedByteError{got: c, exp: '['} + } + it.head += 1 + return skipArrayWithStack(it, c) +} diff --git a/iterator_skip_array_test.go b/iterator_skip_array_test.go new file mode 100644 index 0000000..92e3949 --- /dev/null +++ b/iterator_skip_array_test.go @@ -0,0 +1,120 @@ +package jzon + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Skip_SkipArray(t *testing.T) { + t.Run("eof", func(t *testing.T) { + it := NewIterator() + err := it.SkipArray() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid first byte", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" ]")) + err := it.SkipArray() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after bracket", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" [")) + err := it.SkipArray() + require.Equal(t, io.EOF, err) + }) + t.Run("empty array", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" [ ] ")) + err := it.SkipArray() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid byte after bracket", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" [ ,")) + err := it.SkipArray() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after first element", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" [ 1 ")) + err := it.SkipArray() + require.Equal(t, io.EOF, err) + }) + t.Run("one element", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" [ 1 ] ")) + err := it.SkipArray() + require.NoError(t, err) + }) + t.Run("invalid byte after element", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" [ 1 [ ")) + err := it.SkipArray() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after dot", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" [ 1 , ")) + err := it.SkipArray() + require.Equal(t, io.EOF, err) + }) + t.Run("two elements", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" [ 1 , null ] ")) + err := it.SkipArray() + require.NoError(t, err) + }) + t.Run("nested error eof 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ [ `)) + err := it.SkipArray() + require.Equal(t, io.EOF, err) + }) + t.Run("nested error 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ [ 1 a`)) + err := it.SkipArray() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("nested error eof 2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ [ 1 , `)) + err := it.SkipArray() + require.Equal(t, io.EOF, err) + }) + t.Run("nested error token", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` [ [ 1 , +`)) + err := it.SkipArray() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("nested 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(nestedArray1(100))) + err := it.SkipArray() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("nested 2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(nestedArray2(100))) + err := it.SkipArray() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("nested with object", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(nestedArrayWithObject(100))) + err := it.SkipArray() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) +} diff --git a/iterator_skip_number.go b/iterator_skip_number.go new file mode 100644 index 0000000..023b811 --- /dev/null +++ b/iterator_skip_number.go @@ -0,0 +1,156 @@ +package jzon + +import ( + "io" +) + +func (it *Iterator) skipExponentPart() (err error) { + c, err := it.nextByte() + if err != nil { + return err + } + it.head += 1 + if c == '+' || c == '-' { + c, err = it.nextByte() + if err != nil { + return err + } + it.head += 1 + } + if intDigits[c] == invalidDigit { + return InvalidDigitError{c: c} + } + for { + i := it.head + for ; i < it.tail; i++ { + c = it.buffer[i] + digit := intDigits[c] + if digit == invalidDigit { + it.head = i + return nil + } + } + // i == it.tail + it.head = i + if err = it.readMore(); err != nil { + if err == io.EOF { + return nil + } + return err + } + } +} + +func (it *Iterator) skipFractionPart() (err error) { + c, err := it.nextByte() + if err != nil { + return err + } + if intDigits[c] == invalidDigit { + return InvalidDigitError{c: c} + } + it.head += 1 + for { + i := it.head + for ; i < it.tail; i++ { + c = it.buffer[i] + digit := floatDigits[c] + if digit < 0 { + switch digit { + case expInNumber: + it.head = i + 1 + return it.skipExponentPart() + default: + it.head = i + return nil + } + } + } + // i == it.tail + it.head = i + if err = it.readMore(); err != nil { + if err == io.EOF { + return nil + } + return err + } + } +} + +// make sure that c is '-' or '0'~'9' before calling this method +// (the call of nextToken before this method return NumberValue) +func skipNumber(it *Iterator, c byte) (err error) { + if c == '-' { + c, err = it.nextByte() + if err != nil { + return + } + if intDigits[c] == invalidDigit { + return InvalidDigitError{c: c} + } + it.head += 1 + } + // positive + // here the c can only be '0'~'9' + if c == '0' { + if it.head == it.tail { + if err = it.readMore(); err != nil { + if err == io.EOF { + return nil + } + return err + } + } + switch floatDigits[it.buffer[it.head]] { + case dotInNumber: + it.head += 1 + return it.skipFractionPart() + case expInNumber: + it.head += 1 + return it.skipExponentPart() + default: + return nil + } + } else { + for { + i := it.head + for ; i < it.tail; i++ { + c = it.buffer[i] + digit := floatDigits[c] + if digit < 0 { + switch digit { + case dotInNumber: + it.head = i + 1 + return it.skipFractionPart() + case expInNumber: + it.head = i + 1 + return it.skipExponentPart() + default: + it.head = i + return nil + } + } + } + // i == it.tail + it.head = i + if err = it.readMore(); err != nil { + if err == io.EOF { + return nil + } + return err + } + } + } +} + +func (it *Iterator) SkipNumber() error { + c, vt, err := it.nextToken() + if err != nil { + return err + } + if vt != NumberValue { + return UnexpectedByteError{got: c} + } + it.head += 1 + return skipNumber(it, c) +} diff --git a/iterator_skip_number_test.go b/iterator_skip_number_test.go new file mode 100644 index 0000000..ec1251f --- /dev/null +++ b/iterator_skip_number_test.go @@ -0,0 +1,198 @@ +package jzon + +import ( + "errors" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Skip_SkipNumber(t *testing.T) { + t.Run("eof", func(t *testing.T) { + it := NewIterator() + err := it.SkipNumber() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid first byte", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" +")) + err := it.SkipNumber() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after negative sign", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" -")) + err := it.SkipNumber() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid char after negative sign", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" - 1 ")) + err := it.SkipNumber() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("zero", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" -0")) + err := it.SkipNumber() + require.NoError(t, err) + }) + t.Run("reader error after zero", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte(" -0"), + err: e, + }) + err := it.SkipNumber() + require.Equal(t, e, err) + }) + t.Run("double zero", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" -00 ")) + err := it.SkipNumber() + require.NoError(t, err) + }) + t.Run("zero with fraction", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" -0.1 ")) + err := it.SkipNumber() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("zero with exponent", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" -0e+1 ")) + err := it.SkipNumber() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("non zero", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1 ")) + err := it.SkipNumber() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("non zero with fraction", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1.1 ")) + err := it.SkipNumber() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("non zero with exponent", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1e-1 ")) + err := it.SkipNumber() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("non zero eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 1")) + err := it.SkipNumber() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("non zero reader error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte(" 1"), + err: e, + }) + err := it.SkipNumber() + require.Equal(t, e, err) + }) + t.Run("fraction empty", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0.")) + err := it.SkipNumber() + require.Equal(t, io.EOF, err) + }) + t.Run("fraction invalid", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0.+")) + err := it.SkipNumber() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("fraction with exponent", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" -1.2e+3 ")) + err := it.SkipNumber() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("fraction eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" -1.2")) + err := it.SkipNumber() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("fraction reader error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte(" 1.2"), + err: e, + }) + err := it.SkipNumber() + require.Equal(t, e, err) + }) + t.Run("exponent empty", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0e")) + err := it.SkipNumber() + require.Equal(t, io.EOF, err) + }) + t.Run("exponent eof after sign", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0e+")) + err := it.SkipNumber() + require.Equal(t, io.EOF, err) + }) + t.Run("exponent invalid char after sign", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0e++")) + err := it.SkipNumber() + require.IsType(t, InvalidDigitError{}, err) + }) + t.Run("exponent eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" 0e+1")) + err := it.SkipNumber() + require.NoError(t, err) + }) + t.Run("exponent reader error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + b: []byte(" 0e+1"), + err: e, + }) + err := it.SkipNumber() + require.Equal(t, e, err) + }) +} + +func TestIterator_Skip_SkipNumber_LargeNumber(t *testing.T) { + it := NewIterator() + s := "-" + strings.Repeat("123", 20) + "." + + strings.Repeat("0456", 20) + "e+" + + strings.Repeat("789", 20) + it.ResetBytes([]byte(" " + s + " ")) + err := it.SkipNumber() + require.NoError(t, err) +} diff --git a/iterator_skip_object.go b/iterator_skip_object.go new file mode 100644 index 0000000..f041dc6 --- /dev/null +++ b/iterator_skip_object.go @@ -0,0 +1,69 @@ +package jzon + +func skipObjectWithStack(it *Iterator, _ byte) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == '}' { + it.head += 1 + return nil + } + for { + if c != '"' { + return UnexpectedByteError{got: c, exp: '"'} + } + it.head += 1 + _, err := it.skipObjectField() + if err != nil { + return err + } + c, _, err = it.nextToken() + if err != nil { + return err + } + it.head += 1 + switch c { + case '{': + s := stackPool.Get().(*stack).initObject() + err := skipWithStack(it, stackElementObjectBegin, s) + stackPool.Put(s) + return err + case '[': + s := stackPool.Get().(*stack).initObject() + err := skipWithStack(it, stackElementArrayBegin, s) + stackPool.Put(s) + return err + } + if err = skipFunctions[c](it, c); err != nil { + return err + } + c, _, err = it.nextToken() + if err != nil { + return err + } + it.head += 1 + if c == '}' { + return nil + } + if c != ',' { + return UnexpectedByteError{got: c, exp: '}', exp2: ','} + } + c, _, err = it.nextToken() + if err != nil { + return err + } + } +} + +func (it *Iterator) SkipObject() error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c != '{' { + return UnexpectedByteError{got: c, exp: '{'} + } + it.head += 1 + return skipObjectWithStack(it, c) +} diff --git a/iterator_skip_object_test.go b/iterator_skip_object_test.go new file mode 100644 index 0000000..2ff34c0 --- /dev/null +++ b/iterator_skip_object_test.go @@ -0,0 +1,168 @@ +package jzon + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Skip_SkipObject(t *testing.T) { + t.Run("eof", func(t *testing.T) { + it := NewIterator() + err := it.SkipObject() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid first byte", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" }")) + err := it.SkipObject() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after bracket", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" {")) + err := it.SkipObject() + require.Equal(t, io.EOF, err) + }) + t.Run("empty", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(" { } ")) + err := it.SkipObject() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid field first byte", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { 1`)) + err := it.SkipObject() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("field error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " `)) + err := it.SkipObject() + require.Equal(t, io.EOF, err) + }) + t.Run("value eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : `)) + err := it.SkipObject() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid first value", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : +`)) + err := it.SkipObject() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("eof after first value", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : 1 `)) + err := it.SkipObject() + require.Equal(t, io.EOF, err) + }) + t.Run("only one pair", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : 1 } `)) + err := it.SkipObject() + require.NoError(t, err) + }) + t.Run("non nested second item dot error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : true a `)) + err := it.SkipObject() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("non nested second item eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : false , `)) + err := it.SkipObject() + require.Equal(t, io.EOF, err) + }) + t.Run("non nested second item field error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : false , a`)) + err := it.SkipObject() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("non nested second item", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : false , " " : true } `)) + err := it.SkipObject() + require.NoError(t, err) + }) + t.Run("nested eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : { `)) + err := it.SkipObject() + require.Equal(t, io.EOF, err) + }) + t.Run("nested empty value", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : { } } `)) + err := it.SkipObject() + require.NoError(t, err) + }) + t.Run("nested second item comma error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : { } a `)) + err := it.SkipObject() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("nested second item no field", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : { } , `)) + err := it.SkipObject() + require.Equal(t, io.EOF, err) + }) + t.Run("nested second item bad field", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : { } , a`)) + err := it.SkipObject() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("nested second item field eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : { } , " " `)) + err := it.SkipObject() + require.Equal(t, io.EOF, err) + }) + t.Run("nested second item eof value", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : { } , " " : `)) + err := it.SkipObject() + require.Equal(t, io.EOF, err) + }) + t.Run("nested second item bad value", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : { } , " " : } `)) + err := it.SkipObject() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("nested second item", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` { " " : { } , " " : 0 } `)) + err := it.SkipObject() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("nested", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(nestedObject(100))) + err := it.SkipObject() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("nested with array", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(nestedObjectWithArray(100))) + err := it.SkipObject() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) +} diff --git a/iterator_skip_str.go b/iterator_skip_str.go new file mode 100644 index 0000000..45ff2ee --- /dev/null +++ b/iterator_skip_str.go @@ -0,0 +1,83 @@ +package jzon + +func (it *Iterator) skipU4() error { + remain := 4 + for { + i := it.head + for ; i < it.tail; i++ { + c := it.buffer[i] + u4v := hexValue[c] + if u4v == invalidHex { + return InvalidUnicodeCharError{c: c} + } + if remain == 1 { + it.head = i + 1 + return nil + } + remain-- + } + it.head = i + if err := it.readMore(); err != nil { + return err + } + } +} + +func (it *Iterator) skipEscapedChar() error { + c, err := it.nextByte() + if err != nil { + return err + } + escaped := escapeMap[c] + if escaped != noEscape { + it.head += 1 + return nil + } + if c != 'u' { + return InvalidEscapeCharError{c: c} + } + it.head += 1 + return it.skipU4() +} + +// internal, call only after a '"' is consumed +func skipString(it *Iterator, _ byte) error { + for { + i := it.head + for i < it.tail { + c := it.buffer[i] + if c == '"' { + it.head = i + 1 + return nil + } else if c == '\\' { + it.head = i + 1 + err := it.skipEscapedChar() + if err != nil { + return err + } + i = it.head + } else if c < ' ' { // json.org + return InvalidStringCharError{c: c} + } else { + i++ + } + } + // i == it.tail + it.head = i + if err := it.readMore(); err != nil { + return err + } + } +} + +func (it *Iterator) SkipString() error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c != '"' { + return UnexpectedByteError{got: c, exp: '"'} + } + it.head += 1 + return skipString(it, c) +} diff --git a/iterator_skip_str_test.go b/iterator_skip_str_test.go new file mode 100644 index 0000000..ccd6449 --- /dev/null +++ b/iterator_skip_str_test.go @@ -0,0 +1,83 @@ +package jzon + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Skip_SkipString(t *testing.T) { + t.Run("eof", func(t *testing.T) { + it := NewIterator() + err := it.SkipString() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid first byte", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` a`)) + err := it.SkipString() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("empty string", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` ""`)) + err := it.SkipString() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("eof on escape", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` "\`)) + err := it.SkipString() + require.Equal(t, io.EOF, err) + }) + t.Run("valid escape", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` "\t" `)) + err := it.SkipString() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid escape", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` "\a`)) + err := it.SkipString() + require.IsType(t, InvalidEscapeCharError{}, err) + }) + t.Run("invalid u4", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` "\uX`)) + err := it.SkipString() + require.IsType(t, InvalidUnicodeCharError{}, err) + }) + t.Run("eof u4", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` "\u0`)) + err := it.SkipString() + require.Equal(t, io.EOF, err) + }) + t.Run("valid u4", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` "\u0000" `)) + err := it.SkipString() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + + t.Run("invalid string char", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte{' ', '"', 0}) + err := it.SkipString() + require.IsType(t, InvalidStringCharError{}, err) + }) + t.Run("eof after first byte", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(` " `)) + err := it.SkipString() + require.Equal(t, io.EOF, err) + }) +} diff --git a/iterator_skip_test.go b/iterator_skip_test.go new file mode 100644 index 0000000..342b5eb --- /dev/null +++ b/iterator_skip_test.go @@ -0,0 +1,42 @@ +package jzon + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Skip_Skip(t *testing.T) { + t.Run("eof", func(t *testing.T) { + it := NewIterator() + err := it.Skip() + require.Equal(t, io.EOF, err) + }) + t.Run("skip", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`{ + "string": "string", + "null": null, + "true": true, + "false": false, + "number": -123.0456E+789, + "array": [ "string", null, true, false, + -123.0456E+789, [ ], { } ], + "object": { + "string": "string", + "null": null, + "true": true, + "false": false, + "number": -123.0456E789, + "array": [ "string", null, true, false, + -123.0456E+789, [ ], { } ], + "object": { } + } + }`)) + err := it.Skip() + require.NoError(t, err) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) +} diff --git a/iterator_str.go b/iterator_str.go new file mode 100644 index 0000000..67ce70c --- /dev/null +++ b/iterator_str.go @@ -0,0 +1,255 @@ +package jzon + +import ( + "unicode/utf16" +) + +const ( + noEscape = 0 + invalidHex = -1 +) + +var ( + escapeMap [charNum]byte + hexValue [charNum]int8 +) + +func init() { + // escaped characters + for i := 0; i < charNum; i++ { + escapeMap[i] = noEscape + } + for k, v := range map[byte]byte{ + '"': '"', + '\\': '\\', + '/': '/', + 'b': '\b', + 'f': '\f', + 'n': '\n', + 'r': '\r', + 't': '\t', + } { + escapeMap[k] = v + } + // hex values + for i := 0; i < charNum; i++ { + hexValue[i] = invalidHex + } + for c := '0'; c <= '9'; c++ { + hexValue[c] = int8(c - '0') + } + for c := 'a'; c <= 'f'; c++ { + hexValue[c] = int8(c - 'a' + 10) + } + for c := 'A'; c <= 'F'; c++ { + hexValue[c] = int8(c - 'A' + 10) + } +} + +func (it *Iterator) readU4() (ret rune, err error) { + remain := 4 + for { + i := it.head + for ; i < it.tail; i++ { + c := it.buffer[i] + u4v := hexValue[c] + if u4v == invalidHex { + return 0, InvalidUnicodeCharError{c: c} + } + ret = ret<<4 + int32(u4v) + if remain == 1 { + it.head = i + 1 + return + } + remain-- + } + it.head = i + if err = it.readMore(); err != nil { + return + } + } +} + +func (it *Iterator) readEscapedChar(b []byte) ([]byte, error) { + c, err := it.nextByte() + if err != nil { + return b, err + } + escaped := escapeMap[c] + if escaped != noEscape { + it.head += 1 + return append(b, escaped), nil + } + if c != 'u' { + return b, InvalidEscapeCharError{c: c} + } + it.head += 1 + r, err := it.readU4() + if err != nil { + return b, err + } + if utf16.IsSurrogate(r) { + c, err := it.nextByte() + if err != nil { + return b, err + } + if c != '\\' { + return appendRune(b, r), nil + } + it.head += 1 + c, err = it.nextByte() + if err != nil { + return b, err + } + if c != 'u' { + b = appendRune(b, r) + escaped := escapeMap[c] + if escaped == noEscape { + return b, InvalidEscapeCharError{c: c} + } + it.head += 1 + return append(b, escaped), nil + } + it.head += 1 + r2, err := it.readU4() + if err != nil { + return b, err + } + combined := utf16.DecodeRune(r, r2) + if combined == runeError { + b = appendRune(b, r) + return appendRune(b, r2), nil + } + return appendRune(b, combined), nil + } else { + return appendRune(b, r), nil + } +} + +// internal, call only after a '"' is consumed +func (it *Iterator) readStringAsSlice(buf []byte, caseSensitive bool) (_ []byte, err error) { + for { + i := it.head + for i < it.tail { + c := it.buffer[i] + if c == '"' { + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + 1 + return buf, nil + } else if c == '\\' { + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + 1 + buf, err = it.readEscapedChar(buf) + if err != nil { + return buf, err + } + i = it.head + } else if c < ' ' { // json.org + return buf, InvalidStringCharError{c: c} + } else { + if !caseSensitive { + if c >= 'A' && c <= 'Z' { + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + 1 + buf = append(buf, c-'A'+'a') + } + } + i++ + } + } + // i == it.tail + buf = append(buf, it.buffer[it.head:i]...) + it.head = i + if err = it.readMore(); err != nil { + return buf, err + } + } +} + +func (it *Iterator) expectQuote() error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c != '"' { + return UnexpectedByteError{exp: '"', got: c} + } + it.head += 1 // consume the leading '"' + return nil +} + +func (it *Iterator) ReadStringAsSlice(buf []byte) (_ []byte, err error) { + if err = it.expectQuote(); err != nil { + return + } + return it.readStringAsSlice(buf, true) +} + +// internal, call only after a '"' is consumed +func (it *Iterator) readString() (ret string, err error) { + buf, err := it.readStringAsSlice(it.tmpBuffer[:0], true) + it.tmpBuffer = buf + if err == nil { + ret = string(buf) + } + return +} + +func (it *Iterator) ReadString() (ret string, err error) { + if err = it.expectQuote(); err != nil { + return + } + return it.readString() +} + +// From jsoniter +const ( + t1 = 0x00 // 0000 0000 + tx = 0x80 // 1000 0000 + t2 = 0xC0 // 1100 0000 + t3 = 0xE0 // 1110 0000 + t4 = 0xF0 // 1111 0000 + t5 = 0xF8 // 1111 1000 + + maskx = 0x3F // 0011 1111 + mask2 = 0x1F // 0001 1111 + mask3 = 0x0F // 0000 1111 + mask4 = 0x07 // 0000 0111 + + rune1Max = 1<<7 - 1 + rune2Max = 1<<11 - 1 + rune3Max = 1<<16 - 1 + + surrogateMin = 0xD800 + surrogateMax = 0xDFFF + + maxRune = '\U0010FFFF' // Maximum valid Unicode code point. + runeError = '\uFFFD' // the "error" Rune or "Unicode replacement character" +) + +func appendRune(p []byte, r rune) []byte { + // Negative values are erroneous. Making it unsigned addresses the problem. + switch i := uint32(r); { + case i <= rune1Max: + p = append(p, byte(r)) + return p + case i <= rune2Max: + p = append(p, t2|byte(r>>6)) + p = append(p, tx|byte(r)&maskx) + return p + case i > maxRune, surrogateMin <= i && i <= surrogateMax: + r = runeError + fallthrough + case i <= rune3Max: + p = append(p, t3|byte(r>>12)) + p = append(p, tx|byte(r>>6)&maskx) + p = append(p, tx|byte(r)&maskx) + return p + default: + p = append(p, t4|byte(r>>18)) + p = append(p, tx|byte(r>>12)&maskx) + p = append(p, tx|byte(r>>6)&maskx) + p = append(p, tx|byte(r)&maskx) + return p + } +} diff --git a/iterator_str_test.go b/iterator_str_test.go new file mode 100644 index 0000000..f66262c --- /dev/null +++ b/iterator_str_test.go @@ -0,0 +1,260 @@ +package jzon + +import ( + "errors" + "fmt" + "io" + "reflect" + "testing" + "unsafe" + + "github.com/stretchr/testify/require" +) + +func runeToAscii(s string) string { + var ret string + for _, r := range []rune(s) { + if r < 128 { + ret += string(r) + } else { + ret += fmt.Sprintf("\\u%04x", r) + } + } + return ret +} + +func TestIterator_Str_readU4(t *testing.T) { + origin := `中` + src := []byte(runeToAscii(`"` + origin + `"`)) + t.Run("eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes(src[:3]) + _, err := it.ReadString() + require.Equal(t, io.EOF, err) + }) + t.Run("eof2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes(src[:len(src)-1]) + _, err := it.ReadString() + require.Equal(t, io.EOF, err) + }) + t.Run("invalid_unicode", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\uG`)) + _, err := it.ReadString() + require.IsType(t, InvalidUnicodeCharError{}, err) + }) + t.Run("bytes", func(t *testing.T) { + it := NewIterator() + it.ResetBytes(src) + s, err := it.ReadString() + require.NoError(t, err) + require.Equal(t, origin, s) + }) + t.Run("reader", func(t *testing.T) { + it := NewIterator() + it.Reset(&oneByteReader{ + b: src, + }) + s, err := it.ReadString() + require.NoError(t, err) + require.Equal(t, origin, s) + }) +} + +func TestIterator_Str_readEscapedChar(t *testing.T) { + t.Run("eof", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\`)) + _, err := it.ReadString() + require.Equal(t, io.EOF, err) + }) + t.Run("control", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\n"`)) + s, err := it.ReadString() + require.NoError(t, err) + require.Len(t, s, 1) + require.Equal(t, "\n", s) + }) + t.Run("invalid escape char", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\a"`)) + _, err := it.ReadString() + require.IsType(t, InvalidEscapeCharError{}, err) + }) + t.Run("unicode error 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\u0`)) + _, err := it.ReadString() + require.Equal(t, io.EOF, err) + }) + t.Run("surrogate err 1", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\ud800`)) + _, err := it.ReadString() + require.Equal(t, io.EOF, err) + }) + t.Run("surrogate incomplete", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\ud800"`)) + s, err := it.ReadString() + require.NoError(t, err) + require.Equal(t, string(runeError), s) + }) + t.Run("surrogate err 2", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\ud800\`)) + _, err := it.ReadString() + require.Equal(t, io.EOF, err) + }) + t.Run("surrogate err 3", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\ud800\a`)) + _, err := it.ReadString() + require.IsType(t, InvalidEscapeCharError{}, err) + }) + t.Run("surrogate other escaped char", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\ud800\n"`)) + s, err := it.ReadString() + require.NoError(t, err) + require.Equal(t, string(runeError)+"\n", s) + }) + t.Run("surrogate err 4", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\ud800\u`)) + _, err := it.ReadString() + require.Equal(t, io.EOF, err) + }) + t.Run("surrogate runeError", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\udc00\u0000"`)) + s, err := it.ReadString() + require.NoError(t, err) + require.Equal(t, `"\ufffd\x00"`, fmt.Sprintf("%+q", s)) + }) + t.Run("surrogate", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\uD852\uDF62"`)) + s, err := it.ReadString() + require.NoError(t, err) + require.Equal(t, "𤭢", s) + }) + t.Run("non surrogate", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\u4e2d"`)) // 中 + s, err := it.ReadString() + require.NoError(t, err) + require.Equal(t, "中", s) + }) +} + +func TestIterator_Str_readStringAsSlice(t *testing.T) { + t.Run("reader error", func(t *testing.T) { + it := NewIterator() + e := errors.New("test") + it.Reset(&oneByteReader{ + err: e, + }) + _, err := it.ReadString() + require.Error(t, e, err) + }) + t.Run("bad value type", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`1`)) + _, err := it.ReadString() + require.IsType(t, UnexpectedByteError{}, err) + }) + t.Run("simple", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"abc123"`)) + s, err := it.ReadString() + require.NoError(t, err) + require.Equal(t, "abc123", s) + _, err = it.NextValueType() + require.Equal(t, io.EOF, err) + }) + t.Run("escape error", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\`)) + _, err := it.ReadString() + require.Equal(t, io.EOF, err) + }) + t.Run("escape", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"\n"`)) + s, err := it.ReadString() + require.NoError(t, err) + require.Equal(t, "\n", s) + }) + t.Run("invalid string char", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte("\"\x00\"")) + _, err := it.ReadString() + require.IsType(t, InvalidStringCharError{}, err) + }) + t.Run("reade err", func(t *testing.T) { + it := NewIterator() + it.Reset(&oneByteReader{ + b: []byte(`"\n`), + }) + _, err := it.ReadString() + require.Equal(t, io.EOF, err) + }) + t.Run("reader", func(t *testing.T) { + it := NewIterator() + it.Reset(&oneByteReader{ + b: []byte(`"\n"`), + }) + s, err := it.ReadString() + require.NoError(t, err) + require.Len(t, s, 1) + require.Equal(t, "\n", s) + }) +} + +func TestIterator_Str_ReadStringAsSlice(t *testing.T) { + t.Run("not string", func(t *testing.T) { + it := NewIterator() + _, err := it.ReadStringAsSlice(nil) + require.Equal(t, io.EOF, err) + }) + t.Run("normal", func(t *testing.T) { + it := NewIterator() + it.ResetBytes([]byte(`"abc"`)) + buf := make([]byte, 0, 32) + ret, err := it.ReadStringAsSlice(buf) + require.NoError(t, err) + p1 := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + p2 := (*reflect.SliceHeader)(unsafe.Pointer(&ret)) + require.Equal(t, p1.Data, p2.Data) + require.Equal(t, p1.Cap, p2.Cap) + require.Equal(t, []byte("abc"), ret) + }) +} + +func TestIterator_Str_appendRune(t *testing.T) { + var b []byte + + b = appendRune(b[:0], 0) + require.Equal(t, []byte{0}, b) + + b = appendRune(b[:0], 1<<7) + require.Equal(t, []byte{0xc2, 0x80}, b) + + b = appendRune(b[:0], maxRune+1) + require.Equal(t, []byte{0xef, 0xbf, 0xbd}, b) + + b = appendRune(b[:0], surrogateMin) + require.Equal(t, []byte{0xef, 0xbf, 0xbd}, b) + + b = appendRune(b[:0], surrogateMax) + require.Equal(t, []byte{0xef, 0xbf, 0xbd}, b) + + b = appendRune(b[:0], rune3Max) + require.Equal(t, []byte{0xef, 0xbf, 0xbf}, b) + + b = appendRune(b[:0], rune3Max+1) + require.Equal(t, []byte{0xf0, 0x90, 0x80, 0x80}, b) +} diff --git a/iterator_test.go b/iterator_test.go new file mode 100644 index 0000000..25fdcca --- /dev/null +++ b/iterator_test.go @@ -0,0 +1,107 @@ +package jzon + +import ( + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_New(t *testing.T) { + must := require.New(t) + + it := NewIterator() + must.Nil(it.reader) + must.Nil(it.buffer) + must.Equal(0, it.offset) + must.Equal(0, it.head) + must.Equal(0, it.tail) + // must.Nil(it.Error) +} + +func TestIterator_Reset_Nil(t *testing.T) { + must := require.New(t) + it := NewIterator() + must.Nil(it.reader) + must.Nil(it.buffer) + must.Equal(0, it.offset) + must.Equal(0, it.head) + must.Equal(0, it.tail) + + it.Reset(nil) + must.Nil(it.reader) + must.Nil(it.buffer) + must.Equal(0, it.offset) + must.Equal(0, it.head) + must.Equal(0, it.tail) +} + +func TestIterator_Reset(t *testing.T) { + must := require.New(t) + + // nil -> reader + it := NewIterator() + r := bytes.NewReader(nil) + it.Reset(r) + must.Equal(r, it.reader) + must.NotEmpty(it.buffer) + must.Equal(0, it.head) + must.Equal(0, it.tail) + + // reader -> reader + addr := &it.buffer[0] + r2 := bytes.NewReader(nil) + it.Reset(r2) + must.Equal(r2, it.reader) + must.True(addr == &it.buffer[0]) + must.Equal(0, it.head) + must.Equal(0, it.tail) + + // reader -> byte + b := []byte("abc") + it.ResetBytes(b) + must.Nil(it.reader) + must.True(&b[0] == &it.buffer[0]) + must.Equal(0, it.head) + must.Equal(len(b), it.tail) + + // nil -> byte + it = NewIterator() + b2 := []byte("abc") + it.ResetBytes(b2) + must.Nil(it.reader) + must.True(&b2[0] == &it.buffer[0]) + must.Equal(0, it.head) + must.Equal(len(b2), it.tail) + + // byte -> byte + b3 := []byte("defg") + it.ResetBytes(b3) + must.Nil(it.reader) + must.True(&b3[0] == &it.buffer[0]) + must.Equal(0, it.head) + must.Equal(len(b3), it.tail) + + // byte -> reader + r3 := bytes.NewReader(nil) + it.Reset(r3) + must.Equal(r3, it.reader) + must.Equal(0, it.head) + must.Equal(0, it.tail) +} + +func TestIterator_NextValueType(t *testing.T) { + must := require.New(t) + it := NewIterator() + for c, typ := range valueTypeMap { + it.ResetBytes([]byte{byte(c)}) + next, err := it.NextValueType() + if typ == WhiteSpaceValue { + require.Equal(t, io.EOF, err) + } else { + require.NoError(t, err) + must.Equal(typ, next) + } + } +} diff --git a/iterator_val.go b/iterator_val.go new file mode 100644 index 0000000..db860c7 --- /dev/null +++ b/iterator_val.go @@ -0,0 +1,22 @@ +package jzon + +import ( + "reflect" + "unsafe" +) + +func (it *Iterator) ReadVal(obj interface{}) error { + eface := (*eface)(unsafe.Pointer(&obj)) + if eface.data == nil { + return NilPointerReceiverError + } + dec := it.decoder.getDecoderFromCache(eface.rtype) + if dec == nil { + typ := reflect.TypeOf(obj) + if typ.Kind() != reflect.Ptr { + return PointerReceiverError + } + dec = it.decoder.createDecoder(eface.rtype, typ) + } + return dec.Decode(eface.data, it) +} diff --git a/iterator_val_test.go b/iterator_val_test.go new file mode 100644 index 0000000..3c2a792 --- /dev/null +++ b/iterator_val_test.go @@ -0,0 +1,31 @@ +package jzon + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIterator_Val_ReadVal(t *testing.T) { + t.Run("nil pointer receiver error", func(t *testing.T) { + it := NewIterator() + err := it.ReadVal(nil) + require.Equal(t, NilPointerReceiverError, err) + }) + t.Run("pointer receiver error", func(t *testing.T) { + it := NewIterator() + var o string + err := it.ReadVal(o) + require.Equal(t, PointerReceiverError, err) + }) + t.Run("struct", func(t *testing.T) { + it := NewIterator() + var p struct { + K string `json:"k"` + } + it.ResetBytes([]byte(` { "k": "v" } `)) + err := it.ReadVal(&p) + require.NoError(t, err) + require.Equal(t, "v", p.K) + }) +} diff --git a/jzon.s b/jzon.s new file mode 100644 index 0000000..e69de29 diff --git a/node.go b/node.go new file mode 100644 index 0000000..72ec7ee --- /dev/null +++ b/node.go @@ -0,0 +1,99 @@ +package jzon + +import ( + "sync" +) + +var ( + nodeStackPool = sync.Pool{ + New: func() interface{} { + return &nodeStack{} + }, + } +) + +type readNode struct { + m map[string]interface{} + s []interface{} + field string +} + +type nodeStack struct { + stack []readNode +} + +func (ns *nodeStack) initArray(s []interface{}) *nodeStack { + ns.stack = append(ns.stack[:0], readNode{ + s: s, + }) + return ns +} + +func (ns *nodeStack) initObject(m map[string]interface{}) *nodeStack { + ns.stack = append(ns.stack[:0], readNode{ + m: m, + }) + return ns +} + +func (ns *nodeStack) pushArray(field string) *nodeStack { + ns.stack = append(ns.stack, readNode{ + s: make([]interface{}, 0), + field: field, + }) + return ns +} + +func (ns *nodeStack) pushObject(field string) *nodeStack { + ns.stack = append(ns.stack, readNode{ + m: map[string]interface{}{}, + field: field, + }) + return ns +} + +func (ns *nodeStack) topObject() map[string]interface{} { + return ns.stack[len(ns.stack)-1].m +} + +func (ns *nodeStack) topArray() []interface{} { + return ns.stack[len(ns.stack)-1].s +} + +func (ns *nodeStack) setTopObject(key string, value interface{}) { + ns.stack[len(ns.stack)-1].m[key] = value +} + +func (ns *nodeStack) appendTopArray(value interface{}) { + l := len(ns.stack) - 1 + ns.stack[l].s = append(ns.stack[l].s, value) +} + +func (ns *nodeStack) popObject() { + l := len(ns.stack) - 1 + first := &ns.stack[l] + next := &ns.stack[l-1] + if next.m != nil { + next.m[first.field] = first.m + } else { + next.s = append(next.s, first.m) + } + ns.stack = ns.stack[:l] +} + +func (ns *nodeStack) popArray() { + l := len(ns.stack) - 1 + first := &ns.stack[l] + next := &ns.stack[l-1] + if next.m != nil { + next.m[first.field] = first.s + } else { + next.s = append(next.s, first.s) + } + ns.stack = ns.stack[:l] +} + +func releaseNodeStack(ns *nodeStack) { + ns.stack = ns.stack[:0] + nodeStackPool.Put(ns) +} diff --git a/number.go b/number.go new file mode 100644 index 0000000..672e814 --- /dev/null +++ b/number.go @@ -0,0 +1,7 @@ +package jzon + +import ( + "encoding/json" +) + +type Number = json.Number diff --git a/performance_reset_test.go b/performance_reset_test.go new file mode 100644 index 0000000..82b9d27 --- /dev/null +++ b/performance_reset_test.go @@ -0,0 +1,46 @@ +package jzon + +import ( + "testing" +) + +func resetBytes(it *Iterator, data []byte) { + if it.reader != nil && it.buffer != nil { + releaseByteSlice(it.buffer) + } + it.reader = nil + it.buffer = data + it.offset = 0 + it.head = 0 + it.tail = len(data) +} + +func resetBytes2(it *Iterator, data []byte) { + if it.reader != nil && it.buffer != nil { + releaseByteSlice(it.buffer) + } + *it = Iterator{ + buffer: data, + tail: len(data), + } +} + +func Benchmark_Performance_Reset(b *testing.B) { + data := make([]byte, 128) + b.Run("impl", func(b *testing.B) { + b.ReportAllocs() + it := NewIterator() + b.ResetTimer() + for i := 0; i < b.N; i++ { + resetBytes(it, data) + } + }) + b.Run("alter", func(b *testing.B) { + b.ReportAllocs() + it := NewIterator() + b.ResetTimer() + for i := 0; i < b.N; i++ { + resetBytes2(it, data) + } + }) +} diff --git a/performance_skip_switch_slice_test.go b/performance_skip_switch_slice_test.go new file mode 100644 index 0000000..11a46fd --- /dev/null +++ b/performance_skip_switch_slice_test.go @@ -0,0 +1,51 @@ +package jzon + +import "testing" + +func skip(it *Iterator, c byte, vt ValueType) error { + switch c { + case '"': + return skipString(it, c) + case 'n': + return it.expectBytes("ull") + case 't': + return it.expectBytes("rue") + case 'f': + return it.expectBytes("alse") + case '[': + return skipArrayWithStack(it, c) + case '{': + return skipObjectWithStack(it, c) + default: + if vt != NumberValue { + return UnexpectedByteError{got: c} + } + return skipNumber(it, c) + } +} + +func skip2(it *Iterator, c byte, _ ValueType) error { + return skipFunctions[c](it, c) +} + +func Benchmark_Performance_Skip_Switch(b *testing.B) { + data := []byte(` "s", -123.0456e789, true, false, null, { } ]`) + b.Run("impl", func(b *testing.B) { + b.ReportAllocs() + it := NewIterator() + b.ResetTimer() + for i := 0; i < b.N; i++ { + it.ResetBytes(data) + skip2(it, '[', ArrayValue) + } + }) + b.Run("alter", func(b *testing.B) { + b.ReportAllocs() + it := NewIterator() + b.ResetTimer() + for i := 0; i < b.N; i++ { + it.ResetBytes(data) + skip(it, '[', ArrayValue) + } + }) +} diff --git a/pool.go b/pool.go new file mode 100644 index 0000000..205b0c2 --- /dev/null +++ b/pool.go @@ -0,0 +1,33 @@ +package jzon + +import ( + "sync" +) + +const bufferSize = 64 + +var ( + bytesPool = sync.Pool{ + New: func() interface{} { + return make([]byte, bufferSize) + }, + } +) + +func getByteSlice() []byte { + return bytesPool.Get().([]byte) +} + +func getEmptyByteSlice() []byte { + b := bytesPool.Get().([]byte) + return b[:0] +} + +func getFullByteSlice() []byte { + b := bytesPool.Get().([]byte) + return b[:cap(b)] +} + +func releaseByteSlice(b []byte) { + bytesPool.Put(b) +} diff --git a/reflect.go b/reflect.go new file mode 100644 index 0000000..6fd5f6b --- /dev/null +++ b/reflect.go @@ -0,0 +1,81 @@ +package jzon + +import ( + "reflect" + "unsafe" +) + +/* + * WARNING: + * The linked functions in this file should be used with EXTREMELY careful + */ + +//go:linkname unsafe_New reflect.unsafe_New +func unsafe_New(rtype rtype) unsafe.Pointer + +//go:linkname typedmemclrpartial reflect.typedmemclrpartial +func typedmemclrpartial(t rtype, ptr unsafe.Pointer, off, size uintptr) + +//go:linkname unsafe_NewArray reflect.unsafe_NewArray +func unsafe_NewArray(rtype rtype, length int) unsafe.Pointer + +//go:linkname typedslicecopy reflect.typedslicecopy +func typedslicecopy(rtyp rtype, dst, src reflect.SliceHeader) int + +//go:linkname makemap reflect.makemap +func makemap(rtype rtype, cap int) unsafe.Pointer + +//go:linkname typedmemmove reflect.typedmemmove +func typedmemmove(rtype rtype, dst, src unsafe.Pointer) + +//go:linkname mapassign reflect.mapassign +//go:noescape +func mapassign(t rtype, m, key, val unsafe.Pointer) + +func unsafeMakeSlice(rtype rtype, length, cap int) unsafe.Pointer { + return unsafe.Pointer(&reflect.SliceHeader{ + // TODO: is this safe? + Data: uintptr(unsafe_NewArray(rtype, cap)), + Len: length, + Cap: cap, + }) +} + +func unsafeMakeMap(rtype rtype, cap int) unsafe.Pointer { + m := makemap(rtype, cap) + return unsafe.Pointer(&m) +} + +// see reflect.add +func add(p unsafe.Pointer, x uintptr, whySafe string) unsafe.Pointer { + return unsafe.Pointer(uintptr(p) + x) +} + +// see reflect.grow +func unsafeGrowSlice(rtype, elemRType rtype, ptr unsafe.Pointer, newLength int) unsafe.Pointer { + sh := (*reflect.SliceHeader)(ptr) + if newLength < sh.Cap { + sh.Len = newLength + return ptr + } + newCap := sh.Cap + if sh.Cap == 0 { + newCap = newLength + } else { + for newCap < newLength { + if newCap < 1024 { + newCap <<= 1 + } else { + newCap += newCap >> 2 + } + } + } + newHeader := (*reflect.SliceHeader)(unsafeMakeSlice(rtype, newLength, newCap)) + typedslicecopy(elemRType, *newHeader, *sh) + return unsafe.Pointer(newHeader) +} + +func unsafeSliceChildPtr(ptr unsafe.Pointer, elemSize uintptr, index int) unsafe.Pointer { + sh := (*reflect.SliceHeader)(ptr) + return add(unsafe.Pointer(sh.Data), uintptr(index)*elemSize, "index < len") +} diff --git a/stack.go b/stack.go new file mode 100644 index 0000000..2087101 --- /dev/null +++ b/stack.go @@ -0,0 +1,100 @@ +package jzon + +import ( + "sync" +) + +var ( + stackPool = sync.Pool{ + New: func() interface{} { + return &stack{ + stack: make([]uint64, 1), + } + }, + } +) + +type stackElement = int8 + +const ( + stackElementNone stackElement = -1 + + stackElementObjectBegin stackElement = 0 // 0b00 + stackElementObject stackElement = 2 // 0b10 + + stackElementArrayBegin stackElement = 1 // 0b01 + stackElementArray stackElement = 3 // 0b11 +) + +type stack struct { + stack []uint64 + depth uint +} + +func (s *stack) init() *stack { + s.depth = 0 + return s +} + +func (s *stack) initObject() *stack { + if len(s.stack) == 0 { + s.stack = make([]uint64, 1) + } + s.stack[0] = 0 + s.depth = 1 + return s +} + +func (s *stack) initArray() *stack { + if len(s.stack) == 0 { + s.stack = make([]uint64, 1) + } + s.stack[0] = 1 + s.depth = 1 + return s +} + +func (s *stack) top() stackElement { + if s.depth == 0 { + return stackElementNone + } + depth := s.depth - 1 + div := depth >> 6 + mod := depth & 63 + return stackElement((s.stack[div] >> mod) & 1) +} + +func (s *stack) pop() stackElement { + if s.depth == 0 { + return stackElementNone + } + depth := s.depth - 1 + div := depth >> 6 + mod := depth & 63 + s.depth -= 1 + // stackElementObjectBegin -> stackElementObject + // stackElementArrayBegin -> stackElementArray + return stackElement((s.stack[div]>>mod)&1) | 2 +} + +func (s *stack) pushObject() *stack { + div := s.depth >> 6 + if div == uint(len(s.stack)) { + s.stack = append(s.stack, 0) + } else { + s.stack[div] &= (1 << (s.depth & 63)) - 1 + } + s.depth += 1 + return s +} + +func (s *stack) pushArray() *stack { + div := s.depth >> 6 + if div == uint(len(s.stack)) { + s.stack = append(s.stack, 1) + } else { + s.stack[div] |= 1 << (s.depth & 63) + } + s.depth += 1 + return s +} diff --git a/stack_test.go b/stack_test.go new file mode 100644 index 0000000..25fc751 --- /dev/null +++ b/stack_test.go @@ -0,0 +1,53 @@ +package jzon + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestStack(t *testing.T) { + t.Run("empty", func(t *testing.T) { + s := stackPool.Get().(*stack).init() + require.Equal(t, stackElementNone, s.top()) + require.Equal(t, stackElementNone, s.pop()) + }) + t.Run("array top", func(t *testing.T) { + s := stackPool.Get().(*stack).initArray() + require.Equal(t, stackElementArrayBegin, s.top()) + require.Equal(t, stackElementArray, s.pop()) + require.Equal(t, stackElementNone, s.top()) + require.Equal(t, stackElementNone, s.pop()) + }) + t.Run("object top", func(t *testing.T) { + s := stackPool.Get().(*stack).initObject() + require.Equal(t, stackElementObjectBegin, s.top()) + require.Equal(t, stackElementObject, s.pop()) + require.Equal(t, stackElementNone, s.top()) + require.Equal(t, stackElementNone, s.pop()) + }) + t.Run("nested 1", func(t *testing.T) { + s := stackPool.Get().(*stack) + count := 100 + for i := 0; i < count; i++ { + if i&1 == 0 { + s.pushObject() + require.Equal(t, stackElementObjectBegin, s.top()) + } else { + s.pushArray() + require.Equal(t, stackElementArrayBegin, s.top()) + } + } + for i := count - 1; i >= 0; i-- { + if i&1 == 0 { + require.Equal(t, stackElementObjectBegin, s.top()) + require.Equal(t, stackElementObject, s.pop()) + } else { + require.Equal(t, stackElementArrayBegin, s.top()) + require.Equal(t, stackElementArray, s.pop()) + } + } + require.Equal(t, stackElementNone, s.top()) + require.Equal(t, stackElementNone, s.pop()) + }) +} diff --git a/standard_compatible.go b/standard_compatible.go new file mode 100644 index 0000000..d968469 --- /dev/null +++ b/standard_compatible.go @@ -0,0 +1,12 @@ +package jzon + +func Unmarshal(data []byte, o interface{}) error { + return DefaultDecoder.Unmarshal(data, o) +} + +func Valid(data []byte) bool { + it := NewIterator() + b := it.Valid(data) + ReturnIterator(it) + return b +} diff --git a/standard_compatible_test.go b/standard_compatible_test.go new file mode 100644 index 0000000..159b5f3 --- /dev/null +++ b/standard_compatible_test.go @@ -0,0 +1,100 @@ +package jzon + +import ( + "encoding/json" + "errors" + "os" + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + compatibleOnError = os.Getenv("COMPATIBLE_ON_ERROR") == "1" +) + +func nestedArray1(count int) string { + return strings.Repeat(" [", count) + " [ ] " + + strings.Repeat("] ", count) +} + +func nestedArray2(count int) string { + return strings.Repeat(" [ [ ], ", count) + " [ ] " + + strings.Repeat("] ", count) +} + +func nestedArrayWithObject(count int) string { + return strings.Repeat(" [ { }, ", count) + " [ ] " + + strings.Repeat("] ", count) +} + +func nestedObject(count int) string { + return strings.Repeat(` { "a" : { }, "b": `, count) + " { } " + + strings.Repeat("} ", count) +} + +func nestedObjectWithArray(count int) string { + return strings.Repeat(` { "a" : [ ], "b": `, count) + " [ ] " + + strings.Repeat("} ", count) +} + +func printValue(t *testing.T, prefix string, o interface{}) { + prefix += " " + if o == nil { + t.Logf(prefix + "nil") + return + } + oV := reflect.ValueOf(o) + for indent := prefix; ; indent += " " { + k := oV.Kind() + t.Logf(indent+"%+v %+v", oV.Type(), oV) + if k != reflect.Interface && k != reflect.Ptr { + break + } + if oV.IsNil() { + break + } + oV = oV.Elem() + } +} + +func checkStandard(t *testing.T, decoder *Decoder, data string, ex error, exp, got interface{}) { + b := []byte(data) + expErr := json.Unmarshal(b, exp) + gotErr := decoder.Unmarshal(b, got) + t.Logf("\nexpErr: %+v\ngotErr: %+v", expErr, gotErr) + noError := expErr == nil + if noError { + printValue(t, "exp", reflect.ValueOf(exp).Elem().Interface()) + } + require.Equal(t, noError, gotErr == nil, + "exp %+v\ngot %+v", expErr, gotErr) + require.Equalf(t, noError, ex == nil, "exp:%v\ngot:%v", ex, gotErr) + if ex != nil { + if assert.ObjectsAreEqual(reflect.TypeOf(errors.New("")), reflect.TypeOf(ex)) { + require.Equalf(t, ex, gotErr, "exp:%v\ngot:%v", ex, gotErr) + } else { + require.IsTypef(t, ex, gotErr, "exp:%v\ngot:%v", ex, gotErr) + } + } + if !noError && !compatibleOnError { + return + } + if exp == nil { + require.Equal(t, nil, got) + return + } + expV := reflect.ValueOf(exp) + gotV := reflect.ValueOf(got) + if expV.IsNil() { + require.True(t, gotV.IsNil()) + return + } + expI := expV.Elem().Interface() + gotI := gotV.Elem().Interface() + printValue(t, "got", gotI) + require.Equalf(t, expI, gotI, "exp %+v\ngot %+v", expI, gotI) +} diff --git a/val_decoder.go b/val_decoder.go new file mode 100644 index 0000000..4bab315 --- /dev/null +++ b/val_decoder.go @@ -0,0 +1,98 @@ +package jzon + +import ( + "encoding/json" + "reflect" + "strconv" + "unsafe" +) + +const numKinds = reflect.UnsafePointer + 1 + +var ( + globalValDecoders = map[rtype]ValDecoder{} + kindMap = [numKinds]rtype{} + kindDecoders = [numKinds]ValDecoder{} + keyDecoders = [numKinds]ValDecoder{} +) + +func createGlobalValDecoder(ptr interface{}, dec ValDecoder) { + ef := (*eface)(unsafe.Pointer(&ptr)) + globalValDecoders[ef.rtype] = dec +} + +func mapKind(ptr interface{}, dec ValDecoder) { + ef := (*eface)(unsafe.Pointer(&ptr)) + kind := reflect.TypeOf(ptr).Elem().Kind() + kindMap[kind] = ef.rtype + kindDecoders[kind] = dec +} + +func mapKeyDecoder(ptr interface{}, dec ValDecoder) { + ptrType := reflect.TypeOf(ptr) + kind := ptrType.Elem().Kind() + keyDecoders[kind] = dec +} + +func init() { + // json lib types + createGlobalValDecoder((*json.Number)(nil), (*jsonNumberDecoder)(nil)) + createGlobalValDecoder((*json.RawMessage)(nil), (*jsonRawMessageDecoder)(nil)) + + // kind mapping + mapKind((*bool)(nil), (*boolDecoder)(nil)) + mapKind((*string)(nil), (*stringDecoder)(nil)) + if strconv.IntSize == 32 { + mapKind((*int)(nil), (*int32Decoder)(nil)) + mapKind((*uint)(nil), (*uint32Decoder)(nil)) + } else { + mapKind((*int)(nil), (*int64Decoder)(nil)) + mapKind((*uint)(nil), (*uint64Decoder)(nil)) + } + if unsafe.Sizeof(uintptr(0)) == 4 { + mapKind((*uintptr)(nil), (*uint32Decoder)(nil)) + } else { + mapKind((*uintptr)(nil), (*uint64Decoder)(nil)) + } + mapKind((*int8)(nil), (*int8Decoder)(nil)) + mapKind((*int16)(nil), (*int16Decoder)(nil)) + mapKind((*int32)(nil), (*int32Decoder)(nil)) + mapKind((*int64)(nil), (*int64Decoder)(nil)) + mapKind((*uint8)(nil), (*uint8Decoder)(nil)) + mapKind((*uint16)(nil), (*uint16Decoder)(nil)) + mapKind((*uint32)(nil), (*uint32Decoder)(nil)) + mapKind((*uint64)(nil), (*uint64Decoder)(nil)) + mapKind((*float32)(nil), (*float32Decoder)(nil)) + mapKind((*float64)(nil), (*float64Decoder)(nil)) + + // object key decoders + mapKeyDecoder((*string)(nil), (*stringKeyDecoder)(nil)) + if strconv.IntSize == 32 { + mapKeyDecoder((*int)(nil), (*int32KeyDecoder)(nil)) + } else { + mapKeyDecoder((*int)(nil), (*int64KeyDecoder)(nil)) + } + mapKeyDecoder((*int8)(nil), (*int8KeyDecoder)(nil)) + mapKeyDecoder((*int16)(nil), (*int16KeyDecoder)(nil)) + mapKeyDecoder((*int32)(nil), (*int32KeyDecoder)(nil)) + mapKeyDecoder((*int64)(nil), (*int64KeyDecoder)(nil)) + mapKeyDecoder((*uint8)(nil), (*uint8KeyDecoder)(nil)) + mapKeyDecoder((*uint16)(nil), (*uint16KeyDecoder)(nil)) + mapKeyDecoder((*uint32)(nil), (*uint32KeyDecoder)(nil)) + mapKeyDecoder((*uint64)(nil), (*uint64KeyDecoder)(nil)) + if unsafe.Sizeof(uintptr(0)) == 4 { + mapKeyDecoder((*uintptr)(nil), (*uint32KeyDecoder)(nil)) + } else { + mapKeyDecoder((*uintptr)(nil), (*uint64KeyDecoder)(nil)) + } +} + +type ValDecoder interface { + Decode(ptr unsafe.Pointer, it *Iterator) error +} + +type notSupportedDecoder string + +func (dec notSupportedDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + return TypeNotSupportedError(dec) +} diff --git a/val_decoder_json_number.go b/val_decoder_json_number.go new file mode 100644 index 0000000..115b347 --- /dev/null +++ b/val_decoder_json_number.go @@ -0,0 +1,38 @@ +package jzon + +import "unsafe" + +type jsonNumberDecoder struct { +} + +func (*jsonNumberDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, vt, err := it.nextToken() + if err != nil { + return err + } + var s string + switch vt { + case StringValue: + it.head += 1 + s, err = it.readString() + if err != nil { + return err + } + *((*string)(ptr)) = s + return nil + case NullValue: + // to be compatible with standard lib + it.head += 1 + return it.expectBytes("ull") + case NumberValue: + // do not increase it.head here + s, err = it.readNumberAsString(c) + if err != nil { + return err + } + *((*string)(ptr)) = s + return nil + default: + return UnexpectedByteError{got: c} + } +} diff --git a/val_decoder_json_number_test.go b/val_decoder_json_number_test.go new file mode 100644 index 0000000..2e6b86a --- /dev/null +++ b/val_decoder_json_number_test.go @@ -0,0 +1,51 @@ +package jzon + +import ( + "encoding/json" + "io" + "testing" +) + +func TestValDecoder_JsonNumber(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue json.Number) { + var p1 *json.Number + var p2 *json.Number + if initValue != "" { + b1 := initValue + p1 = &b1 + b2 := initValue + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, "1.23") + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, "") + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `+`, UnexpectedByteError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("invalid string", func(t *testing.T) { + f2(t, `"abc`, io.EOF) + }) + t.Run("string", func(t *testing.T) { + f2(t, `"abc"`, nil) + }) + t.Run("invalid number", func(t *testing.T) { + f2(t, `-0.e`, InvalidDigitError{}) + }) + t.Run("valid number", func(t *testing.T) { + f2(t, `123.456e+0789`, nil) + }) +} diff --git a/val_decoder_json_rawmessage.go b/val_decoder_json_rawmessage.go new file mode 100644 index 0000000..3fd57c2 --- /dev/null +++ b/val_decoder_json_rawmessage.go @@ -0,0 +1,15 @@ +package jzon + +import "unsafe" + +type jsonRawMessageDecoder struct { +} + +func (*jsonRawMessageDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + bytePtr := (*[]byte)(ptr) + b, err := it.AppendRaw((*bytePtr)[:0]) + if err == nil { + *bytePtr = b + } + return err +} diff --git a/val_decoder_json_rawmessage_test.go b/val_decoder_json_rawmessage_test.go new file mode 100644 index 0000000..3c21ba4 --- /dev/null +++ b/val_decoder_json_rawmessage_test.go @@ -0,0 +1,33 @@ +package jzon + +import ( + "encoding/json" + "io" + "testing" +) + +func TestValDecoder_JsonRawMessage(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue string) { + var p1 *json.RawMessage + var p2 *json.RawMessage + if initValue != "" { + b1 := append(json.RawMessage(nil), initValue...) + p1 = &b1 + b2 := append(json.RawMessage(nil), initValue...) + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, "1.23") + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, "") + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("valid", func(t *testing.T) { + f2(t, `null`, nil) + }) +} diff --git a/val_decoder_json_unmarshaler.go b/val_decoder_json_unmarshaler.go new file mode 100644 index 0000000..8fe690f --- /dev/null +++ b/val_decoder_json_unmarshaler.go @@ -0,0 +1,23 @@ +package jzon + +import ( + "encoding/json" + "reflect" + "unsafe" +) + +var ( + jsonUnmarshalerType = reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() +) + +type jsonUnmarshalerDecoder rtype + +func (dec jsonUnmarshalerDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + obj := packEFace(rtype(dec), ptr) + unmarshaler := obj.(json.Unmarshaler) + b, err := it.SkipRaw() + if err != nil { + return err + } + return unmarshaler.UnmarshalJSON(b) +} diff --git a/val_decoder_json_unmarshaler_test.go b/val_decoder_json_unmarshaler_test.go new file mode 100644 index 0000000..f5567f5 --- /dev/null +++ b/val_decoder_json_unmarshaler_test.go @@ -0,0 +1,52 @@ +package jzon + +import ( + "errors" + "io" + "testing" +) + +type testJsonUnmarshaler struct { + data string + err error +} + +func (t *testJsonUnmarshaler) UnmarshalJSON(data []byte) error { + t.data = string(data) + return t.err +} + +func TestValDecoder_JsonUnmarshaler(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue string, initErr error) { + var p1 *testJsonUnmarshaler + var p2 *testJsonUnmarshaler + if initValue != "" { + b1 := testJsonUnmarshaler{ + data: initValue, + err: initErr, + } + p1 = &b1 + b2 := testJsonUnmarshaler{ + data: initValue, + err: initErr, + } + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error, initErr error) { + f(t, data, ex, "dummy", initErr) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, "", nil) + }) + t.Run("eof", func(t *testing.T) { + f2(t, " ", io.EOF, nil) + }) + t.Run("custom error", func(t *testing.T) { + f2(t, ` null `, errors.New("test"), errors.New("test")) + }) + t.Run("no error", func(t *testing.T) { + f2(t, ` null `, nil, nil) + }) +} diff --git a/val_decoder_native.go b/val_decoder_native.go new file mode 100644 index 0000000..fd38035 --- /dev/null +++ b/val_decoder_native.go @@ -0,0 +1,63 @@ +package jzon + +import ( + "unsafe" +) + +// bool decoder +type boolDecoder struct { +} + +func (*boolDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + switch c { + case 'n': + it.head += 1 + return it.expectBytes("ull") + case 't': + it.head += 1 + if err := it.expectBytes("rue"); err != nil { + return err + } + *(*bool)(ptr) = true + return nil + case 'f': + it.head += 1 + if err := it.expectBytes("alse"); err != nil { + return err + } + *(*bool)(ptr) = false + return nil + default: + return UnexpectedByteError{got: c} + } +} + +// string decoder +type stringDecoder struct { +} + +func (*stringDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + switch c { + case '"': + it.head += 1 + s, err := it.readString() + if err != nil { + return err + } + *(*string)(ptr) = s + return nil + case 'n': + it.head += 1 + return it.expectBytes("ull") + default: + return UnexpectedByteError{got: c, exp: '"', exp2: 'n'} + } +} diff --git a/val_decoder_native_array.go b/val_decoder_native_array.go new file mode 100644 index 0000000..918a30f --- /dev/null +++ b/val_decoder_native_array.go @@ -0,0 +1,83 @@ +package jzon + +import ( + "reflect" + "unsafe" +) + +type arrayDecoder struct { + rtype rtype + elemDec ValDecoder + elemRType rtype + elemPtrRType rtype + elemSize uintptr + length int +} + +func newArrayDecoder(arrType reflect.Type) *arrayDecoder { + elem := arrType.Elem() + return &arrayDecoder{ + rtype: rtypeOfType(arrType), + elemRType: rtypeOfType(elem), + elemPtrRType: rtypeOfType(reflect.PtrTo(elem)), + elemSize: elem.Size(), + length: arrType.Len(), + } +} + +func (dec *arrayDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + return it.expectBytes("ull") + } + if c != '[' { + return UnexpectedByteError{got: c, exp: '[', exp2: 'n'} + } + it.head += 1 + c, _, err = it.nextToken() + if err != nil { + return err + } + count := 0 + childPtr := uintptr(ptr) + if c == ']' { + it.head += 1 + } else { + for { + if count < dec.length { + elemPtr := unsafe.Pointer(childPtr) + if err := dec.elemDec.Decode(elemPtr, it); err != nil { + return err + } + count += 1 + childPtr += dec.elemSize + } else { + if err := it.Skip(); err != nil { + return err + } + } + c, _, err = it.nextToken() + if err != nil { + return err + } + it.head += 1 + if c == ']' { + break + } + if c != ',' { + return UnexpectedByteError{got: c, exp: ']', exp2: ','} + } + } + } + if count < dec.length { + // should be safe (?) + typedmemclrpartial(dec.rtype, unsafe.Pointer(childPtr), + uintptr(count)*dec.elemSize, + uintptr(dec.length-count)*dec.elemSize) + } + return nil +} diff --git a/val_decoder_native_array_test.go b/val_decoder_native_array_test.go new file mode 100644 index 0000000..d9098da --- /dev/null +++ b/val_decoder_native_array_test.go @@ -0,0 +1,114 @@ +package jzon + +import ( + "io" + "runtime" + "runtime/debug" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValDecoder_Native_Array(t *testing.T) { + f := func(t *testing.T, data string, ex error, p1, p2 interface{}) { + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, nil, nil) + }) + t.Run("eof", func(t *testing.T) { + arr1 := [...]int{1, 2} + arr2 := [...]int{1, 2} + f(t, "", io.EOF, &arr1, &arr2) + }) + t.Run("null", func(t *testing.T) { + arr1 := [...]int{1, 2} + arr2 := [...]int{1, 2} + f(t, "null", nil, &arr1, &arr2) + }) + t.Run("invalid first byte", func(t *testing.T) { + arr1 := [...]int{1, 2} + arr2 := [...]int{1, 2} + f(t, "+", UnexpectedByteError{}, &arr1, &arr2) + }) + t.Run("eof after bracket", func(t *testing.T) { + arr1 := [...]int{1, 2} + arr2 := [...]int{1, 2} + f(t, "[", io.EOF, &arr1, &arr2) + }) + t.Run("empty", func(t *testing.T) { + arr1 := [...]int{1, 2} + arr2 := [...]int{1, 2} + f(t, " [ ] ", nil, &arr1, &arr2) + }) + t.Run("element error", func(t *testing.T) { + arr1 := [...]int{1, 2} + arr2 := [...]int{1, 2} + f(t, ` [ " `, InvalidDigitError{}, &arr1, &arr2) + }) + t.Run("null element", func(t *testing.T) { + arr1 := [...]int{1, 2} + arr2 := [...]int{1, 2} + f(t, ` [ null ] `, nil, &arr1, &arr2) + }) + t.Run("eof after element", func(t *testing.T) { + arr1 := [...]int{1, 2} + arr2 := [...]int{1, 2} + f(t, ` [ 2 `, io.EOF, &arr1, &arr2) + }) + t.Run("lesser element", func(t *testing.T) { + arr1 := [...]int{1, 2} + arr2 := [...]int{1, 2} + f(t, ` [ 2 ] `, nil, &arr1, &arr2) + }) + t.Run("invalid comma", func(t *testing.T) { + arr1 := [...]int{1, 2} + arr2 := [...]int{1, 2} + f(t, ` [ 2 [ `, UnexpectedByteError{}, &arr1, &arr2) + }) + t.Run("more element error", func(t *testing.T) { + arr1 := [...]int{1, 2} + arr2 := [...]int{1, 2} + f(t, ` [ 2 , 3 , `, io.EOF, &arr1, &arr2) + }) + t.Run("more element", func(t *testing.T) { + arr1 := [...]int{1, 2} + arr2 := [...]int{1, 2} + f(t, ` [ 2 , 3 , "test"]`, nil, &arr1, &arr2) + }) + debug.FreeOSMemory() +} + +func TestValDecoder_Native_Array_Memory(t *testing.T) { + t.Run("test1", func(t *testing.T) { + f := func(i int) [1]*int { + pi := &i + runtime.SetFinalizer(pi, func(_ *int) { + t.Logf("finalizing") + }) + return [1]*int{pi} + } + arr := f(1) + err := Unmarshal([]byte(`[]`), &arr) + require.NoError(t, err) + debug.FreeOSMemory() + t.Logf("please check if the memory has been freed") + }) + t.Run("test2", func(t *testing.T) { + type st struct { + p *int + } + f := func(i int) [1]*st { + pi := &i + runtime.SetFinalizer(pi, func(_ *int) { + t.Logf("finalizing") + }) + return [1]*st{&st{p: pi}} + } + arr := f(1) + err := Unmarshal([]byte(`[]`), &arr) + require.NoError(t, err) + debug.FreeOSMemory() + t.Logf("please check if the memory has been freed") + }) +} diff --git a/val_decoder_native_float.go b/val_decoder_native_float.go new file mode 100644 index 0000000..b80adb5 --- /dev/null +++ b/val_decoder_native_float.go @@ -0,0 +1,47 @@ +package jzon + +import ( + "unsafe" +) + +// float32 decoder +type float32Decoder struct { +} + +func (*float32Decoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + return it.expectBytes("ull") + } + f, err := it.ReadFloat32() + if err != nil { + return err + } + *(*float32)(ptr) = f + return nil +} + +// float64 decoder +type float64Decoder struct { +} + +func (*float64Decoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + return it.expectBytes("ull") + } + f, err := it.ReadFloat64() + if err != nil { + return err + } + *(*float64)(ptr) = f + return nil +} diff --git a/val_decoder_native_float_test.go b/val_decoder_native_float_test.go new file mode 100644 index 0000000..0fd8477 --- /dev/null +++ b/val_decoder_native_float_test.go @@ -0,0 +1,76 @@ +package jzon + +import ( + "io" + "testing" +) + +func TestValDecoder_Native_Float32(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue float32) { + var p1 *float32 + var p2 *float32 + if initValue != 0 { + b1 := initValue + p1 = &b1 + b2 := initValue + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, 1.234) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, 0) + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `true`, InvalidFloatError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("valid", func(t *testing.T) { + f2(t, "0.123e-4", nil) + }) +} + +func TestValDecoder_Native_Float64(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue float64) { + var p1 *float64 + var p2 *float64 + if initValue != 0 { + b1 := initValue + p1 = &b1 + b2 := initValue + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, 1.234) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, 0) + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `true`, InvalidFloatError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("valid", func(t *testing.T) { + f2(t, "0.123e-45", nil) + }) +} diff --git a/val_decoder_native_int.go b/val_decoder_native_int.go new file mode 100644 index 0000000..19ad2ed --- /dev/null +++ b/val_decoder_native_int.go @@ -0,0 +1,89 @@ +package jzon + +import ( + "unsafe" +) + +// int8 decoder +type int8Decoder struct { +} + +func (*int8Decoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + return it.expectBytes("ull") + } + i, err := it.ReadInt8() + if err != nil { + return err + } + *(*int8)(ptr) = i + return nil +} + +// int16 decoder +type int16Decoder struct { +} + +func (*int16Decoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + return it.expectBytes("ull") + } + i, err := it.ReadInt16() + if err != nil { + return err + } + *(*int16)(ptr) = i + return nil +} + +// int32 decoder +type int32Decoder struct { +} + +func (*int32Decoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + return it.expectBytes("ull") + } + i, err := it.ReadInt32() + if err != nil { + return err + } + *(*int32)(ptr) = i + return nil +} + +// int64 decoder +type int64Decoder struct { +} + +func (*int64Decoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + return it.expectBytes("ull") + } + i, err := it.ReadInt64() + if err != nil { + return err + } + *(*int64)(ptr) = i + return nil +} diff --git a/val_decoder_native_int_test.go b/val_decoder_native_int_test.go new file mode 100644 index 0000000..3e79a0f --- /dev/null +++ b/val_decoder_native_int_test.go @@ -0,0 +1,181 @@ +package jzon + +import ( + "io" + "testing" +) + +func TestValDecoder_Native_Int(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue int) { + var p1 *int + var p2 *int + if initValue != 0 { + b1 := initValue + p1 = &b1 + b2 := initValue + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, 1) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, 0) + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `true`, InvalidDigitError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("valid", func(t *testing.T) { + f2(t, "127", nil) + }) +} + +func TestValDecoder_Native_Int8(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue int8) { + var p1 *int8 + var p2 *int8 + if initValue != 0 { + b1 := initValue + p1 = &b1 + b2 := initValue + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, 1) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, 0) + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `true`, InvalidDigitError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("valid", func(t *testing.T) { + f2(t, "127", nil) + }) +} + +func TestValDecoder_Native_Int16(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue int16) { + var p1 *int16 + var p2 *int16 + if initValue != 0 { + b1 := initValue + p1 = &b1 + b2 := initValue + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, 1) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, 0) + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `true`, InvalidDigitError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("valid", func(t *testing.T) { + f2(t, "127", nil) + }) +} + +func TestValDecoder_Native_Int32(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue int32) { + var p1 *int32 + var p2 *int32 + if initValue != 0 { + b1 := initValue + p1 = &b1 + b2 := initValue + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, 1) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, 0) + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `true`, InvalidDigitError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("valid", func(t *testing.T) { + f2(t, "127", nil) + }) +} + +func TestValDecoder_Native_Int64(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue int64) { + var p1 *int64 + var p2 *int64 + if initValue != 0 { + b1 := initValue + p1 = &b1 + b2 := initValue + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, 1) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, 0) + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `true`, InvalidDigitError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("valid", func(t *testing.T) { + f2(t, "127", nil) + }) +} diff --git a/val_decoder_native_interface.go b/val_decoder_native_interface.go new file mode 100644 index 0000000..ef5c578 --- /dev/null +++ b/val_decoder_native_interface.go @@ -0,0 +1,108 @@ +package jzon + +import ( + "reflect" + "unsafe" +) + +/* + * Interface decoder is special, when the object is not nil, + * the internal type cannot be analysed by reflect.TypeOf, + * the value must be used + */ +type efaceDecoder struct { +} + +func (dec *efaceDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + ef := (*eface)(ptr) + if ef.data == nil { + // the pointer is a nil pointer + // or the element is a nil typed pointer (kinda tricky here) + o, err := it.Read() + if err != nil { + return err + } + *(*interface{})(ptr) = o + return nil + } + pObj := (*interface{})(ptr) + obj := *pObj + typ := reflect.TypeOf(obj) + if typ.Kind() != reflect.Ptr { + /* + * Example: + * var o interface{} = 1 + * Unmarshal(`"string"`, &o) + */ + o, err := it.Read() + if err != nil { + return err + } + *pObj = o + return nil + } + // obj is pointer + c, _, err := it.nextToken() + if err != nil { + return err + } + ptrElemType := typ.Elem() + if c == 'n' { + it.head += 1 + if err := it.expectBytes("ull"); err != nil { + return err + } + // we have already check above so that + // obj is not nil + if ptrElemType.Kind() != reflect.Ptr { + /* + * Example: + * i := 1 + * var o interface{} = &i + * Unmarshal(`null`, &o) + */ + *pObj = nil + return nil + } + /* + * Example: + * i := 1 + * pi := &i + * var o interface{} = &pi + * Unmarshal(`null`, &o) + */ + *pObj = reflect.New(ptrElemType).Interface() + return nil + } + // when we arrive here, we have: + // 1 obj is pointer + // 2 obj != nil + if err := it.ReadVal(obj); err != nil { + return err + } + *pObj = obj + return nil +} + +type ifaceDecoder struct { +} + +func (dec *ifaceDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + if err = it.expectBytes("ull"); err != nil { + return err + } + *(*interface{})(ptr) = nil + return nil + } + o := packIFace(ptr) + if o == nil { + return IFaceError + } + return it.ReadVal(o) +} diff --git a/val_decoder_native_interface_test.go b/val_decoder_native_interface_test.go new file mode 100644 index 0000000..ac24992 --- /dev/null +++ b/val_decoder_native_interface_test.go @@ -0,0 +1,306 @@ +package jzon + +import ( + "errors" + "io" + "reflect" + "runtime/debug" + "testing" + + "github.com/stretchr/testify/require" +) + +type testIface interface { + Foo() +} + +type testIfaceImpl struct { + field string +} + +func (testIfaceImpl) Foo() { +} + +func copyValue(t *testing.T, in interface{}) (out interface{}) { + if in == nil { + return nil + } + typ := reflect.TypeOf(in) + switch k := typ.Kind(); k { + case reflect.String, + reflect.Int, + reflect.Uint8: + return in + case reflect.Map: + v := reflect.ValueOf(in) + newV := reflect.MakeMap(typ) + if !v.IsNil() { + iter := v.MapRange() + for iter.Next() { + k := copyValue(t, iter.Key().Interface()) + v := copyValue(t, iter.Value().Interface()) + newV.SetMapIndex(reflect.ValueOf(k), + reflect.ValueOf(v)) + } + } + return newV.Interface() + case reflect.Ptr: + ptrValue := reflect.ValueOf(in) + if ptrValue.IsNil() { + newV := reflect.NewAt(typ.Elem(), nil) + return newV.Interface() + } else { + elem := ptrValue.Elem() + copied := copyValue(t, elem.Interface()) + newV := reflect.New(elem.Type()) + if copied != nil { + newV.Elem().Set(reflect.ValueOf(copied)) + } + return newV.Interface() + } + case reflect.Struct: + oldV := reflect.ValueOf(in) + newV := reflect.New(typ).Elem() + for i := 0; i < oldV.NumField(); i++ { + field := oldV.Field(i) + if field.CanInterface() { + copiedV := copyValue(t, field.Interface()) + newV.Field(i).Set(reflect.ValueOf(copiedV)) + } + } + return newV.Interface() + case reflect.Slice: + v := reflect.ValueOf(in) + l := v.Len() + newV := reflect.MakeSlice(typ, l, l) + for i := 0; i < l; i++ { + copied := copyValue(t, v.Index(i).Interface()) + newV.Index(i).Set(reflect.ValueOf(copied)) + } + return newV.Interface() + case reflect.Interface: + v := reflect.ValueOf(in) + newV := reflect.New(typ) + if !v.IsNil() { + copiedV := v.Elem().Interface() + newV.Elem().Set(reflect.ValueOf(copiedV)) + } + return newV.Elem() + default: + t.Fatalf("%v(%s) not supported", typ, k.String()) + panic("should not reach here") + } +} + +func TestValDecoder_Native_Interface(t *testing.T) { + f := func(t *testing.T, data string, ex error, p1, p2 interface{}) { + t.Log(">>>>> initValues >>>>>") + printValue(t, "p1", p1) + printValue(t, "p2", p2) + t.Log(">>>>>>>>>>>>>>>>>>>>>>") + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + t.Log("<<<<< initValues <<<<<") + printValue(t, "p1", p1) + printValue(t, "p2", p2) + t.Log("<<<<<<<<<<<<<<<<<<<<<<") + } + f2 := func(t *testing.T, data string, ex error, initValues ...interface{}) { + var v1 interface{} + var v2 interface{} + var p1 *interface{} + var p2 *interface{} + if len(initValues) != 0 { + if len(initValues) == 1 { + v1 = copyValue(t, initValues[0]) + v2 = copyValue(t, initValues[0]) + } else { + v1 = initValues[0] + v2 = initValues[1] + } + require.Equal(t, v1, v2) + p1 = &v1 + p2 = &v2 + } + f(t, data, ex, p1, p2) + } + f3 := func(t *testing.T, data string, ex error) { + f2(t, data, ex, "dummy") + } + + // eface + t.Run("nil pointer", func(t *testing.T) { + f2(t, "null", NilPointerReceiverError) + }) + t.Run("eof", func(t *testing.T) { + f3(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `+`, UnexpectedByteError{}, nil) + }) + t.Run("nil init value", func(t *testing.T) { + f2(t, `{"a":"b"}`, nil, nil) + }) + t.Run("nil typed pointer 1", func(t *testing.T) { + f2(t, `{"a":"b"}`, nil, (*int)(nil)) + }) + t.Run("nil typed pointer 2", func(t *testing.T) { + f2(t, `{"a":"b"}`, nil, (**int)(nil)) + }) + t.Run("nil typed pointer 3", func(t *testing.T) { + f2(t, `{"a":"b"}`, nil, (***int)(nil)) + }) + t.Run("non compatible value", func(t *testing.T) { + f2(t, `{"a":"b"}`, nil, 1) + }) + t.Run("non compatible value 2", func(t *testing.T) { + f2(t, `{"a":"b"}`, nil, testJsonUnmarshaler{ + data: "123", + err: errors.New("test"), + }, testJsonUnmarshaler{ + data: "123", + err: errors.New("test"), + }) + }) + t.Run("eof 2", func(t *testing.T) { + f2(t, ``, io.EOF, &testJsonUnmarshaler{ + data: "123", + err: errors.New("test error"), + }, &testJsonUnmarshaler{ + data: "123", + err: errors.New("test error"), + }) + }) + t.Run("non compatible value 3", func(t *testing.T) { + ex := errors.New("test error") + f2(t, `{"a":"b"}`, ex, + &testJsonUnmarshaler{ + data: "123", + err: ex, + }, &testJsonUnmarshaler{ + data: "123", + err: ex, + }) + }) + t.Run("null on non nil init value", func(t *testing.T) { + f2(t, `null`, nil, "1") + }) + t.Run("different type with non nil init value", func(t *testing.T) { + f2(t, `123`, nil, "1") + }) + t.Run("null with nil pointer init value", func(t *testing.T) { + f2(t, `null`, nil, (*int)(nil)) + }) + t.Run("null with nil pointer init value 2", func(t *testing.T) { + f2(t, `null`, nil, (**int)(nil)) + }) + t.Run("invalid null with non nil pointer init value", func(t *testing.T) { + i := 1 + f2(t, `nul`, io.EOF, &i) + }) + t.Run("null with non nil pointer init value", func(t *testing.T) { + i := 1 + f2(t, `null`, nil, &i) + }) + t.Run("non null with non nil pointer init value", func(t *testing.T) { + i := 1 + f2(t, `"test"`, InvalidDigitError{}, &i) + }) + t.Run("null with pt", func(t *testing.T) { + var v interface{} + pv := &v // *interface{} + var ppv interface{} = &pv // **interface{} + f2(t, `null`, nil, ppv) + }) + t.Run("null with ptr 2", func(t *testing.T) { + // var v interface{} + // pv := &v // *interface{} + // var ppv interface{} = &pv // **interface{} + f2(t, `null`, nil, (**interface{})(nil)) + }) + t.Run("non null with ptr 2", func(t *testing.T) { + f2(t, `"test"`, nil, (**interface{})(nil)) + }) + t.Run("null with ptr 3-1", func(t *testing.T) { + var v interface{} + pv := &v // *interface{} + f2(t, `null`, nil, &pv) // **interface{} + }) + t.Run("non null with ptr 3-1", func(t *testing.T) { + var v interface{} + pv := &v // *interface{} + f2(t, `24`, nil, &pv) // **interface{} + }) + t.Run("null with ptr 3-2", func(t *testing.T) { + var v interface{} + pv := &v + ppv := &pv + var pppv interface{} = &ppv + f2(t, `null`, nil, &pppv) + }) + t.Run("non null with ptr 3-2", func(t *testing.T) { + var v interface{} + pv := &v + ppv := &pv + var pppv interface{} = &ppv + f2(t, `"test"`, nil, &pppv) + }) + + // iface + t.Run("iface eof", func(t *testing.T) { + var um1 testIface + var um2 testIface + f(t, ``, io.EOF, &um1, &um2) + }) + t.Run("iface invalid null", func(t *testing.T) { + var um1 testIface + var um2 testIface + f(t, `nul`, io.EOF, &um1, &um2) + }) + t.Run("iface null 1", func(t *testing.T) { + var um1 testIface + var um2 testIface + f(t, `null`, nil, &um1, &um2) + }) + t.Run("iface null 2", func(t *testing.T) { + var um1 testIface = testIfaceImpl{ + field: "test", + } + var um2 testIface = testIfaceImpl{ + field: "test", + } + f(t, `null`, nil, &um1, &um2) + }) + t.Run("iface null 3", func(t *testing.T) { + var um1 testIface = &testIfaceImpl{ + field: "test", + } + var um2 testIface = &testIfaceImpl{ + field: "test", + } + f(t, `null`, nil, &um1, &um2) + }) + t.Run("iface not null 1", func(t *testing.T) { + var um1 testIface + var um2 testIface + f(t, `{}`, IFaceError, &um1, &um2) + }) + t.Run("iface not null 2", func(t *testing.T) { + var um1 testIface = testIfaceImpl{ + field: "test", + } + var um2 testIface = testIfaceImpl{ + field: "test", + } + f(t, `{}`, PointerReceiverError, &um1, &um2) + }) + t.Run("iface not null 3", func(t *testing.T) { + var um1 testIface = &testIfaceImpl{ + field: "test", + } + var um2 testIface = &testIfaceImpl{ + field: "test", + } + f(t, `{}`, nil, &um1, &um2) + }) + debug.FreeOSMemory() +} diff --git a/val_decoder_native_map.go b/val_decoder_native_map.go new file mode 100644 index 0000000..4c9e387 --- /dev/null +++ b/val_decoder_native_map.go @@ -0,0 +1,267 @@ +package jzon + +import ( + "reflect" + "unsafe" +) + +type mapDecoder struct { + rtype rtype + + keyRType rtype + keyDec ValDecoder + + valRType rtype + valPtrRType rtype + valDec ValDecoder +} + +func newMapDecoder(mapType reflect.Type) *mapDecoder { + // Compatible with standard lib + // Map key must either have string kind, have an integer kind, + // or be an encoding.TextUnmarshaler. + keyType := mapType.Key() + var ( + keyDecoder ValDecoder + ) + keyPtrType := reflect.PtrTo(keyType) + keyKind := keyType.Kind() + // the string type is specially treated in order to be + // compatible with the standard lib + if keyKind != reflect.String && keyPtrType.Implements(textUnmarshalerType) { + keyDecoder = textUnmarshalerDecoder(rtypeOfType(keyPtrType)) + } else if keyDecoder = keyDecoders[keyType.Kind()]; keyDecoder == nil { + return nil + } + return &mapDecoder{ + rtype: rtypeOfType(mapType), + keyRType: rtypeOfType(keyType), + keyDec: keyDecoder, + valRType: rtypeOfType(mapType.Elem()), + valPtrRType: rtypeOfType(reflect.PtrTo(mapType.Elem())), + } +} + +func (dec *mapDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + if err = it.expectBytes("ull"); err != nil { + return err + } + *(*unsafe.Pointer)(ptr) = nil + return nil + } + if c != '{' { + return UnexpectedByteError{got: c, exp: '{', exp2: 'n'} + } + it.head += 1 + c, _, err = it.nextToken() + if err != nil { + return err + } + if c == '}' { + it.head += 1 + if *(*unsafe.Pointer)(ptr) == nil { + typedmemmove(dec.rtype, ptr, unsafeMakeMap(dec.rtype, 0)) + } + return nil + } + if *(*unsafe.Pointer)(ptr) == nil { + typedmemmove(dec.rtype, ptr, unsafeMakeMap(dec.rtype, 0)) + } + for { + key := unsafe_New(dec.keyRType) + if err = dec.keyDec.Decode(key, it); err != nil { + return err + } + c, _, err = it.nextToken() + if err != nil { + return err + } + if c != ':' { + return UnexpectedByteError{got: c, exp: ':'} + } + it.head += 1 + val := unsafe_New(dec.valRType) + if err = dec.valDec.Decode(val, it); err != nil { + return err + } + mapassign(dec.rtype, *(*unsafe.Pointer)(ptr), key, val) + c, _, err = it.nextToken() + if err != nil { + return err + } + switch c { + case '}': + it.head += 1 + return nil + case ',': + it.head += 1 + default: + return UnexpectedByteError{got: c, exp: '}', exp2: ','} + } + } +} + +// key decoders +type stringKeyDecoder struct { +} + +func (*stringKeyDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + s, err := it.ReadString() + if err != nil { + return err + } + *(*string)(ptr) = s + return nil +} + +// int key decoders +type int8KeyDecoder struct { +} + +func (*int8KeyDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + if err := it.expectQuote(); err != nil { + return err + } + i, err := it.ReadInt8() + if err != nil { + return err + } + if err := it.expectQuote(); err != nil { + return err + } + *(*int8)(ptr) = i + return nil +} + +type int16KeyDecoder struct { +} + +func (*int16KeyDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + if err := it.expectQuote(); err != nil { + return err + } + i, err := it.ReadInt16() + if err != nil { + return err + } + if err := it.expectQuote(); err != nil { + return err + } + *(*int16)(ptr) = i + return nil +} + +type int32KeyDecoder struct { +} + +func (*int32KeyDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + if err := it.expectQuote(); err != nil { + return err + } + i, err := it.ReadInt32() + if err != nil { + return err + } + if err := it.expectQuote(); err != nil { + return err + } + *(*int32)(ptr) = i + return nil +} + +type int64KeyDecoder struct { +} + +func (*int64KeyDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + if err := it.expectQuote(); err != nil { + return err + } + i, err := it.ReadInt64() + if err != nil { + return err + } + if err := it.expectQuote(); err != nil { + return err + } + *(*int64)(ptr) = i + return nil +} + +// uint key decoders +type uint8KeyDecoder struct { +} + +func (*uint8KeyDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + if err := it.expectQuote(); err != nil { + return err + } + i, err := it.ReadUint8() + if err != nil { + return err + } + if err := it.expectQuote(); err != nil { + return err + } + *(*uint8)(ptr) = i + return nil +} + +type uint16KeyDecoder struct { +} + +func (*uint16KeyDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + if err := it.expectQuote(); err != nil { + return err + } + i, err := it.ReadUint16() + if err != nil { + return err + } + if err := it.expectQuote(); err != nil { + return err + } + *(*uint16)(ptr) = i + return nil +} + +type uint32KeyDecoder struct { +} + +func (*uint32KeyDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + if err := it.expectQuote(); err != nil { + return err + } + i, err := it.ReadUint32() + if err != nil { + return err + } + if err := it.expectQuote(); err != nil { + return err + } + *(*uint32)(ptr) = i + return nil +} + +type uint64KeyDecoder struct { +} + +func (*uint64KeyDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + if err := it.expectQuote(); err != nil { + return err + } + i, err := it.ReadUint64() + if err != nil { + return err + } + if err := it.expectQuote(); err != nil { + return err + } + *(*uint64)(ptr) = i + return nil +} diff --git a/val_decoder_native_map_test.go b/val_decoder_native_map_test.go new file mode 100644 index 0000000..556602e --- /dev/null +++ b/val_decoder_native_map_test.go @@ -0,0 +1,414 @@ +package jzon + +import ( + "io" + "runtime/debug" + "testing" +) + +func TestValDecoder_Native_Map(t *testing.T) { + f := func(t *testing.T, data string, ex error, p1, p2 interface{}) { + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, nil, nil) + }) + t.Run("eof", func(t *testing.T) { + m1 := map[string]int{"a": 1} + m2 := map[string]int{"a": 1} + f(t, "", io.EOF, &m1, &m2) + }) + t.Run("invalid null", func(t *testing.T) { + m1 := map[string]int{"a": 1} + m2 := map[string]int{"a": 1} + f(t, "nul", io.EOF, &m1, &m2) + }) + t.Run("null", func(t *testing.T) { + m1 := map[string]int{"a": 1} + m2 := map[string]int{"a": 1} + f(t, "null", nil, &m1, &m2) + }) + t.Run("invalid first byte", func(t *testing.T) { + m1 := map[string]int{"a": 1} + m2 := map[string]int{"a": 1} + f(t, " } ", UnexpectedByteError{}, &m1, &m2) + }) + t.Run("eof after bracket", func(t *testing.T) { + m1 := map[string]int{"a": 1} + m2 := map[string]int{"a": 1} + f(t, " { ", io.EOF, &m1, &m2) + }) + t.Run("empty", func(t *testing.T) { + m1 := map[string]int{"a": 1} + m2 := map[string]int{"a": 1} + f(t, " { } ", nil, &m1, &m2) + }) + t.Run("empty on nil", func(t *testing.T) { + var m1 map[string]int + var m2 map[string]int + f(t, " { } ", nil, &m1, &m2) + }) + t.Run("value on nil", func(t *testing.T) { + var m1 map[string]int + var m2 map[string]int + f(t, ` { "a" : 1 } `, nil, &m1, &m2) + }) + t.Run("bad key", func(t *testing.T) { + m1 := map[string]int{"a": 1} + m2 := map[string]int{"a": 1} + f(t, ` { "a`, io.EOF, &m1, &m2) + }) + t.Run("eof after key", func(t *testing.T) { + m1 := map[string]int{"a": 1} + m2 := map[string]int{"a": 1} + f(t, ` { "a" `, io.EOF, &m1, &m2) + }) + t.Run("invalid colon", func(t *testing.T) { + m1 := map[string]int{"a": 1} + m2 := map[string]int{"a": 1} + f(t, ` { "a" } `, UnexpectedByteError{}, &m1, &m2) + }) + t.Run("bad value", func(t *testing.T) { + m1 := map[string]int{"a": 1} + m2 := map[string]int{"a": 1} + f(t, ` { "b" : "c" } `, InvalidDigitError{}, &m1, &m2) + }) + t.Run("eof after value", func(t *testing.T) { + m1 := map[string]int{"a": 1} + m2 := map[string]int{"a": 1} + f(t, ` { "b" : 2 `, io.EOF, &m1, &m2) + }) + t.Run("bad comma", func(t *testing.T) { + m1 := map[string]int{"a": 1} + m2 := map[string]int{"a": 1} + f(t, ` { "b" : 2 { `, UnexpectedByteError{}, &m1, &m2) + }) + t.Run("more items", func(t *testing.T) { + m1 := map[string]int{"a": 1} + m2 := map[string]int{"a": 1} + f(t, ` { "b" : 2 , "c" : 3 } `, nil, &m1, &m2) + }) + debug.FreeOSMemory() +} + +type testMapIntKey int + +func (k *testMapIntKey) UnmarshalText(data []byte) error { + *k = testMapIntKey(len(data)) + return nil +} + +type testMapStringKey string + +func (k *testMapStringKey) UnmarshalText(data []byte) error { + *k = testMapStringKey("`" + string(data) + "`") + return nil +} + +func TestValDecoder_Native_Map_KeyDecoder_TextUnmarshaler(t *testing.T) { + f := func(t *testing.T, data string, ex error, p1, p2 interface{}) { + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + t.Run("not supported", func(t *testing.T) { + type key testTextUnmarshaler + m1 := map[key]int{key{ + data: "a", + }: 1} + m2 := map[key]int{key{ + data: "a", + }: 1} + f(t, ` { "b" : 2 } `, TypeNotSupportedError(""), &m1, &m2) + }) + t.Run("string", func(t *testing.T) { + type key = testTextUnmarshaler + m1 := map[key]int{key{ + data: "a", + }: 1} + m2 := map[key]int{key{ + data: "a", + }: 1} + f(t, ` { "b" : 2 } `, nil, &m1, &m2) + }) + t.Run("int key", func(t *testing.T) { + m1 := map[testMapIntKey]testMapIntKey{testMapIntKey(1): 2} + m2 := map[testMapIntKey]testMapIntKey{testMapIntKey(1): 2} + f(t, ` { "3" : "4" } `, nil, &m1, &m2) + }) + t.Run("string key", func(t *testing.T) { + m1 := map[testMapStringKey]testMapStringKey{testMapStringKey("1"): "2"} + m2 := map[testMapStringKey]testMapStringKey{testMapStringKey("1"): "2"} + f(t, ` { "3" : "4" } `, nil, &m1, &m2) + }) +} + +func TestValDecoder_Native_Map_KeyDecoder_String(t *testing.T) { + f := func(t *testing.T, data string, ex error, p1, p2 interface{}) { + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + t.Run("string", func(t *testing.T) { + type key string + m1 := map[key]int{"a": 1} + m2 := map[key]int{"a": 1} + f(t, ` { "b" : 2 } `, nil, &m1, &m2) + }) +} + +func TestValDecoder_Native_Map_KeyDecoder_Int(t *testing.T) { + f := func(t *testing.T, data string, ex error, p1, p2 interface{}) { + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + // int8 + t.Run("int8 invalid", func(t *testing.T) { + type key int8 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "b" : 3 } `, InvalidDigitError{}, &m1, &m2) + }) + t.Run("int8 no leading quote", func(t *testing.T) { + type key int8 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { 2`, UnexpectedByteError{}, &m1, &m2) + }) + t.Run("int8 no trimming quote", func(t *testing.T) { + type key int8 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "2`, io.EOF, &m1, &m2) + }) + t.Run("int8 overflow", func(t *testing.T) { + type key int8 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "128" : 3 } `, IntOverflowError{}, &m1, &m2) + }) + t.Run("int8 valid", func(t *testing.T) { + type key int8 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "2" : 3 } `, nil, &m1, &m2) + }) + // int16 + t.Run("int16 invalid", func(t *testing.T) { + type key int16 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "b" : 3 } `, InvalidDigitError{}, &m1, &m2) + }) + t.Run("int16 no leading quote", func(t *testing.T) { + type key int16 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { 2`, UnexpectedByteError{}, &m1, &m2) + }) + t.Run("int16 no trimming quote", func(t *testing.T) { + type key int16 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "2`, io.EOF, &m1, &m2) + }) + t.Run("int16 overflow", func(t *testing.T) { + type key int16 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "32768" : 3 } `, IntOverflowError{}, &m1, &m2) + }) + t.Run("int16 valid", func(t *testing.T) { + type key int16 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "2" : 3 } `, nil, &m1, &m2) + }) + // int32 + t.Run("int32 invalid", func(t *testing.T) { + type key int32 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "b" : 3 } `, InvalidDigitError{}, &m1, &m2) + }) + t.Run("int32 no leading quote", func(t *testing.T) { + type key int32 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { 2`, UnexpectedByteError{}, &m1, &m2) + }) + t.Run("int32 no trimming quote", func(t *testing.T) { + type key int32 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "2`, io.EOF, &m1, &m2) + }) + t.Run("int32 overflow", func(t *testing.T) { + type key int32 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "2147483649" : 3 } `, IntOverflowError{}, &m1, &m2) + }) + t.Run("int32 valid", func(t *testing.T) { + type key int32 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "2" : 3 } `, nil, &m1, &m2) + }) + // int64 + t.Run("int64 invalid", func(t *testing.T) { + type key int64 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "b" : 3 } `, InvalidDigitError{}, &m1, &m2) + }) + t.Run("int64 no leading quote", func(t *testing.T) { + type key int64 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { 2`, UnexpectedByteError{}, &m1, &m2) + }) + t.Run("int64 no trimming quote", func(t *testing.T) { + type key int64 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "2`, io.EOF, &m1, &m2) + }) + t.Run("int64 overflow", func(t *testing.T) { + type key int64 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "9223372036854775808" : 3 } `, IntOverflowError{}, &m1, &m2) + }) + t.Run("int64 valid", func(t *testing.T) { + type key int64 + m1 := map[key]int{1: 2} + m2 := map[key]int{1: 2} + f(t, ` { "2" : 3 } `, nil, &m1, &m2) + }) +} + +func TestValDecoder_Native_Map_KeyDecoder_Uint(t *testing.T) { + f := func(t *testing.T, data string, ex error, p1, p2 interface{}) { + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + // uint8 + t.Run("uint8 invalid", func(t *testing.T) { + type key uint8 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "b" : 3 } `, InvalidDigitError{}, &m1, &m2) + }) + t.Run("uint8 no leading quote", func(t *testing.T) { + type key uint8 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { 2`, UnexpectedByteError{}, &m1, &m2) + }) + t.Run("uint8 no trimming quote", func(t *testing.T) { + type key uint8 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "2`, io.EOF, &m1, &m2) + }) + t.Run("uint8 overflow", func(t *testing.T) { + type key uint8 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "256" : 3 } `, IntOverflowError{}, &m1, &m2) + }) + t.Run("uint8 valid", func(t *testing.T) { + type key uint8 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "2" : 3 } `, nil, &m1, &m2) + }) + // uint16 + t.Run("uint16 invalid", func(t *testing.T) { + type key uint16 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "b" : 3 } `, InvalidDigitError{}, &m1, &m2) + }) + t.Run("uint16 no leading quote", func(t *testing.T) { + type key uint16 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { 2`, UnexpectedByteError{}, &m1, &m2) + }) + t.Run("uint16 no trimming quote", func(t *testing.T) { + type key uint16 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "2`, io.EOF, &m1, &m2) + }) + t.Run("uint16 overflow", func(t *testing.T) { + type key uint16 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "65536" : 3 } `, IntOverflowError{}, &m1, &m2) + }) + t.Run("uint16 valid", func(t *testing.T) { + type key uint16 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "2" : 3 } `, nil, &m1, &m2) + }) + // uint32 + t.Run("uint32 invalid", func(t *testing.T) { + type key uint32 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "b" : 3 } `, InvalidDigitError{}, &m1, &m2) + }) + t.Run("uint32 no leading quote", func(t *testing.T) { + type key uint32 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { 2`, UnexpectedByteError{}, &m1, &m2) + }) + t.Run("uint32 no trimming quote", func(t *testing.T) { + type key uint32 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "2`, io.EOF, &m1, &m2) + }) + t.Run("uint32 overflow", func(t *testing.T) { + type key uint32 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "4294967296" : 3 } `, IntOverflowError{}, &m1, &m2) + }) + t.Run("uint32 valid", func(t *testing.T) { + type key uint32 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "2" : 3 } `, nil, &m1, &m2) + }) + // uint64 + t.Run("uint64 invalid", func(t *testing.T) { + type key uint64 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "b" : 3 } `, InvalidDigitError{}, &m1, &m2) + }) + t.Run("uint64 no leading quote", func(t *testing.T) { + type key uint64 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { 2`, UnexpectedByteError{}, &m1, &m2) + }) + t.Run("uint64 no trimming quote", func(t *testing.T) { + type key uint64 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "2`, io.EOF, &m1, &m2) + }) + t.Run("uint64 overflow", func(t *testing.T) { + type key uint64 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "18446744073709551616" : 3 } `, IntOverflowError{}, &m1, &m2) + }) + t.Run("uint64 valid", func(t *testing.T) { + type key uint64 + m1 := map[key]uint{1: 2} + m2 := map[key]uint{1: 2} + f(t, ` { "2" : 3 } `, nil, &m1, &m2) + }) +} diff --git a/val_decoder_native_override_test.go b/val_decoder_native_override_test.go new file mode 100644 index 0000000..2391064 --- /dev/null +++ b/val_decoder_native_override_test.go @@ -0,0 +1,36 @@ +package jzon + +import ( + "reflect" + "testing" + "unsafe" + + "github.com/stretchr/testify/require" +) + +type testStringKind string + +type testStringKindDecoder struct { +} + +func (*testStringKindDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + s, err := it.ReadString() + if err != nil { + return err + } + *((*string)(ptr)) = "`" + s + "`" + return nil +} + +func TestValDecoder_Native_Kind_String(t *testing.T) { + decoder := NewDecoder(&DecoderOption{ + ValDecoders: map[reflect.Type]ValDecoder{ + reflect.TypeOf(string("")): (*testStringKindDecoder)(nil), + }, + }) + data := []byte(`"abc"`) + var s testStringKind = "dummy" + err := decoder.Unmarshal(data, &s) + require.NoError(t, err) + require.Equal(t, testStringKind("`abc`"), s) +} diff --git a/val_decoder_native_ptr.go b/val_decoder_native_ptr.go new file mode 100644 index 0000000..4dc2b6f --- /dev/null +++ b/val_decoder_native_ptr.go @@ -0,0 +1,48 @@ +package jzon + +import ( + "reflect" + "unsafe" +) + +type pointerDecoder struct { + elemDec ValDecoder + ptrRType rtype + elemRType rtype +} + +func newPointerDecoder(typ reflect.Type) *pointerDecoder { + return &pointerDecoder{ + ptrRType: rtypeOfType(typ), + elemRType: rtypeOfType(typ.Elem()), + } +} + +func (dec *pointerDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + if err = it.expectBytes("ull"); err != nil { + return err + } + *(*unsafe.Pointer)(ptr) = nil + } else { + elemPtr := *((*unsafe.Pointer)(ptr)) + var tgtPtr unsafe.Pointer + if elemPtr == nil { + tgtPtr = unsafe_New(dec.elemRType) + } else { + tgtPtr = elemPtr + } + if err = dec.elemDec.Decode(tgtPtr, it); err != nil { + return err + } + if elemPtr == nil { + *(*unsafe.Pointer)(ptr) = tgtPtr + } + } + return nil +} diff --git a/val_decoder_native_ptr_test.go b/val_decoder_native_ptr_test.go new file mode 100644 index 0000000..8367f83 --- /dev/null +++ b/val_decoder_native_ptr_test.go @@ -0,0 +1,71 @@ +package jzon + +import ( + "io" + "runtime/debug" + "testing" +) + +func TestValDecoder_Native_Ptr(t *testing.T) { + f := func(t *testing.T, data string, ex error, p1, p2 interface{}) { + t.Log(">>>>> initValues >>>>>") + printValue(t, "p1", p1) + printValue(t, "p2", p2) + t.Log(">>>>>>>>>>>>>>>>>>>>>>") + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + t.Log("<<<<< initValues <<<<<") + printValue(t, "p1", p1) + printValue(t, "p2", p2) + t.Log("<<<<<<<<<<<<<<<<<<<<<<") + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, (*int)(nil), (*int)(nil)) + }) + t.Run("eof", func(t *testing.T) { + pi1 := (*int)(nil) + pi2 := (*int)(nil) + f(t, "", io.EOF, &pi1, &pi2) + }) + t.Run("invalid null", func(t *testing.T) { + pi1 := (*int)(nil) + pi2 := (*int)(nil) + f(t, "nul", io.EOF, &pi1, &pi2) + }) + t.Run("null 1", func(t *testing.T) { + pi1 := (*int)(nil) + pi2 := (*int)(nil) + f(t, "null", nil, &pi1, &pi2) + }) + t.Run("null 2", func(t *testing.T) { + i1 := 1 + i2 := 1 + pi1 := &i1 + pi2 := &i2 + f(t, "null", nil, &pi1, &pi2) + }) + t.Run("not null error 1", func(t *testing.T) { + pi1 := (*int)(nil) + pi2 := (*int)(nil) + f(t, `true`, InvalidDigitError{}, &pi1, &pi2) + }) + t.Run("not null error 2", func(t *testing.T) { + i1 := 1 + i2 := 1 + pi1 := &i1 + pi2 := &i2 + f(t, `true`, InvalidDigitError{}, &pi1, &pi2) + }) + t.Run("not null 1", func(t *testing.T) { + pi1 := (*int)(nil) + pi2 := (*int)(nil) + f(t, `23`, nil, &pi1, &pi2) + }) + t.Run("not null 2", func(t *testing.T) { + i1 := 1 + i2 := 1 + pi1 := &i1 + pi2 := &i2 + f(t, `23`, nil, &pi1, &pi2) + }) + debug.FreeOSMemory() +} diff --git a/val_decoder_native_slice.go b/val_decoder_native_slice.go new file mode 100644 index 0000000..a5cb2b2 --- /dev/null +++ b/val_decoder_native_slice.go @@ -0,0 +1,100 @@ +package jzon + +import ( + "encoding/base64" + "reflect" + "unsafe" +) + +type sliceDecoder struct { + rtype rtype + elemKind reflect.Kind + elemDec ValDecoder + elemRType rtype + elemPtrRType rtype + elemSize uintptr +} + +func newSliceDecoder(sliceType reflect.Type) *sliceDecoder { + elem := sliceType.Elem() + return &sliceDecoder{ + rtype: rtypeOfType(sliceType), + elemKind: elem.Kind(), + elemRType: rtypeOfType(elem), + elemPtrRType: rtypeOfType(reflect.PtrTo(elem)), + elemSize: elem.Size(), + } +} + +func (dec *sliceDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + if err = it.expectBytes("ull"); err != nil { + return err + } + sh := (*reflect.SliceHeader)(ptr) + sh.Len = 0 + sh.Cap = 0 + sh.Data = 0 + return nil + } + if c == '"' { + if dec.elemKind != reflect.Uint8 { + return UnexpectedByteError{got: c, exp: '[', exp2: 'n'} + } + it.head += 1 + // TODO: improve by checking only base64 characters + begin := it.head + oldCapture := it.capture + it.capture = true + if err := skipString(it, c); err != nil { + return err + } + it.capture = oldCapture + buf := it.buffer[begin : it.head-1] + data, err := base64.StdEncoding.DecodeString(*(*string)(unsafe.Pointer(&buf))) + if err != nil { + return err + } + *((*[]byte)(ptr)) = data + return nil + } + if c != '[' { + return UnexpectedByteError{got: c, exp: '[', exp2: 'n'} + } + it.head += 1 + c, _, err = it.nextToken() + if err != nil { + return err + } + newPtr := unsafeMakeSlice(dec.rtype, 0, 0) + if c == ']' { + it.head += 1 + } else { + for length := 1; ; length++ { + newPtr = unsafeGrowSlice(dec.rtype, dec.elemRType, newPtr, length) + // must get the address every time + childPtr := unsafeSliceChildPtr(newPtr, dec.elemSize, length-1) + if err = dec.elemDec.Decode(childPtr, it); err != nil { + return err + } + c, _, err = it.nextToken() + if err != nil { + return err + } + it.head += 1 + if c == ']' { + break + } + if c != ',' { + return UnexpectedByteError{got: c, exp: ']', exp2: ','} + } + } + } + *(*reflect.SliceHeader)(ptr) = *(*reflect.SliceHeader)(newPtr) + return nil +} diff --git a/val_decoder_native_slice_base64_test.go b/val_decoder_native_slice_base64_test.go new file mode 100644 index 0000000..b2550ea --- /dev/null +++ b/val_decoder_native_slice_base64_test.go @@ -0,0 +1,102 @@ +package jzon + +import ( + "encoding/base64" + "io" + "testing" +) + +func TestValDecoder_Native_Base64(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue string) { + var p1 *[]byte + var p2 *[]byte + if initValue != "" { + b1 := append([]byte(nil), initValue...) + p1 = &b1 + b2 := append([]byte(nil), initValue...) + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, "dummy") + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, "") + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `true`, UnexpectedByteError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("invalid string", func(t *testing.T) { + f2(t, `"abc`, io.EOF) + }) + t.Run("invalid base64", func(t *testing.T) { + f2(t, `"abc"`, base64.CorruptInputError(0)) + }) + t.Run("valid base64", func(t *testing.T) { + f2(t, `"`+base64.StdEncoding.EncodeToString( + []byte("abc"))+`"`, nil) + }) + t.Run("invalid array", func(t *testing.T) { + f2(t, `[`, io.EOF) + }) + t.Run("empty array", func(t *testing.T) { + f2(t, `[]`, nil) + }) + t.Run("invalid uint8", func(t *testing.T) { + f2(t, `[256]`, IntOverflowError{}) + }) + t.Run("invalid read more", func(t *testing.T) { + f2(t, `[1`, io.EOF) + }) +} + +type testByte byte + +func (tb *testByte) UnmarshalJSON(data []byte) error { + *tb = testByte(data[0] - '0' + 1) + return nil +} + +func TestValDecoder_Native_Base64_Override(t *testing.T) { + f := func(t *testing.T, data string, ex error, p1, p2 interface{}) { + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + t.Run("array", func(t *testing.T) { + arr1 := [...]testByte{'d', 'u', 'm', 'm', 'y'} + arr2 := [...]testByte{'d', 'u', 'm', 'm', 'y'} + f(t, `"`+base64.StdEncoding.EncodeToString( + []byte("abc"))+`"`, UnexpectedByteError{}, &arr1, &arr2) + }) + t.Run("invalid element type", func(t *testing.T) { + arr1 := []int8{'d', 'u', 'm', 'm', 'y'} + arr2 := []int8{'d', 'u', 'm', 'm', 'y'} + f(t, `"`+base64.StdEncoding.EncodeToString( + []byte("abc"))+`"`, UnexpectedByteError{}, &arr1, &arr2) + }) + t.Run("invalid base64", func(t *testing.T) { + arr1 := []testByte{'d', 'u', 'm', 'm', 'y'} + arr2 := []testByte{'d', 'u', 'm', 'm', 'y'} + f(t, `"abc"`, base64.CorruptInputError(0), &arr1, &arr2) + }) + t.Run("valid base64", func(t *testing.T) { + arr1 := []testByte{'d', 'u', 'm', 'm', 'y'} + arr2 := []testByte{'d', 'u', 'm', 'm', 'y'} + f(t, `"`+base64.StdEncoding.EncodeToString( + []byte("abc"))+`"`, nil, &arr1, &arr2) + }) + t.Run("slice", func(t *testing.T) { + arr1 := []testByte{'d', 'u', 'm', 'm', 'y'} + arr2 := []testByte{'d', 'u', 'm', 'm', 'y'} + f(t, ` [ 1, 2, 3 ] `, nil, &arr1, &arr2) + }) +} diff --git a/val_decoder_native_slice_test.go b/val_decoder_native_slice_test.go new file mode 100644 index 0000000..c65f163 --- /dev/null +++ b/val_decoder_native_slice_test.go @@ -0,0 +1,119 @@ +package jzon + +import ( + "io" + "runtime" + "runtime/debug" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValDecoder_Native_Slice(t *testing.T) { + f := func(t *testing.T, data string, ex error, p1, p2 interface{}) { + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, nil, nil) + }) + t.Run("eof", func(t *testing.T) { + arr1 := []int{1, 2} + arr2 := []int{1, 2} + f(t, "", io.EOF, &arr1, &arr2) + }) + t.Run("invalid null", func(t *testing.T) { + arr1 := []int{1, 2} + arr2 := []int{1, 2} + f(t, "nul", io.EOF, &arr1, &arr2) + }) + t.Run("null", func(t *testing.T) { + arr1 := []int{1, 2} + arr2 := []int{1, 2} + f(t, "null", nil, &arr1, &arr2) + }) + t.Run("invalid first byte", func(t *testing.T) { + arr1 := []int{1, 2} + arr2 := []int{1, 2} + f(t, `+`, UnexpectedByteError{}, &arr1, &arr2) + }) + t.Run("eof after bracket", func(t *testing.T) { + arr1 := []int{1, 2} + arr2 := []int{1, 2} + f(t, ` [ `, io.EOF, &arr1, &arr2) + }) + t.Run("empty", func(t *testing.T) { + arr1 := []int{1, 2} + arr2 := []int{1, 2} + f(t, ` [ ] `, nil, &arr1, &arr2) + }) + t.Run("bad item", func(t *testing.T) { + arr1 := []int{1, 2} + arr2 := []int{1, 2} + f(t, ` [ + `, InvalidDigitError{}, &arr1, &arr2) + }) + t.Run("eof after item", func(t *testing.T) { + arr1 := []int{1, 2} + arr2 := []int{1, 2} + f(t, ` [ 3 `, io.EOF, &arr1, &arr2) + }) + t.Run("lesser item", func(t *testing.T) { + arr1 := []int{1, 2} + arr2 := []int{1, 2} + f(t, ` [ 3 ] `, nil, &arr1, &arr2) + }) + t.Run("bad item", func(t *testing.T) { + arr1 := []int{1, 2} + arr2 := []int{1, 2} + f(t, ` [ "test" ] `, InvalidDigitError{}, &arr1, &arr2) + }) + t.Run("invalid comma", func(t *testing.T) { + arr1 := []int{1, 2} + arr2 := []int{1, 2} + f(t, ` [ 3 [ `, UnexpectedByteError{}, &arr1, &arr2) + }) + t.Run("more item", func(t *testing.T) { + arr1 := []int{1, 2} + arr2 := []int{1, 2} + f(t, ` [ 3 , 4 , 5 ] `, nil, &arr1, &arr2) + }) + t.Run("more item error", func(t *testing.T) { + arr1 := []int{1, 2} + arr2 := []int{1, 2} + f(t, ` [ 3 , 4 , "test" ] `, InvalidDigitError{}, &arr1, &arr2) + }) + debug.FreeOSMemory() +} + +func TestValDecoder_Native_Slice_Memory(t *testing.T) { + t.Run("test1", func(t *testing.T) { + f := func(i int) []*int { + pi := &i + runtime.SetFinalizer(pi, func(_ *int) { + t.Logf("finalizing") + }) + return []*int{pi} + } + arr := f(1) + err := Unmarshal([]byte(`[]`), &arr) + require.NoError(t, err) + debug.FreeOSMemory() + t.Logf("please check if the memory has been freed") + }) + t.Run("test2", func(t *testing.T) { + type st struct { + p *int + } + f := func(i int) []*st { + pi := &i + runtime.SetFinalizer(pi, func(_ *int) { + t.Logf("finalizing") + }) + return []*st{&st{p: pi}} + } + arr := f(1) + err := Unmarshal([]byte(`[]`), &arr) + require.NoError(t, err) + debug.FreeOSMemory() + t.Logf("please check if the memory has been freed") + }) +} diff --git a/val_decoder_native_struct.go b/val_decoder_native_struct.go new file mode 100644 index 0000000..cef0704 --- /dev/null +++ b/val_decoder_native_struct.go @@ -0,0 +1,126 @@ +package jzon + +import ( + "reflect" + "strings" + "unsafe" +) + +type fieldInfo struct { + offset uintptr + ptrType reflect.Type + rtype rtype + decoder ValDecoder +} + +type structDecoder struct { + fields map[string]*fieldInfo +} + +func (dec *Decoder) newStructDecoder(typ reflect.Type) *structDecoder { + var key string + var fields map[string]*fieldInfo + for i := 0; i < typ.NumField(); i++ { + stField := typ.Field(i) + // field name cannot be empty (?) + if stField.Name[0] < 'A' || stField.Name[0] > 'Z' { + continue + } + tagV, ok := stField.Tag.Lookup(dec.tag) + if ok { + if tagV == "-" { + continue + } + // TODO: complete this + key = tagV + } else { + key = stField.Name + } + if !dec.caseSensitive { + key = strings.ToLower(key) + } + if fields == nil { + fields = map[string]*fieldInfo{} + } + fieldPtrType := reflect.PtrTo(stField.Type) + fields[key] = &fieldInfo{ + offset: stField.Offset, + ptrType: fieldPtrType, + rtype: rtypeOfType(fieldPtrType), + } + } + if len(fields) == 0 { + return nil + } + return &structDecoder{ + fields: fields, + } +} + +func (dec *structDecoder) Decode(ptr unsafe.Pointer, it *Iterator) (err error) { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + err = it.expectBytes("ull") + return + } + if c != '{' { + return UnexpectedByteError{got: c, exp2: 'n', exp: '{'} + } + it.head += 1 + c, _, err = it.nextToken() + if err != nil { + return + } + if c == '}' { + it.head += 1 + return + } + if c != '"' { + return UnexpectedByteError{got: c, exp: '}', exp2: '"'} + } + it.head += 1 + for { + _, field, err := it.readObjectFieldAsSlice(it.tmpBuffer[:0], + it.decoder.caseSensitive) + it.tmpBuffer = field + if err != nil { + return err + } + stField := dec.fields[*(*string)(unsafe.Pointer(&field))] + if stField != nil { + fieldPtr := add(ptr, stField.offset, "struct field") + if err = stField.decoder.Decode(fieldPtr, it); err != nil { + return err + } + } else { + if err = it.Skip(); err != nil { + return err + } + } + c, _, err = it.nextToken() + if err != nil { + return err + } + switch c { + case '}': + it.head += 1 + return nil + case ',': + it.head += 1 + c, _, err = it.nextToken() + if err != nil { + return err + } + if c != '"' { + return UnexpectedByteError{got: c, exp: '"'} + } + it.head += 1 + default: + return UnexpectedByteError{got: c, exp: '}', exp2: ','} + } + } +} diff --git a/val_decoder_native_struct_test.go b/val_decoder_native_struct_test.go new file mode 100644 index 0000000..97aa11f --- /dev/null +++ b/val_decoder_native_struct_test.go @@ -0,0 +1,233 @@ +package jzon + +import ( + "io" + "runtime/debug" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValDecoder_Native_Struct_Zero_Field(t *testing.T) { + f := func(t *testing.T, data string, ex error, p1, p2 interface{}) { + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + t.Run("nil receiver", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, nil, nil) + }) + t.Run("eof", func(t *testing.T) { + f(t, "", io.EOF, &struct{}{}, &struct{}{}) + }) + t.Run("null", func(t *testing.T) { + f(t, "null", nil, &struct{}{}, &struct{}{}) + }) + t.Run("invalid first byte", func(t *testing.T) { + f(t, "+", UnexpectedByteError{}, &struct{}{}, &struct{}{}) + }) + debug.FreeOSMemory() +} + +func TestValDecoder_Native_Struct_Mapping(t *testing.T) { + f := func(t *testing.T, data string, ex error, p1, p2 interface{}) { + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + t.Run("unexported field", func(t *testing.T) { + f(t, ` { "a" : "abc" } `, nil, &struct { + a string + }{}, &struct { + a string + }{}) + }) + t.Run("unexported field 2", func(t *testing.T) { + f(t, ` { "a" : "abc" } `, nil, &struct { + a string + B int + }{}, &struct { + a string + B int + }{}) + }) + t.Run("tag ignored 1", func(t *testing.T) { + f(t, ` { "A" : "abc" } `, nil, &struct { + A string `json:"-"` + }{A: "test"}, &struct { + A string `json:"-"` + }{A: "test"}) + }) + t.Run("tag", func(t *testing.T) { + f(t, ` { "b" : "abc" } `, nil, &struct { + A string `json:"B"` + }{A: "test"}, &struct { + A string `json:"B"` + }{A: "test"}) + }) + t.Run("case insensitive", func(t *testing.T) { + f(t, ` { "a" : "abc" } `, nil, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("case insensitive 2", func(t *testing.T) { + f(t, ` { "A" : "abc" } `, nil, &struct { + A string `json:"a"` + }{A: "test"}, &struct { + A string `json:"a"` + }{A: "test"}) + }) + debug.FreeOSMemory() +} + +func TestValDecoder_Native_Struct(t *testing.T) { + f := func(t *testing.T, data string, ex error, p1, p2 interface{}) { + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + t.Run("nil receiver", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, nil, nil) + }) + t.Run("eof", func(t *testing.T) { + f(t, "", io.EOF, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("invalid first byte", func(t *testing.T) { + f(t, "+", UnexpectedByteError{}, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("invalid null", func(t *testing.T) { + f(t, "nul", io.EOF, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("null", func(t *testing.T) { + f(t, "null", nil, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("eof after bracket", func(t *testing.T) { + f(t, "{", io.EOF, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("empty object", func(t *testing.T) { + f(t, " { } ", nil, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("invalid char after bracket", func(t *testing.T) { + f(t, " { { ", UnexpectedByteError{}, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("invalid field", func(t *testing.T) { + f(t, ` { " `, io.EOF, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("invalid field type", func(t *testing.T) { + f(t, ` { "A" : 1 } `, UnexpectedByteError{}, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("field skip error", func(t *testing.T) { + f(t, ` { "b" : } `, UnexpectedByteError{}, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("more field eof", func(t *testing.T) { + f(t, ` { "b" : 1 `, io.EOF, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("non empty", func(t *testing.T) { + f(t, ` { "a" : "abc" } `, nil, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("more field eof after comma", func(t *testing.T) { + f(t, ` { "b" : 1 , `, io.EOF, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("more field invalid byte after comma", func(t *testing.T) { + f(t, ` { "b" : 1 , } `, UnexpectedByteError{}, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("more field invalid comma", func(t *testing.T) { + f(t, ` { "b" : 1 { `, UnexpectedByteError{}, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + t.Run("two fields", func(t *testing.T) { + f(t, ` { "b" : 1 , "a" : "abc" } `, nil, &struct { + A string + }{A: "test"}, &struct { + A string + }{A: "test"}) + }) + debug.FreeOSMemory() +} + +type testStruct struct { + A *testStruct `json:"a"` + C *int `json:"c"` + B *testStruct `json:"b"` +} + +func TestValDecoder_Native_Struct_Nested(t *testing.T) { + f := func(t *testing.T, data string, ex error, p1, p2 interface{}) { + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + t.Run("nested", func(t *testing.T) { + f(t, `{"a":null,"c":1,"b":{}}`, nil, &testStruct{ + A: &testStruct{}, + }, &testStruct{ + A: &testStruct{}, + }) + }) + debug.FreeOSMemory() +} + +func TestValDedocer_Native_Struct_Tag(t *testing.T) { + decoder := NewDecoder(&DecoderOption{ + Tag: "jzon", + }) + var p struct { + A string `jzon:"b"` + } + err := decoder.Unmarshal([]byte(` { "b" : "c" }`), &p) + require.NoError(t, err) + require.Equal(t, "c", p.A) +} diff --git a/val_decoder_native_test.go b/val_decoder_native_test.go new file mode 100644 index 0000000..8dd0699 --- /dev/null +++ b/val_decoder_native_test.go @@ -0,0 +1,97 @@ +package jzon + +import ( + "io" + "testing" +) + +func TestValDecoder_Native_Bool(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue, initBool bool) { + var p1 *bool + var p2 *bool + if initValue { + b1 := initBool + p1 = &b1 + b2 := initBool + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, true, true) + } + f3 := func(t *testing.T, data string, ex error) { + f(t, data, ex, true, false) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, false, false) + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `+`, UnexpectedByteError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("invalid true (init true)", func(t *testing.T) { + f2(t, `tru`, io.EOF) + }) + t.Run("valid true (init true)", func(t *testing.T) { + f2(t, `true`, nil) + }) + t.Run("invalid true (init false)", func(t *testing.T) { + f3(t, `tru`, io.EOF) + }) + t.Run("valid true (init false)", func(t *testing.T) { + f3(t, `true`, nil) + }) + t.Run("invalid false (init true)", func(t *testing.T) { + f2(t, `fals`, io.EOF) + }) + t.Run("valid false (init true)", func(t *testing.T) { + f2(t, `false`, nil) + }) +} + +func TestValDecoder_Native_String(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue string) { + var p1 *string + var p2 *string + if initValue != "" { + b1 := initValue + p1 = &b1 + b2 := initValue + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, "dummy") + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, "") + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `true`, UnexpectedByteError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("invalid string", func(t *testing.T) { + f2(t, `"abc`, io.EOF) + }) + t.Run("valid string", func(t *testing.T) { + f2(t, `"abc"`, nil) + }) +} diff --git a/val_decoder_native_uint.go b/val_decoder_native_uint.go new file mode 100644 index 0000000..9da3a58 --- /dev/null +++ b/val_decoder_native_uint.go @@ -0,0 +1,89 @@ +package jzon + +import ( + "unsafe" +) + +// uint8 decoder +type uint8Decoder struct { +} + +func (*uint8Decoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + return it.expectBytes("ull") + } + i, err := it.ReadUint8() + if err != nil { + return err + } + *(*uint8)(ptr) = i + return nil +} + +// uint16 decoder +type uint16Decoder struct { +} + +func (*uint16Decoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + return it.expectBytes("ull") + } + i, err := it.ReadUint16() + if err != nil { + return err + } + *(*uint16)(ptr) = i + return nil +} + +// uint32 decoder +type uint32Decoder struct { +} + +func (*uint32Decoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + return it.expectBytes("ull") + } + i, err := it.ReadUint32() + if err != nil { + return err + } + *(*uint32)(ptr) = i + return nil +} + +// uint64 decoder +type uint64Decoder struct { +} + +func (*uint64Decoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + if c == 'n' { + it.head += 1 + return it.expectBytes("ull") + } + i, err := it.ReadUint64() + if err != nil { + return err + } + *(*uint64)(ptr) = i + return nil +} diff --git a/val_decoder_native_uint_test.go b/val_decoder_native_uint_test.go new file mode 100644 index 0000000..2b76768 --- /dev/null +++ b/val_decoder_native_uint_test.go @@ -0,0 +1,181 @@ +package jzon + +import ( + "io" + "testing" +) + +func TestValDecoder_Native_Uint(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue uint) { + var p1 *uint + var p2 *uint + if initValue != 0 { + b1 := initValue + p1 = &b1 + b2 := initValue + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, 1) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, 0) + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `true`, InvalidDigitError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("valid", func(t *testing.T) { + f2(t, "127", nil) + }) +} + +func TestValDecoder_Native_Uint8(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue uint8) { + var p1 *uint8 + var p2 *uint8 + if initValue != 0 { + b1 := initValue + p1 = &b1 + b2 := initValue + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, 1) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, 0) + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `true`, InvalidDigitError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("valid", func(t *testing.T) { + f2(t, "127", nil) + }) +} + +func TestValDecoder_Native_Uint16(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue uint16) { + var p1 *uint16 + var p2 *uint16 + if initValue != 0 { + b1 := initValue + p1 = &b1 + b2 := initValue + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, 1) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, 0) + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `true`, InvalidDigitError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("valid", func(t *testing.T) { + f2(t, "127", nil) + }) +} + +func TestValDecoder_Native_Uint32(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue uint32) { + var p1 *uint32 + var p2 *uint32 + if initValue != 0 { + b1 := initValue + p1 = &b1 + b2 := initValue + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, 1) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, 0) + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `true`, InvalidDigitError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("valid", func(t *testing.T) { + f2(t, "127", nil) + }) +} + +func TestValDecoder_Native_Uint64(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue uint64) { + var p1 *uint64 + var p2 *uint64 + if initValue != 0 { + b1 := initValue + p1 = &b1 + b2 := initValue + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex error) { + f(t, data, ex, 1) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, 0) + }) + t.Run("eof", func(t *testing.T) { + f2(t, "", io.EOF) + }) + t.Run("invalid first byte", func(t *testing.T) { + f2(t, `true`, InvalidDigitError{}) + }) + t.Run("invalid null", func(t *testing.T) { + f2(t, "nul", io.EOF) + }) + t.Run("null", func(t *testing.T) { + f2(t, "null", nil) + }) + t.Run("valid", func(t *testing.T) { + f2(t, "127", nil) + }) +} diff --git a/val_decoder_skip.go b/val_decoder_skip.go new file mode 100644 index 0000000..ab36e06 --- /dev/null +++ b/val_decoder_skip.go @@ -0,0 +1,20 @@ +package jzon + +import ( + "unsafe" +) + +type skipDecoder struct { +} + +func (*skipDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + c, _, err := it.nextToken() + if err != nil { + return err + } + it.head += 1 + if c != '{' && c != 'n' { + return UnexpectedByteError{got: c, exp: '{', exp2: 'n'} + } + return skipFunctions[c](it, c) +} diff --git a/val_decoder_text_unmarshaler.go b/val_decoder_text_unmarshaler.go new file mode 100644 index 0000000..95ff1f2 --- /dev/null +++ b/val_decoder_text_unmarshaler.go @@ -0,0 +1,36 @@ +package jzon + +import ( + "encoding" + "reflect" + "unsafe" +) + +var ( + textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() +) + +type textUnmarshalerDecoder rtype + +func (dec textUnmarshalerDecoder) Decode(ptr unsafe.Pointer, it *Iterator) error { + obj := packEFace(rtype(dec), ptr) + unmarshaler := obj.(encoding.TextUnmarshaler) + c, _, err := it.nextToken() + if err != nil { + return err + } + switch c { + case '"': + it.head += 1 + b, err := it.readStringAsSlice(nil, true) + if err != nil { + return err + } + return unmarshaler.UnmarshalText(b) + case 'n': + it.head += 1 + return it.expectBytes("ull") + default: + return UnexpectedByteError{got: c, exp: '"', exp2: 'n'} + } +} diff --git a/val_decoder_text_unmarshaler_test.go b/val_decoder_text_unmarshaler_test.go new file mode 100644 index 0000000..a4e2609 --- /dev/null +++ b/val_decoder_text_unmarshaler_test.go @@ -0,0 +1,61 @@ +package jzon + +import ( + "errors" + "io" + "testing" +) + +type testTextUnmarshaler struct { + data string + err error +} + +func (t *testTextUnmarshaler) UnmarshalText(data []byte) error { + t.data = string(data) + return t.err +} + +func TestValDecoder_TextUnmarshaler(t *testing.T) { + f := func(t *testing.T, data string, ex error, initValue string, initErr error) { + var p1 *testTextUnmarshaler + var p2 *testTextUnmarshaler + if initValue != "" { + b1 := testTextUnmarshaler{ + data: initValue, + err: initErr, + } + p1 = &b1 + b2 := testTextUnmarshaler{ + data: initValue, + err: initErr, + } + p2 = &b2 + } + checkStandard(t, DefaultDecoder, data, ex, p1, p2) + } + f2 := func(t *testing.T, data string, ex, initErr error) { + f(t, data, ex, "dummy", initErr) + } + t.Run("nil pointer", func(t *testing.T) { + f(t, "null", NilPointerReceiverError, "", nil) + }) + t.Run("eof", func(t *testing.T) { + f2(t, " ", io.EOF, nil) + }) + t.Run("invalid byte", func(t *testing.T) { + f2(t, ` + `, UnexpectedByteError{}, nil) + }) + t.Run("null", func(t *testing.T) { + f2(t, ` null `, nil, nil) + }) + t.Run("invalid string", func(t *testing.T) { + f2(t, ` " `, io.EOF, nil) + }) + t.Run("no error", func(t *testing.T) { + f2(t, ` "abc" `, nil, nil) + }) + t.Run("custom error", func(t *testing.T) { + f2(t, ` "abc" `, errors.New("test"), errors.New("test")) + }) +} diff --git a/value_type.go b/value_type.go new file mode 100644 index 0000000..ddd5142 --- /dev/null +++ b/value_type.go @@ -0,0 +1,50 @@ +package jzon + +type ValueType int32 + +const ( + WhiteSpaceValue ValueType = iota + InvalidValue + StringValue + NumberValue + ObjectValue + ArrayValue + BoolValue + NullValue + LastValue +) + +var ( + valueTypeNames [LastValue]string + valueTypeMap [charNum]ValueType +) + +func init() { + // value type names + valueTypeNames[WhiteSpaceValue] = "WhiteSpaceValue" + valueTypeNames[InvalidValue] = "InvalidValue" + valueTypeNames[StringValue] = "StringValue" + valueTypeNames[NumberValue] = "NumberValue" + valueTypeNames[ObjectValue] = "ObjectValue" + valueTypeNames[ArrayValue] = "ArrayValue" + valueTypeNames[BoolValue] = "BoolValue" + valueTypeNames[NullValue] = "NullValue" + + // value type map + for i := 0; i < charNum; i++ { + valueTypeMap[i] = InvalidValue + } + for _, c := range " \n\t\r" { + valueTypeMap[c] = WhiteSpaceValue + } + valueTypeMap['"'] = StringValue + for _, c := range "-0123456789" { + valueTypeMap[c] = NumberValue + } + valueTypeMap['{'] = ObjectValue + valueTypeMap['['] = ArrayValue + for _, c := range "tf" { + valueTypeMap[c] = BoolValue + } + valueTypeMap['n'] = NullValue +}