diff --git a/analysis/benchmark_test.go b/analysis/benchmark_test.go index 4ff00e00a..216e423c2 100644 --- a/analysis/benchmark_test.go +++ b/analysis/benchmark_test.go @@ -15,6 +15,7 @@ package analysis_test import ( + index "github.com/blevesearch/bleve_index_api" "testing" "github.com/blevesearch/bleve/analysis" @@ -32,7 +33,7 @@ func BenchmarkAnalysis(b *testing.B) { } ts := analyzer.Analyze(bleveWikiArticle) - freqs := analysis.TokenFrequency(ts, nil, true) + freqs := analysis.TokenFrequency(ts, nil, index.IncludeTermVectors) if len(freqs) != 511 { b.Errorf("expected %d freqs, got %d", 511, len(freqs)) } diff --git a/analysis/freq.go b/analysis/freq.go index 9f44a6a86..ec40bb6f5 100644 --- a/analysis/freq.go +++ b/analysis/freq.go @@ -18,10 +18,10 @@ import ( index "github.com/blevesearch/bleve_index_api" ) -func TokenFrequency(tokens TokenStream, arrayPositions []uint64, includeTermVectors bool) index.TokenFrequencies { +func TokenFrequency(tokens TokenStream, arrayPositions []uint64, options index.FieldIndexingOptions) index.TokenFrequencies { rv := make(map[string]*index.TokenFreq, len(tokens)) - if includeTermVectors { + if options.IncludeTermVectors() { tls := make([]index.TokenLocation, len(tokens)) tlNext := 0 diff --git a/analysis/freq_test.go b/analysis/freq_test.go index 19d2c82d9..724ff4546 100644 --- a/analysis/freq_test.go +++ b/analysis/freq_test.go @@ -54,7 +54,7 @@ func TestTokenFrequency(t *testing.T) { }, } expectedResult["water"].SetFrequency(2) - result := TokenFrequency(tokens, nil, true) + result := TokenFrequency(tokens, nil, index.IncludeTermVectors) if !reflect.DeepEqual(result, expectedResult) { t.Errorf("expected %#v, got %#v", expectedResult, result) } diff --git a/document/field_boolean.go b/document/field_boolean.go index 7e43348e4..a588bdf9b 100644 --- a/document/field_boolean.go +++ b/document/field_boolean.go @@ -72,7 +72,7 @@ func (b *BooleanField) Analyze() { }) b.length = len(tokens) - b.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options.IncludeTermVectors()) + b.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options) } func (b *BooleanField) Value() []byte { diff --git a/document/field_datetime.go b/document/field_datetime.go index ab182c16f..dbb66a72b 100644 --- a/document/field_datetime.go +++ b/document/field_datetime.go @@ -111,7 +111,7 @@ func (n *DateTimeField) Analyze() { } n.length = len(tokens) - n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors()) + n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options) } func (n *DateTimeField) Value() []byte { diff --git a/document/field_geopoint.go b/document/field_geopoint.go index 5fb940508..35375ff4c 100644 --- a/document/field_geopoint.go +++ b/document/field_geopoint.go @@ -106,7 +106,7 @@ func (n *GeoPointField) Analyze() { } n.length = len(tokens) - n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors()) + n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options) } func (n *GeoPointField) Value() []byte { diff --git a/document/field_numeric.go b/document/field_numeric.go index 2f796eb68..4d2ceb239 100644 --- a/document/field_numeric.go +++ b/document/field_numeric.go @@ -107,7 +107,7 @@ func (n *NumericField) Analyze() { } n.length = len(tokens) - n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors()) + n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options) } func (n *NumericField) Value() []byte { diff --git a/document/field_text.go b/document/field_text.go index 58bebfae2..876d5a634 100644 --- a/document/field_text.go +++ b/document/field_text.go @@ -97,7 +97,7 @@ func (t *TextField) Analyze() { } } t.length = len(tokens) // number of tokens in this doc field - t.frequencies = analysis.TokenFrequency(tokens, t.arrayPositions, t.options.IncludeTermVectors()) + t.frequencies = analysis.TokenFrequency(tokens, t.arrayPositions, t.options) } func (t *TextField) Analyzer() *analysis.Analyzer { diff --git a/go.mod b/go.mod index 78d75a613..99a89277e 100644 --- a/go.mod +++ b/go.mod @@ -4,29 +4,25 @@ go 1.13 require ( github.com/RoaringBitmap/roaring v0.4.23 - github.com/blevesearch/bleve_index_api v0.0.4 - github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040 - github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5 // indirect + github.com/blevesearch/bleve_index_api v0.0.5 github.com/blevesearch/go-porterstemmer v1.0.3 - github.com/blevesearch/scorch_segment_api v0.0.6 + github.com/blevesearch/scorch_segment_api v0.0.7 github.com/blevesearch/segment v0.9.0 github.com/blevesearch/snowballstem v0.9.0 github.com/blevesearch/upsidedown_store_api v0.0.1 - github.com/blevesearch/zapx/v11 v11.1.5 - github.com/blevesearch/zapx/v12 v12.1.5 - github.com/blevesearch/zapx/v13 v13.1.5 - github.com/blevesearch/zapx/v14 v14.1.5 - github.com/blevesearch/zapx/v15 v15.1.5 + github.com/blevesearch/zapx/v11 v11.1.6 + github.com/blevesearch/zapx/v12 v12.1.6 + github.com/blevesearch/zapx/v13 v13.1.6 + github.com/blevesearch/zapx/v14 v14.1.6 + github.com/blevesearch/zapx/v15 v15.1.6 github.com/couchbase/moss v0.1.0 github.com/couchbase/vellum v1.0.2 github.com/golang/protobuf v1.3.2 - github.com/ikawaha/kagome.ipadic v1.1.2 // indirect github.com/kljensen/snowball v0.6.0 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 github.com/spf13/cobra v0.0.5 github.com/steveyen/gtreap v0.1.0 github.com/syndtr/goleveldb v1.0.0 - github.com/tebeka/snowball v0.4.2 // indirect github.com/willf/bitset v1.1.10 go.etcd.io/bbolt v1.3.5 golang.org/x/text v0.3.0 diff --git a/go.sum b/go.sum index 3e3293b68..763ff002f 100644 --- a/go.sum +++ b/go.sum @@ -2,34 +2,30 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/RoaringBitmap/roaring v0.4.23 h1:gpyfd12QohbqhFO4NVDUdoPOCXsyahYRQhINmlHxKeo= github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/blevesearch/bleve_index_api v0.0.4 h1:v+rdWIJ/wmVL81rW0G5cRdIO/Cjbd9NOsmuB0OIbaLs= -github.com/blevesearch/bleve_index_api v0.0.4/go.mod h1:yq9YIbuvEQdUB0h+UiLbpDdLLnvcD5ZkqO0LS1wuSvY= -github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040 h1:SjYVcfJVZoCfBlg+fkaq2eoZHTf5HaJfaTeTkOtyfHQ= -github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040/go.mod h1:WH+MU2F4T0VmSdaPX+Wu5GYoZBrYWdOZWSjzvYcDmqQ= -github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5 h1:/4ikScMMYMqsRFWJjCyzd3CNWB0lxvqDkqa5nEv6NMc= -github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5/go.mod h1:PN0QNTLs9+j1bKy3d/GB/59wsNBFC4sWLWG3k69lWbc= +github.com/blevesearch/bleve_index_api v0.0.5 h1:LdL97X6ZwbCf1FSo6xe5eqT92MDb4X7s2j1chAKRpuY= +github.com/blevesearch/bleve_index_api v0.0.5/go.mod h1:yq9YIbuvEQdUB0h+UiLbpDdLLnvcD5ZkqO0LS1wuSvY= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= -github.com/blevesearch/scorch_segment_api v0.0.6 h1:AL1ZM8rykkd9GQEo4QVPicCyrZmDD1C0DLmKgT2klIQ= -github.com/blevesearch/scorch_segment_api v0.0.6/go.mod h1:65b3su9AXLFeAWQlYHb0qyyPzAzscNmWXMa6qQOMJPo= +github.com/blevesearch/scorch_segment_api v0.0.7 h1:X6eGntic+71Gv77wQnzD2kOWyhEvxQ0x/El6X+NVKaE= +github.com/blevesearch/scorch_segment_api v0.0.7/go.mod h1:gZhO+qjyJ/u0qho9GuhHXtdtT3hpveyA4DfUU9W+kco= github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= github.com/blevesearch/upsidedown_store_api v0.0.1 h1:cf3YQ6c3jvZ/gvHL6k71cshDG+nYTgL5PTWFqYXA3bg= github.com/blevesearch/upsidedown_store_api v0.0.1/go.mod h1:jRZGTYWNqHXvOOjvT0WXHYO5Yin7U/PmTX1nevIY8yo= -github.com/blevesearch/zapx/v11 v11.1.5 h1:junThJcb8Zb9R06FXrSrbGv8w+q0JCeWiCUmBZCzoF0= -github.com/blevesearch/zapx/v11 v11.1.5/go.mod h1:t0iWZlu/G6tiOr2kS4yDQfomnErPOwe7MyTSNfmanZc= -github.com/blevesearch/zapx/v12 v12.1.5 h1:iGVqwKCEvN313f5wxVmgu3G53oIRlzgcStVYmIhzJyg= -github.com/blevesearch/zapx/v12 v12.1.5/go.mod h1:unFlOztl5e37f0shGV/c3O2cwzkQozgoSBwtKz7pYi8= -github.com/blevesearch/zapx/v13 v13.1.5 h1:py2pA1aZhYjmKMngJE13B7eB32zaLz5Ok5117KkIiWA= -github.com/blevesearch/zapx/v13 v13.1.5/go.mod h1:+9llGYb/i+2hl5Sh96AZsiCCfDjMde3zG+MUc8wTvGI= -github.com/blevesearch/zapx/v14 v14.1.5 h1:zDXCsgk0kMQ6oUS8DvNtUuqLvqPZ+5qQE3YBF+hl3lQ= -github.com/blevesearch/zapx/v14 v14.1.5/go.mod h1:SgHVOkbyz7GX53h4mv6jLRWwWW/lfvC+tW8Xz2CiXDw= -github.com/blevesearch/zapx/v15 v15.1.5 h1:YGOuePgbuCM3vS4etn2qQOOxGXr76KFbkO3pFsDELUo= -github.com/blevesearch/zapx/v15 v15.1.5/go.mod h1:0Oh7/luoaUujNBBf5tisBHDwJh0UQ/TFuVH0FUg5XlQ= +github.com/blevesearch/zapx/v11 v11.1.6 h1:8eqdDbWWNX99RS9byGVTiTXrBKfHYdp/XDi5JxWMYrU= +github.com/blevesearch/zapx/v11 v11.1.6/go.mod h1:yKP1ax+HWGb5+Z8SSQ3kRbxJRsJv18tBoap5OWT3nzc= +github.com/blevesearch/zapx/v12 v12.1.6 h1:mce/asG9ov5U3mNiYyLmadJVclDPntd5VLxpTo6Y8bA= +github.com/blevesearch/zapx/v12 v12.1.6/go.mod h1:H/RCE6zfjWofAtniY7qrKFDHB7Kb6piqX3O2s1jtADQ= +github.com/blevesearch/zapx/v13 v13.1.6 h1:aZGnHA1s8JdthJV0voyvgc1/FLz+gqioCf/9MEdRHjc= +github.com/blevesearch/zapx/v13 v13.1.6/go.mod h1:a2NeLQ8RWumX4T+i1FPAUvVPum/QAEi/beubZzcGJsQ= +github.com/blevesearch/zapx/v14 v14.1.6 h1:ldROClIq0ojQEJOAD1FAGOdFc2ciSAJamseZ3WV3r/Y= +github.com/blevesearch/zapx/v14 v14.1.6/go.mod h1:bANjNZhP3TARU6XM89sLAR/wgOPVyPI6zCkeElHoLlM= +github.com/blevesearch/zapx/v15 v15.1.6 h1:afFzPDlyYFJy9K8aWd7ufktJ8G7kViW+t5GQxYEuNf4= +github.com/blevesearch/zapx/v15 v15.1.6/go.mod h1:PUdibSDSmtQMeQX90o9J7zly2RmNJZXet+tfLFgZwng= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -60,8 +56,6 @@ github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORR github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ikawaha/kagome.ipadic v1.1.2 h1:pFxZ1PpMpc6ZoBK712YN5cVK0u/ju2DZ+gRIOriJFFs= -github.com/ikawaha/kagome.ipadic v1.1.2/go.mod h1:DPSBbU0czaJhAb/5uKQZHMc9MTVRpDugJfX+HddPHHg= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -103,8 +97,6 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/tebeka/snowball v0.4.2 h1:ujvgLOr6IHbsvB2Vgz27IcxWqDrNu9/oPhhe74lN/Kc= -github.com/tebeka/snowball v0.4.2/go.mod h1:4IfL14h1lvwZcp1sfXuuc7/7yCsvVffTWxWxCLfFpYg= github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=