Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update analysis.TokenFrequency to use options #1522

Merged
merged 2 commits into from
Dec 16, 2020
Merged

update analysis.TokenFrequency to use options #1522

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5cb6d96
update analysis.TokenFrequency to use options
mschoch Dec 16, 2020
c51d9bb
update to latest apis
mschoch Dec 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion analysis/benchmark_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package analysis_test

import (
index "github.com/blevesearch/bleve_index_api"
"testing"

"github.com/blevesearch/bleve/analysis"
Expand All @@ -32,7 +33,7 @@ func BenchmarkAnalysis(b *testing.B) {
}

ts := analyzer.Analyze(bleveWikiArticle)
freqs := analysis.TokenFrequency(ts, nil, true)
freqs := analysis.TokenFrequency(ts, nil, index.IncludeTermVectors)
if len(freqs) != 511 {
b.Errorf("expected %d freqs, got %d", 511, len(freqs))
}
Expand Down
4 changes: 2 additions & 2 deletions analysis/freq.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ import (
index "github.com/blevesearch/bleve_index_api"
)

func TokenFrequency(tokens TokenStream, arrayPositions []uint64, includeTermVectors bool) index.TokenFrequencies {
func TokenFrequency(tokens TokenStream, arrayPositions []uint64, options index.FieldIndexingOptions) index.TokenFrequencies {
rv := make(map[string]*index.TokenFreq, len(tokens))

if includeTermVectors {
if options.IncludeTermVectors() {
tls := make([]index.TokenLocation, len(tokens))
tlNext := 0

Expand Down
2 changes: 1 addition & 1 deletion analysis/freq_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func TestTokenFrequency(t *testing.T) {
},
}
expectedResult["water"].SetFrequency(2)
result := TokenFrequency(tokens, nil, true)
result := TokenFrequency(tokens, nil, index.IncludeTermVectors)
if !reflect.DeepEqual(result, expectedResult) {
t.Errorf("expected %#v, got %#v", expectedResult, result)
}
Expand Down
2 changes: 1 addition & 1 deletion document/field_boolean.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func (b *BooleanField) Analyze() {
})

b.length = len(tokens)
b.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options.IncludeTermVectors())
b.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options)
}

func (b *BooleanField) Value() []byte {
Expand Down
2 changes: 1 addition & 1 deletion document/field_datetime.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func (n *DateTimeField) Analyze() {
}

n.length = len(tokens)
n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors())
n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options)
}

func (n *DateTimeField) Value() []byte {
Expand Down
2 changes: 1 addition & 1 deletion document/field_geopoint.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ func (n *GeoPointField) Analyze() {
}

n.length = len(tokens)
n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors())
n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options)
}

func (n *GeoPointField) Value() []byte {
Expand Down
2 changes: 1 addition & 1 deletion document/field_numeric.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ func (n *NumericField) Analyze() {
}

n.length = len(tokens)
n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors())
n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options)
}

func (n *NumericField) Value() []byte {
Expand Down
2 changes: 1 addition & 1 deletion document/field_text.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func (t *TextField) Analyze() {
}
}
t.length = len(tokens) // number of tokens in this doc field
t.frequencies = analysis.TokenFrequency(tokens, t.arrayPositions, t.options.IncludeTermVectors())
t.frequencies = analysis.TokenFrequency(tokens, t.arrayPositions, t.options)
}

func (t *TextField) Analyzer() *analysis.Analyzer {
Expand Down
18 changes: 7 additions & 11 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,29 +4,25 @@ go 1.13

require (
github.com/RoaringBitmap/roaring v0.4.23
github.com/blevesearch/bleve_index_api v0.0.4
github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040
github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5 // indirect
github.com/blevesearch/bleve_index_api v0.0.5
github.com/blevesearch/go-porterstemmer v1.0.3
github.com/blevesearch/scorch_segment_api v0.0.6
github.com/blevesearch/scorch_segment_api v0.0.7
github.com/blevesearch/segment v0.9.0
github.com/blevesearch/snowballstem v0.9.0
github.com/blevesearch/upsidedown_store_api v0.0.1
github.com/blevesearch/zapx/v11 v11.1.5
github.com/blevesearch/zapx/v12 v12.1.5
github.com/blevesearch/zapx/v13 v13.1.5
github.com/blevesearch/zapx/v14 v14.1.5
github.com/blevesearch/zapx/v15 v15.1.5
github.com/blevesearch/zapx/v11 v11.1.6
github.com/blevesearch/zapx/v12 v12.1.6
github.com/blevesearch/zapx/v13 v13.1.6
github.com/blevesearch/zapx/v14 v14.1.6
github.com/blevesearch/zapx/v15 v15.1.6
github.com/couchbase/moss v0.1.0
github.com/couchbase/vellum v1.0.2
github.com/golang/protobuf v1.3.2
github.com/ikawaha/kagome.ipadic v1.1.2 // indirect
github.com/kljensen/snowball v0.6.0
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563
github.com/spf13/cobra v0.0.5
github.com/steveyen/gtreap v0.1.0
github.com/syndtr/goleveldb v1.0.0
github.com/tebeka/snowball v0.4.2 // indirect
github.com/willf/bitset v1.1.10
go.etcd.io/bbolt v1.3.5
golang.org/x/text v0.3.0
Expand Down
36 changes: 14 additions & 22 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,34 +2,30 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/RoaringBitmap/roaring v0.4.23 h1:gpyfd12QohbqhFO4NVDUdoPOCXsyahYRQhINmlHxKeo=
github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/blevesearch/bleve_index_api v0.0.4 h1:v+rdWIJ/wmVL81rW0G5cRdIO/Cjbd9NOsmuB0OIbaLs=
github.com/blevesearch/bleve_index_api v0.0.4/go.mod h1:yq9YIbuvEQdUB0h+UiLbpDdLLnvcD5ZkqO0LS1wuSvY=
github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040 h1:SjYVcfJVZoCfBlg+fkaq2eoZHTf5HaJfaTeTkOtyfHQ=
github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040/go.mod h1:WH+MU2F4T0VmSdaPX+Wu5GYoZBrYWdOZWSjzvYcDmqQ=
github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5 h1:/4ikScMMYMqsRFWJjCyzd3CNWB0lxvqDkqa5nEv6NMc=
github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5/go.mod h1:PN0QNTLs9+j1bKy3d/GB/59wsNBFC4sWLWG3k69lWbc=
github.com/blevesearch/bleve_index_api v0.0.5 h1:LdL97X6ZwbCf1FSo6xe5eqT92MDb4X7s2j1chAKRpuY=
github.com/blevesearch/bleve_index_api v0.0.5/go.mod h1:yq9YIbuvEQdUB0h+UiLbpDdLLnvcD5ZkqO0LS1wuSvY=
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0=
github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
github.com/blevesearch/scorch_segment_api v0.0.6 h1:AL1ZM8rykkd9GQEo4QVPicCyrZmDD1C0DLmKgT2klIQ=
github.com/blevesearch/scorch_segment_api v0.0.6/go.mod h1:65b3su9AXLFeAWQlYHb0qyyPzAzscNmWXMa6qQOMJPo=
github.com/blevesearch/scorch_segment_api v0.0.7 h1:X6eGntic+71Gv77wQnzD2kOWyhEvxQ0x/El6X+NVKaE=
github.com/blevesearch/scorch_segment_api v0.0.7/go.mod h1:gZhO+qjyJ/u0qho9GuhHXtdtT3hpveyA4DfUU9W+kco=
github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac=
github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ=
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
github.com/blevesearch/upsidedown_store_api v0.0.1 h1:cf3YQ6c3jvZ/gvHL6k71cshDG+nYTgL5PTWFqYXA3bg=
github.com/blevesearch/upsidedown_store_api v0.0.1/go.mod h1:jRZGTYWNqHXvOOjvT0WXHYO5Yin7U/PmTX1nevIY8yo=
github.com/blevesearch/zapx/v11 v11.1.5 h1:junThJcb8Zb9R06FXrSrbGv8w+q0JCeWiCUmBZCzoF0=
github.com/blevesearch/zapx/v11 v11.1.5/go.mod h1:t0iWZlu/G6tiOr2kS4yDQfomnErPOwe7MyTSNfmanZc=
github.com/blevesearch/zapx/v12 v12.1.5 h1:iGVqwKCEvN313f5wxVmgu3G53oIRlzgcStVYmIhzJyg=
github.com/blevesearch/zapx/v12 v12.1.5/go.mod h1:unFlOztl5e37f0shGV/c3O2cwzkQozgoSBwtKz7pYi8=
github.com/blevesearch/zapx/v13 v13.1.5 h1:py2pA1aZhYjmKMngJE13B7eB32zaLz5Ok5117KkIiWA=
github.com/blevesearch/zapx/v13 v13.1.5/go.mod h1:+9llGYb/i+2hl5Sh96AZsiCCfDjMde3zG+MUc8wTvGI=
github.com/blevesearch/zapx/v14 v14.1.5 h1:zDXCsgk0kMQ6oUS8DvNtUuqLvqPZ+5qQE3YBF+hl3lQ=
github.com/blevesearch/zapx/v14 v14.1.5/go.mod h1:SgHVOkbyz7GX53h4mv6jLRWwWW/lfvC+tW8Xz2CiXDw=
github.com/blevesearch/zapx/v15 v15.1.5 h1:YGOuePgbuCM3vS4etn2qQOOxGXr76KFbkO3pFsDELUo=
github.com/blevesearch/zapx/v15 v15.1.5/go.mod h1:0Oh7/luoaUujNBBf5tisBHDwJh0UQ/TFuVH0FUg5XlQ=
github.com/blevesearch/zapx/v11 v11.1.6 h1:8eqdDbWWNX99RS9byGVTiTXrBKfHYdp/XDi5JxWMYrU=
github.com/blevesearch/zapx/v11 v11.1.6/go.mod h1:yKP1ax+HWGb5+Z8SSQ3kRbxJRsJv18tBoap5OWT3nzc=
github.com/blevesearch/zapx/v12 v12.1.6 h1:mce/asG9ov5U3mNiYyLmadJVclDPntd5VLxpTo6Y8bA=
github.com/blevesearch/zapx/v12 v12.1.6/go.mod h1:H/RCE6zfjWofAtniY7qrKFDHB7Kb6piqX3O2s1jtADQ=
github.com/blevesearch/zapx/v13 v13.1.6 h1:aZGnHA1s8JdthJV0voyvgc1/FLz+gqioCf/9MEdRHjc=
github.com/blevesearch/zapx/v13 v13.1.6/go.mod h1:a2NeLQ8RWumX4T+i1FPAUvVPum/QAEi/beubZzcGJsQ=
github.com/blevesearch/zapx/v14 v14.1.6 h1:ldROClIq0ojQEJOAD1FAGOdFc2ciSAJamseZ3WV3r/Y=
github.com/blevesearch/zapx/v14 v14.1.6/go.mod h1:bANjNZhP3TARU6XM89sLAR/wgOPVyPI6zCkeElHoLlM=
github.com/blevesearch/zapx/v15 v15.1.6 h1:afFzPDlyYFJy9K8aWd7ufktJ8G7kViW+t5GQxYEuNf4=
github.com/blevesearch/zapx/v15 v15.1.6/go.mod h1:PUdibSDSmtQMeQX90o9J7zly2RmNJZXet+tfLFgZwng=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
Expand Down Expand Up @@ -60,8 +56,6 @@ github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORR
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ikawaha/kagome.ipadic v1.1.2 h1:pFxZ1PpMpc6ZoBK712YN5cVK0u/ju2DZ+gRIOriJFFs=
github.com/ikawaha/kagome.ipadic v1.1.2/go.mod h1:DPSBbU0czaJhAb/5uKQZHMc9MTVRpDugJfX+HddPHHg=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
Expand Down Expand Up @@ -103,8 +97,6 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/tebeka/snowball v0.4.2 h1:ujvgLOr6IHbsvB2Vgz27IcxWqDrNu9/oPhhe74lN/Kc=
github.com/tebeka/snowball v0.4.2/go.mod h1:4IfL14h1lvwZcp1sfXuuc7/7yCsvVffTWxWxCLfFpYg=
github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
Expand Down