diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 0000000..94df216 --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,7 @@ +version = 1 + +[[analyzers]] +name = "go" + + [analyzers.meta] + import_root = "github.com/ikawaha/kagome" \ No newline at end of file diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 9f8028d..f016752 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -23,13 +23,13 @@ jobs: uses: actions/checkout@v4 - name: Set up Go 1.x - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version-file: 'go.mod' cache: true - name: Set up Graphviz - uses: ts-graphviz/setup-graphviz@v1 + uses: ts-graphviz/setup-graphviz@v2 - name: Build run: go build -v ./... @@ -46,7 +46,7 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v4 - name: Set up Go 1.x - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version-file: 'go.mod' cache: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 390f550..a2948c8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,7 @@ jobs: # Setup Go - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version-file: 'go.mod' cache: true @@ -38,6 +38,14 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + # Login to GitHub Container Registry before push. + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GO_RELEASER_TOKEN }} + # Build and release - name: Run GoReleaser uses: goreleaser/goreleaser-action@v5 diff --git a/.goreleaser.yml b/.goreleaser.yml index 516049b..31c6c68 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -70,6 +70,7 @@ brews: dockers: - image_templates: - "ikawaha/kagome:{{ .Version }}-amd64" + - "ghcr.io/ikawaha/kagome:{{ .Version }}-amd64" use: buildx goarch: amd64 dockerfile: Dockerfile @@ -78,6 +79,7 @@ dockers: - image_templates: - "ikawaha/kagome:{{ .Version }}-arm64" + - "ghcr.io/ikawaha/kagome:{{ .Version }}-arm64" use: buildx goarch: arm64 dockerfile: Dockerfile @@ -86,6 +88,7 @@ dockers: - image_templates: - "ikawaha/kagome:{{ .Version }}-arm32v5" + - "ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v5" use: buildx goarch: arm goarm: 5 @@ -95,6 +98,7 @@ dockers: - image_templates: - "ikawaha/kagome:{{ .Version }}-arm32v6" + - "ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v6" use: buildx goarch: arm goarm: 6 @@ -104,6 +108,7 @@ dockers: - image_templates: - "ikawaha/kagome:{{ .Version }}-arm32v7" + - "ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v7" use: buildx goarch: arm goarm: 7 @@ -114,6 +119,7 @@ dockers: # Create multiarch manifest file of Docker image. This will bind all the images # above into a single manifest file for "latest" and each version tag. docker_manifests: + # Manifest for Docker Hub - name_template: ikawaha/kagome:{{ .Version }} image_templates: - ikawaha/kagome:{{ .Version }}-amd64 @@ -127,4 +133,20 @@ docker_manifests: - ikawaha/kagome:{{ .Version }}-arm64 - ikawaha/kagome:{{ .Version }}-arm32v5 - ikawaha/kagome:{{ .Version }}-arm32v6 - - ikawaha/kagome:{{ .Version }}-arm32v7 \ No newline at end of file + - ikawaha/kagome:{{ .Version }}-arm32v7 + + # Manifest for GitHub Container Registry + - name_template: ghcr.io/ikawaha/kagome:{{ .Version }} + image_templates: + - ghcr.io/ikawaha/kagome:{{ .Version }}-amd64 + - ghcr.io/ikawaha/kagome:{{ .Version }}-arm64 + - ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v5 + - ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v6 + - ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v7 + - name_template: ghcr.io/ikawaha/kagome:latest + image_templates: + - ghcr.io/ikawaha/kagome:{{ .Version }}-amd64 + - ghcr.io/ikawaha/kagome:{{ .Version }}-arm64 + - ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v5 + - ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v6 + - ghcr.io/ikawaha/kagome:{{ .Version }}-arm32v7 diff --git a/README.md b/README.md index 24b7916..32b0771 100644 --- a/README.md +++ b/README.md @@ -248,6 +248,8 @@ Start a server and try to access the "/tokenize" endpoint. ![webapp](https://raw.githubusercontent.com/wiki/ikawaha/kagome/images/demoapp.gif) +GitHub Page: https://ikawaha.github.io/kagome/ + Start a server and access `http://localhost:6060`. (To draw a lattice, demo application uses graphviz . You need graphviz installed.) @@ -274,16 +276,25 @@ A debug tool of tokenize process outputs a lattice in graphviz dot format. ```sh # Compatible architectures: AMD64, Arm64, Arm32 (Arm v5, v6 and v7) docker pull ikawaha/kagome:latest + +# Alternatively, you can pull from GitHub Container Registry +docker pull ghcr.io/ikawaha/kagome:latest ``` ```sh # Interactive/REPL mode docker run --rm -it ikawaha/kagome:latest + +# If pulling from GitHub Container Registry +docker run --rm -it ghcr.io/ikawaha/kagome:latest ``` ```sh # Server mode (http://localhost:6060) docker run --rm -p 6060:6060 ikawaha/kagome:latest server + +# If pulling from GitHub Container Registry +docker run --rm -p 6060:6060 ghcr.io/ikawaha/kagome:latest server ``` # Building to WebAssembly diff --git a/cmd/lattice/cmd.go b/cmd/lattice/cmd.go index d4d1ddf..dee8c70 100644 --- a/cmd/lattice/cmd.go +++ b/cmd/lattice/cmd.go @@ -59,7 +59,7 @@ func (o *option) parse(args []string) error { } // validations if o.flagSet.NArg() == 0 { - return fmt.Errorf("input is empty") + return errors.New("no option is specified") } if o.dict != "" && o.dict != "ipa" && o.dict != "uni" { return fmt.Errorf("invalid argument: -dict %v", o.dict) @@ -121,7 +121,7 @@ func command(_ context.Context, opt *option) error { } out := Stdout if opt.output != "" { - f, err := os.OpenFile(opt.output, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0o666) + f, err := os.OpenFile(opt.output, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0o600) if err != nil { return err } @@ -153,7 +153,7 @@ func Run(ctx context.Context, args []string) error { if err := opt.parse(args); err != nil { Usage() PrintDefaults(flag.ContinueOnError) - return errors.New("") + return err } return command(ctx, opt) } diff --git a/cmd/server/cmd.go b/cmd/server/cmd.go index da7a16d..cb7b7a9 100644 --- a/cmd/server/cmd.go +++ b/cmd/server/cmd.go @@ -2,7 +2,6 @@ package server import ( "context" - "errors" "flag" "fmt" "io" @@ -11,6 +10,7 @@ import ( "os" "os/signal" "syscall" + "time" "github.com/ikawaha/kagome-dict/dict" "github.com/ikawaha/kagome-dict/ipa" @@ -106,8 +106,9 @@ func command(ctx context.Context, opt *option) error { mux.Handle("/", &TokenizeDemoHandler{tokenizer: t}) mux.Handle("/tokenize", &TokenizeHandler{tokenizer: t}) srv := http.Server{ - Addr: opt.http, - Handler: mux, + Addr: opt.http, + Handler: mux, + ReadHeaderTimeout: 20 * time.Second, } ch := make(chan error) go func() { @@ -132,17 +133,15 @@ func Run(ctx context.Context, args []string) error { if err := opt.parse(args); err != nil { Usage() PrintDefaults(flag.ContinueOnError) - return errors.New("") + return fmt.Errorf("option parse error: %w", err) } ctx, cancel := context.WithCancel(ctx) defer cancel() go func() { c := make(chan os.Signal, 1) signal.Notify(c, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) - select { - case <-c: - cancel() - } + <-c + cancel() }() return command(ctx, opt) } diff --git a/kagome_test.go b/kagome_test.go index 12045f2..0872db0 100644 --- a/kagome_test.go +++ b/kagome_test.go @@ -38,7 +38,7 @@ func TestKagomeTokenizerGolden(t *testing.T) { } defer golden.Close() - dump, err := os.OpenFile(dumpText, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0o666) + dump, err := os.OpenFile(dumpText, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0o600) if err != nil { t.Fatalf("unexpected error, %v", err) } diff --git a/sample/_example/db_search/main.go b/sample/_example/db_search/main.go index bc2ca32..a811a27 100644 --- a/sample/_example/db_search/main.go +++ b/sample/_example/db_search/main.go @@ -24,11 +24,12 @@ Acknowledgements: This example is taken in part from the following book for reference. -- p.372, 9.2 "データーベース登録プログラム", "Go言語プログラミングエッセンス エンジニア選書" +- p.204, 9.2 "データーベース登録プログラム", "Go言語プログラミングエッセンス エンジニア選書" - Written by: Mattn - Published: 2023/3/9 (技術評論社) - ISBN: 4297134195 / 978-4297134198 - ASIN: B0BVZCJQ4F / https://amazon.co.jp/dp/4297134195 + - Original sample code: https://github.com/mattn/aozora-search */ package main @@ -37,6 +38,7 @@ import ( "errors" "fmt" "log" + "os" "strings" "github.com/ikawaha/kagome-dict/ipa" @@ -45,6 +47,13 @@ import ( ) func main() { + if err := run(); err != nil { + log.Println(err) + os.Exit(1) + } +} + +func run() error { // Contents to be inserted into the database. Each element represents a line // of text and will be inserted into a row of the database. lines := []string{ @@ -58,26 +67,30 @@ func main() { // Create a database. In-memory database is used for simplicity. db, err := sql.Open("sqlite3", ":memory:") - PanicOnError(err) - + if err != nil { + return err + } defer db.Close() // Create tables. // The first table "contents_fts" is for storing the original content, and // the second table "fts" is for storing the tokenized content. - _, err = db.Exec(` + if _, err = db.Exec(` CREATE TABLE IF NOT EXISTS contents_fts(docid INTEGER PRIMARY KEY AUTOINCREMENT, content TEXT); CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts4(words); - `) - PanicOnError(err) + `); err != nil { + return err + } // Insert contents for _, line := range lines { rowID, err := insertContent(db, line) - PanicOnError(err) - - err = insertSearchToken(db, rowID, line) - PanicOnError(err) + if err != nil { + return err + } + if err := insertSearchToken(db, rowID, line); err != nil { + return err + } } // Search by word. @@ -98,7 +111,9 @@ func main() { fmt.Println("Searching for:", searchWord) rowIDsFound, err := searchFTS4(db, searchWord) - PanicOnError(err) + if err != nil { + return err + } if len(rowIDsFound) == 0 { fmt.Println(" No results found") @@ -108,8 +123,9 @@ func main() { // Print search results for _, rowID := range rowIDsFound { cont, err := retrieveContent(db, rowID) - PanicOnError(err) - + if err != nil { + return err + } fmt.Printf(" Found content: %s at line: %v\n", cont, rowID) } } @@ -124,6 +140,8 @@ func main() { // Found content: 北方の海の色は、青うございました。 at line: 3 // Searching for: 北 // Found content: 北の海にも棲んでいたのであります。 at line: 2 + + return nil } func insertContent(db *sql.DB, content string) (int64, error) { @@ -142,7 +160,9 @@ func insertSearchToken(db *sql.DB, rowID int64, content string) error { // This example uses the IPA dictionary, but it may be more efficient to use // the 'Uni' dictionary if memory is available. tknzr, err := tokenizer.New(ipa.Dict(), tokenizer.OmitBosEos()) - PanicOnError(err) + if err != nil { + return err + } seg := tknzr.Wakati(content) tokenizedContent := strings.Join(seg, " ") @@ -156,13 +176,6 @@ func insertSearchToken(db *sql.DB, rowID int64, content string) error { return err } -// PanicOnError exits the program with panic if the given error is not nil. -func PanicOnError(err error) { - if err != nil { - log.Panic(err) - } -} - func retrieveContent(db *sql.DB, rowID int) (string, error) { rows, err := db.Query( `SELECT rowid, content FROM contents_fts WHERE rowid=?`,