From 97f8325ccf3826f3e986eaf31ffdca6221637369 Mon Sep 17 00:00:00 2001 From: Ringo Hoffmann Date: Sat, 15 Jun 2024 08:30:51 +0200 Subject: [PATCH 1/5] fix e2e tests on macOS --- e2e/cases/fileimport-parameterized/run.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/e2e/cases/fileimport-parameterized/run.sh b/e2e/cases/fileimport-parameterized/run.sh index c78b18e..98a641e 100644 --- a/e2e/cases/fileimport-parameterized/run.sh +++ b/e2e/cases/fileimport-parameterized/run.sh @@ -2,7 +2,8 @@ set -e # CRLF to LF, because git on windows does git on windows things. # Also '-i' can not be used because sed on OSX does sed on SOX things. -cat body.txt | sed 's/^M$//' | tee body.txt +sed 's/^M$//' < body.txt > body.txt.tmp +mv body.txt.tmp body.txt goat -a "file=body.txt" direct.goat -goat -a "file=body.txt" imported.goat \ No newline at end of file +goat -a "file=body.txt" imported.goat From bf6d7878448b665adab8de017d48b86864db0c50 Mon Sep 17 00:00:00 2001 From: Ringo Hoffmann Date: Mon, 22 Jul 2024 10:02:22 +0200 Subject: [PATCH 2/5] add `jq` builtin --- go.mod | 2 ++ go.sum | 4 ++++ pkg/engine/goja.go | 1 + pkg/engine/goja_builtin.go | 26 ++++++++++++++++++++++++++ 4 files changed, 33 insertions(+) diff --git a/go.mod b/go.mod index 2094b1e..391c32e 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,8 @@ require ( github.com/dlclark/regexp2 v1.11.0 // indirect github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect + github.com/itchyny/gojq v0.12.16 // indirect + github.com/itchyny/timefmt-go v0.1.6 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/go.sum b/go.sum index 62970fe..0410408 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,10 @@ github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/google/pprof v0.0.0-20240528025155-186aa0362fba h1:ql1qNgCyOB7iAEk8JTNM+zJrgIbnyCKX/wdlyPufP5g= github.com/google/pprof v0.0.0-20240528025155-186aa0362fba/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g= +github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM= +github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q= +github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= diff --git a/pkg/engine/goja.go b/pkg/engine/goja.go index d052487..911773f 100644 --- a/pkg/engine/goja.go +++ b/pkg/engine/goja.go @@ -36,6 +36,7 @@ func NewGoja() Engine { t.Set("print", t.builtin_print) t.Set("printf", t.builtin_printf) t.Set("println", t.builtin_println) + t.Set("jq", t.builtin_jq) return &t } diff --git a/pkg/engine/goja_builtin.go b/pkg/engine/goja_builtin.go index 67d1e5d..894591d 100644 --- a/pkg/engine/goja_builtin.go +++ b/pkg/engine/goja_builtin.go @@ -2,6 +2,7 @@ package engine import ( "fmt" + "github.com/itchyny/gojq" "reflect" "strings" @@ -87,3 +88,28 @@ func (t *Goja) builtin_fatalf(format string, v ...any) { func (t *Goja) builtin_printf(format string, v ...any) { fmt.Printf(format, v...) } + +func (t *Goja) builtin_jq(object any, src string) []any { + query, err := gojq.Parse(src) + if err != nil { + panic(t.rt.ToValue(fmt.Sprintf("command parsing failed: %s", err.Error()))) + } + + var results []any + iter := query.Run(object) + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + if err, ok := err.(*gojq.HaltError); ok && err.Value() == nil { + break + } + panic(t.rt.ToValue(err.Error())) + } + results = append(results, v) + } + + return results +} From 54182309af5adea22f3c37fb448ea5238481365d Mon Sep 17 00:00:00 2001 From: Ringo Hoffmann Date: Mon, 22 Jul 2024 10:03:05 +0200 Subject: [PATCH 3/5] update dependencies --- go.mod | 16 ++++++++-------- go.sum | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 391c32e..80b3e67 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,10 @@ module github.com/studio-b12/goat go 1.19 require ( - github.com/alexflint/go-arg v1.5.0 - github.com/dop251/goja v0.0.0-20240610225006-393f6d42497b + github.com/alexflint/go-arg v1.5.1 + github.com/dop251/goja v0.0.0-20240707163329-b1681fb2a2f5 github.com/golang/mock v1.6.0 + github.com/itchyny/gojq v0.12.16 github.com/joho/godotenv v1.5.1 github.com/stretchr/testify v1.8.1 github.com/traefik/paerser v0.2.0 @@ -16,13 +17,12 @@ require ( github.com/BurntSushi/toml v1.4.0 // indirect github.com/alexflint/go-scalar v1.2.0 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect - github.com/charmbracelet/lipgloss v0.11.0 // indirect - github.com/charmbracelet/x/ansi v0.1.2 // indirect + github.com/charmbracelet/lipgloss v0.12.1 // indirect + github.com/charmbracelet/x/ansi v0.1.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dlclark/regexp2 v1.11.0 // indirect + github.com/dlclark/regexp2 v1.11.2 // indirect github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect - github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect - github.com/itchyny/gojq v0.12.16 // indirect + github.com/google/pprof v0.0.0-20240721033354-7089f98c1d14 // indirect github.com/itchyny/timefmt-go v0.1.6 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -31,7 +31,7 @@ require ( github.com/muesli/termenv v0.15.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 0410408..817b4fe 100644 --- a/go.sum +++ b/go.sum @@ -2,27 +2,39 @@ github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0 github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/alexflint/go-arg v1.5.0 h1:rwMKGiaQuRbXfZNyRUvIfke63QvOBt1/QTshlGQHohM= github.com/alexflint/go-arg v1.5.0/go.mod h1:A7vTJzvjoaSTypg4biM5uYNTkJ27SkNTArtYXnlqVO8= +github.com/alexflint/go-arg v1.5.1 h1:nBuWUCpuRy0snAG+uIJ6N0UvYxpxA0/ghA/AaHxlT8Y= +github.com/alexflint/go-arg v1.5.1/go.mod h1:A7vTJzvjoaSTypg4biM5uYNTkJ27SkNTArtYXnlqVO8= github.com/alexflint/go-scalar v1.2.0 h1:WR7JPKkeNpnYIOfHRa7ivM21aWAdHD0gEWHCx+WQBRw= github.com/alexflint/go-scalar v1.2.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/charmbracelet/lipgloss v0.11.0 h1:UoAcbQ6Qml8hDwSWs0Y1cB5TEQuZkDPH/ZqwWWYTG4g= github.com/charmbracelet/lipgloss v0.11.0/go.mod h1:1UdRTH9gYgpcdNN5oBtjbu/IzNKtzVtb7sqN1t9LNn8= +github.com/charmbracelet/lipgloss v0.12.1 h1:/gmzszl+pedQpjCOH+wFkZr/N90Snz40J/NR7A0zQcs= +github.com/charmbracelet/lipgloss v0.12.1/go.mod h1:V2CiwIuhx9S1S1ZlADfOj9HmxeMAORuz5izHb0zGbB8= github.com/charmbracelet/x/ansi v0.1.2 h1:6+LR39uG8DE6zAmbu023YlqjJHkYXDF1z36ZwzO4xZY= github.com/charmbracelet/x/ansi v0.1.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= +github.com/charmbracelet/x/ansi v0.1.4 h1:IEU3D6+dWwPSgZ6HBH+v6oUuZ/nVawMiWj5831KfiLM= +github.com/charmbracelet/x/ansi v0.1.4/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.11.2 h1:/u628IuisSTwri5/UKloiIsH8+qF2Pu7xEQX+yIKg68= +github.com/dlclark/regexp2 v1.11.2/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dop251/goja v0.0.0-20240610225006-393f6d42497b h1:fMKDnOAKCGXSZBphY/ilLtu7cmwMnjqE+xJxUkfkpCY= github.com/dop251/goja v0.0.0-20240610225006-393f6d42497b/go.mod h1:o31y53rb/qiIAONF7w3FHJZRqqP3fzHUr1HqanthByw= +github.com/dop251/goja v0.0.0-20240707163329-b1681fb2a2f5 h1:ZRqTaoW9WZ2DqeOQGhK9q73eCb47SEs30GV2IRHT9bo= +github.com/dop251/goja v0.0.0-20240707163329-b1681fb2a2f5/go.mod h1:o31y53rb/qiIAONF7w3FHJZRqqP3fzHUr1HqanthByw= github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q= github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/google/pprof v0.0.0-20240528025155-186aa0362fba h1:ql1qNgCyOB7iAEk8JTNM+zJrgIbnyCKX/wdlyPufP5g= github.com/google/pprof v0.0.0-20240528025155-186aa0362fba/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20240721033354-7089f98c1d14 h1:m2fdPWWX/0mdyA1X3XbVTag5NEwmWv0mieoVuRq14A4= +github.com/google/pprof v0.0.0-20240721033354-7089f98c1d14/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g= github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM= github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q= @@ -75,6 +87,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 2d587e8a14a31938e0115a0afbca3fd066a7a1fd Mon Sep 17 00:00:00 2001 From: Ringo Hoffmann Date: Mon, 22 Jul 2024 10:14:15 +0200 Subject: [PATCH 4/5] add `jq` builtin documentation --- docs/book/src/scripting/builtins.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/book/src/scripting/builtins.md b/docs/book/src/scripting/builtins.md index e156617..205e10c 100644 --- a/docs/book/src/scripting/builtins.md +++ b/docs/book/src/scripting/builtins.md @@ -16,6 +16,7 @@ The following built-in functions are available in each script instance. - [`errorf`](#errorf) - [`fatalf`](#fatalf) - [`debugf`](#debugf) +- [`jq`](#jq) ## `assert` @@ -213,3 +214,24 @@ Logs a *debug* log entry to the output logger(s) with the given `format` formatt ```js debugf("Hello %s!", "World"); ``` + +## `jq` + +```ts +function jq(object: any, src: string): any[]; +``` + +For more complex analysis on JSON responses, `jq` can be used to perform JQ commands on any passed `object`. Goat uses [itchyny/gojq](https://github.com/itchyny/gojq) as implementation of JQ. [Here](https://jqlang.github.io/jq/manual) you can find more information about the JQ syntax. + +The results are always returned as a list. If there are no results, an empty list is returned. When the command compilation fails or the JQ execution fails, the function will throw an exception. + +**Example** + +*Explanation: Lists all instances recursively where the value of `href` is an object and is empty.* + +```js +jq(response.Body, `.. + | if type == "object" then .href else empty end + | if type == "object" then . else empty end + | select( . | length == 0 )`); +``` From ecb6e4eba45c8b9e4ed98406d534208541408e0c Mon Sep 17 00:00:00 2001 From: Ringo Hoffmann Date: Mon, 22 Jul 2024 10:17:33 +0200 Subject: [PATCH 5/5] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ab97e8..7e89ba8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ - Added a new script builtin [`assert_eq`](https://studio-b12.github.io/goat/scripting/builtins.html#assert_eq), where you can pass two values which are compared and output for better error clarification. +- Added a new script builtin [`jq`](https://studio-b12.github.io/goat/scripting/builtins.html#jq), to perform JQ commands on any object in script blocks. + # Minor Changes and Bug Fixes - Fixed an issue when parsing file descriptors on Windows systems. \ No newline at end of file