Skip to content

Commit

Permalink
Add jq executor
Browse files Browse the repository at this point in the history
  • Loading branch information
yottahmd committed Mar 2, 2023
1 parent be80c7c commit bd23df7
Show file tree
Hide file tree
Showing 7 changed files with 185 additions and 4 deletions.
45 changes: 45 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ Dagu is a tool for scheduling and running tasks based on a directed acyclic grap
- [Executors](#executors)
- [Running Docker Containers](#running-docker-containers)
- [Making HTTP Requests](#making-http-requests)
- [Executing jq Command](#executing-jq-command)
- [Command Execution over SSH](#command-execution-over-ssh)
- [Admin Configuration](#admin-configuration)
- [Environment Variable](#environment-variable)
Expand Down Expand Up @@ -505,6 +506,50 @@ steps:
}
```

### Executing jq Command

The `jq` executor can be used to transform, query, and format JSON.

Query Example:

```yaml
steps:
- name: run query
executor: jq
command: '{(.id): .["10"].b}'
script: |
{"id": "sample", "10": {"b": 42}}
```

output:
```json
{
"sample": 42
}
```

Formatting JSON:

```yaml
steps:
- name: format json
executor: jq
script: |
{"id": "sample", "10": {"b": 42}}
```

output:
```json
{
"10": {
"b": 42
},
"id": "sample"
}
```

The `jq` result can be used in following steps via [Output](#output) or [Stdout Redirection](#stdout-and-stderr-redirection).

### Command Execution over SSH

The `ssh` executor allows us to execute commands on remote hosts over SSH.
Expand Down
38 changes: 38 additions & 0 deletions examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
- [How to run docker containers inside a `dagu` container](#how-to-run-docker-containers-inside-a-dagu-container)
- [Command Execution over SSH](#command-execution-over-ssh)
- [Making HTTP Requests](#making-http-requests)
- [Querying JSON with jq](#querying-json-with-jq)
- [Format JSON with jq](#format-json-with-jq)
- [Sending Email Notification](#sending-email-notification)
- [Customizing Signal on Stop](#customizing-signal-on-stop)
- [How to contribute?](#how-to-contribute)
Expand Down Expand Up @@ -202,6 +204,42 @@ steps:
}
```

## Querying JSON with jq
```yaml
steps:
- name: run query
executor: jq
command: '{(.id): .["10"].b}'
script: |
{"id": "sample", "10": {"b": 42}}
```

log output:
```json
{
"sample": 42
}
```

## Format JSON with jq
```yaml
steps:
- name: format json
executor: jq
script: |
{"id": "sample", "10": {"b": 42}}
```

log output:
```json
{
"10": {
"b": 42
},
"id": "sample"
}
```

## Sending Email Notification

Email example
Expand Down
10 changes: 10 additions & 0 deletions examples/jq.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
steps:
- name: run query
executor: jq
command: '{(.id): .["10"].b}'
script: |
{"id": "sample", "10": {"b": 42}}
- name: format json
executor: jq
script: |
{"id": "sample", "10": {"b": 42}}
9 changes: 6 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ require (
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/itchyny/gojq v0.12.12 // indirect
github.com/itchyny/timefmt-go v0.1.5 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
Expand All @@ -38,13 +41,13 @@ require (
require (
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.3.4 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/samber/lo v1.27.0
golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b
golang.org/x/net v0.0.0-20220812174116-3211cb980234 // indirect
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab
golang.org/x/sys v0.5.0
gopkg.in/yaml.v3 v3.0.1 // indirect
)
12 changes: 12 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,21 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
github.com/itchyny/gojq v0.12.12 h1:x+xGI9BXqKoJQZkr95ibpe3cdrTbY8D9lonrK433rcA=
github.com/itchyny/gojq v0.12.12/go.mod h1:j+3sVkjxwd7A7Z5jrbKibgOLn0ZfLWkV+Awxr/pyzJE=
github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE=
github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
github.com/jedib0t/go-pretty/v6 v6.3.6 h1:A6w2BuyPMtf7M82BGRBys9bAba2C26ZX9lrlrZ7uH6U=
github.com/jedib0t/go-pretty/v6 v6.3.6/go.mod h1:MgmISkTWDSFu0xOqiZ0mKNntMQ2mDgOcwOkwBEkMDJI=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
Expand All @@ -50,6 +58,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.3.4 h1:3Z3Eu6FGHZWSfNKJTOUiPatWwfc7DzJRU04jFUqJODw=
github.com/rivo/uniseg v0.3.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
Expand Down Expand Up @@ -102,6 +112,8 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
Expand Down
2 changes: 1 addition & 1 deletion internal/dag/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -564,7 +564,7 @@ func assertStepDef(def *stepDef) error {
if def.Name == "" {
return fmt.Errorf("step name must be specified")
}
if def.Command == "" {
if def.Command == "" && def.Executor != "jq" {
return fmt.Errorf("step command must be specified")
}
return nil
Expand Down
73 changes: 73 additions & 0 deletions internal/executor/jq.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package executor

import (
"context"
"encoding/json"
"fmt"
"io"
"os"

"github.com/itchyny/gojq"
"github.com/yohamta/dagu/internal/dag"
)

type JqExecutor struct {
stdout io.Writer
stderr io.Writer
query string
input map[string]interface{}
}

func (e *JqExecutor) SetStdout(out io.Writer) {
e.stdout = out
}

func (e *JqExecutor) SetStderr(out io.Writer) {
e.stderr = out
}

func (e *JqExecutor) Kill(sig os.Signal) error {
return nil
}

func (e *JqExecutor) Run() error {
query, err := gojq.Parse(e.query)
if err != nil {
return err
}
iter := query.Run(e.input)
for {
v, ok := iter.Next()
if !ok {
break
}
if err, ok := v.(error); ok {
e.stderr.Write([]byte(fmt.Sprintf("%#v", err)))
continue
}
val, err := json.MarshalIndent(v, "", " ")
if err != nil {
e.stderr.Write([]byte(fmt.Sprintf("%#v", err)))
continue
}
e.stdout.Write(val)
}
return nil
}

func CreateJqExecutor(ctx context.Context, step *dag.Step) (Executor, error) {
s := os.ExpandEnv(step.Script)
input := map[string]interface{}{}
if err := json.Unmarshal([]byte(s), &input); err != nil {
return nil, err
}
return &JqExecutor{
stdout: os.Stdout,
input: input,
query: step.CmdWithArgs,
}, nil
}

func init() {
Register("jq", CreateJqExecutor)
}

0 comments on commit bd23df7

Please sign in to comment.