Merge pull request #107 from dagu-go/feat/output

feat: output to environment variable

README.md

@@ -31,6 +31,7 @@
     - [Conditional Logic](#conditional-logic)
     - [State Handlers](#state-handlers)
     - [Redirection](#redirection)
+    - [Output](#output)
     - [Repeating Task](#repeating-task)
     - [All Available Fields](#all-available-fields)
   - [Admin Configuration](#admin-configuration)
@@ -201,7 +202,7 @@ steps:
 
 ### Redirection
 
-Sometimes you want to redirect standard out to a file to use in subsequent tasks. You can use `stdout` field to do so.
+`stdout` field can be used to write standard output to a file.
 
 ```yaml
 name: example
@@ -211,6 +212,18 @@ steps:
     stdout: "/tmp/hello" # the content will be "hello\n"
 ```
 
+### Output
+
+`output` field can be used to write standard output to a environment variable. Leading and trailing space will be trimmed automatically.
+
+```yaml
+name: example
+steps:
+  - name: step 1
+    command: "echo foo"
+    output: FOO # will contain "foo"
+```
+
 ### State Handlers
 
 It is often desirable to take action when a specific event happens, for example, when a workflow fails. To achieve this, you can use `handlerOn` fields.

cmd/retry_test.go

@@ -33,7 +33,7 @@ func Test_retryCommand(t *testing.T) {
 	require.NoError(t, err)
 
 	for _, n := range status.Status.Nodes {
-		n.Command = "true"
+		n.CmdWithArgs = "echo parameter is $1"
 	}
 	err = dw.Write(status.Status)
 	require.NoError(t, err)

internal/agent/agent_test.go

@@ -156,7 +156,7 @@ func TestRetry(t *testing.T) {
 	assert.Equal(t, scheduler.SchedulerStatus_Error, status.Status)
 
 	for _, n := range status.Nodes {
-		n.Command = "true"
+		n.CmdWithArgs = "true"
 	}
 	a := &Agent{
 		Config: &Config{

internal/config/config.go

@@ -371,8 +371,10 @@ func buildStep(variables []string, def *stepDef) (*Step, error) {
 	step := &Step{}
 	step.Name = def.Name
 	step.Description = def.Description
-	step.Command, step.Args = utils.SplitCommand(def.Command)
+	step.CmdWithArgs = def.Command
+	step.Command, step.Args = utils.SplitCommand(step.CmdWithArgs) // Will be eva
 	step.Stdout = os.ExpandEnv(def.Stdout)
+	step.Output = def.Output
 	step.Dir = os.ExpandEnv(def.Dir)
 	step.Variables = variables
 	step.Depends = def.Depends

internal/config/definition.go

@@ -37,6 +37,7 @@ type stepDef struct {
 	Dir           string
 	Command       string
 	Stdout        string
+	Output        string
 	Depends       []string
 	ContinueOn    *continueOnDef
 	RetryPolicy   *retryPolicyDef

internal/config/loader_test.go

@@ -23,11 +23,12 @@ func TestLoadConfig(t *testing.T) {
 
 	steps := []*Step{
 		{
-			Name:      "1",
-			Dir:       testHomeDir,
-			Command:   "true",
-			Args:      []string{},
-			Variables: testEnv,
+			Name:        "1",
+			Dir:         testHomeDir,
+			CmdWithArgs: "true",
+			Command:     "true",
+			Args:        []string{},
+			Variables:   testEnv,
 			Preconditions: []*Condition{
 				{
 					Condition: "`echo test`",
@@ -50,6 +51,7 @@ func TestLoadConfig(t *testing.T) {
 		{
 			Name:          "2",
 			Dir:           testDir,
+			CmdWithArgs:   "false",
 			Command:       "false",
 			Args:          []string{},
 			Variables:     testEnv,
@@ -65,10 +67,12 @@ func TestLoadConfig(t *testing.T) {
 	}
 
 	makeTestStepFunc := func(name string) *Step {
+		c := fmt.Sprintf("%s.sh", name)
 		return &Step{
 			Name:          name,
 			Dir:           testDir,
-			Command:       fmt.Sprintf("%s.sh", name),
+			CmdWithArgs:   c,
+			Command:       c,
 			Args:          []string{},
 			Variables:     testEnv,
 			Preconditions: []*Condition{},

internal/config/step.go

@@ -11,8 +11,10 @@ type Step struct {
 	Description   string
 	Variables     []string
 	Dir           string
+	CmdWithArgs   string
 	Command       string
 	Stdout        string
+	Output        string
 	Args          []string
 	Depends       []string
 	ContinueOn    ContinueOn

internal/scheduler/node.go

@@ -2,13 +2,15 @@ package scheduler
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	"fmt"
 	"io"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
@@ -58,6 +60,8 @@ type Node struct {
 	logWriter    *bufio.Writer
 	stdoutFile   *os.File
 	stdoutWriter *bufio.Writer
+	outputWriter *os.File
+	outputReader *os.File
 }
 
 type NodeState struct {
@@ -73,6 +77,9 @@ type NodeState struct {
 func (n *Node) Execute() error {
 	ctx, fn := context.WithCancel(context.Background())
 	n.cancelFunc = fn
+	if n.CmdWithArgs != "" {
+		n.Command, n.Args = utils.SplitCommand(os.ExpandEnv(n.CmdWithArgs))
+	}
 	n.cmd = exec.CommandContext(ctx, n.Command, n.Args...)
 	cmd := n.cmd
 	cmd.Dir = n.Dir
@@ -94,7 +101,24 @@ func (n *Node) Execute() error {
 		cmd.Stdout = io.MultiWriter(n.logWriter, n.stdoutWriter)
 	}
 
+	if n.Output != "" {
+		var err error
+		if n.outputReader, n.outputWriter, err = os.Pipe(); err != nil {
+			return err
+		}
+		cmd.Stdout = io.MultiWriter(cmd.Stdout, n.outputWriter)
+	}
+
 	n.Error = cmd.Run()
+
+	if n.outputReader != nil && n.Output != "" {
+		utils.LogIgnoreErr("close pipe writer", n.outputWriter.Close())
+		var buf bytes.Buffer
+		_, _ = io.Copy(&buf, n.outputReader)
+		ret := buf.String()
+		os.Setenv(n.Output, strings.TrimSpace(ret))
+	}
+
 	return n.Error
 }
 

internal/scheduler/node_test.go

@@ -50,7 +50,7 @@ func TestSignal(t *testing.T) {
 	require.Equal(t, n.Status, NodeStatus_Cancel)
 }
 
-func TestOutputLogAndStdout(t *testing.T) {
+func TestLogAndStdout(t *testing.T) {
 	n := &Node{
 		Step: &config.Step{
 			Command: "echo",
@@ -77,3 +77,30 @@ func TestOutputLogAndStdout(t *testing.T) {
 	dat, _ = os.ReadFile(n.logFile.Name())
 	require.Equal(t, "done\n", string(dat))
 }
+
+func TestOutput(t *testing.T) {
+	n := &Node{
+		Step: &config.Step{
+			Command: "echo",
+			Args:    []string{"hello"},
+			Dir:     os.Getenv("HOME"),
+			Output:  "OUTPUT_TEST",
+		},
+	}
+	err := n.setup(os.Getenv("HOME"), "test-request-id-output")
+	require.NoError(t, err)
+	defer func() {
+		_ = n.teardown()
+	}()
+
+	err = n.Execute()
+	require.NoError(t, err)
+	err = n.teardown()
+	require.NoError(t, err)
+
+	dat, _ := os.ReadFile(n.logFile.Name())
+	require.Equal(t, "hello\n", string(dat))
+
+	val := os.Getenv("OUTPUT_TEST")
+	require.Equal(t, "hello", val)
+}

internal/utils/utils.go

@@ -55,7 +55,7 @@ func FormatDuration(t time.Duration, defaultVal string) string {
 }
 
 func SplitCommand(cmd string) (program string, args []string) {
-	vals := strings.SplitN(os.ExpandEnv(cmd), " ", 2)
+	vals := strings.SplitN(cmd, " ", 2)
 	if len(vals) > 1 {
 		return vals[0], strings.Split(vals[1], " ")
 	}

tests/testdata/cmd_retry.yaml

@@ -1,4 +1,4 @@
-name: "agent retry"
+name: "retry"
 params: "param-value"
 steps:
   - name: "1"