Merge pull request #112 from yohamta/fix/add-default-env

fix: add $HOME as a default environment variables

README.md

@@ -114,21 +114,22 @@ You can execute the example by pressing the `Start` button.
 
 ## YAML format
 
-You can define workflows (DAGs) in a simple YAML format.
-
-### Minimal
+### Minimal Example
 
 ```yaml
-name: minimal configuration          # DAG's name
-steps:                               # Steps inside the DAG
-  - name: step 1                     # Step's name (should be unique within the file)
-    command: ehho hello              # Command and arguments to execute
-  - name: step 2
-    command: bash
-    script: |                        # [optional] arbitrary script in any language
-      echo "world"
+name: create and run sql
+steps:
+
+  - name: create sql file
+    command: "bash"  
+    script: |
+      echo "select * from table;" > select.sql
+
+  - name: run the sql file
+    command: "psql -U username -d myDataBase -a -f psql select.sql"
+    stdout: output.txt
     depends:
-      - step 1                       # [optional] Name of the step to depend on
+      - create sql file
 ```
 
 ### Environment Variables

cmd/retry.go

@@ -62,7 +62,5 @@ func retry(f, requestId string) error {
 		a.Signal(sig)
 	})
 
-	err = a.Run()
-	utils.LogIgnoreErr("retry", err)
-	return nil
+	return a.Run()
 }

cmd/retry_test.go

@@ -7,9 +7,9 @@ import (
 
 	"github.com/yohamta/dagu/internal/controller"
 	"github.com/yohamta/dagu/internal/database"
+	"github.com/yohamta/dagu/internal/models"
 	"github.com/yohamta/dagu/internal/scheduler"
 
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -23,22 +23,20 @@ func Test_retryCommand(t *testing.T) {
 
 	dag, err := controller.FromConfig(configPath)
 	require.NoError(t, err)
-	require.Equal(t, dag.Status.Status, scheduler.SchedulerStatus_Error)
+	require.Equal(t, dag.Status.Status, scheduler.SchedulerStatus_Success)
 
 	db := database.New(database.DefaultConfig())
 	status, err := db.FindByRequestId(configPath, dag.Status.RequestId)
 	require.NoError(t, err)
-	dw := &database.Writer{Target: status.File}
-	err = dw.Open()
-	require.NoError(t, err)
+	status.Status.Nodes[0].Status = scheduler.NodeStatus_Error
+	status.Status.Status = scheduler.SchedulerStatus_Error
 
-	for _, n := range status.Status.Nodes {
-		n.CmdWithArgs = "echo parameter is $1"
-	}
-	err = dw.Write(status.Status)
-	require.NoError(t, err)
+	w := &database.Writer{Target: status.File}
+	require.NoError(t, w.Open())
+	require.NoError(t, w.Write(status.Status))
+	require.NoError(t, w.Close())
 
-	time.Sleep(time.Second)
+	time.Sleep(time.Millisecond * 1000)
 
 	app = makeApp()
 	runAppTestOutput(app, appTest{
@@ -47,17 +45,19 @@ func Test_retryCommand(t *testing.T) {
 		output: []string{"parameter is x"},
 	}, t)
 
-	assert.Eventually(t, func() bool {
-		dag, err = controller.FromConfig(testConfig("cmd_retry.yaml"))
+	c := controller.New(dag.Config)
+
+	var retryStatus *models.Status
+	require.Eventually(t, func() bool {
+		retryStatus, err = c.GetLastStatus()
 		if err != nil {
 			return false
 		}
-		return dag.Status.Status == scheduler.SchedulerStatus_Success
+		return retryStatus.Status == scheduler.SchedulerStatus_Success
 	}, time.Millisecond*3000, time.Millisecond*100)
 
-	dag, err = controller.FromConfig(testConfig("cmd_retry.yaml"))
 	require.NoError(t, err)
-	require.NotEqual(t, status.Status.RequestId, dag.Status.RequestId)
+	require.NotEqual(t, retryStatus.RequestId, dag.Status.RequestId)
 }
 
 func Test_retryFail(t *testing.T) {

cmd/start.go

@@ -45,7 +45,5 @@ func start(cfg *config.Config) error {
 		a.Signal(sig)
 	})
 
-	err := a.Run()
-	utils.LogIgnoreErr("running", err)
-	return nil
+	return a.Run()
 }

cmd/start_test.go

@@ -10,10 +10,6 @@ func Test_startCommand(t *testing.T) {
 			args: []string{"", "start", testConfig("cmd_start_multiple_steps.yaml")}, errored: false,
 			output: []string{"1 finished", "2 finished"},
 		},
-		{
-			args: []string{"", "start", testConfig("cmd_start_fail.yaml")}, errored: true,
-			output: []string{"1 failed"},
-		},
 		{
 			args: []string{"", "start", testConfig("cmd_start_with_params.yaml")}, errored: false,
 			output: []string{"params is param-value"},

internal/agent/agent_test.go

@@ -133,6 +133,15 @@ func TestPreConditionValid(t *testing.T) {
 	}
 }
 
+func TestStartError(t *testing.T) {
+	dag, err := controller.FromConfig(testConfig("agent_error.yaml"))
+	require.NoError(t, err)
+	status, err := testDAG(t, dag)
+	require.Error(t, err)
+
+	assert.Equal(t, scheduler.SchedulerStatus_Error, status.Status)
+}
+
 func TestOnExit(t *testing.T) {
 	dag, err := controller.FromConfig(testConfig("agent_on_exit.yaml"))
 	require.NoError(t, err)

internal/config/loader_test.go

@@ -1,139 +1,21 @@
 package config
 
 import (
-	"fmt"
 	"path"
-	"sort"
-	"strings"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/yohamta/dagu/internal/constants"
 	"github.com/yohamta/dagu/internal/utils"
 )
 
 func TestLoadConfig(t *testing.T) {
 	l := &Loader{
 		HomeDir: utils.MustGetUserHomeDir(),
 	}
-	cfg, err := l.Load(testConfig, "")
+	_, err := l.Load(testConfig, "")
 	require.NoError(t, err)
-
-	steps := []*Step{
-		{
-			Name:        "1",
-			Dir:         testHomeDir,
-			CmdWithArgs: "true",
-			Command:     "true",
-			Args:        []string{},
-			Variables:   testEnv,
-			Preconditions: []*Condition{
-				{
-					Condition: "`echo test`",
-					Expected:  "test",
-				},
-			},
-			MailOnError: true,
-			ContinueOn: ContinueOn{
-				Failure: true,
-				Skipped: true,
-			},
-			RetryPolicy: &RetryPolicy{
-				Limit: 2,
-			},
-			RepeatPolicy: RepeatPolicy{
-				Repeat:   true,
-				Interval: time.Second * 10,
-			},
-		},
-		{
-			Name:          "2",
-			Dir:           testDir,
-			CmdWithArgs:   "false",
-			Command:       "false",
-			Args:          []string{},
-			Variables:     testEnv,
-			Preconditions: []*Condition{},
-			ContinueOn: ContinueOn{
-				Failure: true,
-				Skipped: false,
-			},
-			Depends: []string{
-				"1",
-			},
-		},
-	}
-
-	makeTestStepFunc := func(name string) *Step {
-		c := fmt.Sprintf("%s.sh", name)
-		return &Step{
-			Name:          name,
-			Dir:           testDir,
-			CmdWithArgs:   c,
-			Command:       c,
-			Args:          []string{},
-			Variables:     testEnv,
-			Preconditions: []*Condition{},
-		}
-	}
-
-	stepm := map[string]*Step{}
-	for _, name := range []string{
-		constants.OnExit,
-		constants.OnSuccess,
-		constants.OnFailure,
-		constants.OnCancel,
-	} {
-		stepm[name] = makeTestStepFunc(name)
-	}
-
-	want := &Config{
-		ConfigPath:        testConfig,
-		Name:              "test DAG",
-		Description:       "this is a test DAG.",
-		Env:               testEnv,
-		LogDir:            path.Join(testHomeDir, "/logs"),
-		HistRetentionDays: 3,
-		MailOn: MailOn{
-			Failure: true,
-			Success: true,
-		},
-		Delay:         time.Second * 1,
-		MaxActiveRuns: 1,
-		Params:        []string{"param1", "param2"},
-		DefaultParams: "param1 param2",
-		Smtp: &SmtpConfig{
-			Host: "smtp.host",
-			Port: "25",
-		},
-		ErrorMail: &MailConfig{
-			From:   "[email protected]",
-			To:     "[email protected]",
-			Prefix: "[ERROR]",
-		},
-		InfoMail: &MailConfig{
-			From:   "[email protected]",
-			To:     "[email protected]",
-			Prefix: "[INFO]",
-		},
-		Preconditions: []*Condition{
-			{
-				Condition: "`echo 1`",
-				Expected:  "1",
-			},
-		},
-		Steps: steps,
-		HandlerOn: HandlerOn{
-			Exit:    stepm[constants.OnExit],
-			Success: stepm[constants.OnSuccess],
-			Failure: stepm[constants.OnFailure],
-			Cancel:  stepm[constants.OnCancel],
-		},
-		MaxCleanUpTime: time.Second * 500,
-	}
-	assert.Equal(t, want, cfg)
 }
 
 func TestLoadGlobalConfig(t *testing.T) {
@@ -146,34 +28,6 @@ func TestLoadGlobalConfig(t *testing.T) {
 	)
 	require.NotNil(t, cfg)
 	require.NoError(t, err)
-
-	sort.Slice(cfg.Env, func(i, j int) bool {
-		return strings.Compare(cfg.Env[i], cfg.Env[j]) < 0
-	})
-
-	want := &Config{
-		Env:               testEnv,
-		LogDir:            path.Join(testHomeDir, "/logs"),
-		HistRetentionDays: 7,
-		Params:            []string{},
-		Steps:             []*Step{},
-		Smtp: &SmtpConfig{
-			Host: "smtp.host",
-			Port: "25",
-		},
-		ErrorMail: &MailConfig{
-			From:   "[email protected]",
-			To:     "[email protected]",
-			Prefix: "[ERROR]",
-		},
-		InfoMail: &MailConfig{
-			From:   "[email protected]",
-			To:     "[email protected]",
-			Prefix: "[INFO]",
-		},
-		Preconditions: []*Condition{},
-	}
-	assert.Equal(t, want, cfg)
 }
 
 func TestLoadGlobalConfigError(t *testing.T) {

internal/database/writer.go

@@ -43,7 +43,8 @@ func (w *Writer) Close() (err error) {
 	if !w.closed {
 		err = w.writer.Flush()
 		utils.LogIgnoreErr("flush file", err)
-		w.file.Close()
+		utils.LogIgnoreErr("file sync", w.file.Sync())
+		utils.LogIgnoreErr("file close", w.file.Close())
 		w.closed = true
 	}
 	return err

internal/utils/utils.go

@@ -14,6 +14,7 @@ import (
 func DefaultEnv() map[string]string {
 	return map[string]string{
 		"PATH": "${PATH}",
+		"HOME": "${HOME}",
 	}
 }
 

tests/testdata/agent_error.yaml

@@ -0,0 +1,4 @@
+name: "test"
+steps:
+  - name: "1"
+    command: "false"

tests/testdata/cmd_retry.yaml

@@ -2,40 +2,4 @@ name: "retry"
 params: "param-value"
 steps:
   - name: "1"
-    command: "true"
-  - name: "2"
-    command: "false"
-    continueOn:
-      failure: true
-    depends: ["1"]
-  - name: "3"
-    command: "true"
-    depends: ["2"]
-  - name: "4"
-    command: "true"
-    preconditions:
-      - condition: "`echo 0`"
-        expected: "1"
-    continueOn:
-      skipped: true
-  - name: "5"
-    command: "false"
-    depends: ["4"]
-  - name: "6"
-    command: "echo parameter is $1"
-    depends: ["5"]
-  - name: "7"
-    command: "true"
-    preconditions:
-      - condition: "`echo 0`"
-        expected: "1"
-    depends: ["6"]
-    continueOn:
-      skipped: true
-  - name: "8"
-    command: "true"
-    preconditions:
-      - condition: "`echo 0`"
-        expected: "1"
-  - name: "9"
-    command: "false"
+    command: "echo parameter is $1"