TODO Apply "symflower fix" to a "write-test" result of a model when i…

…t errors, so model responses can possibly be fixed

Part of #213

evaluate/task/symflower-fix.go

@@ -0,0 +1,50 @@
+package task
+
+import (
+	"context"
+	"time"
+
+	pkgerrors "github.com/pkg/errors"
+	"github.com/symflower/eval-dev-quality/evaluate/metrics"
+	"github.com/symflower/eval-dev-quality/language"
+	"github.com/symflower/eval-dev-quality/log"
+	"github.com/symflower/eval-dev-quality/tools"
+	"github.com/symflower/eval-dev-quality/util"
+)
+
+// symflowerFix runs the "symflower fix" command and returns the corresponding assessments.
+func symflowerFix(logger *log.Logger, modelAssessment metrics.Assessments, repositoryPath string, language language.Language) (assessments metrics.Assessments, problems []error, err error) {
+	assessments = metrics.NewAssessments()
+
+	start := time.Now()
+	output, err := util.CommandWithResult(context.Background(), logger, &util.Command{
+		Command: []string{
+			tools.SymflowerPath, "fix",
+			"--language", language.ID(),
+			"--workspace", repositoryPath,
+		},
+
+		Directory: repositoryPath,
+	})
+	if err != nil {
+		return nil, nil, pkgerrors.WithStack(err)
+	}
+	duration := time.Since(start)
+	logger.Printf("with symflower repair: %s", output)
+
+	// The processing time is summed with the one from the model.
+	assessments[metrics.AssessmentKeyProcessingTime] = modelAssessment[metrics.AssessmentKeyProcessingTime] + uint64(duration.Milliseconds())
+
+	coverage, ps, err := language.Execute(logger, repositoryPath)
+	problems = append(problems, ps...)
+	if err != nil {
+		return nil, ps, pkgerrors.WithStack(err)
+	}
+	logger.Printf("with symflower repair: Executes tests with %d coverage objects", coverage)
+
+	assessments.Award(metrics.AssessmentKeyFilesExecuted)
+	assessments.Award(metrics.AssessmentKeyResponseNoError)
+	assessments.AwardPoints(metrics.AssessmentKeyCoverage, coverage)
+
+	return assessments, problems, err
+}

evaluate/task/task-write-test.go

@@ -63,6 +63,7 @@ func (t *TaskWriteTests) Run(repository evaltask.Repository) (repositoryAssessme
 	}
 
 	modelAssessment := metrics.NewAssessments()
+	symflowerAssessment := metrics.NewAssessments()
 	for _, filePath := range filePaths {
 		if err := repository.Reset(t.Logger); err != nil {
 			t.Logger.Panicf("ERROR: unable to reset temporary repository path: %s", err)
@@ -93,7 +94,18 @@ func (t *TaskWriteTests) Run(repository evaltask.Repository) (repositoryAssessme
 		if err != nil {
 			problems = append(problems, pkgerrors.WithMessage(err, filePath))
 
-			continue
+			// Run "symflower fix"  if the model response fails to execute.
+			if t.Language.ID() == "golang" { // Currently we only support Go for "symflower fix".
+				assessments, ps, err := symflowerFix(log, modelAssessment, dataPath, t.Language)
+				problems = append(problems, ps...)
+				if err != nil {
+					problems = append(problems, err)
+
+					continue
+				}
+
+				symflowerAssessment.Add(assessments)
+			}
 		}
 		log.Printf("Executes tests with %d coverage objects", coverage)
 		modelAssessment.Award(metrics.AssessmentKeyFilesExecuted)
@@ -103,6 +115,9 @@ func (t *TaskWriteTests) Run(repository evaltask.Repository) (repositoryAssessme
 	repositoryAssessment = map[evaltask.Identifier]metrics.Assessments{
 		IdentifierWriteTests: modelAssessment,
 	}
+	if len(symflowerAssessment) > 0 {
+		repositoryAssessment[IdentifierWriteTestsSymflowerFix] = symflowerAssessment
+	}
 
 	return repositoryAssessment, problems, nil
 }

evaluate/task/task-write-test_test.go

@@ -14,6 +14,8 @@ import (
 	"github.com/symflower/eval-dev-quality/log"
 	modeltesting "github.com/symflower/eval-dev-quality/model/testing"
 	"github.com/symflower/eval-dev-quality/task"
+	"github.com/zimmski/osutil"
+	"github.com/zimmski/osutil/bytesutil"
 )
 
 func TestTaskWriteTestsRun(t *testing.T) {
@@ -77,4 +79,73 @@ func TestTaskWriteTestsRun(t *testing.T) {
 			},
 		})
 	})
+
+	t.Run("Symflower Fix", func(t *testing.T) {
+		t.Run("Go", func(t *testing.T) {
+			validateGo := func(t *testing.T, testName string, testFileContent string, expectedAssessments map[task.Identifier]metrics.Assessments) {
+				temporaryDirectoryPath := t.TempDir()
+				repositoryPath := filepath.Join(temporaryDirectoryPath, "golang", "plain")
+				require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "golang", "plain"), repositoryPath))
+
+				modelMock := modeltesting.NewMockModelNamed(t, "mocked-model")
+				modelMock.RegisterGenerateSuccess(t, IdentifierWriteTests, "plain_test.go", testFileContent, metricstesting.AssessmentsWithProcessingTime).Once()
+
+				validate(t, &tasktesting.TestCaseTask{
+					Name: testName,
+
+					Model:          modelMock,
+					Language:       &golang.Language{},
+					TestDataPath:   temporaryDirectoryPath,
+					RepositoryPath: filepath.Join("golang", "plain"),
+
+					ExpectedRepositoryAssessment: expectedAssessments,
+					ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
+						filepath.Join(string(IdentifierWriteTests), "mocked-model", "golang", "golang", "plain.log"): func(t *testing.T, filePath, data string) {
+							assert.Contains(t, data, "Evaluating model \"mocked-model\"")
+							assert.Contains(t, data, "PASS: TestPlain")
+						},
+					},
+				})
+			}
+			{
+				expectedAssessments := map[task.Identifier]metrics.Assessments{
+					IdentifierWriteTests: metrics.Assessments{
+						metrics.AssessmentKeyFilesExecuted:   1,
+						metrics.AssessmentKeyResponseNoError: 1,
+						metrics.AssessmentKeyCoverage:        10,
+					},
+				}
+				validateGo(t, "Model generated correct test", bytesutil.StringTrimIndentations(`
+					package plain
+
+					import "testing"
+
+					func TestPlain(t *testing.T) {
+						   plain()
+					}
+				`), expectedAssessments)
+			}
+			{
+				expectedAssessments := map[task.Identifier]metrics.Assessments{
+					IdentifierWriteTests: metrics.Assessments{
+						metrics.AssessmentKeyFilesExecuted:   0, // TODO update the assessments.
+						metrics.AssessmentKeyResponseNoError: 1,
+						metrics.AssessmentKeyCoverage:        0, // TODO update the assessments.
+					},
+				}
+				validateGo(t, "Model generated test with unused import", bytesutil.StringTrimIndentations(`
+					package plain
+
+					import (
+						"testing"
+						"strings"
+					)
+
+					func TestPlain(t *testing.T) {
+					   	plain()
+					}
+				`), expectedAssessments)
+			}
+		})
+	})
 }

evaluate/task/task.go

@@ -33,6 +33,8 @@ func registerIdentifier(name string) (identifier evaltask.Identifier) {
 var (
 	// IdentifierWriteTests holds the identifier for the "write test" task.
 	IdentifierWriteTests = registerIdentifier("write-tests")
+	// IdentifierWriteTestsSymflowerFix holds the identifier for the "write test" task with the "symflower fix" applied.
+	IdentifierWriteTestsSymflowerFix = registerIdentifier("write-tests-symflower-fix")
 	// IdentifierCodeRepair holds the identifier for the "code repair" task.
 	IdentifierCodeRepair = registerIdentifier("code-repair")
 )