TODO Apply "symflower repair" to a "write-test" result of a model whe…

…n it errors, so model responses can possibly be fixed

Part of #213

evaluate/task/symflower-repair.go

@@ -0,0 +1,49 @@
+package task
+
+import (
+	"context"
+	"time"
+
+	pkgerrors "github.com/pkg/errors"
+	"github.com/symflower/eval-dev-quality/evaluate/metrics"
+	"github.com/symflower/eval-dev-quality/language"
+	"github.com/symflower/eval-dev-quality/log"
+	"github.com/symflower/eval-dev-quality/tools"
+	"github.com/symflower/eval-dev-quality/util"
+)
+
+// symflowerRepair runs the "symflower repair" command.
+func symflowerRepair(logger *log.Logger, modelAssessment metrics.Assessments, repositoryPath string, language language.Language) (assessments metrics.Assessments, problems []error, err error) {
+	assessments = metrics.NewAssessments()
+
+	start := time.Now()
+	if _, err = util.CommandWithResult(context.Background(), logger, &util.Command{
+		Command: []string{
+			tools.SymflowerPath, "repair",
+			"--language", language.ID(),
+			"--workspace", repositoryPath,
+		},
+
+		Directory: repositoryPath,
+	}); err != nil {
+		return nil, nil, pkgerrors.WithStack(err)
+	}
+	duration := time.Since(start)
+
+	assessments[metrics.AssessmentKeyProcessingTime] = uint64(duration.Milliseconds())
+	assessments.Add(modelAssessment)
+
+	coverage, ps, err := language.Execute(logger, repositoryPath)
+	problems = append(problems, ps...)
+	if err != nil {
+		problems = append(problems, pkgerrors.WithStack(err))
+
+		return assessments, ps, err
+	}
+
+	logger.Printf("Executes tests with %d coverage objects", coverage)
+	assessments.Award(metrics.AssessmentKeyFilesExecuted)
+	assessments.AwardPoints(metrics.AssessmentKeyCoverage, coverage)
+
+	return assessments, problems, err
+}

evaluate/task/task-write-test.go

@@ -62,6 +62,7 @@ func (t *TaskWriteTests) Run(repository evaltask.Repository) (repositoryAssessme
 		return nil, problems, pkgerrors.WithStack(err)
 	}
 
+	repositoryAssessment = map[evaltask.Identifier]metrics.Assessments{}
 	modelAssessment := metrics.NewAssessments()
 	for _, filePath := range filePaths {
 		if err := repository.Reset(t.Logger); err != nil {
@@ -92,17 +93,21 @@ func (t *TaskWriteTests) Run(repository evaltask.Repository) (repositoryAssessme
 		problems = append(problems, ps...)
 		if err != nil {
 			problems = append(problems, pkgerrors.WithMessage(err, filePath))
+			symflowerAssessment, ps, err := symflowerRepair(log, modelAssessment, dataPath, t.Language)
+			if err != nil {
+				problems = append(problems, ps...)
 
-			continue
+				continue
+			}
+
+			repositoryAssessment[IdentifierWriteTestsSymflowerRepair] = symflowerAssessment
 		}
 		log.Printf("Executes tests with %d coverage objects", coverage)
 		modelAssessment.Award(metrics.AssessmentKeyFilesExecuted)
 		modelAssessment.AwardPoints(metrics.AssessmentKeyCoverage, coverage)
 	}
 
-	repositoryAssessment = map[evaltask.Identifier]metrics.Assessments{
-		IdentifierWriteTests: modelAssessment,
-	}
+	repositoryAssessment[IdentifierWriteTests] = modelAssessment
 
 	return repositoryAssessment, problems, nil
 }

evaluate/task/task-write-test_test.go

@@ -14,6 +14,8 @@ import (
 	"github.com/symflower/eval-dev-quality/log"
 	modeltesting "github.com/symflower/eval-dev-quality/model/testing"
 	"github.com/symflower/eval-dev-quality/task"
+	"github.com/zimmski/osutil"
+	"github.com/zimmski/osutil/bytesutil"
 )
 
 func TestTaskWriteTestsRun(t *testing.T) {
@@ -77,4 +79,73 @@ func TestTaskWriteTestsRun(t *testing.T) {
 			},
 		})
 	})
+
+	t.Run("Symflower Repair", func(t *testing.T) {
+		t.Run("Go", func(t *testing.T) {
+			validateGo := func(t *testing.T, testName string, testFileContent string, expectedAssessments map[task.Identifier]metrics.Assessments) {
+				temporaryDirectoryPath := t.TempDir()
+				repositoryPath := filepath.Join(temporaryDirectoryPath, "golang", "plain")
+				require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "golang", "plain"), repositoryPath))
+
+				modelMock := modeltesting.NewMockModelNamed(t, "mocked-model")
+				modelMock.RegisterGenerateSuccess(t, IdentifierWriteTests, "plain_test.go", testFileContent, metricstesting.AssessmentsWithProcessingTime).Once()
+
+				validate(t, &tasktesting.TestCaseTask{
+					Name: testName,
+
+					Model:          modelMock,
+					Language:       &golang.Language{},
+					TestDataPath:   temporaryDirectoryPath,
+					RepositoryPath: filepath.Join("golang", "plain"),
+
+					ExpectedRepositoryAssessment: expectedAssessments,
+					ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
+						filepath.Join(string(IdentifierWriteTests), "mocked-model", "golang", "golang", "plain.log"): func(t *testing.T, filePath, data string) {
+							assert.Contains(t, data, "Evaluating model \"mocked-model\"")
+							assert.Contains(t, data, "PASS: TestTaskB")
+						},
+					},
+				})
+			}
+			{
+				expectedAssessments := map[task.Identifier]metrics.Assessments{
+					IdentifierWriteTests: metrics.Assessments{
+						metrics.AssessmentKeyFilesExecuted:   1,
+						metrics.AssessmentKeyResponseNoError: 1,
+						metrics.AssessmentKeyCoverage:        10,
+					},
+				}
+				validateGo(t, "Model generated correct test", bytesutil.StringTrimIndentations(`
+					package plain
+
+					import "testing"
+
+					func TestPlain(t *testing.T) {
+						   plain()
+					}
+				`), expectedAssessments)
+			}
+			{
+				expectedAssessments := map[task.Identifier]metrics.Assessments{
+					IdentifierWriteTests: metrics.Assessments{
+						metrics.AssessmentKeyFilesExecuted:   0, // TODO update the assessments.
+						metrics.AssessmentKeyResponseNoError: 1,
+						metrics.AssessmentKeyCoverage:        0, // TODO update the assessments.
+					},
+				}
+				validateGo(t, "Model generated test with unused import", bytesutil.StringTrimIndentations(`
+					package plain
+
+					import (
+						"testing"
+						"strings"
+					)
+
+					func TestPlain(t *testing.T) {
+					   	plain()
+					}
+				`), expectedAssessments)
+			}
+		})
+	})
 }

evaluate/task/task.go

@@ -33,6 +33,8 @@ func registerIdentifier(name string) (identifier evaltask.Identifier) {
 var (
 	// IdentifierWriteTests holds the identifier for the "write test" task.
 	IdentifierWriteTests = registerIdentifier("write-tests")
+	// IdentifierWriteTestsSymflowerRepair holds the identifier for the "write test" task with the "symflower repair" applied.
+	IdentifierWriteTestsSymflowerRepair = registerIdentifier("write-tests-symflower-repair")
 	// IdentifierCodeRepair holds the identifier for the "code repair" task.
 	IdentifierCodeRepair = registerIdentifier("code-repair")
 )