Validate the "plain" and "light" repositories, to ensure the testdata…

… for the "write-tests" task is well-formed Part of #263
symflower · Jul 15, 2024 · d160d04 · d160d04
1 parent ec85d15
commit d160d04
Show file tree

Hide file tree

Showing 3 changed files with 114 additions and 0 deletions.
diff --git a/evaluate/task/repository.go b/evaluate/task/repository.go
@@ -94,6 +94,8 @@ func (r *Repository) Validate(logger *log.Logger, language language.Language) (e
 		switch taskIdentifier {
 		case IdentifierCodeRepair:
 			return validateCodeRepairRepository(logger, r.DataPath(), language)
+		case IdentifierWriteTests:
+			return validateWriteTestsRepository(logger, r.DataPath(), language)
 		}
 	}
 

diff --git a/evaluate/task/task-write-test.go b/evaluate/task/task-write-test.go
@@ -4,9 +4,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 
 	pkgerrors "github.com/pkg/errors"
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
+	"github.com/symflower/eval-dev-quality/language"
+	"github.com/symflower/eval-dev-quality/log"
 	"github.com/symflower/eval-dev-quality/model"
 	evaltask "github.com/symflower/eval-dev-quality/task"
 )
@@ -123,3 +126,31 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva
 
 	return repositoryAssessment, problems, nil
 }
+
+// validateWriteTestsRepository checks if the repository for the "write-tests" task is well-formed.
+func validateWriteTestsRepository(logger *log.Logger, repositoryPath string, language language.Language) (err error) {
+	logger.Printf("validating repository %q", repositoryPath)
+
+	files, err := language.Files(logger, repositoryPath)
+	if err != nil {
+		return pkgerrors.WithStack(err)
+	}
+
+	var sourceFiles []string
+	var testFiles []string
+	for _, file := range files {
+		if strings.HasSuffix(file, language.DefaultTestFileSuffix()) {
+			testFiles = append(testFiles, file)
+		} else if strings.HasSuffix(file, language.DefaultFileExtension()) {
+			sourceFiles = append(sourceFiles, file)
+		}
+	}
+
+	if len(sourceFiles) == 0 {
+		return pkgerrors.Errorf("the repository %q must contain at least one %s source file, but found none", repositoryPath, language.Name())
+	} else if len(testFiles) > 0 {
+		return pkgerrors.Errorf("the repository %q must contain only %s source files, but found %+v", repositoryPath, language.Name(), testFiles)
+	}
+
+	return nil
+}
diff --git a/evaluate/task/task-write-test_test.go b/evaluate/task/task-write-test_test.go
@@ -14,6 +14,7 @@ import (
 	tasktesting "github.com/symflower/eval-dev-quality/evaluate/task/testing"
 	"github.com/symflower/eval-dev-quality/language"
 	"github.com/symflower/eval-dev-quality/language/golang"
+	"github.com/symflower/eval-dev-quality/language/java"
 	languagetesting "github.com/symflower/eval-dev-quality/language/testing"
 	"github.com/symflower/eval-dev-quality/log"
 	modeltesting "github.com/symflower/eval-dev-quality/model/testing"
@@ -216,3 +217,83 @@ func TestTaskWriteTestsRun(t *testing.T) {
 		})
 	})
 }
+
+func TestValidateWriteTestsRepository(t *testing.T) {
+	validate := func(t *testing.T, tc *tasktesting.TestCaseValidateRepository) {
+		t.Run(tc.Name, func(t *testing.T) {
+			tc.Validate(t, validateWriteTestsRepository)
+		})
+	}
+
+	t.Run("Go", func(t *testing.T) {
+		t.Run("Plain", func(t *testing.T) {
+			validate(t, &tasktesting.TestCaseValidateRepository{
+				Name: "Well-formed",
+
+				TestdataPath:   filepath.Join("..", "..", "testdata"),
+				RepositoryPath: filepath.Join("golang", "plain"),
+				Language:       &golang.Language{},
+			})
+		})
+		t.Run("Light", func(t *testing.T) {
+			validate(t, &tasktesting.TestCaseValidateRepository{
+				Name: "Repository with test files",
+
+				Before: func(repositoryPath string) {
+					fileATest, err := os.Create(filepath.Join(repositoryPath, "fileA_test.go"))
+					require.NoError(t, err)
+					fileATest.Close()
+				},
+
+				TestdataPath:          filepath.Join("..", "..", "testdata"),
+				RepositoryPath:        filepath.Join("golang", "light"),
+				Language:              &golang.Language{},
+				ExpectedErrorContains: "must contain only Go source files, but found [fileA_test.go]",
+			})
+			validate(t, &tasktesting.TestCaseValidateRepository{
+				Name: "Well-formed",
+
+				TestdataPath:   filepath.Join("..", "..", "testdata"),
+				RepositoryPath: filepath.Join("golang", "light"),
+				Language:       &golang.Language{},
+			})
+		})
+	})
+	t.Run("Java", func(t *testing.T) {
+		t.Run("Plain", func(t *testing.T) {
+			validate(t, &tasktesting.TestCaseValidateRepository{
+				Name: "Well-formed",
+
+				TestdataPath:   filepath.Join("..", "..", "testdata"),
+				RepositoryPath: filepath.Join("java", "plain"),
+				Language:       &java.Language{},
+			})
+		})
+		t.Run("Light", func(t *testing.T) {
+			validate(t, &tasktesting.TestCaseValidateRepository{
+				Name: "Repository with test files",
+
+				Before: func(repositoryPath string) {
+					somePackage := filepath.Join(repositoryPath, "src", "test", "java", "com", "eval")
+					require.NoError(t, os.MkdirAll(somePackage, 0700))
+
+					fileATest, err := os.Create(filepath.Join(somePackage, "FileATest.java"))
+					require.NoError(t, err)
+					fileATest.Close()
+				},
+
+				TestdataPath:          filepath.Join("..", "..", "testdata"),
+				RepositoryPath:        filepath.Join("java", "light"),
+				Language:              &java.Language{},
+				ExpectedErrorContains: "must contain only Java source files, but found [src/test/java/com/eval/FileATest.java]",
+			})
+			validate(t, &tasktesting.TestCaseValidateRepository{
+				Name: "Well-formed",
+
+				TestdataPath:   filepath.Join("..", "..", "testdata"),
+				RepositoryPath: filepath.Join("java", "light"),
+				Language:       &java.Language{},
+			})
+		})
+	})
+}