Validate the "plain" and "light" repositories, to ensure the testdata…

… for the "write-tests" task is well-formed Part of #263
symflower · Jul 12, 2024 · 23bef30 · 23bef30
1 parent 4791bd9
commit 23bef30
Show file tree

Hide file tree

Showing 3 changed files with 165 additions and 0 deletions.
diff --git a/evaluate/task/repository.go b/evaluate/task/repository.go
@@ -94,6 +94,8 @@ func (r *Repository) Validate(logger *log.Logger, language language.Language) (e
 		switch taskIdentifier {
 		case IdentifierCodeRepair:
 			return validateCodeRepairRepository(logger, r.DataPath(), language)
+		case IdentifierWriteTests:
+			return validateWriteTestsRepository(logger, r.DataPath(), language)
 		}
 	}
 

diff --git a/evaluate/task/task-write-test.go b/evaluate/task/task-write-test.go
@@ -4,9 +4,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 
 	pkgerrors "github.com/pkg/errors"
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
+	"github.com/symflower/eval-dev-quality/language"
+	"github.com/symflower/eval-dev-quality/log"
 	"github.com/symflower/eval-dev-quality/model"
 	evaltask "github.com/symflower/eval-dev-quality/task"
 )
@@ -123,3 +126,26 @@ func (t *TaskWriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[eva
 
 	return repositoryAssessment, problems, nil
 }
+
+// validateWriteTestsRepository checks if the repository for the "write-tests" task is well-formed.
+func validateWriteTestsRepository(logger *log.Logger, repositoryPath string, language language.Language) (err error) {
+	logger.Printf("validating repository %q", repositoryPath)
+
+	files, err := language.Files(logger, repositoryPath)
+	if err != nil {
+		return pkgerrors.WithStack(err)
+	}
+
+	var testFiles []string
+	for _, file := range files {
+		if strings.HasSuffix(file, language.DefaultTestFileSuffix()) {
+			testFiles = append(testFiles, file)
+		}
+	}
+
+	if len(testFiles) > 0 {
+		return pkgerrors.Errorf("the repository %q must contain only %s source files, but found %+v", repositoryPath, language.Name(), testFiles)
+	}
+
+	return nil
+}
diff --git a/evaluate/task/task-write-test_test.go b/evaluate/task/task-write-test_test.go
@@ -14,6 +14,7 @@ import (
 	tasktesting "github.com/symflower/eval-dev-quality/evaluate/task/testing"
 	"github.com/symflower/eval-dev-quality/language"
 	"github.com/symflower/eval-dev-quality/language/golang"
+	"github.com/symflower/eval-dev-quality/language/java"
 	languagetesting "github.com/symflower/eval-dev-quality/language/testing"
 	"github.com/symflower/eval-dev-quality/log"
 	modeltesting "github.com/symflower/eval-dev-quality/model/testing"
@@ -216,3 +217,139 @@ func TestTaskWriteTestsRun(t *testing.T) {
 		})
 	})
 }
+
+func TestValidateWriteTestsRepository(t *testing.T) {
+	type testCase struct {
+		Name string
+
+		Before func(repositoryPath string)
+
+		RepositoryPath string
+		Language       language.Language
+
+		ExpectedErr string
+	}
+
+	validate := func(t *testing.T, tc *testCase) {
+		t.Run(tc.Name, func(t *testing.T) {
+			logOutput, logger := log.Buffer()
+			defer func() {
+				if t.Failed() {
+					t.Logf("Logging output: %s", logOutput.String())
+				}
+			}()
+
+			temporaryDirectoryPath := t.TempDir()
+			repositoryPath := filepath.Join(temporaryDirectoryPath, "testdata", tc.RepositoryPath)
+			require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", tc.RepositoryPath), repositoryPath))
+
+			if tc.Before != nil {
+				tc.Before(repositoryPath)
+			}
+
+			actualErr := validateWriteTestsRepository(logger, repositoryPath, tc.Language)
+			if len(tc.ExpectedErr) > 0 {
+				require.ErrorContains(t, actualErr, tc.ExpectedErr)
+			} else {
+				require.NoError(t, actualErr)
+			}
+		})
+	}
+
+	t.Run("Go", func(t *testing.T) {
+		t.Run("Plain", func(t *testing.T) {
+			validate(t, &testCase{
+				Name: "Repository with test files",
+
+				Before: func(repositoryPath string) {
+					// Create test files to trigger an error.
+					_, err := os.Create(filepath.Join(repositoryPath, "foo_test.go"))
+					require.NoError(t, err)
+					_, err = os.Create(filepath.Join(repositoryPath, "bar_test.go"))
+					require.NoError(t, err)
+				},
+
+				RepositoryPath: filepath.Join("golang", "plain"),
+				Language:       &golang.Language{},
+				ExpectedErr:    "must contain only Go source files, but found [bar_test.go foo_test.go]",
+			})
+			validate(t, &testCase{
+				Name: "Well-formed",
+
+				RepositoryPath: filepath.Join("golang", "plain"),
+				Language:       &golang.Language{},
+			})
+		})
+		t.Run("Light", func(t *testing.T) {
+			validate(t, &testCase{
+				Name: "Repository with test files",
+
+				Before: func(repositoryPath string) {
+					// Create test files to trigger an error.
+					_, err := os.Create(filepath.Join(repositoryPath, "foo_test.go"))
+					require.NoError(t, err)
+					_, err = os.Create(filepath.Join(repositoryPath, "bar_test.go"))
+					require.NoError(t, err)
+				},
+
+				RepositoryPath: filepath.Join("golang", "light"),
+				Language:       &golang.Language{},
+				ExpectedErr:    "must contain only Go source files, but found [bar_test.go foo_test.go]",
+			})
+			validate(t, &testCase{
+				Name: "Well-formed",
+
+				RepositoryPath: filepath.Join("golang", "light"),
+				Language:       &golang.Language{},
+			})
+		})
+	})
+	t.Run("Java", func(t *testing.T) {
+		t.Run("Plain", func(t *testing.T) {
+			validate(t, &testCase{
+				Name: "Repository with test files",
+
+				Before: func(repositoryPath string) {
+					// Create test files to trigger an error.
+					_, err := os.Create(filepath.Join(repositoryPath, "FooTest.java"))
+					require.NoError(t, err)
+					_, err = os.Create(filepath.Join(repositoryPath, "BarTest.java"))
+					require.NoError(t, err)
+				},
+
+				RepositoryPath: filepath.Join("java", "plain"),
+				Language:       &java.Language{},
+				ExpectedErr:    "must contain only Java source files, but found [BarTest.java FooTest.java]",
+			})
+			validate(t, &testCase{
+				Name: "Well-formed",
+
+				RepositoryPath: filepath.Join("java", "plain"),
+				Language:       &java.Language{},
+			})
+		})
+		t.Run("Light", func(t *testing.T) {
+			validate(t, &testCase{
+				Name: "Repository with test files",
+
+				Before: func(repositoryPath string) {
+					// Create test files to trigger an error.
+					_, err := os.Create(filepath.Join(repositoryPath, "FooTest.java"))
+					require.NoError(t, err)
+					_, err = os.Create(filepath.Join(repositoryPath, "BarTest.java"))
+					require.NoError(t, err)
+				},
+
+				RepositoryPath: filepath.Join("java", "light"),
+				Language:       &java.Language{},
+				ExpectedErr:    "must contain only Java source files, but found [BarTest.java FooTest.java]",
+			})
+			validate(t, &testCase{
+				Name: "Well-formed",
+
+				RepositoryPath: filepath.Join("java", "light"),
+				Language:       &java.Language{},
+			})
+		})
+	})
+}