Check if the testdata repository is valid before running the evaluati…

…on, so it is checked just once

Part of #263

evaluate/evaluate.go

@@ -93,6 +93,8 @@ func Evaluate(ctx *Context) (assessments *report.AssessmentStore, totalScore uin
 			temporaryRepository, cleanup, err := evaluatetask.TemporaryRepository(ctx.Log, ctx.TestdataPath, repositoryPath)
 			if err != nil {
 				ctx.Log.Panicf("ERROR: unable to create temporary repository path: %+v", err)
+			} else if err = temporaryRepository.Validate(ctx.Log, language); err != nil {
+				ctx.Log.Panicf("ERROR: malformed repository %q: %+v", temporaryRepository.Name(), err)
 			}
 
 			defer cleanup()
@@ -189,6 +191,8 @@ func Evaluate(ctx *Context) (assessments *report.AssessmentStore, totalScore uin
 			temporaryRepository, cleanup, err := evaluatetask.TemporaryRepository(ctx.Log, ctx.TestdataPath, repositoryPath)
 			if err != nil {
 				ctx.Log.Panicf("ERROR: unable to create temporary repository path: %s", err)
+			} else if err = temporaryRepository.Validate(ctx.Log, l); err != nil {
+				ctx.Log.Panicf("ERROR: malformed repository %q: %+v", temporaryRepository.Name(), err)
 			}
 
 			defer cleanup()

evaluate/task/repository.go

@@ -12,6 +12,7 @@ import (
 	"github.com/zimmski/osutil"
 	"github.com/zimmski/osutil/bytesutil"
 
+	"github.com/symflower/eval-dev-quality/language"
 	"github.com/symflower/eval-dev-quality/log"
 	"github.com/symflower/eval-dev-quality/task"
 	"github.com/symflower/eval-dev-quality/util"
@@ -87,6 +88,18 @@ func (r *Repository) SupportedTasks() (tasks []task.Identifier) {
 	return r.Tasks
 }
 
+// Validate checks it the repository is well-formed.
+func (r *Repository) Validate(logger *log.Logger, language language.Language) (err error) {
+	for _, taskIdentifier := range r.SupportedTasks() {
+		switch taskIdentifier {
+		case IdentifierCodeRepair:
+			return validateCodeRepairRepository(logger, r.DataPath(), language)
+		}
+	}
+
+	return nil
+}
+
 // Reset resets a repository back to its "initial" commit.
 func (r *Repository) Reset(logger *log.Logger) (err error) {
 	out, err := util.CommandWithResult(context.Background(), logger, &util.Command{

evaluate/task/task-code-repair.go

@@ -8,6 +8,7 @@ import (
 
 	pkgerrors "github.com/pkg/errors"
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
+	"github.com/symflower/eval-dev-quality/language"
 	"github.com/symflower/eval-dev-quality/log"
 	"github.com/symflower/eval-dev-quality/model"
 	evaltask "github.com/symflower/eval-dev-quality/task"
@@ -121,10 +122,60 @@ func (t *TaskCodeRepair) unpackCodeRepairPackage(ctx evaltask.Context, fileLogge
 		return "", nil, pkgerrors.Errorf("package %q in repository %q must contain source files with compilation errors", packagePath, ctx.Repository.Name())
 	}
 
-	sourceFilePath, err = packageHasSourceAndTestFile(fileLogger, ctx.Repository.Name(), packagePath, ctx.Language)
+	sourceFilePath, err = packageSourceFile(fileLogger, packagePath, ctx.Language)
 	if err != nil {
 		return "", nil, err
 	}
 
 	return sourceFilePath, mistakes, nil
 }
+
+// validateCodeRepairRepository checks if the repository for the "code-repair" task is well-formed.
+func validateCodeRepairRepository(logger *log.Logger, repositoryPath string, language language.Language) (err error) {
+	logger.Printf("validating repository %q", repositoryPath)
+
+	files, err := os.ReadDir(repositoryPath)
+	if err != nil {
+		return pkgerrors.WithStack(err)
+	}
+
+	var packagePaths []string
+	var otherFiles []string
+	for _, file := range files {
+		if file.Name() == "repository.json" {
+			continue
+		} else if file.IsDir() {
+			packagePaths = append(packagePaths, filepath.Join(repositoryPath, file.Name()))
+		} else {
+			otherFiles = append(otherFiles, file.Name())
+		}
+	}
+
+	if len(otherFiles) > 0 {
+		return pkgerrors.Errorf("the code repair repository %q must contain only packages, but found %+v", repositoryPath, otherFiles)
+	}
+
+	for _, packagePath := range packagePaths {
+		files, err := language.Files(logger, packagePath)
+		if err != nil {
+			return pkgerrors.WithStack(err)
+		}
+
+		sourceFiles := []string{}
+		testFiles := []string{}
+		for _, file := range files {
+			if strings.HasSuffix(file, language.DefaultTestFileSuffix()) {
+				testFiles = append(testFiles, file)
+			} else if strings.HasSuffix(file, language.DefaultFileExtension()) {
+				sourceFiles = append(sourceFiles, file)
+			}
+		}
+		if len(sourceFiles) != 1 {
+			return pkgerrors.Errorf("the code repair package %q in repository %q must contain exactly one %s source file, but found %+v", packagePath, repositoryPath, language.Name(), sourceFiles)
+		} else if len(testFiles) != 1 {
+			return pkgerrors.Errorf("the code repair repository %q must contain exactly one %s test file, but found %+v", repositoryPath, language.Name(), testFiles)
+		}
+	}
+
+	return nil
+}

evaluate/task/task-code-repair_test.go

@@ -1,6 +1,7 @@
 package task
 
 import (
+	"os"
 	"path/filepath"
 	"testing"
 
@@ -9,6 +10,7 @@ import (
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
 	metricstesting "github.com/symflower/eval-dev-quality/evaluate/metrics/testing"
 	tasktesting "github.com/symflower/eval-dev-quality/evaluate/task/testing"
+	"github.com/symflower/eval-dev-quality/language"
 	"github.com/symflower/eval-dev-quality/language/golang"
 	"github.com/symflower/eval-dev-quality/language/java"
 	"github.com/symflower/eval-dev-quality/log"
@@ -275,3 +277,223 @@ func TestTaskCodeRepairRun(t *testing.T) {
 		}
 	})
 }
+
+func TestValidateCodeRepairRepository(t *testing.T) {
+	type testCase struct {
+		Name string
+
+		Before func(repositoryPath string)
+
+		RepositoryPath string
+		Language       language.Language
+
+		ExpectedErr string
+	}
+
+	validate := func(t *testing.T, tc *testCase) {
+		t.Run(tc.Name, func(t *testing.T) {
+			logOutput, logger := log.Buffer()
+			defer func() {
+				if t.Failed() {
+					t.Logf("Logging output: %s", logOutput.String())
+				}
+			}()
+
+			temporaryDirectory := t.TempDir()
+			repositoryPath := filepath.Join(temporaryDirectory, "testdata", tc.RepositoryPath)
+			require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", tc.RepositoryPath), repositoryPath))
+
+			if tc.Before != nil {
+				tc.Before(repositoryPath)
+			}
+
+			actualErr := validateCodeRepairRepository(logger, repositoryPath, tc.Language)
+			if len(tc.ExpectedErr) > 0 {
+				assert.ErrorContains(t, actualErr, tc.ExpectedErr)
+			} else {
+				require.NoError(t, actualErr)
+			}
+		})
+	}
+
+	validate(t, &testCase{
+		Name: "Repository root path contains source files",
+
+		Before: func(repositoryPath string) {
+			// Create source files on the repository root path to trigger an error.
+			file, err := os.Create(filepath.Join(repositoryPath, "foo.go"))
+			require.NoError(t, err)
+			defer file.Close()
+			file, err = os.Create(filepath.Join(repositoryPath, "bar.go"))
+			require.NoError(t, err)
+			defer file.Close()
+		},
+
+		RepositoryPath: filepath.Join("golang", "mistakes"),
+		Language:       &golang.Language{},
+
+		ExpectedErr: "must contain only packages, but found [bar.go foo.go]",
+	})
+	t.Run("Go", func(t *testing.T) {
+		validate(t, &testCase{
+			Name: "Package does not contain source file",
+
+			Before: func(repositoryPath string) {
+				// Create a new package without any source file to trigger an error.
+				require.NoError(t, os.MkdirAll(filepath.Join(repositoryPath, "foobar"), 0700))
+			},
+
+			RepositoryPath: filepath.Join("golang", "mistakes"),
+			Language:       &golang.Language{},
+
+			ExpectedErr: "must contain exactly one Go source file, but found []",
+		})
+		validate(t, &testCase{
+			Name: "Package contains multiple source files",
+
+			Before: func(repositoryPath string) {
+				// Create a new package without any source file to trigger an error.
+				newPackage := filepath.Join(repositoryPath, "foobar")
+				require.NoError(t, os.MkdirAll(newPackage, 0700))
+				file, err := os.Create(filepath.Join(newPackage, "foo.go"))
+				require.NoError(t, err)
+				defer file.Close()
+				file, err = os.Create(filepath.Join(newPackage, "bar.go"))
+				require.NoError(t, err)
+				defer file.Close()
+			},
+
+			RepositoryPath: filepath.Join("golang", "mistakes"),
+			Language:       &golang.Language{},
+
+			ExpectedErr: "must contain exactly one Go source file, but found [bar.go foo.go]",
+		})
+		validate(t, &testCase{
+			Name: "Package does not contain test file",
+
+			Before: func(repositoryPath string) {
+				newPackage := filepath.Join(repositoryPath, "foobar")
+				require.NoError(t, os.MkdirAll(newPackage, 0700))
+				file, err := os.Create(filepath.Join(newPackage, "foo.go"))
+				require.NoError(t, err)
+				defer file.Close()
+			},
+
+			RepositoryPath: filepath.Join("golang", "mistakes"),
+			Language:       &golang.Language{},
+
+			ExpectedErr: "must contain exactly one Go test file, but found []",
+		})
+		validate(t, &testCase{
+			Name: "Package contains multiple test files",
+
+			Before: func(repositoryPath string) {
+				// Create a new package without any source file to trigger an error.
+				newPackage := filepath.Join(repositoryPath, "foobar")
+				require.NoError(t, os.MkdirAll(newPackage, 0700))
+				file, err := os.Create(filepath.Join(newPackage, "foo.go"))
+				require.NoError(t, err)
+				defer file.Close()
+				file, err = os.Create(filepath.Join(newPackage, "foo_test.go"))
+				require.NoError(t, err)
+				defer file.Close()
+				file, err = os.Create(filepath.Join(newPackage, "bar_test.go"))
+				require.NoError(t, err)
+				defer file.Close()
+			},
+
+			RepositoryPath: filepath.Join("golang", "mistakes"),
+			Language:       &golang.Language{},
+
+			ExpectedErr: "must contain exactly one Go test file, but found [bar_test.go foo_test.go]",
+		})
+		validate(t, &testCase{
+			Name: "Well-formed",
+
+			RepositoryPath: filepath.Join("golang", "mistakes"),
+			Language:       &golang.Language{},
+		})
+	})
+	t.Run("Java", func(t *testing.T) {
+		validate(t, &testCase{
+			Name: "Package does not contain source file",
+
+			Before: func(repositoryPath string) {
+				// Create a new package without any source file to trigger an error.
+				require.NoError(t, os.MkdirAll(filepath.Join(repositoryPath, "foobar"), 0700))
+			},
+
+			RepositoryPath: filepath.Join("java", "mistakes"),
+			Language:       &java.Language{},
+
+			ExpectedErr: "must contain exactly one Java source file, but found []",
+		})
+		validate(t, &testCase{
+			Name: "Package contains multiple source files",
+
+			Before: func(repositoryPath string) {
+				// Create a new package without any source file to trigger an error.
+				newPackage := filepath.Join(repositoryPath, "foobar", "src", "main", "java", "com", "eval")
+				require.NoError(t, os.MkdirAll(newPackage, 0700))
+				file, err := os.Create(filepath.Join(newPackage, "Foo.java"))
+				require.NoError(t, err)
+				defer file.Close()
+				file, err = os.Create(filepath.Join(newPackage, "Bar.java"))
+				require.NoError(t, err)
+				defer file.Close()
+			},
+
+			RepositoryPath: filepath.Join("java", "mistakes"),
+			Language:       &java.Language{},
+
+			ExpectedErr: "must contain exactly one Java source file, but found [src/main/java/com/eval/Bar.java src/main/java/com/eval/Foo.java]",
+		})
+		validate(t, &testCase{
+			Name: "Package does not contain test file",
+
+			Before: func(repositoryPath string) {
+				newPackage := filepath.Join(repositoryPath, "foobar", "src", "main", "java", "com", "eval")
+				require.NoError(t, os.MkdirAll(newPackage, 0700))
+				file, err := os.Create(filepath.Join(newPackage, "Foo.java"))
+				require.NoError(t, err)
+				defer file.Close()
+			},
+
+			RepositoryPath: filepath.Join("java", "mistakes"),
+			Language:       &java.Language{},
+
+			ExpectedErr: "must contain exactly one Java test file, but found []",
+		})
+		validate(t, &testCase{
+			Name: "Package contains multiple test files",
+
+			Before: func(repositoryPath string) {
+				// Create a new package without any source file to trigger an error.
+				sourcePackage := filepath.Join(repositoryPath, "foobar", "src", "main", "java", "com", "eval")
+				testPackage := filepath.Join(repositoryPath, "foobar", "src", "test", "java", "com", "eval")
+				require.NoError(t, os.MkdirAll(sourcePackage, 0700))
+				require.NoError(t, os.MkdirAll(testPackage, 0700))
+				file, err := os.Create(filepath.Join(sourcePackage, "Foo.java"))
+				require.NoError(t, err)
+				defer file.Close()
+				file, err = os.Create(filepath.Join(testPackage, "FooTest.java"))
+				require.NoError(t, err)
+				defer file.Close()
+				file, err = os.Create(filepath.Join(testPackage, "BarTest.java"))
+				require.NoError(t, err)
+				defer file.Close()
+			},
+
+			RepositoryPath: filepath.Join("java", "mistakes"),
+			Language:       &java.Language{},
+
+			ExpectedErr: "must contain exactly one Java test file, but found [src/test/java/com/eval/BarTest.java src/test/java/com/eval/FooTest.java]",
+		})
+		validate(t, &testCase{
+			Name: "Well-formed",
+
+			RepositoryPath: filepath.Join("java", "mistakes"),
+			Language:       &java.Language{},
+		})
+	})
+}

evaluate/task/task.go

@@ -86,27 +86,20 @@ func (t *taskLogger) finalize(problems []error) {
 	t.logClose()
 }
 
-// packageHasSourceAndTestFile checks if a package as a source file and the corresponding test file for the given language, and returns the source file path.
-func packageHasSourceAndTestFile(log *log.Logger, repositoryName string, packagePath string, language language.Language) (sourceFilePath string, err error) {
+// packageSourceFile returns the source file of a package.
+func packageSourceFile(log *log.Logger, packagePath string, language language.Language) (sourceFilePath string, err error) {
 	filePaths, err := language.Files(log, packagePath)
 	if err != nil {
 		return "", pkgerrors.WithStack(err)
-	} else if len(filePaths) != 2 {
-		return "", pkgerrors.Errorf("package %q in repository %q must only contain an implementation file and the corresponding test file, but found %#v", packagePath, repositoryName, filePaths)
 	}
-	var hasTestFile bool
+
 	for _, file := range filePaths {
 		if strings.HasSuffix(file, language.DefaultTestFileSuffix()) {
-			hasTestFile = true
+			continue
 		} else if filepath.Ext(file) == language.DefaultFileExtension() {
 			sourceFilePath = file
 		}
 	}
-	if sourceFilePath == "" {
-		return "", pkgerrors.Errorf("package %q in repository %q does not contain a source file", packagePath, repositoryName)
-	} else if !hasTestFile {
-		return "", pkgerrors.Errorf("package %q in repository %q does not contain a test file", packagePath, repositoryName)
-	}
 
 	return sourceFilePath, nil
 }

task/task.go

@@ -54,6 +54,9 @@ type Repository interface {
 	// SupportedTasks returns the list of task identifiers the repository supports.
 	SupportedTasks() (tasks []Identifier)
 
+	// Validate checks it the repository is well-formed.
+	Validate(logger *log.Logger, language language.Language) (err error)
+
 	// Reset resets the repository to its initial state.
 	Reset(logger *log.Logger) (err error)
 }