fix, Default to all repositories if none are selected in CLI

Fixes #163

cmd/eval-dev-quality/cmd/evaluate.go

@@ -223,31 +223,41 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.
 
 	// Gather repositories and update language selection accordingly.
 	{
-		commandRepositories := map[string]bool{}
-		commandRepositoriesLanguages := map[string]bool{}
-		for _, r := range command.Repositories {
-			languageIDOfRepository := strings.SplitN(r, string(os.PathSeparator), 2)[0]
-			if _, ok := languagesSelected[languageIDOfRepository]; ok {
-				commandRepositories[r] = true
-				commandRepositoriesLanguages[languageIDOfRepository] = true
-			} else {
-				command.logger.Printf("Excluded repository %s because its language %q is not enabled for this evaluation", r, languageIDOfRepository)
+		if len(command.Repositories) == 0 {
+			for _, l := range command.Languages {
+				repositories, err := language.RepositoriesForLanguage(language.Languages[l], command.TestdataPath)
+				if err != nil {
+					command.logger.Panicf("ERROR: %s", err)
+				}
+				command.Repositories = repositories
 			}
-		}
-		for languageID := range languagesSelected {
-			if len(command.Repositories) == 0 || commandRepositoriesLanguages[languageID] {
-				commandRepositories[filepath.Join(languageID, evaluate.RepositoryPlainName)] = true
-			} else {
-				command.Languages = slices.DeleteFunc(command.Languages, func(l string) bool {
-					return l == languageID
-				})
-				delete(languagesSelected, languageID)
-				command.logger.Printf("Excluded language %q because it is not part of the selected repositories", languageID)
+		} else {
+			commandRepositories := map[string]bool{}
+			commandRepositoriesLanguages := map[string]bool{}
+			for _, r := range command.Repositories {
+				languageIDOfRepository := strings.SplitN(r, string(os.PathSeparator), 2)[0]
+				if _, ok := languagesSelected[languageIDOfRepository]; ok {
+					commandRepositories[r] = true
+					commandRepositoriesLanguages[languageIDOfRepository] = true
+				} else {
+					command.logger.Printf("Excluded repository %s because its language %q is not enabled for this evaluation", r, languageIDOfRepository)
+				}
+			}
+			for languageID := range languagesSelected {
+				if commandRepositoriesLanguages[languageID] { // Also add the plain repository in case we already have repositories for this language.
+					commandRepositories[filepath.Join(languageID, evaluate.RepositoryPlainName)] = true
+				} else {
+					command.Languages = slices.DeleteFunc(command.Languages, func(l string) bool {
+						return l == languageID
+					})
+					delete(languagesSelected, languageID)
+					command.logger.Printf("Excluded language %q because it is not part of the selected repositories", languageID)
+				}
 			}
 
+			command.Repositories = maps.Keys(commandRepositories)
+			sort.Strings(command.Repositories)
 		}
-		command.Repositories = maps.Keys(commandRepositories)
-		sort.Strings(command.Repositories)
 		evaluationContext.RepositoryPaths = command.Repositories
 	}
 

cmd/eval-dev-quality/cmd/evaluate_test.go

@@ -191,6 +191,8 @@ func TestEvaluateExecute(t *testing.T) {
 			Arguments: []string{
 				"--language", "golang",
 				"--model", "symflower/symbolic-execution",
+				"--repository", filepath.Join("golang", "plain"),
+				"--repository", filepath.Join("java", "plain"),
 			},
 
 			ExpectedOutputValidate: func(t *testing.T, output string, resultPath string) {
@@ -270,6 +272,8 @@ func TestEvaluateExecute(t *testing.T) {
 
 			Arguments: []string{
 				"--model", "symflower/symbolic-execution",
+				"--repository", filepath.Join("golang", "plain"),
+				"--repository", filepath.Join("java", "plain"),
 			},
 
 			ExpectedOutputValidate: func(t *testing.T, output string, resultPath string) {
@@ -720,6 +724,7 @@ func TestEvaluateExecute(t *testing.T) {
 		Arguments: []string{
 			"--language", "golang",
 			"--model", "symflower/symbolic-execution",
+			"--repository", filepath.Join("golang", "plain"),
 		},
 
 		ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
@@ -742,6 +747,7 @@ func TestEvaluateExecute(t *testing.T) {
 		Arguments: []string{
 			"--language", "golang",
 			"--model", "symflower/symbolic-execution",
+			"--repository", filepath.Join("golang", "plain"),
 		},
 
 		ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
@@ -918,4 +924,25 @@ func TestEvaluateInitialize(t *testing.T) {
 			}, context.Languages)
 		},
 	})
+	validate(t, &testCase{
+		Name: "Selecting no repository defaults to all",
+
+		Command: makeValidCommand(func(command *Evaluate) {
+			command.Repositories = []string{}
+		}),
+
+		ValidateCommand: func(t *testing.T, command *Evaluate) {
+			// Check if all Go repositories are indeed selected.
+			directories, err := os.ReadDir(filepath.Join("..", "..", "..", "testdata", "golang"))
+			require.NoError(t, err)
+			relativeRepositoryPath := make([]string, len(directories))
+			for i, directory := range directories {
+				relativeRepositoryPath[i] = filepath.Join("golang", directory.Name())
+			}
+
+			for _, golangRepository := range relativeRepositoryPath {
+				assert.Contains(t, command.Repositories, golangRepository)
+			}
+		},
+	})
 }

evaluate/evaluate.go

@@ -3,8 +3,10 @@ package evaluate
 import (
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/symflower/eval-dev-quality/evaluate/report"
+	"github.com/symflower/eval-dev-quality/language"
 	evallanguage "github.com/symflower/eval-dev-quality/language"
 	"github.com/symflower/eval-dev-quality/log"
 	evalmodel "github.com/symflower/eval-dev-quality/model"
@@ -145,17 +147,15 @@ func Evaluate(ctx *Context) (assessments report.AssessmentPerModelPerLanguagePer
 	ctx.Log.Printf("Evaluating models and languages")
 	// Create temporary repositories for each language so the repository is copied only once per language.
 	temporaryRepositories := map[string]*Repository{}
-	for _, language := range ctx.Languages {
-		languagePath := filepath.Join(ctx.TestdataPath, language.ID())
-		repositories, err := os.ReadDir(languagePath)
+	for _, l := range ctx.Languages {
+		relativeRepositoryPaths, err := language.RepositoriesForLanguage(l, ctx.TestdataPath)
 		if err != nil {
-			ctx.Log.Panicf("ERROR: language path %q cannot be accessed: %s", languagePath, err)
+			ctx.Log.Panicf("ERROR: %s", err)
 		}
-		for _, repository := range repositories {
-			repositoryPath := filepath.Join(language.ID(), repository.Name())
+		for _, repositoryPath := range relativeRepositoryPaths {
 
 			// Do not include "plain" repositories in this step of the evaluation, because they have been checked with the common check before.
-			if !repositoriesLookup[repositoryPath] || repository.Name() == RepositoryPlainName {
+			if !repositoriesLookup[repositoryPath] || strings.HasSuffix(repositoryPath, RepositoryPlainName) {
 				continue
 			}
 

language/language.go

@@ -1,6 +1,10 @@
 package language
 
 import (
+	"os"
+	"path/filepath"
+	"sort"
+
 	pkgerrors "github.com/pkg/errors"
 
 	"github.com/symflower/eval-dev-quality/log"
@@ -40,3 +44,23 @@ func Register(language Language) {
 
 	Languages[id] = language
 }
+
+// RepositoriesForLanguage returns the relative repository paths for a language.
+func RepositoriesForLanguage(language Language, testdataPath string) (relativeRepositoryPaths []string, err error) {
+	languagePath := filepath.Join(testdataPath, language.ID())
+	languageRepositories, err := os.ReadDir(languagePath)
+	if err != nil {
+		pkgerrors.WithMessagef(err, "language path %q cannot be accessed", languagePath)
+	}
+
+	for _, repository := range languageRepositories {
+		if !repository.IsDir() {
+			continue
+		}
+		relativeRepositoryPaths = append(relativeRepositoryPaths, filepath.Join(language.ID(), repository.Name()))
+	}
+
+	sort.Strings(relativeRepositoryPaths)
+
+	return relativeRepositoryPaths, nil
+}