Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Write available and selected models into a configuration file for documentation/reproducibility #285

Merged
merged 1 commit into from
Jul 25, 2024
Merged

Write available and selected models into a configuration file for documentation/reproducibility #285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions cmd/eval-dev-quality/cmd/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package cmd

import (
"encoding/json"
"io"

pkgerrors "github.com/pkg/errors"
)

// EvaluationConfiguration holds data of how an evaluation was configured.
type EvaluationConfiguration struct {
// Models holds model configuration data.
Models ModelsConfiguration
}

// ModelsConfiguration holds model data of how an evaluation was configured.
type ModelsConfiguration struct {
// Selected holds the models selected for an evaluation.
Selected []string
// Available holds the models that were available at the time of an evaluation.
Available []string
}

// Write stores the configuration in JSON format.
func (c *EvaluationConfiguration) Write(writer io.Writer) error {
encoder := json.NewEncoder(writer)
if err := encoder.Encode(c); err != nil {
return pkgerrors.Wrap(err, "writing configuration")
}

return nil
}

// NewEvaluationConfiguration creates an empty configuration.
func NewEvaluationConfiguration() *EvaluationConfiguration {
return &EvaluationConfiguration{}
}
20 changes: 17 additions & 3 deletions cmd/eval-dev-quality/cmd/evaluate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,9 @@ func (command *Evaluate) SetArguments(args []string) {
}

// Initialize initializes the command according to the arguments.
func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.Context) {
func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.Context, evaluationConfiguration *EvaluationConfiguration) {
evaluationContext = &evaluate.Context{}
evaluationConfiguration = NewEvaluationConfiguration()

// Check and validate common options.
{
Expand Down Expand Up @@ -369,6 +370,7 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.
for _, m := range ms {
models[m.ID()] = m
evaluationContext.ProviderForModel[m] = p
evaluationConfiguration.Models.Available = append(evaluationConfiguration.Models.Available, m.ID())
}
}
modelIDs := maps.Keys(models)
Expand All @@ -391,19 +393,31 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.
evaluationContext.Models = make([]model.Model, len(command.Models))
for i, modelID := range command.Models {
evaluationContext.Models[i] = modelsSelected[modelID]
evaluationConfiguration.Models.Selected = append(evaluationConfiguration.Models.Selected, modelID)
}
}

return evaluationContext
return evaluationContext, evaluationConfiguration
}

// Execute executes the command.
func (command *Evaluate) Execute(args []string) (err error) {
command.timestamp = time.Now()

evaluationContext := command.Initialize(args)
evaluationContext, evaluationConfiguration := command.Initialize(args)
if evaluationContext == nil {
command.logger.Panic("ERROR: empty evaluation context")
} else if evaluationConfiguration == nil {
command.logger.Panic("ERROR: empty evaluation configuration")
}

configurationFile, err := os.Create(filepath.Join(evaluationContext.ResultPath, "config.json"))
if err != nil {
command.logger.Panicf("ERROR: cannot create configuration file: %s", err)
}
defer configurationFile.Close()
if err := evaluationConfiguration.Write(configurationFile); err != nil {
command.logger.Panicf("ERROR: %s", err)
}

switch command.Runtime {
Expand Down
39 changes: 31 additions & 8 deletions cmd/eval-dev-quality/cmd/evaluate_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ func TestEvaluateExecute(t *testing.T) {
filepath.Join("result-directory", "categories.svg"): func(t *testing.T, filePath, data string) {
validateSVGContent(t, data, []*metrics.AssessmentCategory{metrics.AssessmentCategoryCodeNoExcess}, 1)
},
filepath.Join("result-directory", "config.json"): nil,
filepath.Join("result-directory", "evaluation.csv"): func(t *testing.T, filePath, data string) {
actualAssessments := validateMetrics(t, extractMetricsCSVMatch, data, []metrics.Assessments{
metrics.Assessments{
Expand Down Expand Up @@ -285,6 +286,7 @@ func TestEvaluateExecute(t *testing.T) {
filepath.Join("result-directory", "categories.svg"): func(t *testing.T, filePath, data string) {
validateSVGContent(t, data, []*metrics.AssessmentCategory{metrics.AssessmentCategoryCodeNoExcess}, 1)
},
filepath.Join("result-directory", "config.json"): nil,
filepath.Join("result-directory", "evaluation.csv"): func(t *testing.T, filePath, data string) {
actualAssessments := validateMetrics(t, extractMetricsCSVMatch, data, []metrics.Assessments{
metrics.Assessments{
Expand Down Expand Up @@ -380,6 +382,7 @@ func TestEvaluateExecute(t *testing.T) {
filepath.Join("result-directory", "categories.svg"): func(t *testing.T, filePath, data string) {
validateSVGContent(t, data, []*metrics.AssessmentCategory{metrics.AssessmentCategoryCodeNoExcess}, 1)
},
filepath.Join("result-directory", "config.json"): nil,
filepath.Join("result-directory", "evaluation.csv"): func(t *testing.T, filePath, data string) {
actualAssessments := validateMetrics(t, extractMetricsCSVMatch, data, []metrics.Assessments{
metrics.Assessments{
Expand Down Expand Up @@ -430,6 +433,7 @@ func TestEvaluateExecute(t *testing.T) {
filepath.Join("result-directory", "categories.svg"): func(t *testing.T, filePath, data string) {
validateSVGContent(t, data, []*metrics.AssessmentCategory{metrics.AssessmentCategoryCodeNoExcess}, 1)
},
filepath.Join("result-directory", "config.json"): nil,
filepath.Join("result-directory", "evaluation.csv"): func(t *testing.T, filePath, data string) {
actualAssessments := validateMetrics(t, extractMetricsCSVMatch, data, []metrics.Assessments{
metrics.Assessments{
Expand Down Expand Up @@ -516,6 +520,7 @@ func TestEvaluateExecute(t *testing.T) {

ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
filepath.Join("result-directory", "categories.svg"): nil,
filepath.Join("result-directory", "config.json"): nil,
filepath.Join("result-directory", "evaluation.csv"): nil,
filepath.Join("result-directory", "evaluation.log"): func(t *testing.T, filePath, data string) {
// Since the model is non-deterministic, we can only assert that the model did at least not error.
Expand Down Expand Up @@ -545,6 +550,7 @@ func TestEvaluateExecute(t *testing.T) {

ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
filepath.Join("result-directory", "categories.svg"): nil,
filepath.Join("result-directory", "config.json"): nil,
filepath.Join("result-directory", "evaluation.csv"): nil,
filepath.Join("result-directory", "evaluation.log"): nil,
filepath.Join("result-directory", "README.md"): nil,
Expand Down Expand Up @@ -590,6 +596,7 @@ func TestEvaluateExecute(t *testing.T) {

ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
filepath.Join("result-directory", "categories.svg"): nil,
filepath.Join("result-directory", "config.json"): nil,
filepath.Join("result-directory", "evaluation.csv"): nil,
filepath.Join("result-directory", "evaluation.log"): func(t *testing.T, filePath, data string) {
// Since the model is non-deterministic, we can only assert that the model did at least not error.
Expand Down Expand Up @@ -634,6 +641,7 @@ func TestEvaluateExecute(t *testing.T) {
},
ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
filepath.Join("result-directory", "categories.svg"): nil,
filepath.Join("result-directory", "config.json"): nil,
filepath.Join("result-directory", "evaluation.csv"): func(t *testing.T, filePath, data string) {
actualAssessments := validateMetrics(t, extractMetricsCSVMatch, data, []metrics.Assessments{
metrics.Assessments{
Expand Down Expand Up @@ -808,9 +816,11 @@ func TestEvaluateExecute(t *testing.T) {
},
ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
filepath.Join("result-directory", "evaluation.log"): nil,
filepath.Join("result-directory", "config.json"): nil,

// Parallel run 1
filepath.Join("result-directory", "symflower_symbolic-execution", "categories.svg"): nil,
filepath.Join("result-directory", "symflower_symbolic-execution", "config.json"): nil,
filepath.Join("result-directory", "symflower_symbolic-execution", "evaluation.csv"): func(t *testing.T, filePath, data string) {
actualAssessments := validateMetrics(t, extractMetricsCSVMatch, data, []metrics.Assessments{
metrics.Assessments{
Expand Down Expand Up @@ -862,6 +872,7 @@ func TestEvaluateExecute(t *testing.T) {

// Parallel run 2
filepath.Join("result-directory", "symflower_symbolic-execution_1", "categories.svg"): nil,
filepath.Join("result-directory", "symflower_symbolic-execution_1", "config.json"): nil,
filepath.Join("result-directory", "symflower_symbolic-execution_1", "evaluation.csv"): func(t *testing.T, filePath, data string) {
actualAssessments := validateMetrics(t, extractMetricsCSVMatch, data, []metrics.Assessments{
metrics.Assessments{
Expand Down Expand Up @@ -913,6 +924,7 @@ func TestEvaluateExecute(t *testing.T) {

// Parallel run 3
filepath.Join("result-directory", "symflower_symbolic-execution_2", "categories.svg"): nil,
filepath.Join("result-directory", "symflower_symbolic-execution_2", "config.json"): nil,
filepath.Join("result-directory", "symflower_symbolic-execution_2", "evaluation.csv"): func(t *testing.T, filePath, data string) {
actualAssessments := validateMetrics(t, extractMetricsCSVMatch, data, []metrics.Assessments{
metrics.Assessments{
Expand Down Expand Up @@ -991,6 +1003,7 @@ func TestEvaluateExecute(t *testing.T) {

ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
filepath.Join("result-directory", "categories.svg"): nil,
filepath.Join("result-directory", "config.json"): nil,
filepath.Join("result-directory", "evaluation.csv"): nil,
filepath.Join("result-directory", "evaluation.log"): nil,
filepath.Join("result-directory", "README.md"): nil,
Expand All @@ -1012,6 +1025,7 @@ func TestEvaluateExecute(t *testing.T) {

ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
filepath.Join("result-directory-0", "categories.svg"): nil,
filepath.Join("result-directory-0", "config.json"): nil,
filepath.Join("result-directory-0", "evaluation.csv"): nil,
filepath.Join("result-directory-0", "evaluation.log"): nil,
filepath.Join("result-directory-0", "README.md"): nil,
Expand All @@ -1026,10 +1040,11 @@ func TestEvaluateInitialize(t *testing.T) {

Command *Evaluate

ValidateCommand func(t *testing.T, command *Evaluate)
ValidateContext func(t *testing.T, context *evaluate.Context)
ValidateResults func(t *testing.T, resultsPath string)
ValidatePanic string
ValidateCommand func(t *testing.T, command *Evaluate)
ValidateContext func(t *testing.T, context *evaluate.Context)
ValidateConfiguration func(t *testing.T, config *EvaluationConfiguration)
ValidateResults func(t *testing.T, resultsPath string)
ValidatePanic string
}

validate := func(t *testing.T, tc *testCase) {
Expand All @@ -1049,18 +1064,18 @@ func TestEvaluateInitialize(t *testing.T) {
tc.Command.logger = logger
tc.Command.ResultPath = strings.ReplaceAll(tc.Command.ResultPath, "$TEMP_PATH", temporaryDirectory)

var actualEvaluationContext *evaluate.Context

if tc.ValidatePanic != "" {
assert.PanicsWithValue(t, tc.ValidatePanic, func() {
actualEvaluationContext = tc.Command.Initialize([]string{})
_, _ = tc.Command.Initialize([]string{})
})

return
}

var actualEvaluationContext *evaluate.Context
var actualEvaluationConfiguration *EvaluationConfiguration
assert.NotPanics(t, func() {
actualEvaluationContext = tc.Command.Initialize([]string{})
actualEvaluationContext, actualEvaluationConfiguration = tc.Command.Initialize([]string{})
})

if tc.ValidateCommand != nil {
Expand All @@ -1070,6 +1085,10 @@ func TestEvaluateInitialize(t *testing.T) {
require.NotNil(t, actualEvaluationContext)
tc.ValidateContext(t, actualEvaluationContext)
}
if tc.ValidateConfiguration != nil {
require.NotNil(t, actualEvaluationConfiguration)
tc.ValidateConfiguration(t, actualEvaluationConfiguration)
}
if tc.ValidateResults != nil {
tc.ValidateResults(t, temporaryDirectory)
}
Expand Down Expand Up @@ -1144,6 +1163,10 @@ func TestEvaluateInitialize(t *testing.T) {
}
assert.Contains(t, modelIDs, "symflower/symbolic-execution")
},
ValidateConfiguration: func(t *testing.T, config *EvaluationConfiguration) {
assert.Contains(t, config.Models.Available, "symflower/symbolic-execution")
assert.Contains(t, config.Models.Selected, "symflower/symbolic-execution")
},
})
validate(t, &testCase{
Name: "Remove repository if language is not selected",
Expand Down