TODO, Combined changes to test cases because of logging changes

Sorry but I won't split these...

cmd/eval-dev-quality/cmd/evaluate_test.go

@@ -314,10 +314,10 @@ func TestEvaluateExecute(t *testing.T) {
 					validateREADME(t, data)
 				},
 				filepath.Join("result-directory", string(evaluatetask.IdentifierWriteTests), "symflower_symbolic-execution", "golang", "golang", "plain", "evaluation.log"): func(t *testing.T, filePath, data string) {
-					assert.Contains(t, data, "coverage objects: [{")
+					assert.Contains(t, data, "\"msg\":\"coverage objects\",\"objects\":\"[{")
 				},
 				filepath.Join("result-directory", string(evaluatetask.IdentifierWriteTests), "symflower_symbolic-execution", "java", "java", "plain", "evaluation.log"): func(t *testing.T, filePath, data string) {
-					assert.Contains(t, data, "coverage objects: [{")
+					assert.Contains(t, data, "\"msg\":\"coverage objects\",\"objects\":\"[{")
 				},
 			},
 		})
@@ -462,7 +462,7 @@ func TestEvaluateExecute(t *testing.T) {
 
 				ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
 					filepath.Join("result-directory", "evaluation.log"): func(t *testing.T, filePath string, data string) {
-						assert.Contains(t, data, "Skipping unavailable provider \"openrouter\"")
+						assert.Contains(t, data, "\"msg\":\"skipping unavailable provider\",\"provider\":\"openrouter\"")
 					},
 				},
 				ExpectedPanicContains: "ERROR: model openrouter/auto does not exist",
@@ -730,13 +730,13 @@ func TestEvaluateExecute(t *testing.T) {
 					})
 				},
 				filepath.Join("result-directory", "evaluation.log"): func(t *testing.T, filePath, data string) {
-					assert.Contains(t, data, "Run 1/3")
-					assert.Contains(t, data, "Run 2/3")
-					assert.Contains(t, data, "Run 3/3")
+					assert.Contains(t, data, "\"msg\":\"starting run\",\"count\":1,\"total\":3}")
+					assert.Contains(t, data, "\"msg\":\"starting run\",\"count\":2,\"total\":3}")
+					assert.Contains(t, data, "\"msg\":\"starting run\",\"count\":3,\"total\":3}")
 				},
 				filepath.Join("result-directory", "README.md"): nil,
 				filepath.Join("result-directory", string(evaluatetask.IdentifierWriteTests), "symflower_symbolic-execution", "golang", "golang", "plain", "evaluation.log"): func(t *testing.T, filePath, data string) {
-					assert.Equal(t, 3, strings.Count(data, `Evaluating model "symflower/symbolic-execution"`))
+					assert.Equal(t, 3, strings.Count(data, "\"msg\":\"evaluating model\",\"model\":\"symflower/symbolic-execution\""))
 				},
 			},
 		})

evaluate/evaluate_test.go

@@ -394,7 +394,7 @@ func TestEvaluate(t *testing.T) {
 				ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
 					"evaluation.log": nil,
 					filepath.Join(string(evaluatetask.IdentifierWriteTests), log.CleanModelNameForFileSystem(mockedModelID), "golang", "golang", "plain", "evaluation.log"): func(t *testing.T, filePath, data string) {
-						assert.Contains(t, data, "Attempt 1/3: "+ErrEmptyResponseFromModel.Error())
+						assert.Contains(t, data, "\"msg\":\"query attempt\",\"count\":1,\"total\":3,\"error\":\""+ErrEmptyResponseFromModel.Error())
 					},
 					filepath.Join(string(evaluatetask.IdentifierWriteTests), log.CleanModelNameForFileSystem(mockedModelID), "golang", "golang", "plain", "response-1.log"): nil,
 					"evaluation.csv": nil,
@@ -1253,12 +1253,12 @@ func TestEvaluate(t *testing.T) {
 				},
 				ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
 					"evaluation.log": func(t *testing.T, filePath string, data string) {
-						assert.Contains(t, data, "Run 1/3")
-						assert.Contains(t, data, "Run 2/3")
-						assert.Contains(t, data, "Run 3/3")
-						assert.NotRegexp(t, `Run \d+/\d+ for model`, data)
+						assert.Contains(t, data, "\"msg\":\"starting run\",\"count\":1,\"total\":3}")
+						assert.Contains(t, data, "\"msg\":\"starting run\",\"count\":2,\"total\":3}")
+						assert.Contains(t, data, "\"msg\":\"starting run\",\"count\":3,\"total\":3}")
+						assert.NotRegexp(t, `\\\"msg\\\":\\\"starting run\\\",\\\"count\\\":\d+,\\\"total\\\":\d+,`, data)
 
-						assert.Equal(t, 1, strings.Count(data, "Creating temporary repository"), "create only one temporary repository")
+						assert.Equal(t, 1, strings.Count(data, "creating temporary repository"), "create only one temporary repository")
 					},
 					filepath.Join(string(evaluatetask.IdentifierWriteTests), log.CleanModelNameForFileSystem(mockedModelID), "golang", "golang", "plain", "evaluation.log"): nil,
 					"evaluation.csv": nil,
@@ -1431,13 +1431,13 @@ func TestEvaluate(t *testing.T) {
 				},
 				ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
 					"evaluation.log": func(t *testing.T, filePath string, data string) {
-						assert.Equal(t, 1, strings.Count(data, "Creating temporary repository"), "create only one temporary repository")
+						assert.Equal(t, 1, strings.Count(data, "creating temporary repository"), "create only one temporary repository")
 					},
 					filepath.Join(string(evaluatetask.IdentifierWriteTests), log.CleanModelNameForFileSystem(mockedModelID), "golang", "golang", "plain", "evaluation.log"): func(t *testing.T, filePath string, data string) {
-						assert.Contains(t, data, "Run 1/3 for model")
-						assert.Contains(t, data, "Run 2/3 for model")
-						assert.Contains(t, data, "Run 3/3 for model")
-						assert.NotRegexp(t, `Run \d+/\d+$`, data)
+						assert.Contains(t, data, "\"msg\":\"starting run\",\"count\":1,\"total\":3}")
+						assert.Contains(t, data, "\"msg\":\"starting run\",\"count\":2,\"total\":3}")
+						assert.Contains(t, data, "\"msg\":\"starting run\",\"count\":3,\"total\":3}")
+						assert.NotRegexp(t, `\\\"msg\\\":\\\"starting run\\\",\\\"count\\\":\d+,\\\"total\\\":\d+,`, data)
 					},
 					"evaluation.csv": nil,
 				},

evaluate/task/test-integration/task_test.go

@@ -85,10 +85,10 @@ func TestWriteTestsRun(t *testing.T) {
 			},
 		},
 		ValidateLog: func(t *testing.T, data string) {
-			assert.Contains(t, data, "Evaluating model \"symflower/symbolic-execution\"")
+			assert.Contains(t, data, "msg=\"evaluating model\" model=symflower/symbolic-execution")
 			assert.Contains(t, data, "Generated 1 test")
 			assert.Contains(t, data, "PASS: TestSymflowerPlain")
-			assert.Contains(t, data, "Evaluated model \"symflower/symbolic-execution\"")
+			assert.Contains(t, data, "msg=\"evaluated model\" model=symflower/symbolic-execution")
 		},
 	})
 }

evaluate/task/write-test_test.go

@@ -53,7 +53,7 @@ func TestWriteTestsRun(t *testing.T) {
 		// Generate invalid code for caseA (with and without template).
 		modelMock.RegisterGenerateSuccess(t, "caseA_test.go", "does not compile", metricstesting.AssessmentsWithProcessingTime).Twice()
 		// Generate valid code for caseB (with and without template).
-		modelMock.RegisterGenerateSuccess(t, "caseB_test.go", "package plain\n\nimport \"testing\"\n\nfunc TestCaseB(t *testing.T){}", metricstesting.AssessmentsWithProcessingTime).Twice()
+		modelMock.RegisterGenerateSuccess(t, "caseB_test.go", "package plain\n\nimport \\\"testing\\\"\n\nfunc TestCaseB(t *testing.T){}", metricstesting.AssessmentsWithProcessingTime).Twice()
 
 		validate(t, &tasktesting.TestCaseTask{
 			Name: "Plain",
@@ -93,7 +93,7 @@ func TestWriteTestsRun(t *testing.T) {
 				"exit status 1",                  // Symflower fix not applicable (overwrote template).
 			},
 			ValidateLog: func(t *testing.T, data string) {
-				assert.Equal(t, 1, strings.Count(data, "Evaluating model \"mocked-model\""))
+				assert.Equal(t, 1, strings.Count(data, "msg=\"evaluating model\" model=mocked-model"))
 				assert.Equal(t, 4, strings.Count(data, "PASS: TestCaseB")) // Bare model result, with fix, with template, with template and fix.
 			},
 		})
@@ -120,7 +120,7 @@ func TestWriteTestsRun(t *testing.T) {
 					ExpectedRepositoryAssessment: expectedAssessments,
 					ExpectedProblemContains:      expectedProblems,
 					ValidateLog: func(t *testing.T, data string) {
-						assert.Contains(t, data, "Evaluating model \"mocked-model\"")
+						assert.Contains(t, data, "msg=\"evaluating model\" model=mocked-model")
 						if assertTestsPass {
 							assert.Contains(t, data, "PASS: TestPlain")
 						}
@@ -295,7 +295,7 @@ func TestWriteTestsRun(t *testing.T) {
 			},
 			ExpectedProblemContains: nil,
 			ValidateLog: func(t *testing.T, data string) {
-				assert.Contains(t, data, "Evaluating model \"mocked-model\"")
+				assert.Contains(t, data, "msg=\"evaluating model\" model=mocked-model")
 			},
 		})
 	}
@@ -390,7 +390,7 @@ func TestWriteTestsRun(t *testing.T) {
 				IdentifierWriteTestsSymflowerTemplateSymflowerFix: metrics.Assessments{},
 			},
 			ValidateLog: func(t *testing.T, data string) {
-				assert.Contains(t, data, "Ignoring file \"plain.go\" (as configured by the repository)")
+				assert.Contains(t, data, "msg=\"ignoring file (as configured by the repository)\" path=plain.go")
 			},
 		})
 	}

tools/install_test.go

@@ -36,7 +36,7 @@ func ValidateInstallTool(t *testing.T, tool Tool) {
 		logOutput, logger := log.Buffer()
 		require.NoError(t, InstallTool(logger, tool, temporaryPath))
 
-		require.Contains(t, logOutput.String(), fmt.Sprintf(`Install %q to`, tool.ID()))
+		require.Contains(t, logOutput.String(), fmt.Sprintf(`msg="installing tool" tool=%s`, tool.ID()))
 		toolPath, err := exec.LookPath(tool.BinaryName())
 		require.NoError(t, err)
 		require.NotEmpty(t, toolPath)
@@ -46,7 +46,7 @@ func ValidateInstallTool(t *testing.T, tool Tool) {
 		logOutput, logger := log.Buffer()
 		require.NoError(t, InstallTool(logger, tool, temporaryPath))
 
-		require.NotContains(t, logOutput.String(), fmt.Sprintf(`Install %q to`, tool.ID()))
+		require.NotContains(t, logOutput.String(), fmt.Sprintf(`msg="installing tool" tool=%s`, tool.ID()))
 		toolPath, err := exec.LookPath(tool.BinaryName())
 		require.NoError(t, err)
 		require.NotEmpty(t, toolPath)