Merge pull request #234 from symflower/docker-runtime-follow-up

Docker runtime follow up

.github/workflows/test.yml

@@ -22,7 +22,7 @@ jobs:
           java-version: '11'
 
       - name: Set up Maven
-        uses: stCarolas/setup-maven@v4.5
+        uses: stCarolas/setup-maven@v5
         with:
           maven-version: '3.9.1'
 
@@ -36,6 +36,11 @@ jobs:
         run: make install
         shell: bash # Explicitly use Bash because otherwise failing Windows jobs are not erroring.
 
+      - name: Install required tools
+        run: |
+          make install-tools
+        shell: bash # Explicitly use Bash because otherwise failing Windows jobs are not erroring.
+
       - name: Install testing tools
         run: |
           make install-tools-testing

Dockerfile

@@ -1,40 +1,62 @@
+# Builder image.
+FROM golang:latest as builder
+
+WORKDIR /app
+COPY ./ ./
+
+# Build the binary.
+RUN go mod tidy
+RUN CGO_ENABLED=0 go build -o eval-dev-quality ./cmd/eval-dev-quality
+
+# Actual running image.
 FROM ubuntu:noble
 RUN apt-get update && apt-get install -y ca-certificates wget unzip git make && update-ca-certificates
 
-WORKDIR /home/ubuntu/eval-dev-quality
-COPY ./ ./
-RUN chown -R ubuntu:ubuntu ./
+# Non-root ollama need a hardcoded directory to store ssh-key.
+RUN mkdir -p /.ollama && chmod 777 /.ollama
+# Same for symflower
+RUN mkdir -p /.config && chmod 777 /.config
+RUN mkdir -p /.eval-dev-quality && chmod 777 /.eval-dev-quality
+RUN mkdir -p /.cache && chmod 777 /.cache
 
+# Switch to the ubuntu user as we want it to run as non-root.
 USER ubuntu
-RUN mkdir -p ~/.eval-dev-quality
+WORKDIR /app
+COPY --chown=ubuntu:ubuntu ./testdata ./testdata
+COPY --chown=ubuntu:ubuntu ./Makefile ./Makefile
+RUN mkdir -p .eval-dev-quality
 
 # Install Maven
 RUN wget https://archive.apache.org/dist/maven/maven-3/3.9.1/binaries/apache-maven-3.9.1-bin.tar.gz && \
-	tar -xf apache-maven-3.9.1-bin.tar.gz -C ~/.eval-dev-quality/ && \
+	tar -xf apache-maven-3.9.1-bin.tar.gz -C /app/.eval-dev-quality/ && \
 	rm apache-maven-3.9.1-bin.tar.gz
-ENV PATH="${PATH}:/home/ubuntu/.eval-dev-quality/apache-maven-3.9.1/bin"
+ENV PATH="${PATH}:/app/.eval-dev-quality/apache-maven-3.9.1/bin"
 
 # Install Gradle
 RUN wget https://services.gradle.org/distributions/gradle-8.0.2-bin.zip && \
-	unzip gradle-8.0.2-bin.zip -d ~/.eval-dev-quality/ && \
+	unzip gradle-8.0.2-bin.zip -d /app/.eval-dev-quality/ && \
 	rm gradle-8.0.2-bin.zip
-ENV PATH="${PATH}:/home/ubuntu/.eval-dev-quality/gradle-8.0.2/bin"
+ENV PATH="${PATH}:/app/.eval-dev-quality/gradle-8.0.2/bin"
 
 # Install Java
 RUN wget https://corretto.aws/downloads/latest/amazon-corretto-11-x64-linux-jdk.tar.gz && \
-	tar -xf amazon-corretto-11-x64-linux-jdk.tar.gz -C ~/.eval-dev-quality/ && \
+	tar -xf amazon-corretto-11-x64-linux-jdk.tar.gz -C /app/.eval-dev-quality/ && \
 	rm amazon-corretto-11-x64-linux-jdk.tar.gz
-ENV JAVA_HOME="/home/ubuntu/.eval-dev-quality/amazon-corretto-11.0.23.9.1-linux-x64"
+ENV JAVA_HOME="/app/.eval-dev-quality/amazon-corretto-11.0.23.9.1-linux-x64"
 ENV PATH="${PATH}:${JAVA_HOME}/bin"
 
 # Install Go
 RUN wget https://go.dev/dl/go1.21.5.linux-amd64.tar.gz && \
-	tar -xf go1.21.5.linux-amd64.tar.gz -C ~/.eval-dev-quality/ && \
+	tar -xf go1.21.5.linux-amd64.tar.gz -C /app/.eval-dev-quality/ && \
 	rm go1.21.5.linux-amd64.tar.gz
-ENV PATH="${PATH}:/home/ubuntu/.eval-dev-quality/go/bin"
-ENV PATH="${PATH}:/home/ubuntu/go/bin"
+ENV PATH="${PATH}:/app/.eval-dev-quality/go/bin"
+RUN go env -w GOPATH=/app/.eval-dev-quality/go
 
-# Setup the evaluation
+# Install the binary
+COPY --from=builder --chown=ubuntu:ubuntu /app/eval-dev-quality /app/.eval-dev-quality/bin/
+ENV PATH="${PATH}:/app/.eval-dev-quality/bin"
+RUN make install-tools-testing
+RUN make install-tools /app/.eval-dev-quality/bin
 
-RUN make install-all
-ENV PATH="${PATH}:/home/ubuntu/.eval-dev-quality/bin"
+# CHMOD everything because of non-root user ids
+RUN chmod -R 777 /app

Makefile

@@ -1,7 +1,7 @@
 export ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
 export PACKAGE_BASE := github.com/symflower/eval-dev-quality
-export UNIT_TEST_TIMEOUT := 480
+export UNIT_TEST_TIMEOUT := 720
 
 ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
 $(eval $(ARGS):;@:) # turn arguments into do-nothing targets
@@ -44,13 +44,16 @@ install: # [<Go package] - # Build and install everything, or only the specified
 	go install -v $(PACKAGE)
 .PHONY: install
 
-install-all: install install-tools-testing # Install everything for and of this repository.
+install-all: install install-tools install-tools-testing # Install everything for and of this repository.
 .PHONY: install-all
 
+install-tools: # Install tools that are required for running the evaluation.
+	eval-dev-quality install-tools $(if $(ARGS), --install-tools-path $(word 1,$(ARGS)))
+.PHONY: install-tools
+
 install-tools-testing: # Install tools that are used for testing.
 	go install -v github.com/vektra/mockery/[email protected]
 	go install -v gotest.tools/[email protected]
-	eval-dev-quality install-tools
 .PHONY: install-tools-testing
 
 generate: # Run code generation.

README.md

@@ -135,9 +135,16 @@ Total coverage 100.000000%
 
 The execution by default also creates an report file `REPORT.md` that contains additional evaluation results and links to individual result files.
 
-# Docker
+# Containerized use
 
-## Setup
+## Notes
+
+The following parameters do have a special behavior when using a containerized runtime.
+- `--testdata`: The check if the path exists is ignored on the host system but still enforced inside the container because the paths of the host and inside the container might differ.
+
+## Docker
+
+### Setup
 
 Ensure that docker is installed on the system.
 
@@ -158,8 +165,7 @@ docker run -v ./:/home/ubuntu/evaluation --user $(id -u):$(id -g) eval-dev-quali
 docker run -v ./:/home/ubuntu/evaluation --user $(id -u):$(id -g) ghcr.io/symflower/eval-dev-quality:latest eval-dev-quality evaluate --model symflower/symbolic-execution --result-path /home/ubuntu/evaluation/%datetime%
 ```
 
-
-# Kubernetes
+## Kubernetes
 
 Please check the [Kubernetes](./docs/kubernetes/README.md) documentation.
 

cmd/eval-dev-quality/cmd/command.go

@@ -31,6 +31,10 @@ func Execute(logger *log.Logger, arguments []string) {
 			c.SetLogger(logger)
 		}
 
+		if c, ok := command.(SetArguments); ok {
+			c.SetArguments(arguments)
+		}
+
 		return command.Execute(args)
 	}
 
@@ -51,3 +55,9 @@ type SetLogger interface {
 	// SetLogger sets the logger of the command.
 	SetLogger(logger *log.Logger)
 }
+
+// SetArguments defines a command that allows to set its arguments.
+type SetArguments interface {
+	// SetArguments sets the commands arguments.
+	SetArguments(args []string)
+}

cmd/eval-dev-quality/cmd/evaluate.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"os/user"
 	"path/filepath"
 	"slices"
 	"sort"
@@ -82,6 +83,8 @@ type Evaluate struct {
 	// Namespace the namespace under which the kubernetes resources should be created.
 	Namespace string `long:"namespace" description:"The Namespace which should be used for kubernetes resources." default:"eval-dev-quality"`
 
+	// args holds a list of all the passed arguments.
+	args []string
 	// logger holds the logger of the command.
 	logger *log.Logger
 	// timestamp holds the timestamp of the command execution.
@@ -95,6 +98,14 @@ func (command *Evaluate) SetLogger(logger *log.Logger) {
 	command.logger = logger
 }
 
+var _ SetArguments = (*Evaluate)(nil)
+
+// SetArguments sets the commands arguments.
+func (command *Evaluate) SetArguments(args []string) {
+	availableFlags := util.Flags(command)
+	command.args = util.FilterArgsKeep(args, availableFlags)
+}
+
 // Initialize initializes the command according to the arguments.
 func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.Context, cleanup func()) {
 	// Ensure the cleanup always runs in case there is a panic.
@@ -172,15 +183,17 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.
 
 	// Ensure the "testdata" path exists and make it absolute.
 	{
-		if err := osutil.DirExists(command.TestdataPath); err != nil {
-			command.logger.Panicf("ERROR: testdata path %q cannot be accessed: %s", command.TestdataPath, err)
-		}
-		testdataPath, err := filepath.Abs(command.TestdataPath)
-		if err != nil {
-			command.logger.Panicf("ERROR: could not resolve testdata path %q to an absolute path: %s", command.TestdataPath, err)
+		if command.Runtime == "local" { // Ignore testdata path during containerized execution.
+			if err := osutil.DirExists(command.TestdataPath); err != nil {
+				command.logger.Panicf("ERROR: testdata path %q cannot be accessed: %s", command.TestdataPath, err)
+			}
+			testdataPath, err := filepath.Abs(command.TestdataPath)
+			if err != nil {
+				command.logger.Panicf("ERROR: could not resolve testdata path %q to an absolute path: %s", command.TestdataPath, err)
+			}
+			command.TestdataPath = testdataPath
+			evaluationContext.TestdataPath = testdataPath
 		}
-		command.TestdataPath = testdataPath
-		evaluationContext.TestdataPath = testdataPath
 	}
 
 	// Setup evaluation result directory.
@@ -446,18 +459,31 @@ func (command *Evaluate) evaluateLocal(evaluationContext *evaluate.Context) (err
 
 // evaluateDocker executes the evaluation for each model inside a docker container.
 func (command *Evaluate) evaluateDocker(ctx *evaluate.Context) (err error) {
-	availableFlags := util.Flags(command)
 	ignoredFlags := []string{
 		"model",
 		"parallel",
 		"result-path",
+		"runtime-image",
 		"runtime",
 	}
 
-	// Filter all the args to only contain flags which can be used.
-	args := util.FilterArgsKeep(os.Args[2:], availableFlags)
 	// Filter the args to remove all flags unsuited for running the container.
-	args = util.FilterArgsRemove(args, ignoredFlags)
+	args := util.FilterArgsRemove(command.args, ignoredFlags)
+
+	// Get current user for volume ID mapping.
+	user, err := user.Current()
+	if err != nil {
+		return pkgerrors.WithStack(err)
+	}
+
+	resultPath, err := filepath.Abs(command.ResultPath)
+	if err != nil {
+		return pkgerrors.WithStack(err)
+	}
+	// Set permission 777 so the non-root docker image is able to store its results inside the result path.
+	if err := os.Chmod(resultPath, 0777); err != nil {
+		return pkgerrors.WithStack(err)
+	}
 
 	parallel := util.NewParallel(command.Parallel)
 
@@ -470,22 +496,14 @@ func (command *Evaluate) evaluateDocker(ctx *evaluate.Context) (err error) {
 			continue
 		}
 
-		// Create for each model a dedicated subfolder inside the results path.
-		resultPath, err := filepath.Abs(command.ResultPath)
-		if err != nil {
-			return err
-		}
-		// Set permission 777 so the non-root docker image is able to store its results inside the result path.
-		if err := os.Chmod(resultPath, 0777); err != nil {
-			return err
-		}
-
 		// Commands regarding the docker runtime.
 		dockerCommand := []string{
 			"docker",
 			"run",
 			"-v", // bind volume
-			resultPath + ":/home/ubuntu/evaluation",
+			resultPath + ":/app/evaluation",
+			"--user",
+			user.Uid + ":" + user.Gid,
 			"--rm", // automatically remove container after it finished
 			command.RuntimeImage,
 		}
@@ -497,7 +515,7 @@ func (command *Evaluate) evaluateDocker(ctx *evaluate.Context) (err error) {
 			"--model",
 			model.ID(),
 			"--result-path",
-			"/home/ubuntu/evaluation/" + model.ID(),
+			"/app/evaluation/" + model.ID(),
 		}
 
 		cmd := append(dockerCommand, evaluationCommand...)
@@ -531,6 +549,7 @@ func (command *Evaluate) evaluateKubernetes(ctx *evaluate.Context) (err error) {
 		"model",
 		"parallel",
 		"result-path",
+		"runtime-image",
 		"runtime",
 	}