Copy cluster data and documentation update

README.md

@@ -151,23 +151,24 @@ Ensure that docker is installed on the system.
 ### Build or pull the image
 
 ```bash
-docker build . -t eval-dev-quality
+docker build . -t eval-dev-quality:dev
 ```
 
 ```bash
-docker pull ghcr.io/symflower/eval-dev-quality:latest
+docker pull ghcr.io/symflower/eval-dev-quality:main
 ```
 
 ### Run the evaluation either with the built or pulled image
 
 The following command will run the model `symflower/symbolic-execution` and stores the the results of the run inside the local directory `evaluation`.
 
 ```bash
-docker run -v ./:/home/ubuntu/evaluation --user $(id -u):$(id -g) eval-dev-quality:latest eval-dev-quality evaluate --model symflower/symbolic-execution --result-path /home/ubuntu/evaluation/%datetime%
+eval-dev-quality evaluate --runtime docker --runtime-image eval-dev-quality:dev --model symflower/symbolic-execution
 ```
 
+Omitting the `--runtime-image` parameter will default to the image from the `main` branch. `ghcr.io/symflower/eval-dev-quality:main`
 ```bash
-docker run -v ./:/home/ubuntu/evaluation --user $(id -u):$(id -g) ghcr.io/symflower/eval-dev-quality:latest eval-dev-quality evaluate --model symflower/symbolic-execution --result-path /home/ubuntu/evaluation/%datetime%
+eval-dev-quality evaluate --runtime docker --model symflower/symbolic-execution
 ```
 
 ## Kubernetes

cmd/eval-dev-quality/cmd/evaluate.go

@@ -596,7 +596,7 @@ func (command *Evaluate) evaluateDocker(ctx *evaluate.Context) (err error) {
 
 // evaluateKubernetes executes the evaluation for each model inside a kubernetes run container.
 func (command *Evaluate) evaluateKubernetes(ctx *evaluate.Context) (err error) {
-	tmpl, err := template.ParseFiles(filepath.Join("conf", "kubernetes-job.yml"))
+	jobTmpl, err := template.ParseFiles(filepath.Join("conf", "kube", "job.yml"))
 	if err != nil {
 		return pkgerrors.Wrap(err, "could not create kubernetes job template")
 	}
@@ -661,7 +661,7 @@ func (command *Evaluate) evaluateKubernetes(ctx *evaluate.Context) (err error) {
 
 		parallel.Execute(func() {
 			var tmplData bytes.Buffer
-			tmpl.Execute(&tmplData, data)
+			jobTmpl.Execute(&tmplData, data)
 
 			commandOutput, err := util.CommandWithResult(context.Background(), command.logger, &util.Command{
 				Command: kubeCommand,
@@ -710,5 +710,82 @@ func (command *Evaluate) evaluateKubernetes(ctx *evaluate.Context) (err error) {
 	}
 	parallel.Wait()
 
+	// Copy data from volume back to host.
+	{
+		storageTmpl, err := template.ParseFiles(filepath.Join("conf", "kube", "storage-access.yml"))
+		if err != nil {
+			return pkgerrors.Wrap(err, "could not create kubernetes storage access template")
+		}
+
+		data := map[string]string{
+			"name":      "eval-storage-access",
+			"namespace": command.Namespace,
+		}
+
+		var tmplData bytes.Buffer
+		storageTmpl.Execute(&tmplData, data)
+
+		// Create the storage access pod.
+		output, err := util.CommandWithResult(context.Background(), command.logger, &util.Command{
+			Command: []string{
+				"kubectl",
+				"apply",
+				"-f",
+				"-", // apply STDIN
+			},
+			Stdin: tmplData.String(),
+		})
+		if err != nil {
+			return pkgerrors.WithMessage(pkgerrors.WithStack(err), output)
+		}
+
+		// Fetch the container name.
+		output, err = util.CommandWithResult(context.Background(), command.logger, &util.Command{
+			Command: []string{
+				"kubectl",
+				"get",
+				"pods",
+				"--namespace", command.Namespace,
+				"-l", "app=eval-storage-access",
+				"-o", "custom-columns=:metadata.name",
+			},
+			Stdin: tmplData.String(),
+		})
+		if err != nil {
+			return pkgerrors.WithMessage(pkgerrors.WithStack(err), output)
+		}
+		podName := strings.TrimSpace(output)
+
+		// Copy data from volume to filesystem.
+		output, err = util.CommandWithResult(context.Background(), command.logger, &util.Command{
+			Command: []string{
+				"kubectl",
+				"cp",
+				"--namespace", command.Namespace,
+				command.Namespace + "/" + podName + ":/var/evaluations/.",
+				command.ResultPath,
+			},
+		})
+		if err != nil {
+			return pkgerrors.WithMessage(pkgerrors.WithStack(err), output)
+		}
+
+		// Remove the data from the cluster volume
+		output, err = util.CommandWithResult(context.Background(), command.logger, &util.Command{
+			Command: []string{
+				"kubectl",
+				"exec",
+				"--namespace", command.Namespace,
+				podName,
+				"--",
+				"sh", "-c",
+				"rm -rf /var/evaluations/*",
+			},
+		})
+		if err != nil {
+			return pkgerrors.WithMessage(pkgerrors.WithStack(err), output)
+		}
+	}
+
 	return nil
 }

docs/kubernetes/connect.yml → conf/kube/storage-access.yml

@@ -1,16 +1,16 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: evaluation-storage-access
-  namespace: eval-dev-quality
+  name: {{.name}}
+  namespace: {{.namespace}}
 spec:
   selector:
     matchLabels:
-      app: eval-dev-quality-storage
+      app: eval-storage-access
   template:
     metadata:
       labels:
-        app: eval-dev-quality-storage
+        app: eval-storage-access
     spec:
       containers:
       - name: storage-access

docs/kubernetes/README.md

@@ -1,35 +1,19 @@
 # Running "eval-dev-quality" on Kubernetes
 
-
 ### Prerequisite Checklist
 
 - `kubectl` installed and configured with authentication to the cluster.
 - Dedicated Namespace to run the jobs.
 - RWX volume to store the evaluation results (check `volume.yml` for inspiration).
 
-### Running a evaluation without the eval-dev-quality kubernetes runtime
-
-- Change the `command` in `job.yml` to the desired command.
-- Run it with `kubectl --namespace $NAMESPACE apply -f job.yml`.
-- Check the job with `kubectl get pods --namespace $NAMESPACE` until status shows `completed`.
-- Remove the job with `kubectl --namespace $NAMESPACE delete --force -f job.yml`.
-- [Getting the evaluation data](#getting-the-evaluation-data)
-
 ### Running multiple evaluations with the eval-dev-quality kubernetes runtime
 
 - Define all the models with `--model` which should be run inside the containerized workload.
 - Define the parameter `--runtime kubernetes` to indicate that jobs should run inside a kubernetes cluster.
 - Define the parameter `--parallel 20` to indicate how many jobs should run simultaneously.
-- [Getting the evaluation data](#getting-the-evaluation-data)
 
 Example:
 ```bash
 eval-dev-quality evaluate --runtime kubernetes --runs 5 --model symflower/symbolic-execution --model symflower/symbolic-execution --model symflower/symbolic-execution --repository golang/plain --parallel 2
 ```
 This commands run 3x the `symflower/symbolic-execution` model with 5 runs of each model inside a containerized workload on the kubernetes cluster, it will limit the parallel execution to 2 containers at the same time.
-
-### Getting the evaluation data
-
-- Run `kubectl --namespace $NAMESPACE apply -f connect.yml` to start a busybox container with the PVC mount.
-- Use `kubectl cp eval-dev-quality/evaluation-storage-access-XXXXXXX:/var/evaluations ./evaluations` to copy all evaluations to a local folder.
-- Delete the pod with `kubectl --namespace $NAMESPACE delete --force -f connect.yml` when not needed anymore.