diff --git a/README.md b/README.md index da7d7bca..e9ac2a20 100644 --- a/README.md +++ b/README.md @@ -151,11 +151,11 @@ Ensure that docker is installed on the system. ### Build or pull the image ```bash -docker build . -t eval-dev-quality +docker build . -t eval-dev-quality:dev ``` ```bash -docker pull ghcr.io/symflower/eval-dev-quality:latest +docker pull ghcr.io/symflower/eval-dev-quality:main ``` ### Run the evaluation either with the built or pulled image @@ -163,11 +163,12 @@ docker pull ghcr.io/symflower/eval-dev-quality:latest The following command will run the model `symflower/symbolic-execution` and stores the the results of the run inside the local directory `evaluation`. ```bash -docker run -v ./:/home/ubuntu/evaluation --user $(id -u):$(id -g) eval-dev-quality:latest eval-dev-quality evaluate --model symflower/symbolic-execution --result-path /home/ubuntu/evaluation/%datetime% +eval-dev-quality evaluate --runtime docker --runtime-image eval-dev-quality:dev --model symflower/symbolic-execution ``` +Omitting the `--runtime-image` parameter will default to the image from the `main` branch. `ghcr.io/symflower/eval-dev-quality:main` ```bash -docker run -v ./:/home/ubuntu/evaluation --user $(id -u):$(id -g) ghcr.io/symflower/eval-dev-quality:latest eval-dev-quality evaluate --model symflower/symbolic-execution --result-path /home/ubuntu/evaluation/%datetime% +eval-dev-quality evaluate --runtime docker --model symflower/symbolic-execution ``` ## Kubernetes diff --git a/cmd/eval-dev-quality/cmd/evaluate.go b/cmd/eval-dev-quality/cmd/evaluate.go index cbac8a1d..a446238a 100644 --- a/cmd/eval-dev-quality/cmd/evaluate.go +++ b/cmd/eval-dev-quality/cmd/evaluate.go @@ -596,7 +596,7 @@ func (command *Evaluate) evaluateDocker(ctx *evaluate.Context) (err error) { // evaluateKubernetes executes the evaluation for each model inside a kubernetes run container. func (command *Evaluate) evaluateKubernetes(ctx *evaluate.Context) (err error) { - tmpl, err := template.ParseFiles(filepath.Join("conf", "kubernetes-job.yml")) + jobTmpl, err := template.ParseFiles(filepath.Join("conf", "kube", "job.yml")) if err != nil { return pkgerrors.Wrap(err, "could not create kubernetes job template") } @@ -661,7 +661,7 @@ func (command *Evaluate) evaluateKubernetes(ctx *evaluate.Context) (err error) { parallel.Execute(func() { var tmplData bytes.Buffer - tmpl.Execute(&tmplData, data) + jobTmpl.Execute(&tmplData, data) commandOutput, err := util.CommandWithResult(context.Background(), command.logger, &util.Command{ Command: kubeCommand, @@ -710,5 +710,82 @@ func (command *Evaluate) evaluateKubernetes(ctx *evaluate.Context) (err error) { } parallel.Wait() + // Copy data from volume back to host. + { + storageTmpl, err := template.ParseFiles(filepath.Join("conf", "kube", "storage-access.yml")) + if err != nil { + return pkgerrors.Wrap(err, "could not create kubernetes storage access template") + } + + data := map[string]string{ + "name": "eval-storage-access", + "namespace": command.Namespace, + } + + var tmplData bytes.Buffer + storageTmpl.Execute(&tmplData, data) + + // Create the storage access pod. + output, err := util.CommandWithResult(context.Background(), command.logger, &util.Command{ + Command: []string{ + "kubectl", + "apply", + "-f", + "-", // apply STDIN + }, + Stdin: tmplData.String(), + }) + if err != nil { + return pkgerrors.WithMessage(pkgerrors.WithStack(err), output) + } + + // Fetch the container name. + output, err = util.CommandWithResult(context.Background(), command.logger, &util.Command{ + Command: []string{ + "kubectl", + "get", + "pods", + "--namespace", command.Namespace, + "-l", "app=eval-storage-access", + "-o", "custom-columns=:metadata.name", + }, + Stdin: tmplData.String(), + }) + if err != nil { + return pkgerrors.WithMessage(pkgerrors.WithStack(err), output) + } + podName := strings.TrimSpace(output) + + // Copy data from volume to filesystem. + output, err = util.CommandWithResult(context.Background(), command.logger, &util.Command{ + Command: []string{ + "kubectl", + "cp", + "--namespace", command.Namespace, + command.Namespace + "/" + podName + ":/var/evaluations/.", + command.ResultPath, + }, + }) + if err != nil { + return pkgerrors.WithMessage(pkgerrors.WithStack(err), output) + } + + // Remove the data from the cluster volume + output, err = util.CommandWithResult(context.Background(), command.logger, &util.Command{ + Command: []string{ + "kubectl", + "exec", + "--namespace", command.Namespace, + podName, + "--", + "sh", "-c", + "rm -rf /var/evaluations/*", + }, + }) + if err != nil { + return pkgerrors.WithMessage(pkgerrors.WithStack(err), output) + } + } + return nil } diff --git a/conf/kubernetes-job.yml b/conf/kube/job.yml similarity index 100% rename from conf/kubernetes-job.yml rename to conf/kube/job.yml diff --git a/docs/kubernetes/connect.yml b/conf/kube/storage-access.yml similarity index 79% rename from docs/kubernetes/connect.yml rename to conf/kube/storage-access.yml index 939a23a2..d6a6e50f 100644 --- a/docs/kubernetes/connect.yml +++ b/conf/kube/storage-access.yml @@ -1,16 +1,16 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: evaluation-storage-access - namespace: eval-dev-quality + name: {{.name}} + namespace: {{.namespace}} spec: selector: matchLabels: - app: eval-dev-quality-storage + app: eval-storage-access template: metadata: labels: - app: eval-dev-quality-storage + app: eval-storage-access spec: containers: - name: storage-access diff --git a/docs/kubernetes/README.md b/docs/kubernetes/README.md index d980a19e..fc928312 100644 --- a/docs/kubernetes/README.md +++ b/docs/kubernetes/README.md @@ -1,35 +1,19 @@ # Running "eval-dev-quality" on Kubernetes - ### Prerequisite Checklist - `kubectl` installed and configured with authentication to the cluster. - Dedicated Namespace to run the jobs. - RWX volume to store the evaluation results (check `volume.yml` for inspiration). -### Running a evaluation without the eval-dev-quality kubernetes runtime - -- Change the `command` in `job.yml` to the desired command. -- Run it with `kubectl --namespace $NAMESPACE apply -f job.yml`. -- Check the job with `kubectl get pods --namespace $NAMESPACE` until status shows `completed`. -- Remove the job with `kubectl --namespace $NAMESPACE delete --force -f job.yml`. -- [Getting the evaluation data](#getting-the-evaluation-data) - ### Running multiple evaluations with the eval-dev-quality kubernetes runtime - Define all the models with `--model` which should be run inside the containerized workload. - Define the parameter `--runtime kubernetes` to indicate that jobs should run inside a kubernetes cluster. - Define the parameter `--parallel 20` to indicate how many jobs should run simultaneously. -- [Getting the evaluation data](#getting-the-evaluation-data) Example: ```bash eval-dev-quality evaluate --runtime kubernetes --runs 5 --model symflower/symbolic-execution --model symflower/symbolic-execution --model symflower/symbolic-execution --repository golang/plain --parallel 2 ``` This commands run 3x the `symflower/symbolic-execution` model with 5 runs of each model inside a containerized workload on the kubernetes cluster, it will limit the parallel execution to 2 containers at the same time. - -### Getting the evaluation data - -- Run `kubectl --namespace $NAMESPACE apply -f connect.yml` to start a busybox container with the PVC mount. -- Use `kubectl cp eval-dev-quality/evaluation-storage-access-XXXXXXX:/var/evaluations ./evaluations` to copy all evaluations to a local folder. -- Delete the pod with `kubectl --namespace $NAMESPACE delete --force -f connect.yml` when not needed anymore. diff --git a/docs/kubernetes/job.yml b/docs/kubernetes/job.yml deleted file mode 100644 index e4520395..00000000 --- a/docs/kubernetes/job.yml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: eval-dev-quality - namespace: eval-dev-quality -spec: - template: - spec: - containers: - - name: eval-dev-quality - image: ghcr.io/symflower/eval-dev-quality:docker-image - command: ["eval-dev-quality", "evaluate", "--model", "symflower/symbolic-execution", "--result-path","/var/evaluations/%datetime%"] - volumeMounts: - - mountPath: "/var/evaluations" - name: evaluations - securityContext: - fsGroup: 1000 - restartPolicy: Never - volumes: - - name: evaluations - persistentVolumeClaim: - claimName: eval-dev-quality - backoffLimit: 1