From b70e6b5640a7ca54b8a7603da1f0ceda9f037c4d Mon Sep 17 00:00:00 2001 From: Yilin Xu Date: Fri, 15 Oct 2021 22:00:42 -0400 Subject: [PATCH] test: add pi test A simple pi calculation example is added. The pi program can be deployed with default scheduler and kubeflux scheduler. Those can be used to test kubeflux scheduler. --- examples/pi/.gitignore | 2 + examples/pi/Dockerfile | 11 ++ examples/pi/Dockerfile.segfault | 11 ++ examples/pi/Makefile | 56 ++++++++ examples/pi/README.md | 81 +++++++++++ examples/pi/clean_pods.sh | 3 + examples/pi/demo_failed_pod_cancellation.sh | 6 + examples/pi/init_kind_cluster.sh | 44 ++++++ examples/pi/kubesched.yaml | 38 +++++ examples/pi/pi-job-default.yaml | 20 +++ examples/pi/pi-job-kubeflux-segfault.yaml | 21 +++ examples/pi/pi-job-kubeflux.yaml | 21 +++ examples/pi/pi.yaml | 9 ++ examples/pi/rbac.yaml | 152 ++++++++++++++++++++ 14 files changed, 475 insertions(+) create mode 100644 examples/pi/.gitignore create mode 100644 examples/pi/Dockerfile create mode 100644 examples/pi/Dockerfile.segfault create mode 100644 examples/pi/Makefile create mode 100644 examples/pi/README.md create mode 100755 examples/pi/clean_pods.sh create mode 100755 examples/pi/demo_failed_pod_cancellation.sh create mode 100755 examples/pi/init_kind_cluster.sh create mode 100644 examples/pi/kubesched.yaml create mode 100644 examples/pi/pi-job-default.yaml create mode 100644 examples/pi/pi-job-kubeflux-segfault.yaml create mode 100644 examples/pi/pi-job-kubeflux.yaml create mode 100644 examples/pi/pi.yaml create mode 100644 examples/pi/rbac.yaml diff --git a/examples/pi/.gitignore b/examples/pi/.gitignore new file mode 100644 index 0000000..ef469c2 --- /dev/null +++ b/examples/pi/.gitignore @@ -0,0 +1,2 @@ +logs/ +demo/ diff --git a/examples/pi/Dockerfile b/examples/pi/Dockerfile new file mode 100644 index 0000000..a8839c3 --- /dev/null +++ b/examples/pi/Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu:latest +MAINTAINER Eduardo Arango + +RUN apt-get update +RUN apt-get install -y bash wget build-essential gcc g++ git +RUN git clone https://github.com/ArangoGutierrez/Pi.git +WORKDIR Pi +RUN g++ -fopenmp pi.c -o pi +RUN cp pi /bin + +ENTRYPOINT ["pi","5000"] diff --git a/examples/pi/Dockerfile.segfault b/examples/pi/Dockerfile.segfault new file mode 100644 index 0000000..c686dc2 --- /dev/null +++ b/examples/pi/Dockerfile.segfault @@ -0,0 +1,11 @@ +FROM ubuntu:latest +MAINTAINER Eduardo Arango + +RUN apt-get update +RUN apt-get install -y bash wget build-essential gcc g++ git +RUN git clone https://github.com/ArangoGutierrez/Pi.git +WORKDIR Pi +RUN g++ -fopenmp pi.c -o pi +RUN cp pi /bin + +ENTRYPOINT ["pi"] diff --git a/examples/pi/Makefile b/examples/pi/Makefile new file mode 100644 index 0000000..9b2be42 --- /dev/null +++ b/examples/pi/Makefile @@ -0,0 +1,56 @@ +GITVERSION := $(shell git rev-parse --short HEAD) + +DOCKER_IMAGE_NAME=pi +DOCKER_IMAGE_TAG=${GITVERSION} + +DOCKER_IMAGE=${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} + +DOCKER_SEGFAULT_IMAGE_NAME=pi-segfault +DOCKER_SEGFAULT_IMAGE=${DOCKER_SEGFAULT_IMAGE_NAME}:${DOCKER_IMAGE_TAG} + +all:: build-image + +clean:: clean-image + +build-image: + @echo "Building Docker image..."; + docker build -t ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} . + docker tag ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} ${DOCKER_IMAGE_NAME}:latest + docker tag ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} localhost:5000/${DOCKER_IMAGE_NAME}:latest + +build-segfault-image: + @echo "Building Docker image (pi-segfault)..."; + docker build -t ${DOCKER_SEGFAULT_IMAGE_NAME}:${DOCKER_IMAGE_TAG} -f Dockerfile.segfault . + docker tag ${DOCKER_SEGFAULT_IMAGE_NAME}:${DOCKER_IMAGE_TAG} ${DOCKER_SEGFAULT_IMAGE_NAME}:latest + docker tag ${DOCKER_SEGFAULT_IMAGE_NAME}:${DOCKER_IMAGE_TAG} localhost:5000/${DOCKER_SEGFAULT_IMAGE_NAME}:latest + +push-image-local: + @echo "Pushing Docker image..."; + docker push localhost:5000/${DOCKER_IMAGE_NAME}:latest + +push-segfault-image-local: + @echo "Pushing Docker image (pi-segfault)..."; + docker push localhost:5000/${DOCKER_SEGFAULT_IMAGE_NAME}:latest + +push-image: + @echo "Pushing Docker image..."; + docker push ${DOCKER_IMAGE_NAME}:latest + +clean-image: + @echo "Cleaning Docker image..."; + docker rmi -f ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} ${DOCKER_IMAGE_NAME}:latest localhost:5000/${DOCKER_IMAGE_NAME}:latest + +setup-kubeflux: + kubectl create -f ./rbac.yaml + kubectl create -f ./kubesched.yaml + +reset-kubeflux: + kubectl delete -f ./kubesched.yaml + kubectl create -f ./kubesched.yaml + +check-kubeflux: + kubectl get pod -l app=kubeflux -n kube-system + +delete-kind: + kind delete cluster --name xyl-kind + diff --git a/examples/pi/README.md b/examples/pi/README.md new file mode 100644 index 0000000..244c7fa --- /dev/null +++ b/examples/pi/README.md @@ -0,0 +1,81 @@ +# A Minimum Working Example for Kubeflux + +- [source](https://stackoverflow.com/questions/42564058/how-to-use-local-docker-images-with-minikube) +- [Pi](https://github.com/ArangoGutierrez/Pi) +- [Multi-nodes solution](https://minikube.sigs.k8s.io/docs/handbook/registry/) +- [reference](https://hasura.io/blog/sharing-a-local-registry-for-minikube-37c7240d0615/) +## Setup + - local cluster: minikube + - Kubeflux: `kubesched.yaml` + +## Using local docker image in minikube (multi-node can not use this method) + +```bash +# Start minikube +minikube start --kuberbetes-version=1.19.8 + +# Set docker env +eval $(minikube docker-env) # unix shells +minikube docker-env | Invoke-Expression # PowerShell + +# Build image +docker build -t pi:latest . + +# Run in minikube +kubectl run hello-foo --image=pi:latest --restart=Never --image-pull-policy=Never + +# Check that it's running +kubectl get pods + +eval $(minikube docker-env -u) +``` + +## Pushing to an in-cluster using Registry addon + +- [sources](https://minikube.sigs.k8s.io/docs/handbook/pushing/) +- [refer](https://github.com/kubernetes/minikube/issues/6012) + +## Steps: + +```bash + # we can find plugin version by reading from logs command's output and find corresponding k8s version from https://github.com/cmisale/scheduler-plugins + $ minikube start --kubernetes-version=v1.19.8 + $ kubectl create -f rbac.yaml + $ kubectl create -f ./kubesched.yaml + # find correct pod name with command minikube kubectl -- get pods -A + $ kubectl logs -n kube-system kubeflux-plugin-57c56dd87f-9c5km + $ kubectl create -f pi.yaml + + $ POD=$(kubectl get pod -l app=kubeflux -n kube-system -o jsonpath="{.items[0].metadata.name}") + $ kubectl exec -it -n kube-system -- /bin/bash + + $ kubectl exec -it $(kubectl get pod -l app=kubeflux -n kube-system -o jsonpath="{.items[0].metadata.name}") -n kube-system -- /bin/bash + + $ kubectl cp kube-system/$(kubectl get pod -l app=kubeflux -n kube-system -o jsonpath="{.items[0].metadata.name}"):/home/data/jgf/kubecluster.json kubecluster_cp.json +``` + +### How to copy file from pods to local + + - [refer](https://stackoverflow.com/questions/52407277/how-to-copy-files-from-kubernetes-pods-to-local-system) + + ```bash + kubectl cp kube-system/kubeflux-plugin-7b5fb86f5f-8wqq2:/home/data/jgf/kubecluster.json kubecluster_before.json + ``` + + ### On CPU limit + + - [refer](https://learnk8s.io/setting-cpu-memory-limits-requests) + + + ### Resource query + + - [refer](https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_4.html) + + ### Kind local registry + + - [refer](https://kind.sigs.k8s.io/docs/user/local-registry/) + + + ### Kind cluster with metrics server + + - [refer](https://www.scmgalaxy.com/tutorials/kubernetes-metrics-server-error-readiness-probe-failed-http-probe-failed-with-statuscode/) \ No newline at end of file diff --git a/examples/pi/clean_pods.sh b/examples/pi/clean_pods.sh new file mode 100755 index 0000000..130f726 --- /dev/null +++ b/examples/pi/clean_pods.sh @@ -0,0 +1,3 @@ +#!/bin/sh +kubectl delete -f ./pi-job-kubeflux-segfault.yaml +kubectl delete -f ./pi-job-kubeflux.yaml diff --git a/examples/pi/demo_failed_pod_cancellation.sh b/examples/pi/demo_failed_pod_cancellation.sh new file mode 100755 index 0000000..4996bb8 --- /dev/null +++ b/examples/pi/demo_failed_pod_cancellation.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +kubectl create -f ./pi-job-kubeflux-segfault.yaml +echo "wait 5 seconds" +sleep 5 +kubectl create -f ./pi-job-kubeflux.yaml diff --git a/examples/pi/init_kind_cluster.sh b/examples/pi/init_kind_cluster.sh new file mode 100755 index 0000000..ff65662 --- /dev/null +++ b/examples/pi/init_kind_cluster.sh @@ -0,0 +1,44 @@ +#!/bin/sh +set -o errexit + +# create registry container unless it already exists +reg_name='kind-registry' +reg_port='5000' +running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" +if [ "${running}" != 'true' ]; then + docker run \ + -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \ + registry:2 +fi + +# create a cluster with the local registry enabled in containerd +cluster_name=${USER}-kind +cat <