From bfbc88f2cae427bb804870a71d46d58569512bd6 Mon Sep 17 00:00:00 2001 From: Brent Gardner Date: Wed, 5 Oct 2022 14:55:47 -0600 Subject: [PATCH 01/12] Add helm chart --- helm/ballista/.gitignore | 3 + helm/ballista/.helmignore | 23 ++++ helm/ballista/Chart.lock | 6 + helm/ballista/Chart.yaml | 29 +++++ helm/ballista/templates/NOTES.txt | 22 ++++ helm/ballista/templates/_helpers.tpl | 62 +++++++++++ helm/ballista/templates/executor.yaml | 89 +++++++++++++++ helm/ballista/templates/hpa.yaml | 28 +++++ helm/ballista/templates/ingress.yaml | 61 ++++++++++ helm/ballista/templates/scheduler.yaml | 88 +++++++++++++++ helm/ballista/templates/serviceaccount.yaml | 12 ++ .../templates/tests/test-connection.yaml | 15 +++ helm/ballista/values.yaml | 104 ++++++++++++++++++ 13 files changed, 542 insertions(+) create mode 100644 helm/ballista/.gitignore create mode 100644 helm/ballista/.helmignore create mode 100644 helm/ballista/Chart.lock create mode 100644 helm/ballista/Chart.yaml create mode 100644 helm/ballista/templates/NOTES.txt create mode 100644 helm/ballista/templates/_helpers.tpl create mode 100644 helm/ballista/templates/executor.yaml create mode 100644 helm/ballista/templates/hpa.yaml create mode 100644 helm/ballista/templates/ingress.yaml create mode 100644 helm/ballista/templates/scheduler.yaml create mode 100644 helm/ballista/templates/serviceaccount.yaml create mode 100644 helm/ballista/templates/tests/test-connection.yaml create mode 100644 helm/ballista/values.yaml diff --git a/helm/ballista/.gitignore b/helm/ballista/.gitignore new file mode 100644 index 000000000..f263fee85 --- /dev/null +++ b/helm/ballista/.gitignore @@ -0,0 +1,3 @@ +*.tar.gz +*.tgz + diff --git a/helm/ballista/.helmignore b/helm/ballista/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/helm/ballista/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/ballista/Chart.lock b/helm/ballista/Chart.lock new file mode 100644 index 000000000..f322a648d --- /dev/null +++ b/helm/ballista/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: minio + repository: https://charts.bitnami.com/bitnami + version: 11.10.2 +digest: sha256:0135f0f1c4e0ecf5b539210e1780d81d20d537ef008737b5f854e62f8c786d89 +generated: "2022-09-16T12:20:09.802584178-06:00" diff --git a/helm/ballista/Chart.yaml b/helm/ballista/Chart.yaml new file mode 100644 index 000000000..e7b5d4589 --- /dev/null +++ b/helm/ballista/Chart.yaml @@ -0,0 +1,29 @@ +apiVersion: v2 +name: ballista +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" + +dependencies: + - name: minio + version: 11.10.x + repository: https://charts.bitnami.com/bitnami diff --git a/helm/ballista/templates/NOTES.txt b/helm/ballista/templates/NOTES.txt new file mode 100644 index 000000000..404805d8c --- /dev/null +++ b/helm/ballista/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.scheduler.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "ballista.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.scheduler.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "ballista.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "ballista.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.scheduler.port }} +{{- else if contains "ClusterIP" .Values.service.scheduler.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "ballista.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit 127.0.0.1:50050 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 50050:$CONTAINER_PORT +{{- end }} diff --git a/helm/ballista/templates/_helpers.tpl b/helm/ballista/templates/_helpers.tpl new file mode 100644 index 000000000..924ad73f3 --- /dev/null +++ b/helm/ballista/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "ballista.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ballista.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ballista.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "ballista.labels" -}} +helm.sh/chart: {{ include "ballista.chart" . }} +{{ include "ballista.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "ballista.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ballista.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ballista.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "ballista.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/helm/ballista/templates/executor.yaml b/helm/ballista/templates/executor.yaml new file mode 100644 index 000000000..7c8d811b8 --- /dev/null +++ b/helm/ballista/templates/executor.yaml @@ -0,0 +1,89 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ballista-executor + labels: + {{- include "ballista.labels" . | nindent 4 }} +spec: + serviceName: executor + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + run: ballista-executor + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + run: ballista-executor + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "ballista.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-executor + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}/{{ .Values.image.executor }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/root/ballista-executor", "--scheduler-host=ballista-scheduler"] + env: + - name: AWS_DEFAULT_REGION + value: {{.Values.config.s3.region}} + - name: AWS_ENDPOINT + value: {{.Values.config.s3.endpoint}} + - name: AWS_ALLOW_HTTP + value: {{.Values.config.s3.allowHttp | quote}} + - name: AWS_ACCESS_KEY_ID + value: {{.Values.minio.auth.rootUser}} + - name: AWS_SECRET_ACCESS_KEY + value: {{.Values.minio.auth.rootPassword}} + ports: + - name: flight + containerPort: 50051 + protocol: TCP + livenessProbe: + tcpSocket: + port: 50051 + readinessProbe: + tcpSocket: + port: 50051 + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: ballista-executor + labels: + run: ballista-executor +spec: + type: {{ .Values.service.executor.type }} + ports: + - port: {{ .Values.service.executor.port }} + targetPort: 50051 + protocol: TCP + name: flight + selector: + run: ballista-executor diff --git a/helm/ballista/templates/hpa.yaml b/helm/ballista/templates/hpa.yaml new file mode 100644 index 000000000..de7832dd1 --- /dev/null +++ b/helm/ballista/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "ballista.fullname" . }} + labels: + {{- include "ballista.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "ballista.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/helm/ballista/templates/ingress.yaml b/helm/ballista/templates/ingress.yaml new file mode 100644 index 000000000..28a5d1ef2 --- /dev/null +++ b/helm/ballista/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "ballista.fullname" . -}} +{{- $svcPort := .Values.service.scheduler.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "ballista.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/ballista/templates/scheduler.yaml b/helm/ballista/templates/scheduler.yaml new file mode 100644 index 000000000..9d420a554 --- /dev/null +++ b/helm/ballista/templates/scheduler.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ballista-scheduler + labels: + {{- include "ballista.labels" . | nindent 4 }} +spec: + serviceName: scheduler + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + run: ballista-scheduler + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + run: ballista-scheduler + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "ballista.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-scheduler + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}/{{ .Values.image.scheduler }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: AWS_DEFAULT_REGION + value: {{.Values.config.s3.region}} + - name: AWS_ENDPOINT + value: {{.Values.config.s3.endpoint}} + - name: AWS_ALLOW_HTTP + value: {{.Values.config.s3.allowHttp | quote}} + - name: AWS_ACCESS_KEY_ID + value: {{.Values.minio.auth.rootUser}} + - name: AWS_SECRET_ACCESS_KEY + value: {{.Values.minio.auth.rootPassword}} + ports: + - name: flightsql + containerPort: 50050 + protocol: TCP + livenessProbe: + tcpSocket: + port: 50050 + readinessProbe: + tcpSocket: + port: 50050 + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: ballista-scheduler + labels: + run: ballista-scheduler +spec: + type: {{ .Values.service.scheduler.type }} + ports: + - port: {{ .Values.service.scheduler.port }} + targetPort: 50050 + protocol: TCP + name: flightsql + selector: + run: ballista-scheduler diff --git a/helm/ballista/templates/serviceaccount.yaml b/helm/ballista/templates/serviceaccount.yaml new file mode 100644 index 000000000..53875df1b --- /dev/null +++ b/helm/ballista/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ballista.serviceAccountName" . }} + labels: + {{- include "ballista.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/ballista/templates/tests/test-connection.yaml b/helm/ballista/templates/tests/test-connection.yaml new file mode 100644 index 000000000..2c7aef1b7 --- /dev/null +++ b/helm/ballista/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "ballista.fullname" . }}-test-connection" + labels: + {{- include "ballista.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "ballista.fullname" . }}:{{ .Values.service.scheduler.port }}'] + restartPolicy: Never diff --git a/helm/ballista/values.yaml b/helm/ballista/values.yaml new file mode 100644 index 000000000..69c722dec --- /dev/null +++ b/helm/ballista/values.yaml @@ -0,0 +1,104 @@ +# Default values for ballista. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: dev-registry:36556 + scheduler: arrow-ballista_scheduler + executor: arrow-ballista_executor + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +config: + s3: + allowHttp: true + region: "minio" + endpoint: "http://ballista-minio:9000" + +imagePullSecrets: + - name: regcred +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + scheduler: + type: ClusterIP + port: 50050 + executor: + type: ClusterIP + port: 50051 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: ballista-scheduler.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +minio: + persistence: + enabled: false + storageClass: azurefile-csi + size: 1Gi + existingClaim: "" + auth: + rootUser: minio From 71304c3da7126f8e9490c4d54837231250184d05 Mon Sep 17 00:00:00 2001 From: Brent Gardner Date: Fri, 7 Oct 2022 14:15:06 -0600 Subject: [PATCH 02/12] Documentation --- helm/README.md | 37 ++++++++++++++++++++++++++ helm/ballista/Chart.lock | 6 ++--- helm/ballista/templates/executor.yaml | 2 +- helm/ballista/templates/scheduler.yaml | 2 +- helm/ballista/values.yaml | 8 +++--- 5 files changed, 46 insertions(+), 9 deletions(-) create mode 100644 helm/README.md diff --git a/helm/README.md b/helm/README.md new file mode 100644 index 000000000..91732d0c1 --- /dev/null +++ b/helm/README.md @@ -0,0 +1,37 @@ +# Ballista Helm Chart + +Helm is a convenient way to install applications into Kubernetes. It can work against any Kubeneretes environement, +including all the major cloud providers. For the purposes of this documentation, we will use microk8s to install locally. + +## Prerequisites + +```shell +sudo apt install kubectl docker.io docker-compose +sudo snap install go --classic +~/go/bin/kind create cluster +``` + +## Build + +```shell +dev/build-ballista-docker.sh +``` + +## Deploy + +```shell +~/go/bin/kind load docker-image ballista-scheduler:latest +~/go/bin/kind load docker-image ballista-executor:latest + +cd helm/ballista +helm repo add bitnami https://charts.bitnami.com/bitnami +helm dep update +helm dep build +helm install ballista . +``` + +## Teardown + +```shell +helm uninstall ballista +``` \ No newline at end of file diff --git a/helm/ballista/Chart.lock b/helm/ballista/Chart.lock index f322a648d..62875df04 100644 --- a/helm/ballista/Chart.lock +++ b/helm/ballista/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: minio repository: https://charts.bitnami.com/bitnami - version: 11.10.2 -digest: sha256:0135f0f1c4e0ecf5b539210e1780d81d20d537ef008737b5f854e62f8c786d89 -generated: "2022-09-16T12:20:09.802584178-06:00" + version: 11.10.5 +digest: sha256:53448ec4e280c4a77c33fe944da96c38bb877f3777c9a9dcfc364fc9012a1b09 +generated: "2022-10-07T13:05:49.084667348-06:00" diff --git a/helm/ballista/templates/executor.yaml b/helm/ballista/templates/executor.yaml index 7c8d811b8..4eeac6b1c 100644 --- a/helm/ballista/templates/executor.yaml +++ b/helm/ballista/templates/executor.yaml @@ -32,7 +32,7 @@ spec: - name: {{ .Chart.Name }}-executor securityContext: {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}/{{ .Values.image.executor }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + image: "{{ .Values.image.repository }}{{ .Values.image.executor }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} command: ["/root/ballista-executor", "--scheduler-host=ballista-scheduler"] env: diff --git a/helm/ballista/templates/scheduler.yaml b/helm/ballista/templates/scheduler.yaml index 9d420a554..ab78fe7e9 100644 --- a/helm/ballista/templates/scheduler.yaml +++ b/helm/ballista/templates/scheduler.yaml @@ -32,7 +32,7 @@ spec: - name: {{ .Chart.Name }}-scheduler securityContext: {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}/{{ .Values.image.scheduler }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + image: "{{ .Values.image.repository }}{{ .Values.image.scheduler }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: AWS_DEFAULT_REGION diff --git a/helm/ballista/values.yaml b/helm/ballista/values.yaml index 69c722dec..479d01343 100644 --- a/helm/ballista/values.yaml +++ b/helm/ballista/values.yaml @@ -5,10 +5,10 @@ replicaCount: 1 image: - repository: dev-registry:36556 - scheduler: arrow-ballista_scheduler - executor: arrow-ballista_executor - pullPolicy: IfNotPresent + repository: "" + scheduler: ballista-scheduler + executor: ballista-executor + pullPolicy: Never # Overrides the image tag whose default is the chart appVersion. tag: "latest" From 2a349745395c72c1fb5f06b4e5e2f2a826f83d78 Mon Sep 17 00:00:00 2001 From: Brent Gardner Date: Fri, 7 Oct 2022 14:15:31 -0600 Subject: [PATCH 03/12] Documentation --- helm/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/README.md b/helm/README.md index 91732d0c1..5b7cfd16c 100644 --- a/helm/README.md +++ b/helm/README.md @@ -20,6 +20,7 @@ dev/build-ballista-docker.sh ## Deploy ```shell +# See https://iximiuz.com/en/posts/kubernetes-kind-load-docker-image/ ~/go/bin/kind load docker-image ballista-scheduler:latest ~/go/bin/kind load docker-image ballista-executor:latest From c7ef91647a851c6a2f16b478c2e6b115b6b385d6 Mon Sep 17 00:00:00 2001 From: Brent Gardner Date: Fri, 7 Oct 2022 14:16:40 -0600 Subject: [PATCH 04/12] Documentation --- helm/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/README.md b/helm/README.md index 5b7cfd16c..4ca916801 100644 --- a/helm/README.md +++ b/helm/README.md @@ -21,6 +21,7 @@ dev/build-ballista-docker.sh ```shell # See https://iximiuz.com/en/posts/kubernetes-kind-load-docker-image/ +# https://kind.sigs.k8s.io/docs/user/quick-start/#loading-an-image-into-your-cluster ~/go/bin/kind load docker-image ballista-scheduler:latest ~/go/bin/kind load docker-image ballista-executor:latest From 54028fb8e2e17715188accb4bb06fbb075309958 Mon Sep 17 00:00:00 2001 From: Brent Gardner Date: Fri, 7 Oct 2022 14:19:26 -0600 Subject: [PATCH 05/12] Install kind --- helm/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/README.md b/helm/README.md index 4ca916801..85f4de398 100644 --- a/helm/README.md +++ b/helm/README.md @@ -8,6 +8,7 @@ including all the major cloud providers. For the purposes of this documentation, ```shell sudo apt install kubectl docker.io docker-compose sudo snap install go --classic +go install sigs.k8s.io/kind@v0.16.0 ~/go/bin/kind create cluster ``` From 42aecbc179b2c30e34d5ac1e254420f4c4b874d6 Mon Sep 17 00:00:00 2001 From: Brent Gardner Date: Fri, 7 Oct 2022 15:22:23 -0600 Subject: [PATCH 06/12] Port forward --- helm/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/helm/README.md b/helm/README.md index 85f4de398..ac562bbb2 100644 --- a/helm/README.md +++ b/helm/README.md @@ -33,6 +33,13 @@ helm dep build helm install ballista . ``` +## Connect + +```shell +kubectl port-forward ballista-scheduler-0 50050:50050 +sqline # ... see FlightSQL instructions +``` + ## Teardown ```shell From d236633d0109b1b5e1cdce87ed6ee927098bb787 Mon Sep 17 00:00:00 2001 From: Brent Gardner Date: Fri, 7 Oct 2022 15:52:11 -0600 Subject: [PATCH 07/12] Update text --- helm/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helm/README.md b/helm/README.md index ac562bbb2..8adcdcff6 100644 --- a/helm/README.md +++ b/helm/README.md @@ -1,7 +1,8 @@ # Ballista Helm Chart Helm is a convenient way to install applications into Kubernetes. It can work against any Kubeneretes environement, -including all the major cloud providers. For the purposes of this documentation, we will use microk8s to install locally. +including all the major cloud providers. +For the purposes of this documentation, we will use Kubernetes-in-Docker (kind) to install locally. ## Prerequisites From a91d96fb6c4a60f7eea42811bedbf407c1450a9d Mon Sep 17 00:00:00 2001 From: Brent Gardner Date: Mon, 10 Oct 2022 10:42:08 -0600 Subject: [PATCH 08/12] Update helm/README.md Co-authored-by: Andy Grove --- helm/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/helm/README.md b/helm/README.md index 8adcdcff6..98a1f8b1d 100644 --- a/helm/README.md +++ b/helm/README.md @@ -34,6 +34,12 @@ helm dep build helm install ballista . ``` +## Access Scheduler Web UI + +Run the following command to redirect localhost port 8080 to port 80 in the scheduler container and then view the scheduler UI at http://localhost:8080. + +kubectl port-forward ballista-scheduler-0 8080:80 + ## Connect ```shell From eb9026344f1375254f934572ba4c9d534af9d71f Mon Sep 17 00:00:00 2001 From: Brent Gardner Date: Mon, 10 Oct 2022 10:43:06 -0600 Subject: [PATCH 09/12] Update text --- helm/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/README.md b/helm/README.md index 98a1f8b1d..23bb2e4c0 100644 --- a/helm/README.md +++ b/helm/README.md @@ -7,8 +7,8 @@ For the purposes of this documentation, we will use Kubernetes-in-Docker (kind) ## Prerequisites ```shell -sudo apt install kubectl docker.io docker-compose -sudo snap install go --classic +sudo apt install docker.io docker-compose +sudo snap install go kubectl helm --classic go install sigs.k8s.io/kind@v0.16.0 ~/go/bin/kind create cluster ``` From 21c3d885aee43f9176b4cc20024591800312b3e8 Mon Sep 17 00:00:00 2001 From: Brent Gardner Date: Mon, 10 Oct 2022 10:48:23 -0600 Subject: [PATCH 10/12] Fix some RAT errors --- helm/ballista/Chart.yaml | 16 +++++++++++++++ helm/ballista/templates/executor.yaml | 16 +++++++++++++++ helm/ballista/templates/hpa.yaml | 16 +++++++++++++++ helm/ballista/templates/ingress.yaml | 16 +++++++++++++++ helm/ballista/templates/scheduler.yaml | 16 +++++++++++++++ helm/ballista/templates/serviceaccount.yaml | 16 +++++++++++++++ .../templates/tests/test-connection.yaml | 16 +++++++++++++++ helm/ballista/values.yaml | 20 +++++++++++++++---- 8 files changed, 128 insertions(+), 4 deletions(-) diff --git a/helm/ballista/Chart.yaml b/helm/ballista/Chart.yaml index e7b5d4589..913a954a1 100644 --- a/helm/ballista/Chart.yaml +++ b/helm/ballista/Chart.yaml @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. apiVersion: v2 name: ballista description: A Helm chart for Kubernetes diff --git a/helm/ballista/templates/executor.yaml b/helm/ballista/templates/executor.yaml index 4eeac6b1c..61c2d7e68 100644 --- a/helm/ballista/templates/executor.yaml +++ b/helm/ballista/templates/executor.yaml @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/helm/ballista/templates/hpa.yaml b/helm/ballista/templates/hpa.yaml index de7832dd1..85d82aa65 100644 --- a/helm/ballista/templates/hpa.yaml +++ b/helm/ballista/templates/hpa.yaml @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. {{- if .Values.autoscaling.enabled }} apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler diff --git a/helm/ballista/templates/ingress.yaml b/helm/ballista/templates/ingress.yaml index 28a5d1ef2..a404bb39f 100644 --- a/helm/ballista/templates/ingress.yaml +++ b/helm/ballista/templates/ingress.yaml @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. {{- if .Values.ingress.enabled -}} {{- $fullName := include "ballista.fullname" . -}} {{- $svcPort := .Values.service.scheduler.port -}} diff --git a/helm/ballista/templates/scheduler.yaml b/helm/ballista/templates/scheduler.yaml index ab78fe7e9..96d58cade 100644 --- a/helm/ballista/templates/scheduler.yaml +++ b/helm/ballista/templates/scheduler.yaml @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/helm/ballista/templates/serviceaccount.yaml b/helm/ballista/templates/serviceaccount.yaml index 53875df1b..f62982e97 100644 --- a/helm/ballista/templates/serviceaccount.yaml +++ b/helm/ballista/templates/serviceaccount.yaml @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. {{- if .Values.serviceAccount.create -}} apiVersion: v1 kind: ServiceAccount diff --git a/helm/ballista/templates/tests/test-connection.yaml b/helm/ballista/templates/tests/test-connection.yaml index 2c7aef1b7..9fe7660c6 100644 --- a/helm/ballista/templates/tests/test-connection.yaml +++ b/helm/ballista/templates/tests/test-connection.yaml @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. apiVersion: v1 kind: Pod metadata: diff --git a/helm/ballista/values.yaml b/helm/ballista/values.yaml index 479d01343..bbf316271 100644 --- a/helm/ballista/values.yaml +++ b/helm/ballista/values.yaml @@ -1,7 +1,19 @@ -# Default values for ballista. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. replicaCount: 1 image: From 7db92cc0395128d352d507778b034deef25a9615 Mon Sep 17 00:00:00 2001 From: Brent Gardner Date: Mon, 10 Oct 2022 11:22:14 -0600 Subject: [PATCH 11/12] RAT --- dev/release/rat_exclude_files.txt | 1 + helm/README.md | 19 +++++++++++++++++++ helm/ballista/.helmignore | 20 +++++++++++++++++--- helm/ballista/templates/NOTES.txt | 18 ++++++++++++++++++ helm/ballista/templates/_helpers.tpl | 17 ++++++++++++++++- 5 files changed, 71 insertions(+), 4 deletions(-) diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 2f847b952..9ef3f94ce 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -94,6 +94,7 @@ dev/tasks/linux-packages/apache-arrow/debian/source/format dev/tasks/linux-packages/apache-arrow/debian/watch dev/tasks/requirements*.txt dev/tasks/conda-recipes/* +helm/Chart.lock pax_global_header MANIFEST.in __init__.pxd diff --git a/helm/README.md b/helm/README.md index 23bb2e4c0..7be9832b7 100644 --- a/helm/README.md +++ b/helm/README.md @@ -1,3 +1,22 @@ + + # Ballista Helm Chart Helm is a convenient way to install applications into Kubernetes. It can work against any Kubeneretes environement, diff --git a/helm/ballista/.helmignore b/helm/ballista/.helmignore index 0e8a0eb36..bd2d4649d 100644 --- a/helm/ballista/.helmignore +++ b/helm/ballista/.helmignore @@ -1,6 +1,20 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + .DS_Store # Common VCS dirs .git/ diff --git a/helm/ballista/templates/NOTES.txt b/helm/ballista/templates/NOTES.txt index 404805d8c..a63344279 100644 --- a/helm/ballista/templates/NOTES.txt +++ b/helm/ballista/templates/NOTES.txt @@ -1,3 +1,21 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} 1. Get the application URL by running these commands: {{- if .Values.ingress.enabled }} {{- range $host := .Values.ingress.hosts }} diff --git a/helm/ballista/templates/_helpers.tpl b/helm/ballista/templates/_helpers.tpl index 924ad73f3..e575aaa6d 100644 --- a/helm/ballista/templates/_helpers.tpl +++ b/helm/ballista/templates/_helpers.tpl @@ -1,5 +1,20 @@ {{/* -Expand the name of the chart. + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. */}} {{- define "ballista.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} From c0e2e17164906a01b5cb852e55c716c08d92962a Mon Sep 17 00:00:00 2001 From: Brent Gardner Date: Mon, 10 Oct 2022 11:37:59 -0600 Subject: [PATCH 12/12] Fix path --- dev/release/rat_exclude_files.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 9ef3f94ce..d1ee673d5 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -94,7 +94,7 @@ dev/tasks/linux-packages/apache-arrow/debian/source/format dev/tasks/linux-packages/apache-arrow/debian/watch dev/tasks/requirements*.txt dev/tasks/conda-recipes/* -helm/Chart.lock +helm/ballista/Chart.lock pax_global_header MANIFEST.in __init__.pxd