Add Minikube quickstart guide

quickstart/README.md

@@ -0,0 +1,77 @@
+# Minikube quickstart
+
+### Install Minikube
+Follow instructions in https://minikube.sigs.k8s.io/docs/start/ to install the minikube.
+
+### Start Minikube
+```
+minikube start
+```
+
+### Enable Minikube `ingress` addon
+```
+minikube addons enable ingress
+```
+
+### Apply quickstart `.yml` configuration
+Assuming you cloned Lighter repo to `~/git/lighter`:
+1. Create main resources
+```
+minikube kubectl -- apply -f git/lighter/quickstart/lighter.yml
+minikube kubectl -- apply -f git/lighter/quickstart/jupyterlab.yml
+```
+2. Create `ingress` resources
+```
+minikube kubectl -- apply -f git/lighter/quickstart/nginx/lighter-nginx.yml
+minikube kubectl -- apply -f git/lighter/quickstart/nginx/jupyterlab-nginx.yml
+```
+
+### Verify deployment
+**_NOTE:_** Jupyterlab can take few minutes to start because the container is being altered on runtime. 
+```
+minikube kubectl -- -n spark get all
+```
+```
+NAME                              READY   STATUS    RESTARTS   AGE
+pod/jupyterlab-5685dcf5c7-ggcpl   1/1     Running   0          4m34s
+pod/lighter-5484769777-7r9ln      1/1     Running   0          5m12s
+
+NAME                 TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)              AGE
+service/jupyterlab   ClusterIP   10.100.99.140   <none>        8888/TCP             4m34s
+service/lighter      ClusterIP   10.98.245.141   <none>        8080/TCP,25333/TCP   5m12s
+
+NAME                         READY   UP-TO-DATE   AVAILABLE   AGE
+deployment.apps/jupyterlab   1/1     1            1           4m34s
+deployment.apps/lighter      1/1     1            1           5m12s
+
+NAME                                    DESIRED   CURRENT   READY   AGE
+replicaset.apps/jupyterlab-5685dcf5c7   1         1         1       4m34s
+replicaset.apps/lighter-5484769777      1         1         1       5m12s
+```
+CDRs (from `ingress` addon) needs separate call:
+```
+minikube kubectl -- -n spark get ingress
+```
+```
+NAME         CLASS   HOSTS         ADDRESS        PORTS   AGE
+jupyterlab   nginx   spark.local   192.168.49.2   80      87s
+lighter      nginx   spark.local   192.168.49.2   80      87s
+```
+
+### Optional: Windows with WSL2 with Ubuntu
+1. Run this in WSL2
+```
+minikube tunnel
+```
+2. Add this entry to your `C:\Windows\System32\Drivers\etc\hosts` file on Windows:
+```
+127.0.0.1 spark.local
+```
+
+### Try it out!
+
+Lighter URL: http://spark.local/lighter/
+
+Jupyterlab URL: http://spark.local/jupyterlab/
+
+**_NOTE:_** Jupyterlab will already have `quickstart.ipynb` notebook with two cells. Execute them and that's it. 

quickstart/jupyterlab.yml

@@ -0,0 +1,191 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: jupyterlab
+  namespace: spark
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: jupyterlab
+  namespace: spark
+rules:
+  - apiGroups: [ "" ]
+    resources: [ "pods", "services", "configmaps", "pods/log" ]
+    verbs: [ "*" ]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: jupyterlab
+  namespace: spark
+subjects:
+  - kind: ServiceAccount
+    name: jupyterlab
+    namespace: spark
+roleRef:
+  kind: Role
+  name: jupyterlab
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: jupyterlab
+  namespace: spark
+  labels:
+    run: jupyterlab
+spec:
+  ports:
+    - port: 8888
+      protocol: TCP
+  selector:
+    run: jupyterlab
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: spark
+  name: jupyterlab
+spec:
+  selector:
+    matchLabels:
+      run: jupyterlab
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: 1
+  template:
+    metadata:
+      labels:
+        run: jupyterlab
+    spec:
+      containers:
+        - image: jupyter/base-notebook:lab-3.5.2
+          name: jupyterlab
+          resources:
+            requests:
+              cpu: "0.25"
+              memory: "512Mi"
+          ports:
+            - containerPort: 8888
+          command: [ "start-notebook.sh" ]
+          args:
+            - "--NotebookApp.token="
+            - "--NotebookApp.base_url=jupyterlab"
+            - "--KernelSpecManager.ensure_native_kernel=False"
+            - "--NotebookApp.notebook_dir=/home/jovyan/work"
+          lifecycle:
+            postStart:
+              exec:
+                command:
+                  - /bin/sh
+                  - -c
+                  - |
+                    conda install sparkmagic -y
+                    jupyter kernelspec uninstall sparkkernel -y
+                    jupyter kernelspec uninstall sparkrkernel -y
+                    jupyter kernelspec uninstall python3 -y
+                    mkdir -p /home/jovyan/.sparkmagic/
+                    cp /tmp/sparkmagic.json /home/jovyan/.sparkmagic/config.json
+                    cp /tmp/quickstart.ipynb /home/jovyan/work/quickstart.ipynb
+          volumeMounts:
+            - name: sparkmagic
+              mountPath: /tmp/sparkmagic.json
+              subPath: sparkmagic
+            - name: quickstart
+              mountPath: /tmp/quickstart.ipynb
+              subPath: quickstart
+      serviceAccountName: jupyterlab
+      volumes:
+        - name: sparkmagic
+          configMap:
+            name: sparkmagic
+        - name: quickstart
+          configMap:
+            name: quickstart
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sparkmagic
+  namespace: spark
+data:
+  sparkmagic: |
+    {
+      "kernel_python_credentials" : {
+        "username": "",
+        "password": "",
+        "url": "http://lighter.spark:8080/lighter/api",
+        "auth": "None"
+      },
+      "livy_session_startup_timeout_seconds": 600,
+      "custom_headers": {
+        "X-Compatibility-Mode": "sparkmagic"
+      }
+    }
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: quickstart
+  namespace: spark
+data:
+  quickstart: |
+    {
+     "cells": [
+      {
+       "cell_type": "code",
+       "execution_count": null,
+       "id": "c72ef05c-13ee-4199-a002-b2be815ce419",
+       "metadata": {},
+       "outputs": [],
+       "source": [
+        "%%configure -f\n",
+        "{\n",
+        "    \"name\": \"Test Lighter\",\n",
+        "    \"conf\":{\n",
+        "        \"spark.kubernetes.container.image\": \"apache/spark-py:v3.3.1\"\n",
+        "    }\n",
+        "}"
+       ]
+      },
+      {
+       "cell_type": "code",
+       "execution_count": null,
+       "id": "94d68783-5c86-440e-84da-aa8be288cdd5",
+       "metadata": {},
+       "outputs": [],
+       "source": [
+        "df = spark.createDataFrame(\n",
+        "    [\n",
+        "        (1, \"foo\"),\n",
+        "        (2, \"bar\"),\n",
+        "    ],\n",
+        "    [\"id\", \"label\"]\n",
+        ")\n",
+        "df.collect()"
+       ]
+      }
+     ],
+     "metadata": {
+      "kernelspec": {
+       "display_name": "PySpark",
+       "language": "python",
+       "name": "pysparkkernel"
+      },
+      "language_info": {
+       "codemirror_mode": {
+        "name": "python",
+        "version": 3
+       },
+       "file_extension": ".py",
+       "mimetype": "text/x-python",
+       "name": "pyspark",
+       "pygments_lexer": "python3"
+      }
+     },
+     "nbformat": 4,
+     "nbformat_minor": 5
+    }

quickstart/lighter.yml

@@ -0,0 +1,95 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: spark
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: lighter
+  namespace: spark
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: lighter
+  namespace: spark
+rules:
+  - apiGroups: [""]
+    resources: ["pods", "services", "configmaps", "pods/log"]
+    verbs: ["*"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: lighter
+  namespace: spark
+subjects:
+  - kind: ServiceAccount
+    name: lighter
+    namespace: spark
+roleRef:
+  kind: Role
+  name: lighter
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: lighter
+  namespace: spark
+  labels:
+    run: lighter
+spec:
+  ports:
+    -   name: api
+        port: 8080
+        protocol: TCP
+    -   name: javagw
+        port: 25333
+        protocol: TCP
+  selector:
+    run: lighter
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: spark
+  name: lighter
+spec:
+  selector:
+    matchLabels:
+      run: lighter
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: 1
+  template:
+    metadata:
+      labels:
+        run: lighter
+    spec:
+      containers:
+        -   image: ghcr.io/exacaster/lighter:0.0.41-spark3.3.1
+            name: lighter
+            readinessProbe:
+              httpGet:
+                path: /health/readiness
+                port: 8080
+              initialDelaySeconds: 15
+              periodSeconds: 15
+            resources:
+              requests:
+                cpu: "0.25"
+                memory: "512Mi"
+            ports:
+              -   containerPort: 8080
+            env:
+              -   name: LIGHTER_KUBERNETES_ENABLED
+                  value: "true"
+              -   name: LIGHTER_MAX_RUNNING_JOBS
+                  value: "15"
+              -   name: LIGHTER_KUBERNETES_SERVICE_ACCOUNT
+                  value: lighter
+      serviceAccountName: lighter

quickstart/nginx/jupyterlab-nginx.yml

@@ -0,0 +1,19 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: jupyterlab
+  namespace: spark
+  annotations:
+    nginx.ingress.kubernetes.io/rewrite-target: /jupyterlab/$2
+spec:
+  rules:
+    - host: spark.local
+      http:
+        paths:
+          - path: /jupyterlab(/|$)(.*)
+            pathType: Prefix
+            backend:
+              service:
+                name: jupyterlab
+                port:
+                  number: 8888