more changes

Signed-off-by: Yuvraj <[email protected]>

charts/flyte-core/values-sandbox.yaml

@@ -58,7 +58,7 @@ configmap:
         #  DEFAULT_ENV_VAR: VALUE
         default-cpus: 100m
         default-env-vars:
-          - FLYTE_AWS_ENDPOINT: "http://minio:9000"
+          - FLYTE_AWS_ENDPOINT: "http://minio.flyte.svc.cluster.local:9000"
           - FLYTE_AWS_ACCESS_KEY_ID: minio
           - FLYTE_AWS_SECRET_ACCESS_KEY: flyte
         default-memory: 200Mi

charts/flyte-deps/README.md

@@ -115,5 +115,5 @@ helm upgrade -f values.yaml flyte .
 | redoc.tolerations | list | `[]` | tolerations for Minio deployment |
 | sparkoperator | object | `{"enabled":false}` | Optional: Spark Plugin using the Spark Operator |
 | sparkoperator.enabled | bool | `false` | - enable or disable Sparkoperator deployment installation |
-| webhook.enabled | bool | `false` |  |
+| webhook.enabled | bool | `true` |  |
 | webhook.service | object | `{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"}` | Service settings for the webhook |

charts/flyte-deps/values.yaml

@@ -3,7 +3,7 @@ redis:
   enabled: false
 
 webhook:
-  enabled: false
+  enabled: true
   # -- Service settings for the webhook
   service:
     annotations:

deployment/sandbox/flyte_sandbox_deps_helm_generated.yaml

@@ -549,6 +549,18 @@ spec:
     app.kubernetes.io/name: redoc
     app.kubernetes.io/instance: flyte
 ---
+# Source: flyte-deps/templates/webhook/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: flyte-pod-webhook
+  namespace: flyte
+  annotations: 
+    projectcontour.io/upstream-protocol.h2c: grpc
+spec:
+  type: ExternalName
+  externalName: host.docker.internal
+---
 # Source: flyte-deps/charts/kubernetes-dashboard/templates/deployment.yaml
 # Copyright 2017 The Kubernetes Authors.
 #

docker/sandbox/Dockerfile

@@ -35,6 +35,7 @@ RUN wget -q -O - https://raw.githubusercontent.com/flyteorg/flytectl/master/inst
 # Copy flyte chart
 COPY charts/flyte/ /flyteorg/share/flyte
 COPY charts/flyte-core/ /flyteorg/share/flyte-core
+COPY charts/flyte-deps/ /flyteorg/share/flyte-deps
 
 # Copy scripts
 COPY docker/sandbox/kubectl docker/sandbox/cgroup-v2-hack.sh docker/sandbox/wait-for-flyte.sh /flyteorg/bin/

docker/sandbox/flyte-entrypoint-default.sh

@@ -43,24 +43,42 @@ then
 fi
 
 # Deploy flyte
-echo "Deploying Flyte..."
+helm repo add flyteorg https://flyteorg.github.io/flyte
+
+echo "Deploying Flyte-deps..."
+version=""
+charts="/flyteorg/share/flyte-deps"
+
+if [[ $FLYTE_TEST = "release" ]]
+then
+  version="--version $FLYTE_VERSION"
+  charts="flyteorg/flyte-deps"
+fi
+
+if [[ $FLYTE_TEST = "local" ]]
+then
+  helm dep update $charts
+fi
+
+helm upgrade -n flyte --create-namespace flyte-deps $charts --kubeconfig /etc/rancher/k3s/k3s.yaml --install $version --set webhook.enabled=false,contour.enabled=true
+
 version=""
-charts="/flyteorg/share/flyte"
+charts="/flyteorg/share/flyte-core"
 
 if [[ $FLYTE_TEST = "release" ]]
 then
-  helm repo add flyteorg https://flyteorg.github.io/flyte
-  helm fetch flyteorg/flyte --version=$FLYTE_VERSION
+  rm /flyteorg/share/flyte-core
+  helm fetch -untar --untardir /flyteorg/share/ flyteorg/flyte-core --version=$FLYTE_VERSION
   version="--version $FLYTE_VERSION"
-  charts="flyteorg/flyte"
+  charts="flyteorg/flyte-core"
 fi
 
 if [[ $FLYTE_TEST = "local" ]]
 then
   helm dep update $charts
 fi
 
-helm upgrade -n flyte --create-namespace flyte $charts --kubeconfig /etc/rancher/k3s/k3s.yaml --install $version
+helm upgrade -n flyte --create-namespace flyte-core $charts --kubeconfig /etc/rancher/k3s/k3s.yaml --install $version -f /flyteorg/share/flyte-core/values-sandbox.yaml
 
 wait-for-flyte.sh
 

rsts/deployment/index.rst

@@ -52,15 +52,6 @@ solution). The following pages will help you effectively deploy and manage an en
 
     ---
 
-    .. link-button:: deployment-sandbox
-       :type: ref
-       :text: Sandbox
-       :classes: btn-block stretched-link
-    ^^^^^^^^^^^^
-    Deployment guides with detailed instructions for sandbox deployment.
-
-    ---
-
     .. link-button:: multicluster
        :type: ref
        :text: Multi-cluster setup
@@ -102,7 +93,6 @@ solution). The following pages will help you effectively deploy and manage an en
     overview
     aws/index
     gcp/index
-    sandbox/index
     multicluster
     cluster_config/index
     sandbox

rsts/deployment/plugin_setup/k8s/index.rst

@@ -15,7 +15,7 @@ This guide gives an overview of setting up the K8s Operator backend plugin in yo
 
 2. Setup the cluster
 
-  * Make sure you have up and running flyte cluster in `AWS <https://docs.flyte.org/en/latest/deployment/aws/index.html#deployment-aws>`__ / `GCP <https://docs.flyte.org/en/latest/deployment/gcp/index.html#deployment-gcp>`__ /  `Sandbox <https://docs.flyte.org/en/latest/deployment/gcp/index.html#deployment-sandbox>`__
+  * Make sure you have up and running flyte cluster in `AWS <https://docs.flyte.org/en/latest/deployment/aws/index.html#deployment-aws>`__ / `GCP <https://docs.flyte.org/en/latest/deployment/gcp/index.html#deployment-gcp>`__ /  `Sandbox <https://docs.flyte.org/en/latest/deployment/sandbox/index.html#deployment-sandbox>`__
   * Make sure you have correct kubeconfig and selected the correct kubernetes context
   * make sure you have the correct flytectl config at ~/.flyte/config.yaml
 

rsts/deployment/plugin_setup/webapi/snowflake.rst

@@ -13,7 +13,7 @@ This guide gives an overview of how to set up Snowflake in your Flyte deployment
 
 2. Setup the cluster
 
-  * Make sure you have up and running flyte cluster in `AWS <https://docs.flyte.org/en/latest/deployment/aws/index.html#deployment-aws>`__ / `GCP <https://docs.flyte.org/en/latest/deployment/gcp/index.html#deployment-gcp>`__ /  `Sandbox <https://docs.flyte.org/en/latest/deployment/gcp/index.html#deployment-sandbox>`__
+  * Make sure you have up and running flyte cluster in `AWS <https://docs.flyte.org/en/latest/deployment/aws/index.html#deployment-aws>`__ / `GCP <https://docs.flyte.org/en/latest/deployment/gcp/index.html#deployment-gcp>`__ /  `Sandbox <https://docs.flyte.org/en/latest/deployment/sandbox/index.html#deployment-sandbox>`__
   * Make sure you have correct kubeconfig and selected the correct kubernetes context
   * make sure you have the correct flytectl config at ~/.flyte/config.yaml
 

rsts/deployment/sandbox.rst

@@ -67,100 +67,16 @@ This installs all the dependencies as Kubernetes deployments. We call this a San
 Deploy Flyte Sandbox environment laptop / workstation / single machine
 =======================================================================
 
+
 Ensure ``kubectl`` is installed. Follow `kubectl installation docs <https://kubernetes.io/docs/tasks/tools/install-kubectl/>`__. On Mac::
 
     brew install kubectl
 
-.. tabbed:: Docker Image
-
-    Recommend using ``flytectl sandbox`` as describe in :ref:`getting-started`
-
-    .. prompt:: bash $
-
-          docker run --rm --privileged -p 30081:30081 -p 30084:30084 -p 30088:30088 cr.flyte.org/flyteorg/flyte-sandbox
-
-.. tabbed:: k3d
-
-    .. TODO::
-
-      Allow installing latest version of k3d once this `issue is fixed <https://github.com/rancher/k3d/issues/380>`__
-
-    #. Install k3d Using ``curl``::
-
-        curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | TAG=v4.2.0 bash
-
-       Or Using ``wget`` ::
-
-        wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | TAG=v4.2.0 bash
-
-
-    #. Start a new K3s cluster called Flyte ::
-
-        k3d cluster create flyte -p 30081:30081 --no-lb  --k3s-server-arg '–no-deploy=traefik' --k3s-server-arg '–no-deploy=servicelb'
-
-
-    #. Ensure the context is set to the new cluster::
-
-        kubectl config set-context flyte
-
-    #. Install Flyte::
-
-        kubectl create ns flyte
-        kubectl create -f  https://raw.githubusercontent.com/flyteorg/flyte/master/deployment/sandbox/flyte_helm_generated.yaml
-
-
-    #. Connect to `FlyteConsole <localhost:30081/console>`__
-    #. [Optional] You can delete the cluster once you are done with the tutorial using - ::
-
-        k3d cluster delete flyte
-
-    .. note::
+Recommend using ``flytectl sandbox start`` as describe in :ref:`getting-started`
 
-        #. Sometimes Flyteconsole will not open up. This is probably because your docker networking is impacted. One solution is to restart docker and repeat the previous steps.
-        #. To debug you can try a simple exercise - run nginx as follows::
-
-            docker run -it --rm -p 8083:80 nginx
-
-           Now connect to `locahost:8083 <localhost:8083>`__. If this does not work, then the networking is most probably impacted. Please restart docker daemon.
-
-.. tabbed:: Docker-Mac + K8s
-
-    .. TODO::
-
-        These instructions currently still rely on the old kustomize setup, and will be moved over to the Helm chart soon.
-
-    #. Install Docker for mac with Kubernetes as explained `here <https://www.docker.com/blog/docker-mac-kubernetes/>`_
-    #. Make sure Kubernetes is started and once started make sure your kubectx is set to the `docker-desktop` cluster, typically ::
-
-            kubectl config set-context docker-desktop
-
-    #. Install Flyte::
-
-        kubectl create -f https://raw.githubusercontent.com/flyteorg/flyte/master/deployment/sandbox/flyte_generated.yaml
-
-    #. Connect to `FlyteConsole <localhost/console>`__
-
-.. tabbed::  Using Minikube (Not recommended)
-
-    .. TODO::
-
-        These instructions currently still rely on the old kustomize setup, and will be moved over to the Helm chart soon.
-
-    #. Install `Minikube <https://kubernetes.io/docs/tasks/tools/install-minikube/>`_
-
-    #. Install Flyte::
-
-        kubectl create -f https://raw.githubusercontent.com/flyteorg/flyte/master/deployment/sandbox/flyte_generated.yaml
-
-    .. note::
-
-        - Minikube runs in a Virtual Machine on your host
-        - So if you try to access the flyte console on localhost, that will not work, because the Virtual Machine has a different IP address.
-        - Flyte runs within Kubernetes (minikube), so to access FlyteConsole, you cannot just use https://localhost:30081/console. You need to use the IP address of the minikube VM instead of the localhost
-        - Refer to https://kubernetes.io/docs/tutorials/hello-minikube/ to understand how to run a sample app on kubernetes using minikube and Katacoda. To  register workflows, tasks, etc. or use the CLI to query Flyte services, you have to use the IP address.
-        - If you are building an image locally and want to execute on Minikube hosted Flyte environment, please push the image to docker registry running on the Minikube VM.
-        - Another alternative is to change the docker host, to build the docker image on the Minikube hosted docker daemon. https://minikube.sigs.k8s.io/docs/handbook/pushing/ provides more detailed information about this process. Flyte can only run images that are accessible to Kubernetes. To make an image accessible, you could either push it to a remote registry or to a registry that is available to Kubernetes. In the minikube case, this registry is the one that is running on the VM.
+.. prompt:: bash $
 
+        docker run --rm --privileged -p 30081:30081 -p 30084:30084 -p 30088:30088 cr.flyte.org/flyteorg/flyte-sandbox
 
 .. _deployment-sandbox-dedicated-k8s-cluster:
 
@@ -186,31 +102,38 @@ We'll proceed like with :ref:`locally hosted flyte <deploy-sandbox-local>` with
 Flyte configuration on your remote cluster.
 
 
-#. The Flyte sandbox can be deployed via a helm chart. From the root dir of the flyte repo run ::
+#. Add Helm repo for flyte 
 
-    helm repo add flyte https://flyteorg.github.io/flyte
-    helm install -n flyte -f values.yaml --create-namespace flyte flyte/flyte
+    helm repo add flyteorg https://helm.flyte.org
 
-#. For customizations instructions, see ``/charts/flyte/README.md`` in the flyte repo.
+#. Install Flyte dependency helm chart(It will install the minio,postgres,kubernetes-dashboard and contour) ::
 
-#. You can now port-forward (or if you have load-balancer enabled then get an LB) to connect to remote FlyteConsole, as follows ::
+    helm install -n flyte flyte-deps flyteorg/flyte-deps --create-namespace --set webhook.enabled=false,minio.service.type=LoadBalancer,contour.enabled=true,contour.envoy.service.type=LoadBalancer,kubernetes-dashboard.service.type=LoadBalancer
 
-    kubectl port-forward --address 0.0.0.0 svc/flyte-contour-envoy 30081:80 -n flyte
+#. Install flyte-core chart ::
 
-#. Open the console http://localhost:30081/console.
+    helm install flyte flyteorg/flyte-core -n flyte -f https://raw.githubusercontent.com/flyteorg/flyte/master/charts/flyte-core/values-sandbox.yaml --wait
+
+#. Verify Flyte deployment ::
+
+    kubect get pods -n flyte
+
+#. Get Flyte deployment URL ::
+
+    kubect get service -n flyte
 
 #. In order to interact with your Flyte instance using ``flytectl``, initialise your configuration to point to this host ::
 
-    flytectl config init --host='localhost:30081' --insecure
+    flytectl config init --host='<CONTOUR_URL>' --insecure
 
-#. Open the minio console http://localhost:30088. Your minio username is `minio` and password is `miniostorage`.
+#. Open the minio console http://<MINIO_URL>. Your minio username is `minio` and password is `miniostorage`.
 
-#. Open the Kubernetes dashboard http://localhost:30082.
+#. Open the Kubernetes dashboard http://<K8S_DASHBOARD_URL>.
 
 #. You can port-forward to connect postgres using ::
 
     kubectl port-forward --address 0.0.0.0 svc/postgres 5432:5432 -n flyte
-
+    
 #. Now use these credentials for postgres
 
    .. code-block::