Merge pull request #8559 from sleshchenko/k8s-infra

Added kubernetes infrastructure

assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/WsMasterModule.java

@@ -77,7 +77,10 @@
 import org.eclipse.che.security.PasswordEncryptor;
 import org.eclipse.che.workspace.infrastructure.docker.DockerInfraModule;
 import org.eclipse.che.workspace.infrastructure.docker.local.LocalDockerModule;
+import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesInfraModule;
+import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesInfrastructure;
 import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftInfraModule;
+import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftInfrastructure;
 import org.eclipse.persistence.config.PersistenceUnitProperties;
 import org.flywaydb.core.internal.util.PlaceholderReplacer;
 
@@ -229,8 +232,10 @@ protected void configure() {
             .properties(persistenceProperties));
 
     String infrastructure = System.getenv("CHE_INFRASTRUCTURE_ACTIVE");
-    if ("openshift".equals(infrastructure)) {
+    if (OpenShiftInfrastructure.NAME.equals(infrastructure)) {
       install(new OpenShiftInfraModule());
+    } else if (KubernetesInfrastructure.NAME.equals(infrastructure)) {
+      install(new KubernetesInfraModule());
     } else {
       install(new LocalDockerModule());
       install(new DockerInfraModule());

assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties

@@ -317,83 +317,112 @@ db.schema.flyway.scripts.locations=classpath:che-schema
 # Note that this property is needed for backward compatibility and will be removed soon.
 che.predefined.stacks.reload_on_start=false
 
-### OpenShift Infra parameters
-
-che.infra.openshift.master_url=
-che.infra.openshift.username=
-che.infra.openshift.password=
-che.infra.openshift.oauth_token=
-che.infra.openshift.trust_certs=
+### Kubernetes Infra parameters
 
-# Create routes with Transport Layer Security (TLS) enabled
-che.infra.openshift.tls_enabled=false
+# Configuration of Kubernetes client that Infra will use
+che.infra.kubernetes.master_url=
+che.infra.kubernetes.username=
+che.infra.kubernetes.password=
+che.infra.kubernetes.oauth_token=
+che.infra.kubernetes.trust_certs=
 
-# Defines OpenShift namespace in which all workspaces will be created.
-# If not set, every workspace will be created in a new project, where project name = workspace id
-che.infra.openshift.project=
+# Defines Kubernetes namespace in which all workspaces will be created.
+# If not set, every workspace will be created in a new namespace, where namespace = workspace id
+#
+# Ignored for OpenShift infra. Use `che.infra.openshift.project` instead
+che.infra.kubernetes.namespace=
 
-che.infra.openshift.machine_start_timeout_min=5
+che.infra.kubernetes.machine_start_timeout_min=5
 
-che.infra.openshift.bootstrapper.binary_url=http://${CHE_HOST}:${CHE_PORT}/agent-binaries/linux_amd64/bootstrapper/bootstrapper
-che.infra.openshift.bootstrapper.timeout_min=10
-che.infra.openshift.bootstrapper.installer_timeout_sec=180
-che.infra.openshift.bootstrapper.server_check_period_sec=3
+che.infra.kubernetes.bootstrapper.binary_url=http://${CHE_HOST}:${CHE_PORT}/agent-binaries/linux_amd64/bootstrapper/bootstrapper
+che.infra.kubernetes.bootstrapper.timeout_min=10
+che.infra.kubernetes.bootstrapper.installer_timeout_sec=180
+che.infra.kubernetes.bootstrapper.server_check_period_sec=3
 
 # Defines whether use the Persistent Volume Claim for che workspace needs
 # e.g backup projects, logs etc or disable it.
-che.infra.openshift.pvc.enabled=true
+che.infra.kubernetes.pvc.enabled=true
 
 # Defined which strategy will be used while choosing PVC for workspaces.
 #
 # Supported strategies:
 # - 'common'
-#        PVC with name that is configured by 'che.infra.openshift.pvc.name'
+#        PVC with name that is configured by 'che.infra.kubernetes.pvc.name'
 #        property will be used for storing workspaces data of each workspace
 #        in an OpenShift project.
 #        Existing PVC will be used or new one will be created if it doesn't exist.
 #
 # - 'unique'
-#        PVC with name that is evaluated as '{che.infra.openshift.pvc.name} + '-' + {generated_8_chars}'
+#        PVC with name that is evaluated as '{che.infra.kubernetes.pvc.name} + '-' + {generated_8_chars}'
 #        will be used for storing of workspaces data.
 #        Existing PVC will be used or a new one will be created if it doesn't exist.
-che.infra.openshift.pvc.strategy=common
+che.infra.kubernetes.pvc.strategy=common
 
 # Defines whether to run a job that creates workspace's subpath directories in persistent volume for the 'common' strategy before launching a workspace.
 # Necessary in some versions of OpenShift/Kubernetes as workspace subpath volume mounts are created with root permissions,
 # and thus cannot be modified by workspaces running as a user (presents an error importing projects into a workspace in Che).
 # The default is "true", but should be set to false if the version of Openshift/Kubernetes creates subdirectories with user permissions.
 # Relevant issue: https://github.com/kubernetes/kubernetes/issues/41638
 # Note that this property has effect only if the 'common' PVC strategy used.
-che.infra.openshift.pvc.precreate_subpaths=true
+che.infra.kubernetes.pvc.precreate_subpaths=true
 
 # Defines the name of Persistent Volume Claim for che workspaces.
-che.infra.openshift.pvc.name=claim-che-workspace
+che.infra.kubernetes.pvc.name=claim-che-workspace
 
 # Defines the size of Persistent Volume Claim of che workspace.
 # Format described here:
 # https://docs.openshift.com/container-platform/latest/dev_guide/compute_resources.html#dev-compute-resources
-che.infra.openshift.pvc.quantity=10Gi
+che.infra.kubernetes.pvc.quantity=10Gi
 
 # Pod that is launched when performing persistent volume claim maintenance jobs on OpenShift
-che.infra.openshift.pvc.jobs.image=centos:centos7
+che.infra.kubernetes.pvc.jobs.image=centos:centos7
 
 # Defines pod memory limit for persistent volume claim maintenance jobs
-che.infra.openshift.pvc.jobs.memorylimit=250Mi
+che.infra.kubernetes.pvc.jobs.memorylimit=250Mi
 
 # Defines Persistent Volume Claim access mode.
 # Note that for common PVC strategy changing of access mode affects the number of simultaneously running workspaces.
 # If OpenShift flavor where che running is using PVs with RWX access mode then a limit of running workspaces at the same time
 # bounded only by che limits configuration like(RAM, CPU etc).
 # Detailed information about access mode is described here:
 # https://docs.openshift.com/container-platform/latest/architecture/additional_concepts/storage.html#pv-access-modes
-che.infra.openshift.pvc.access_mode=ReadWriteOnce
+che.infra.kubernetes.pvc.access_mode=ReadWriteOnce
 
 # Defined range of ports for installers servers
 #
 # By default, installer will use own port, but if it conflicts with another installer servers
 # then OpenShift infrastructure will reconfigure installer to use first available from this range
-che.infra.openshift.installer_server_min_port=10000
-che.infra.openshift.installer_server_max_port=20000
+che.infra.kubernetes.installer_server_min_port=10000
+che.infra.kubernetes.installer_server_max_port=20000
+
+# Defines annotations for ingresses which are used for servers exposing. Value depends on ingress controller.
+# For example for nginx ingress controller 0.9.0-beta17 the following value is recommended:
+# {"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false",\
+#     "ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"}
+#
+# OpenShift infrastructure ignores this property because it uses Routes instead of ingresses.
+che.infra.kubernetes.ingress.annotations_json=NULL
+
+# Defines security context for pods that will be created by Kubernetes Infra
+#
+# This is ignored by OpenShift infra
+che.infra.kubernetes.pod.security_context.run_as_user=NULL
+che.infra.kubernetes.pod.security_context.fs_group=NULL
+
+
+### OpenShift Infra parameters
+#
+# Since OpenShift infrastructure reuse Kubernetes infrastructure components
+# OpenShift infrastructure reuse most of the Kubernetes configuration attributes.
+#
+
+# Defines OpenShift namespace in which all workspaces will be created.
+# If not set, every workspace will be created in a new project, where project name = workspace id
+che.infra.openshift.project=
+
+# Create routes with Transport Layer Security (TLS) enabled
+che.infra.openshift.tls_enabled=false
+
 
 # Single port mode wildcard domain host & port. nip.io is used by default
 che.singleport.wildcard_domain.host=NULL

assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che_aliases.properties

@@ -10,3 +10,24 @@
 #
 
 # File with a map new_property_name=old_property_name1,old_property_name2
+
+che.infra.kubernetes.master_url=che.infra.openshift.master_url
+che.infra.kubernetes.username=che.infra.openshift.username
+che.infra.kubernetes.password=che.infra.openshift.password
+che.infra.kubernetes.oauth_token=che.infra.openshift.oauth_token
+che.infra.kubernetes.trust_certs=che.infra.openshift.trust_certs
+che.infra.kubernetes.machine_start_timeout_min=che.infra.openshift.machine_start_timeout_min
+che.infra.kubernetes.bootstrapper.binary_url=che.infra.openshift.bootstrapper.binary_url
+che.infra.kubernetes.bootstrapper.timeout_min=che.infra.openshift.bootstrapper.timeout_min
+che.infra.kubernetes.bootstrapper.installer_timeout_sec=che.infra.openshift.bootstrapper.installer_timeout_sec
+che.infra.kubernetes.bootstrapper.server_check_period_sec=che.infra.openshift.bootstrapper.server_check_period_sec
+che.infra.kubernetes.pvc.enabled=che.infra.openshift.pvc.enabled
+che.infra.kubernetes.pvc.strategy=che.infra.openshift.pvc.strategy
+che.infra.kubernetes.pvc.precreate_subpaths=che.infra.openshift.pvc.precreate_subpaths
+che.infra.kubernetes.pvc.name=che.infra.openshift.pvc.name
+che.infra.kubernetes.pvc.quantity=che.infra.openshift.pvc.quantity
+che.infra.kubernetes.pvc.jobs.image=che.infra.openshift.pvc.jobs.image
+che.infra.kubernetes.pvc.jobs.memorylimit=che.infra.openshift.pvc.jobs.memorylimit
+che.infra.kubernetes.pvc.access_mode=che.infra.openshift.pvc.access_mode
+che.infra.kubernetes.installer_server_min_port=che.infra.openshift.installer_server_min_port
+che.infra.kubernetes.installer_server_max_port=che.infra.openshift.installer_server_max_port

dockerfiles/init/manifests/che.env

@@ -436,59 +436,89 @@ CHE_SINGLE_PORT=false
 
 ########################################################################################
 #####                                                                              #####
-#####                               Openshift Infrastructure                       #####
+#####                              Kubernetes Infrastructure                       #####
 #####                                                                              #####
 #
-#CHE_INFRA_OPENSHIFT_MASTER__URL=
-#CHE_INFRA_OPENSHIFT_USERNAME=
-#CHE_INFRA_OPENSHIFT_PASSWORD=
-#CHE_INFRA_OPENSHIFT_OAUTH__TOKEN=
-#CHE_INFRA_OPENSHIFT_TRUST__CERTS=
+#Configuration of Kubernetes client that Infra will use
+#CHE_INFRA_KUBERNETES_MASTER__URL=
+#CHE_INFRA_KUBERNETES_USERNAME=
+#CHE_INFRA_KUBERNETES_PASSWORD=
+#CHE_INFRA_KUBERNETES_OAUTH__TOKEN=
+#CHE_INFRA_KUBERNETES_TRUST__CERTS=
+
+# Defines Kubernetes namespace in which all workspaces will be created.
+# If not set, every workspace will be created in a new namespace, where namespace = workspace id
+#
+# Ignored for OpenShift infra. Use `CHE_INFRA_OPENSHIFT_PROJECT` instead
+#CHE_INFRA_KUBERNETES_NAMESPACE=
 
-#CHE_INFRA_OPENSHIFT_MACHINE__START__TIMEOUT__MIN=5
+#CHE_INFRA_KUBERNETES_MACHINE__START__TIMEOUT__MIN=5
 
 # Defines whether use the Persistent Volume Claim for che workspace needs
 # e.g backup projects, logs etc or disable it.
-#CHE_INFRA_OPENSHIFT_PVC_ENABLED=true
+#CHE_INFRA_KUBERNETES_PVC_ENABLED=true
 
 # Defined which strategy will be used while choosing PVC for workspaces.
 #
 # Supported strategies:
 # - 'common'
-#        PVC with name that is configured by 'che.infra.openshift.pvc.name'
+#        PVC with name that is configured by 'che.infra.kubernetes.pvc.name'
 #        the property will be used for storing workspaces data of each workspace
 #        in an OpenShift project.
 #        Existing PVC will be used or new one will be created if it doesn't exist.
 #
 # - 'unique'
-#        PVC with name that is evaluated as '{che.infra.openshift.pvc.name} + '-' + {generated_8_chars}'
+#        PVC with name that is evaluated as '{che.infra.kubernetes.pvc.name} + '-' + {generated_8_chars}'
 #        will be used for storing of workspaces data.
 #        Existing PVC will be used or a new one will be created if it doesn't exist.
-#CHE_INFRA_OPENSHIFT_PVC_STRATEGY=common
+#CHE_INFRA_KUBERNETES_PVC_STRATEGY=common
 
 # Defines whether to run a job that creates workspace's subpath directories in persistent volume for the 'common' strategy before launching a workspace.
 # Necessary in some versions of OpenShift/Kubernetes as workspace subpath volume mounts are created with root permissions,
 # and thus cannot be modified by workspaces running as a user (presents an error importing projects into a workspace in Che).
-# The default is "true", but should be set to false if the version of Openshift/Kubernetes creates subdirectories with user permissions.
+# The default is "true", but should be set to false if the version of Kubernetes/Kubernetes creates subdirectories with user permissions.
 # Relevant issue: https://github.com/kubernetes/kubernetes/issues/41638
 # Note that this property has effect only if the 'common' PVC strategy used.
-#CHE_INFRA_OPENSHIFT_PVC_PRECREATE__SUBPATHS=true
+#CHE_INFRA_KUBERNETES_PVC_PRECREATE__SUBPATHS=true
 
 # Defines the name of Persistent Volume Claim for che workspace.
-#CHE_INFRA_OPENSHIFT_PVC_NAME=claim-che-workspace
+#CHE_INFRA_KUBERNETES_PVC_NAME=claim-che-workspace
 
 # Defines the size of Persistent Volume Claim of che workspace.
 # Format described here:
-# https://docs.openshift.com/container-platform/latest/dev_guide/compute_resources.html#dev-compute-resources
-#CHE_INFRA_OPENSHIFT_PVC_QUANTITY=10Gi
+# https://docs.kubernetes.com/container-platform/latest/dev_guide/compute_resources.html#dev-compute-resources
+#CHE_INFRA_KUBERNETES_PVC_QUANTITY=10Gi
 
 # Defines Persistent Volume Claim access mode.
 # Note that for common PVC strategy changing of access mode affects the number of simultaneously running workspaces.
 # If OpenShift flavor where che running is using PVs with RWX access mode then a limit of running workspaces at the same time
 # bounded only by che limits configuration like(RAM, CPU etc).
 # Detailed information about access mode is described here:
-# https://docs.openshift.com/container-platform/latest/architecture/additional_concepts/storage.html#pv-access-modes
-#CHE_INFRA_OPENSHIFT_PVC_ACCESS__MODE=ReadWriteOnce
+# https://docs.kubernetes.com/container-platform/latest/architecture/additional_concepts/storage.html#pv-access-modes
+#CHE_INFRA_KUBERNETES_PVC_ACCESS__MODE=ReadWriteOnce
+
+# Defines annotations for ingresses which are used for servers exposing. Value depends on ingress controller.
+# For example for nginx ingress controller 0.9.0-beta17 the following value is recommended:
+# {"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false",\
+#     "ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"}
+#
+# OpenShift infrastructure ignores this property because it uses Routes instead of ingresses.
+#CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON=NULL
+
+########################################################################################
+#####                                                                              #####
+#####                               Openshift Infrastructure                       #####
+#####                                                                              #####
+#
+# Since OpenShift infrastructure reuse Kubernetes infrastructure components
+# OpenShift infrastructure reuse most of the Kubernetes configuration attributes.
+
+# Defines OpenShift namespace in which all workspaces will be created.
+# If not set, every workspace will be created in a new project, where project name = workspace id
+#CHE_INFRA_OPENSHIFT_PROJECT=
+
+# Create routes with Transport Layer Security (TLS) enabled
+CHE_INFRA_OPENSHIFT_TLS_ENABLED=false
 
 
 ########################################################################################

dockerfiles/init/modules/kubernetes/Deploy Che.md

@@ -0,0 +1,35 @@
+# Deploy single user Che to k8s
+Tested on minikube with vm providers Virtualbox and kvm2. Note that Che with workspaces requires quite a lot
+of RAM. Initial tests were done with 10GB, but it is definitely more than it is needed to start Che 
+and couple of workspaces.
+
+IP of VM is supposed to be `192.168.99.100`. `nip.io` is also used for handling hosts resolution.
+If you have another IP or DNS replace these values in k8s.yml file.
+
+Services are exposed using ingress controller approach.
+We added ingress annotations to customize ingress controller behavior -
+not to break websocket connections.
+In particular testing environment was setup with NginX ingress controller 0.9.0.
+So we added annotations specific to this implementation and version:
+- nginx.ingress.kubernetes.io/rewrite-target: /
+- nginx.ingress.kubernetes.io/ssl-redirect": "false"
+- nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+- nginx.ingress.kubernetes.io/proxy-connect-timeout: "3600"
+
+If you use another ingress controller implementation or version you need to customize
+Che master ingress and value of environment variable `CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON` stored in ConfigMap.
+Value of the map should be expressed as a stringified JSON.
+
+And environment variable would be: `'{"nginx.ingress.kubernetes.io/rewrite-target": "/","nginx.ingress.kubernetes.io/ssl-redirect": "false","nginx.ingress.kubernetes.io/proxy-connect-timeout": "3600","nginx.ingress.kubernetes.io/proxy-read-timeout": "3600"}'`
+
+###Prerequisites:
+- Ingress controller is running. Note: you can start it on minikube with `minikube addons enable ingress`.
+- Currently Che workspaces work with NginX ingress controller only. Note: it is default ingress controller on minikube.
+- DNS discovery should be enabled. Note: enabled by default in minikube.
+### Deployment process:
+Note: despite the fact that it is not necessary to use a separate namespace for Che
+we use it to simplify development operations such as cleaning of spoiled environment
+and clean redeploy of Che.
+- Create namespace `che`: `kubectl create namespace che`
+- Deploy Che: `kubectl --namespace=che apply -f che-kubernetes.yaml`
+- Check Che pod status until it become `Running`: `kubectl get --namespace=che pods`