resources.yaml

kind: List
apiVersion: v1
metadata:
  name: streaming-lab-list
  description: a custom object list for the radanalytics.io streaming lab

items:

  - apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: oshinko
      annotations:
        serviceaccounts.openshift.io/oauth-redirectreference.first: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"oshinko-web-oaproxy"}}'

  - apiVersion: v1
    kind: RoleBinding
    metadata:
      name: oshinko-edit
    roleRef:
      name: edit
    subjects:
      - kind: ServiceAccount
        name: oshinko

  - apiVersion: v1
    kind: ConfigMap
    metadata:
      name: streaming-lab
    data:
      sparkimage: docker.io/elmiko/openshift-spark:2.2.1-streaming-lab
      spark-defaults.conf: |
        spark.jars.packages org.apache.spark:spark-sql-kafka-0-10_2.11:2.2.1

  - apiVersion: v1
    kind: Template
    labels:
      application: oshinko-python-spark
      createdBy: template-oshinko-python-spark-build-dc
    metadata:
      annotations:
        description: Create a buildconfig, imagestream and deploymentconfig using source-to-image and Python Spark source files hosted in git
        openshift.io/display-name: Apache Spark Python
      name: oshinko-python-spark-build-dc
    objects:
    - apiVersion: v1
      kind: ImageStream
      metadata:
        name: ${APPLICATION_NAME}
        labels:
          app: ${APPLICATION_NAME}
      spec:
        dockerImageRepository: ${APPLICATION_NAME}
        tags:
        - name: latest
    - apiVersion: v1
      kind: BuildConfig
      metadata:
        name: ${APPLICATION_NAME}
        labels:
          app: ${APPLICATION_NAME}
      spec:
        output:
          to:
            kind: ImageStreamTag
            name: ${APPLICATION_NAME}:latest
        source:
          contextDir: ${CONTEXT_DIR}
          git:
            ref: ${GIT_REF}
            uri: ${GIT_URI}
          type: Git
        strategy:
          sourceStrategy:
            env:
            - name: APP_FILE
              value: ${APP_FILE}
            forcePull: true
            from:
              kind: DockerImage
              name: docker.io/elmiko/radanalytics-pyspark:2.2.1-streaming-lab
          type: Source
        triggers:
        - imageChange: {}
          type: ImageChange
        - type: ConfigChange
        - github:
            secret: ${APPLICATION_NAME}
          type: GitHub
        - generic:
            secret: ${APPLICATION_NAME}
          type: Generic
    - apiVersion: v1
      kind: DeploymentConfig
      metadata:
        name: ${APPLICATION_NAME}
        labels:
          deploymentConfig: ${APPLICATION_NAME}
          app: ${APPLICATION_NAME}
      spec:
        replicas: 1
        selector:
          deploymentConfig: ${APPLICATION_NAME}
        strategy:
          type: Rolling
        template:
          metadata:
            labels:
              deploymentConfig: ${APPLICATION_NAME}
              app: ${APPLICATION_NAME}
          spec:
            containers:
            - env:
              - name: OSHINKO_CLUSTER_NAME
                value: ${OSHINKO_CLUSTER_NAME}
              - name: APP_ARGS
                value: ${APP_ARGS}
              - name: SPARK_OPTIONS
                value: ${SPARK_OPTIONS}
              - name: OSHINKO_DEL_CLUSTER
                value: ${OSHINKO_DEL_CLUSTER}
              - name: APP_EXIT
                value: "true"
              - name: OSHINKO_NAMED_CONFIG
                value: ${OSHINKO_NAMED_CONFIG}
              - name: OSHINKO_SPARK_DRIVER_CONFIG
                value: ${OSHINKO_SPARK_DRIVER_CONFIG}
              - name: POD_NAME
                valueFrom:
                  fieldRef:
                    fieldPath: metadata.name
              image: ${APPLICATION_NAME}
              imagePullPolicy: IfNotPresent
              name: ${APPLICATION_NAME}
              resources: {}
              terminationMessagePath: /dev/termination-log
              volumeMounts:
              - mountPath: /etc/podinfo
                name: podinfo
                readOnly: false
            dnsPolicy: ClusterFirst
            restartPolicy: Always
            serviceAccount: oshinko
            volumes:
            - downwardAPI:
                items:
                - fieldRef:
                    fieldPath: metadata.labels
                  path: labels
              name: podinfo
        triggers:
        - imageChangeParams:
            automatic: true
            containerNames:
              - ${APPLICATION_NAME}
            from:
              kind: ImageStreamTag
              name: ${APPLICATION_NAME}:latest
          type: ImageChange
        - type: ConfigChange
    - apiVersion: v1
      kind: Service
      metadata:
        name: ${APPLICATION_NAME}
        labels:
          app: ${APPLICATION_NAME}
      spec:
        ports:
        - name: 8080-tcp
          port: 8080
          protocol: TCP
          targetPort: 8080
        selector:
          deploymentConfig: ${APPLICATION_NAME}
    parameters:
    - description: 'The name to use for the buildconfig, imagestream and deployment components'
      from: 'python-spark-[a-z0-9]{4}'
      generate: expression
      name: APPLICATION_NAME
      required: true
    - description: The URL of the repository with your application source code
      displayName: Git Repository URL
      name: GIT_URI
    - description: Optional branch, tag or commit
      displayName: Git Reference
      name: GIT_REF
    - description: Git sub-directory path
      name: CONTEXT_DIR
    - description: The name of the main py file to run. If this is not specified and there is a single py file at top level of the git respository, that file will be chosen.
      name: APP_FILE
    - description: Command line arguments to pass to the Spark application
      name: APP_ARGS
    - description: List of additional Spark options to pass to spark-submit (for exmaple --conf property=value --conf property=value). Note, --master and --class are set by the launcher and should not be set here
      name: SPARK_OPTIONS
    - description: The name of the Spark cluster to run against. The cluster will be created if it does not exist, and a random cluster name will be chosen if this value is left blank.
      name: OSHINKO_CLUSTER_NAME
    - description: The name of a stored cluster configuration to use if a cluster is created, default is 'default'.
      name: OSHINKO_NAMED_CONFIG
      value: streaming-lab
    - description: The name of a configmap to use for the Spark configuration of the driver. If this configmap is empty the default Spark configuration will be used.
      name: OSHINKO_SPARK_DRIVER_CONFIG
      value: streaming-lab
    - description: If a cluster is created on-demand, delete the cluster when the application finishes if this option is set to 'true'
      name: OSHINKO_DEL_CLUSTER
      required: true
      value: 'true'

  - apiVersion: v1
    kind: Template
    template: oshinko-webui
    metadata:
      name: oshinko-webui
    objects:
      - kind: Service
        apiVersion: v1
        metadata:
          name: ${OSHINKO_WEB_NAME}-proxy
          labels:
            name: ${OSHINKO_WEB_NAME}-proxy
        spec:
          ports:
          - name: oc-proxy-port
            protocol: TCP
            port: 8001
            targetPort: 8001
          selector:
            name: ${OSHINKO_WEB_NAME}
      - kind: Service
        apiVersion: v1
        metadata:
          name: ${OSHINKO_WEB_NAME}
          labels:
            name: ${OSHINKO_WEB_NAME}
        spec:
          ports:
            - name: o-web-port
              protocol: TCP
              port: 8080
              targetPort: 8080
          selector:
            name: ${OSHINKO_WEB_NAME}
      - kind: Route
        apiVersion: v1
        metadata:
          name: ${OSHINKO_WEB_NAME}
        spec:
          host: ${OSHINKO_WEB_ROUTE_HOSTNAME}
          path: /webui
          to:
            kind: Service
            name: ${OSHINKO_WEB_NAME}
          alternateBackends:
            - kind: Service
              name: ${OSHINKO_WEB_NAME}
      - kind: DeploymentConfig
        apiVersion: v1
        metadata:
          name: ${OSHINKO_WEB_NAME}
        spec:
          strategy:
            type: Rolling
          triggers:
            - type: ConfigChange
          replicas: 1
          selector:
            name: ${OSHINKO_WEB_NAME}
          template:
            metadata:
              labels:
                name: ${OSHINKO_WEB_NAME}
            spec:
              containers:
                - name: ${OSHINKO_WEB_NAME}
                  image: ${OSHINKO_WEB_IMAGE}
                  imagePullPolicy: Always
                  ports:
                    - name: o-web-port
                      containerPort: 8080
                      protocol: TCP
                  env:
                    - name: SPARK_DEFAULT
                      value: docker.io/elmiko/openshift-spark:2.2.1-streaming-lab
                    - name: OSHINKO_REFRESH_INTERVAL
                      value: ${OSHINKO_REFRESH_INTERVAL}
                    - name: WEB_ROUTE_NAME
                      value: ${OSHINKO_WEB_NAME}
                    - name: INSECURE_WEBUI
                      value: "true"
                    - name: CURRENT_NAMESPACE
                      valueFrom:
                        fieldRef:
                          fieldPath: metadata.namespace
                  readinessProbe:
                    failureThreshold: 3
                    httpGet:
                      path: /webui
                      port: 8080
                      scheme: HTTP
                    periodSeconds: 10
                    successThreshold: 1
                    timeoutSeconds: 1
                    initialDelaySeconds: 20
                  livenessProbe:
                    failureThreshold: 3
                    httpGet:
                      path: /webui
                      port: 8080
                      scheme: HTTP
                    periodSeconds: 10
                    successThreshold: 1
                    timeoutSeconds: 1
                    initialDelaySeconds: 20
                - name: oc-proxy
                  image: radanalyticsio/oc-proxy:stable
                  imagePullPolicy: IfNotPresent
                  args:
                    - proxy
                    - "-p"
                    - '8001'
                    - "--address=0.0.0.0"
                    - "--disable-filter=true"
                    - "--api-prefix=/proxy"
                  ports:
                    - name: oc-proxy-port
                      containerPort: 8001
                      protocol: TCP
              serviceAccount: oshinko
    parameters:
      - name: OSHINKO_WEB_NAME
        description: Name of the oshinko web service
        value: "oshinko-web"
      - name: OSHINKO_WEB_IMAGE
        description: Full name of the oshinko web image
        required: true
        value: radanalyticsio/oshinko-webui:stable
      - name: OSHINKO_WEB_ROUTE_HOSTNAME
        description: The hostname used to create the external route for the webui
      - name: OSHINKO_REFRESH_INTERVAL
        value: "5"
        description: Refresh interval for updating cluster list in seconds

  - apiVersion: v1
    kind: Template
    template: streaming-lab-notebook
    metadata:
      name: streaming-lab-notebook
    objects:
    - kind: Service
      apiVersion: v1
      metadata:
        name: ${APPLICATION_NAME}
        labels:
          name: ${APPLICATION_NAME}
      spec:
        ports:
          - protocol: TCP
            port: 8888
            targetPort: 8888
        selector:
          name: ${APPLICATION_NAME}
    - kind: Route
      apiVersion: v1
      metadata:
        name: ${APPLICATION_NAME}
      spec:
        host: ${ROUTE_HOSTNAME}
        to:
          kind: Service
          name: ${APPLICATION_NAME}
    - kind: DeploymentConfig
      apiVersion: v1
      metadata:
        name: ${APPLICATION_NAME}
      spec:
        strategy:
          type: Rolling
        triggers:
          - type: ConfigChange
        replicas: 1
        selector:
          name: ${APPLICATION_NAME}
        template:
          metadata:
            labels:
              name: ${APPLICATION_NAME}
          spec:
            containers:
              - name: ${APPLICATION_NAME}
                image: quay.io/willbenton/streaming-lab-notebooks:madrid-lab
                env:
                  - name: JUPYTER_NOTEBOOK_PASSWORD
                    value: developer
                ports:
                  - containerPort: 8888
                    protocol: TCP
    parameters:
      - name: APPLICATION_NAME
        description: the application name
        value: streaming-lab-notebook
      - name: ROUTE_HOSTNAME
        description: a hostname for the route