Add openshift specific files based on pipeline v0.22.0

OWNERS

@@ -1,11 +1,7 @@
 # The OWNERS file is used by prow to automatically merge approved PRs.
 
 approvers:
-- bobcatfish
-- chmouel
-- dlorenc
-- ImJasonH
-- kimsterv
-- vdemeester
-- sbwsg
-- vinamra28
+- tektoncd-approvers
+
+reviewers:
+- tektoncd-reviewers

OWNERS_ALIASES

@@ -0,0 +1,14 @@
+aliases:
+  tektoncd-approvers:
+  - chmouel
+  - sthaha
+  - vdemeester
+  tektoncd-reviewers:
+  - chmouel
+  - khrm
+  - nikhil-thomas
+  - piyush-garg
+  - pradeepitm12
+  - savitaashture
+  - sthaha
+  - vdemeester

openshift/Dockerfile

@@ -0,0 +1,30 @@
+FROM openshift/origin-release:golang-1.13
+
+# download, verify and install openshift client tools (oc and kubectl)
+# upgrade the openssl to the recent version ie 1.1.1g
+WORKDIR /tmp
+RUN OPENSHIFT_CLIENT_VERSION=$(curl -s https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/release.txt|sed -n '/Version:/ { s/[ ]*Version:[ ]*// ;p}') \
+    && curl -L -O -s https://mirror.openshift.com/pub/openshift-v4/clients/ocp/$OPENSHIFT_CLIENT_VERSION/openshift-client-linux-$OPENSHIFT_CLIENT_VERSION.tar.gz \
+    && curl -LO https://www.openssl.org/source/openssl-1.1.1g.tar.gz \
+    && curl -L -s https://mirror.openshift.com/pub/openshift-v4/clients/ocp/$OPENSHIFT_CLIENT_VERSION/sha256sum.txt | \
+       grep openshift-client-linux-$OPENSHIFT_CLIENT_VERSION.tar.gz > openshift-origin-client-tools.sha256 \
+    && sha256sum -c openshift-origin-client-tools.sha256 \
+    && mkdir openshift-origin-client-tools \
+    && tar xzf openshift-client-linux-$OPENSHIFT_CLIENT_VERSION.tar.gz --directory openshift-origin-client-tools \
+    && tar xf openssl-1.1.1g.tar.gz \
+    && mv /tmp/openshift-origin-client-tools/oc /usr/bin/oc \
+    && mv /tmp/openshift-origin-client-tools/kubectl /usr/bin/kubectl \
+    && rm -rf ./openshift* \
+    && cd openssl-1.1.1g \
+    && ./config --prefix=/usr/local/ssl --openssldir=/usr/local/ssl enable-ec_nistp_64_gcc_128 no-ssl2 no-ssl3 no-comp no-idea no-dtls no-dtls1 no-shared no-psk no-srp no-ec2m no-weak-ssl-ciphers \
+    && make install \
+    && cd .. \
+    && rm -rf openssl-1.1.1g.tar.gz openssl-1.1.1g \
+    && mv /usr/local/ssl/bin/openssl /usr/local/bin/openssl \
+    && oc version
+
+# upgrade to latest PyYAML
+RUN python3 -mpip install -U PyYAML
+
+# Allow runtime users to add entries to /etc/passwd
+RUN chmod g+rw /etc/passwd

openshift/Makefile

@@ -0,0 +1,16 @@
+DETECT_RELEASE_YAML=$(shell bash pipeline-latest-release.sh)
+STABLE_RELEASE_YAML=$(shell bash pipeline-latest-release.sh --only-stable-release)
+RELEASE_YAML=
+
+# Temporary hack to use the stable release if nightly doesn't exist, in case release fails
+test-e2e-downstream-nightly:
+	make test-e2e-downstream RELEASE_YAML=$(DETECT_RELEASE_YAML)
+.PHONY: test-e2e-downstream-nightly
+
+test-e2e-downstream-stable:
+	@make test-e2e-downstream RELEASE_YAML=$(STABLE_RELEASE_YAML)
+.PHONY: test-e2e-downstream-stable
+
+test-e2e-downstream:
+	@env RELEASE_YAML=$(RELEASE_YAML) ./e2e-tests-openshift.sh
+.PHONY: test-e2e-downstream

openshift/e2e-add-service-account.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# This will add a serviceAccount to a TaskRun/PipelineRun with pyyaml via
+# STDIN/STDOUT eg:
+#
+# python openshift/e2e-add-service-account-tr.py \
+#   SERVICE_ACCOUNT < run.yaml > newfile.yaml
+#
+import yaml
+import sys
+data = list(yaml.safe_load_all(sys.stdin))
+for x in data:
+    if x['kind'] in ('PipelineRun', 'TaskRun'):
+        x['spec']['serviceAccountName'] = sys.argv[1]
+print(yaml.dump_all(data, default_flow_style=False))

openshift/e2e-tests-openshift.sh

@@ -0,0 +1,169 @@
+#!/usr/bin/env bash
+#
+# This will runs the E2E tests on OpenShift
+#
+set -e
+
+# Maximin number of parallel tasks run at the same time
+# # start from 0 so 4 => 5
+MAX_NUMBERS_OF_PARALLEL_TASKS=4
+
+# This is needed on openshift CI since HOME is read only and if we don't cache,
+# it takes over 15s every kubectl query without caching.
+KUBECTL_CMD="kubectl --cache-dir=/tmp/cache"
+
+# Give these tests the priviliged rights
+PRIVILEGED_TESTS="buildah buildpacks buildpacks-phases jib-gradle kaniko kythe-go orka-init orka-teardown s2i"
+
+# Skip those tests when they really can't work in OpenShift
+SKIP_TESTS="docker-build orka-full orka-deploy"
+
+# Orka Tasks which can be tested as privileged but existing SA needs to be used
+# to give privileged access
+ORKA_TASKS="orka-init orka-teardown"
+
+# Service Account used for image builder
+SERVICE_ACCOUNT=builder
+
+function check-service-endpoints() {
+  service=${1}
+  namespace=${2}
+  echo "-----------------------"
+  echo "checking ${namespace}/${service} service endpoints"
+  count=0
+  while [[ -z $(${KUBECTL_CMD} get endpoints ${service} -n ${namespace} -o jsonpath='{.subsets}') ]]; do
+    # retry for 15 mins
+    sleep 10
+    if [[ $count -gt 90 ]]; then
+      echo ${namespace}/${service} endpoints unavailable
+      exit 1
+    fi
+    echo waiting for ${namespace}/${service} endpoints
+    count=$(( count+1 ))
+  done
+}
+
+# Create some temporary file to work with, we will delete them right after exiting
+TMPF2=$(mktemp /tmp/.mm.XXXXXX)
+TMPF=$(mktemp /tmp/.mm.XXXXXX)
+clean() { rm -f ${TMP} ${TMPF2}; }
+trap clean EXIT
+
+source $(dirname $0)/../test/e2e-common.sh
+cd $(dirname $(readlink -f $0))/..
+
+# Install CI
+[[ -z ${LOCAL_CI_RUN} ]] && install_pipeline_crd
+
+# list tekton-pipelines-webhook service endpoints
+check-service-endpoints "tekton-pipelines-webhook" "tekton-pipelines"
+
+CURRENT_TAG=$(git describe --tags 2>/dev/null || true)
+
+# in_array function: https://www.php.net/manual/en/function.in-array.php :-D
+function in_array() {
+    param=$1;shift
+    for elem in $@;do
+        [[ $param == $elem ]] && return 0;
+    done
+    return 1
+}
+
+function test_privileged {
+    local cnt=0
+    local task_to_tests=""
+
+    # Run the privileged tests
+    for runtest in $@;do
+        in_array ${runtest} ${SKIP_TESTS} && { echo "Skipping: ${runtest}"; continue ;}
+
+        # Add here the pre-apply-taskrun-hook function so we can do our magic to add the serviceAccount on the TaskRuns,
+        function pre-apply-taskrun-hook() {
+            btest=$(basename $(dirname $(dirname $runtest)))
+            if $(in_array ${btest} ${ORKA_TASKS}); then
+                oc adm policy add-scc-to-user privileged system:serviceaccount:${tns}:orka-svc || true
+            else
+                cp ${TMPF} ${TMPF2}
+                python3 openshift/e2e-add-service-account.py ${SERVICE_ACCOUNT} < ${TMPF2} > ${TMPF}
+                oc adm policy add-scc-to-user privileged system:serviceaccount:${tns}:${SERVICE_ACCOUNT} || true
+            fi
+        }
+        unset -f pre-apply-task-hook || true
+
+        task_to_tests="${task_to_tests} task/${runtest}/*/tests"
+
+        if [[ ${cnt} == "${MAX_NUMBERS_OF_PARALLEL_TASKS}" ]];then
+            echo "---"
+            echo "Running privileged test: ${task_to_tests}"
+            echo "---"
+
+            test_task_creation ${task_to_tests}
+
+            cnt=0
+            task_to_tests=""
+            continue
+        fi
+
+        cnt=$((cnt+1))
+    done
+
+    # Remaining task
+    if [[ -n ${task_to_tests} ]];then
+        echo "---"
+        echo "Running privileged test: ${task_to_tests}"
+        echo "---"
+
+        test_task_creation ${task_to_tests}
+    fi
+}
+
+function test_non_privileged {
+    local cnt=0
+    local task_to_tests=""
+
+    # Run the non privileged tests
+    for runtest in $@;do
+        btest=$(basename $(dirname $(dirname $runtest)))
+        in_array ${btest} ${SKIP_TESTS} && { echo "Skipping: ${btest}"; continue ;}
+        in_array ${btest} ${PRIVILEGED_TESTS} && continue # We did them previously
+
+        # Make sure the functions are not set anymore here or this will get run.
+        unset -f pre-apply-taskrun-hook || true
+        unset -f pre-apply-task-hook || true
+
+        task_to_tests="${task_to_tests} ${runtest}"
+
+        if [[ ${cnt} == "${MAX_NUMBERS_OF_PARALLEL_TASKS}" ]];then
+            echo "---"
+            echo "Running non privileged test: ${task_to_tests}"
+            echo "---"
+
+            test_task_creation ${task_to_tests}
+
+            cnt=0
+            task_to_tests=""
+            continue
+        fi
+
+        cnt=$((cnt+1))
+    done
+
+    # Remaining task
+    if [[ -n ${task_to_tests} ]];then
+        echo "---"
+        echo "Running non privileged test: ${task_to_tests}"
+        echo "---"
+
+        test_task_creation ${task_to_tests}
+    fi
+}
+
+# Test if yamls can install
+until test_yaml_can_install; do
+  echo "-----------------------"
+  echo 'retry test_yaml_can_install'
+  echo "-----------------------"
+  sleep 5
+done
+test_non_privileged $(\ls -1 -d task/*/*/tests)
+test_privileged ${PRIVILEGED_TESTS}

openshift/pipeline-latest-release.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+#
+# Detect which version of pipeline should be installed
+# First it tries nightly
+# If that doesn't work it tries previous releases (until the MAX_SHIFT variable)
+# If not it exit 1
+# It can take the argument --only-stable-release to not do nightly but only detect the pipeline version
+
+MAX_SHIFT=2
+NIGHTLY_RELEASE="https://raw.githubusercontent.com/openshift/tektoncd-pipeline/release-next/openshift/release/tektoncd-pipeline-nightly.yaml"
+STABLE_RELEASE_URL='https://raw.githubusercontent.com/openshift/tektoncd-pipeline/${version}/openshift/release/tektoncd-pipeline-${version}.yaml'
+
+function get_version {
+    local shift=${1} # 0 is latest, increase is the version before etc...
+    local version=$(curl -s https://api.github.com/repos/tektoncd/pipeline/releases | python -c "from pkg_resources import parse_version;import sys, json;jeez=json.load(sys.stdin);print(sorted([x['tag_name'] for x in jeez], key=parse_version, reverse=True)[${shift}])")
+    echo $(eval echo ${STABLE_RELEASE_URL})
+}
+
+function tryurl {
+    curl -s -o /dev/null -f ${1} || return 1
+}
+
+if [[ ${1} != "--only-stable-release" ]];then
+   if tryurl ${NIGHTLY_RELEASE};then
+       echo ${NIGHTLY_RELEASE}
+       exit
+   fi
+fi
+
+for shifted in `seq 0 ${MAX_SHIFT}`;do
+    versionyaml=$(get_version ${shifted})
+    if tryurl ${versionyaml};then
+        echo ${versionyaml}
+        exit 0
+    fi
+done
+
+exit 1

openshift/release/README.md

@@ -0,0 +1,35 @@
+# Release creation
+
+## Branching
+
+As far as branching goes, we have two use-cases:
+
+1. Creating a branch based off an upstream release tag.
+2. Having a branch that follow upstream's HEAD and serves as a vehicle for continuous integration.
+
+A prerequisite for both scripts is that your local clone of the repository has a remote "upstream"
+that points to the upstream repository and a remote "openshift" that points to the openshift fork.
+
+Run the scripts from the root of the repository.
+
+### Creating a branch based off an upstream release tag
+
+To create a clean branch from an upstream release tag, use the `create-release-branch.sh` script:
+
+```bash
+$ ./openshift/release/create-release-branch.sh v0.4.1 release-0.4
+```
+
+This will create a new branch "release-0.4" based off the tag "v0.4.1" and add all OpenShift specific
+files that we need to run CI on top of it.
+
+### Updating the release-next branch that follow upstream's HEAD
+
+To update a branch to the latest HEAD of upstream use the `update-to-head.sh` script:
+
+```bash
+$ ./openshift/release/update-to-head.sh
+```
+
+That will pull the latest master from upstream, rebase the current fixes on the release-next branch
+on top of it, update the Openshift specific files if necessary, and then trigger CI.

openshift/release/create-release-branch.sh

@@ -0,0 +1,44 @@
+#!/bin/bash -e
+set -eu
+
+tag=$1
+tag_regexp="^v([0-9]+)\.([0-9]+)\.([0-9]+)"
+
+if [[ -z ${tag} ]];then
+    echo "You need specify a tag like v0.9.1"
+    exit 1
+fi
+
+if [[ ! $tag =~ $tag_regexp ]];then
+    echo "\"$tag\" is wrong format. Must have proper format like v1.2.3"
+    exit 1
+fi
+
+release=release-v${BASH_REMATCH[1]}.${BASH_REMATCH[2]}
+
+echo "===== Resetting branch ${release} based on ${tag}"
+
+# Fetch the latest tags and checkout a new branch from the wanted tag.
+git fetch upstream --tags
+
+echo "===== Checkout upstream/master as base"
+git checkout --no-track -B "${release}" upstream/master
+
+echo "===== Adding openshift specific files from openshift/master"
+git fetch openshift master
+git checkout openshift/master -- openshift OWNERS_ALIASES OWNERS
+
+git add openshift OWNERS_ALIASES OWNERS
+git commit -m "Add openshift specific files based on pipeline ${tag}"
+
+echo "===== Creating tag ${tag}"
+git tag --force ${tag}
+
+echo "===== Pushing branch '${release}' to openshift remote"
+git push openshift ${release}
+
+echo "===== Pushing tag '${tag}' to openshift remote"
+git push --tags openshift ${tag}
+
+echo "===== Done"
+echo "$(git remote get-url openshift)/tree/${release}"