Add fedora28 testing container, run container tests in parallel

Jenkinsfile

@@ -13,7 +13,8 @@ timestamps {
 
     // Needed for podTemplate()
     env.SLAVE_TAG = env.SLAVE_TAG ?: 'stable'
-    env.ENVSETUPTESTC7_TAG = env.ENVSETUPTESTC7_TAG ?: 'stable'
+    env.ENVSETUPTEST_C7_TAG = env.ENVSETUPTEST_C7_TAG ?: 'stable'
+    env.ENVSETUPTEST_F28_TAG = env.ENVSETUPTEST_F28_TAG ?: 'stable'
 
     env.DOCKER_REPO_URL = env.DOCKER_REPO_URL ?: '172.30.254.79:5000'
     env.OPENSHIFT_NAMESPACE = env.OPENSHIFT_NAMESPACE ?: 'continuous-infra'
@@ -62,7 +63,7 @@ timestamps {
                                     extensions: [
                                             [
                                                     $class: 'org.jenkinsci.plugins.ghprb.extensions.status.GhprbSimpleStatus',
-                                                    commitStatusContext: 'Contra Env Setup Stage Job',
+                                                    commitStatusContext: 'Contra Env Setup Testing Job',
                                                     showMatrixStatus: false,
                                                     triggeredStatus: 'Starting job...',
                                                     startedStatus: 'Testing...',
@@ -93,9 +94,12 @@ timestamps {
                                     string(name: 'SLAVE_TAG',
                                             defaultValue: 'stable',
                                             description: 'Tag for slave image'),
-                                    string(name: 'ENVSETUPTESTC7_TAG',
+                                    string(name: 'ENVSETUPTEST_C7_TAG',
                                             defaultValue: 'stable',
                                             description: 'Tag for contra-env-setup-test-c7 image'),
+                                    string(name: 'ENVSETUPTEST_F28_TAG',
+                                            defaultValue: 'stable',
+                                            description: 'Tag for contra-env-setup-test-f28 image'),
                                     string(name: 'DOCKER_REPO_URL',
                                             defaultValue: '172.30.254.79:5000',
                                             description: 'Docker repo url for Openshift instance'),
@@ -127,7 +131,14 @@ timestamps {
                     // This adds the contra-env-setup-test-c7 container to the pod.
                     containerTemplate(name: 'contra-env-setup-test-c7',
                             alwaysPullImage: true,
-                            image: DOCKER_REPO_URL + '/' + OPENSHIFT_NAMESPACE + '/contra-env-setup-test-c7:' + ENVSETUPTESTC7_TAG,
+                            image: DOCKER_REPO_URL + '/' + OPENSHIFT_NAMESPACE + '/contra-env-setup-test-c7:' + ENVSETUPTEST_C7_TAG,
+                            ttyEnabled: true,
+                            privileged: true,
+                            workingDir: '/workDir'),
+                    // This adds the contra-env-setup-test-f28 container to the pod.
+                    containerTemplate(name: 'contra-env-setup-test-f28',
+                            alwaysPullImage: true,
+                            image: DOCKER_REPO_URL + '/' + OPENSHIFT_NAMESPACE + '/contra-env-setup-test-f28:' + ENVSETUPTEST_F28_TAG,
                             ttyEnabled: true,
                             privileged: true,
                             workingDir: '/workDir'),
@@ -176,14 +187,33 @@ timestamps {
 
                                 currentStage = "test-env-setup"
                                 stage(currentStage) {
-
-                                    envsetupUtils.timedPipelineStep(stepName: currentStage, debug: true) {
-                                        // Set stage specific vars
-                                        envsetupUtils.setStageEnvVars(currentStage)
-
-                                        // Run contra-env-setup test
-                                        pipelineUtils.executeInContainer(currentStage, "contra-env-setup-test-c7", "/home/prepare_and_test.sh")
-                                    }
+                                    parallel (
+                                        'centos7': {
+                                            stage("${currentStage}-centos7") {
+                                                envsetupUtils.timedPipelineStep(stepName: "${currentStage}-centos7", debug: true) {
+                                                    // Set stage specific vars
+                                                    envsetupUtils.setStageEnvVars(currentStage)
+
+                                                    // Run contra-env-setup test
+                                                    pipelineUtils.executeInContainer("${currentStage}-centos7", "contra-env-setup-test-c7", "/home/prepare_and_test.sh")
+                                                }
+                                            }
+                                        },
+                                        'fedora28': {
+                                            stage("${currentStage}-fedora28") {
+                                                envsetupUtils.timedPipelineStep(stepName: "${currentStage}-fedora28", debug: true) {
+                                                    // Set stage specific vars
+                                                    envsetupUtils.setStageEnvVars(currentStage)
+
+                                                    // Fix for minishift cgrups/cpuset bug
+                                                    pipelineUtils.executeInContainer("${currentStage}-fedora28", "contra-env-setup-test-f28", "/home/fix-minishift.sh || true")
+
+                                                    // Run contra-env-setup test
+                                                    pipelineUtils.executeInContainer("${currentStage}-fedora28", "contra-env-setup-test-f28", "/home/prepare_and_test.sh")
+                                                }
+                                            }
+                                        }
+                                    )
                                 }
                             } catch (e) {
                                 // Set build result
@@ -208,6 +238,9 @@ timestamps {
                                 ciMetrics.setMetricTag(jobMeasurement, 'build_result', currentBuild.result)
                                 ciMetrics.setMetricField(jobMeasurement, 'build_time', currentBuild.getDuration())
 
+                                // Archive JUnit results
+                                step([$class: 'JUnitResultArchiver', testResults: '*.xml'])
+
                             }
                         }
                     }

test/config/Dockerfiles/contra-env-setup-test-c7/prepare_and_test.sh

@@ -2,17 +2,25 @@
 
 base_dir=/home
 
+# Set log dir to the WORKSPACE where it can be archived
+if [ ! -z "${WORKSPACE}" ]; then
+    log_dir=${WORKSPACE}
+else
+    log_dir=${base_dir}/logs
+fi
+
 # Set environment to use ara with ansible
 export ara_location=$(python -c "import os,ara; print(os.path.dirname(ara.__file__))")
 export ANSIBLE_CALLBACK_PLUGINS=$ara_location/plugins/callbacks
 export ANSIBLE_ACTION_PLUGINS=$ara_location/plugins/actions
 export ANSIBLE_LIBRARY=$ara_location/plugins/modules
 
 export USER=$(whoami)
+
 cd ${base_dir}
 
 # Prepare repo and logs directories
-mkdir -p ${base_dir}/logs
+mkdir -p ${log_dir}
 
 git clone https://github.com/CentOS-PaaS-SIG/contra-env-setup.git
 
@@ -30,18 +38,10 @@ fi
 
 popd
 
-# Temporary fix for skipping nested virtualization setup
-sed -i 's/- import_tasks: setup_nested_virt.yml/#- import_tasks: setup_nested_virt.yml/g' ${base_dir}/contra-env-setup/playbooks/roles/prereqs/tasks/main.yml
-
 # Run the playbook locally with added hook for debugging variables needed for testing
 /usr/bin/ansible-playbook -vv -i "localhost," ${base_dir}/contra-env-setup/playbooks/setup.yml -e user=root \
-                          -e ansible_connection=local -e setup_playbook_hooks=true \
+                          -e ansible_connection=local -e setup_nested_virt=false -e setup_playbook_hooks=true \
                           --extra-vars='{"hooks": ["/home/debug_vars.yml"]}'
 
 # Run tests with pytest
-python -m pytest ${base_dir}/test_contra_env_setup.py -v > ${base_dir}/logs/contra_env_setup.log
-
-# Copy logs to the work directory where they can be archived
-if [ ! -z "${WORKSPACE}" ]; then
-    cp ${base_dir}/logs/* ${WORKSPACE}
-fi
+python -m pytest ${base_dir}/test_contra_env_setup.py -v --junitxml=${log_dir}/contra_env_setup_centos7.xml > ${log_dir}/contra_env_setup_centos7.log

test/config/Dockerfiles/contra-env-setup-test-c7/test_contra_env_setup.py

@@ -13,7 +13,7 @@ def run_info():
     profile = None
     minishift_bin = None
     oc_bin = None
-    
+
     # Get Playbook ID from ara
     playbook_list_cmd = '/usr/bin/ara playbook list -f json'
     playbook_list = json.loads(check_output(playbook_list_cmd.split()))
@@ -33,7 +33,7 @@ def run_info():
         if result.get('Name') == 'playbook_hooks : debug-vars':
             debug_result_id = result.get('ID')
             break
-    
+
     if debug_result_id is not None:
         debug_result_cmd = '/usr/bin/ara result show %s --raw -f json' % debug_result_id
         debug_result = json.loads(check_output(debug_result_cmd.split()))
@@ -44,11 +44,11 @@ def run_info():
             oc_bin = test_run_info.get('oc_bin')
 
     return {
-               'playbook_id': playbook_id,
-               'profile': profile,
-               'minishift_bin': minishift_bin,
-               'oc_bin': oc_bin,
-           }
+        'playbook_id': playbook_id,
+        'profile': profile,
+        'minishift_bin': minishift_bin,
+        'oc_bin': oc_bin,
+    }
 
 
 def test_playbook_success(run_info):
@@ -70,7 +70,7 @@ def test_playbook_success(run_info):
             tasks_ok = stats.get('Ok')
             tasks_changed = stats.get('Changed')
             tasks_failed = stats.get('Failed')
-    
+
     assert(tasks_failed == 0)
     assert(tasks_changed > 0)
     assert(tasks_ok > 0)
@@ -79,7 +79,7 @@ def test_playbook_success(run_info):
 def test_binary_locations(run_info):
     minishift_bin = run_info.get('minishift_bin')
     oc_bin = run_info.get('oc_bin')
- 
+
     assert(minishift_bin is not None)
     assert(os.path.isfile(minishift_bin))
     assert(oc_bin is not None)
@@ -113,7 +113,7 @@ def test_buildconfigs(run_info):
 
     assert(oc_result and 'ansible-executor' in oc_result)
     assert(oc_result and 'linchpin-executor' in oc_result)
-    assert(oc_result and 'jenkins-contra-slave' in oc_result)
+    assert(oc_result and 'jenkins-contra-sample-project-slave' in oc_result)
     assert(oc_result and 'jenkins ' in oc_result)
 
 
@@ -134,9 +134,9 @@ def test_builds(run_info):
                 linchpin_executor_success = True
             if 'jenkins-' in line and 'Complete' in line and 'slave' not in line:
                 jenkins_success = True
-            if 'jenkins-contra-slave-' in line and 'Complete' in line:
+            if 'jenkins-contra-sample-project-slave-' in line and 'Complete' in line:
                 jenkins_contra_slave_success = True
- 
+
 
     assert(ansible_executor_success)
     assert(linchpin_executor_success)
@@ -154,7 +154,7 @@ def test_imagestreams(run_info):
 
     assert(oc_result and 'ansible-executor' in oc_result)
     assert(oc_result and 'linchpin-executor' in oc_result)
-    assert(oc_result and 'jenkins-contra-slave' in oc_result)
+    assert(oc_result and 'jenkins-contra-sample-project-slave' in oc_result)
     assert(oc_result and 'jenkins' in oc_result)
 
 
@@ -179,7 +179,7 @@ def test_jenkins_master_pod(run_info):
         oc_result = check_output(oc_cmd.split())
 
         for line in oc_result.splitlines():
-            if 'jenkins' in line and 'slave' not in line and 'Running' in line:
+            if 'jenkins' in line and 'Running' in line:
                 jenkins_master_pod_running = True
 
     assert(jenkins_master_pod_running)

test/config/Dockerfiles/contra-env-setup-test-f28/Dockerfile

@@ -0,0 +1,59 @@
+FROM fedora:28
+MAINTAINER "Krunoslav Pavic" <[email protected]>
+
+ENV container docker
+ENV HOME=/home
+
+WORKDIR $HOME
+
+COPY prepare_and_test.sh $HOME/
+COPY test_contra_env_setup.py $HOME/
+COPY debug_vars.yml $HOME/
+
+RUN dnf -y install libguestfs libguestfs-tools-c \
+                   git libvirt-client \
+                   PyYAML python2-setuptools virt-install \
+                   python-pip python-devel gcc net-tools \
+                   openssh-clients sshpass \
+                   libselinux-python libffi-devel\
+                   openssl-devel openssh-server libyaml-devel \
+                   python-lxml libvirt libvirt-devel \
+                   libvirt-python ansible pytest \
+                   && dnf clean all
+
+RUN dnf -y update && dnf clean all
+RUN dnf -y install systemd
+
+RUN ssh-keygen -f /root/.ssh/id_rsa -t rsa -N ''
+RUN printf "Host *\n    StrictHostKeyChecking no" > /root/.ssh/config
+
+RUN dnf -y install libvirt-daemon-driver-* libvirt-daemon libvirt-daemon-kvm qemu-kvm socat && dnf clean all; \
+(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \
+rm -f /lib/systemd/system/multi-user.target.wants/*;\
+rm -f /etc/systemd/system/*.wants/*;\
+rm -f /lib/systemd/system/local-fs.target.wants/*; \
+rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+rm -f /lib/systemd/system/basic.target.wants/*;\
+rm -f /lib/systemd/system/anaconda.target.wants/*; \
+systemctl  enable libvirtd; \
+systemctl enable virtlockd
+#systemctl enable debug-shell.service
+
+RUN pip install ara
+
+# The default.xml needs to be have an unique network if there are multiple containers running
+COPY default.xml /etc/libvirt/qemu/networks/
+COPY docker-machines.xml /etc/libvirt/qemu/networks/
+
+# Edit the service file which includes ExecStartPost to chmod /dev/kvm
+RUN sed -i "/Service/a ExecStartPost=\/bin\/chmod 666 /dev/kvm" /usr/lib/systemd/system/libvirtd.service
+
+# Fix for libvirt namespaces
+RUN echo "namespaces = []" >> /etc/libvirt/qemu.conf
+
+RUN curl -o /root/minishift.tgz -L https://github.com/minishift/minishift/releases/download/v1.23.0/minishift-1.23.0-linux-amd64.tgz && tar xvzf /root/minishift.tgz -C /root/
+RUN curl -o /usr/local/bin/docker-machine-driver-kvm -L https://github.com/dhiltgen/docker-machine-kvm/releases/download/v0.10.0/docker-machine-driver-kvm-centos7 && chmod 755 /usr/local/bin/docker-machine-driver-kvm
+COPY fix-minishift.sh $HOME/
+
+CMD ["/usr/sbin/init"]

test/config/Dockerfiles/contra-env-setup-test-f28/debug_vars.yml

@@ -0,0 +1,10 @@
+- name: "Set debug vars"
+  set_fact:
+    test_run_info:
+      minishift_bin: "{{ minishift_bin }}"
+      oc_bin: "{{ oc_bin }}"
+      profile: "{{ profile }}"
+
+- name: "debug-vars"
+  debug:
+    var: test_run_info

test/config/Dockerfiles/contra-env-setup-test-f28/default.xml

@@ -0,0 +1,12 @@
+<network>
+  <name>default</name>
+  <uuid>f487971c-469a-4752-ba15-ff85024834a8</uuid>
+  <forward mode='nat'/>
+  <bridge name='virbr4' stp='on' delay='0'/>
+  <ip address='192.168.126.1' netmask='255.255.255.0'>
+    <dhcp>
+      <range start='192.168.126.2' end='192.168.126.254'/>
+    </dhcp>
+  </ip>
+</network>
+

test/config/Dockerfiles/contra-env-setup-test-f28/docker-machines.xml

@@ -0,0 +1,11 @@
+<network>
+  <name>docker-machines</name>
+  <uuid>f487971c-469a-4752-ba15-ff85024834a9</uuid>
+  <forward mode='nat'/>
+  <bridge name='virbr1' stp='on' delay='0'/>
+  <ip address='192.168.43.1' netmask='255.255.255.0'>
+    <dhcp>
+      <range start='192.168.43.2' end='192.168.43.254'/>
+    </dhcp>
+  </ip>
+</network>

test/config/Dockerfiles/contra-env-setup-test-f28/fix-minishift.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+systemctl start libvirtd || true
+
+/root/minishift-1.23.0-linux-amd64/minishift start -v 5 --profile minishift --disk-size 40gb --memory 8092mb --openshift-version v3.10.0 || /root/minishift-1.23.0-linux-amd64/minishift delete -f
+
+/root/minishift-1.23.0-linux-amd64/minishift delete -f

test/config/Dockerfiles/contra-env-setup-test-f28/prepare_and_test.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+base_dir=/home
+
+# Set log dir to the WORKSPACE where it can be archived
+if [ ! -z "${WORKSPACE}" ]; then
+    log_dir=${WORKSPACE}
+else
+    log_dir=${base_dir}/logs
+fi
+
+# Set environment to use ara with ansible
+export ara_location=$(python -c "import os,ara; print(os.path.dirname(ara.__file__))")
+export ANSIBLE_CALLBACK_PLUGINS=$ara_location/plugins/callbacks
+export ANSIBLE_ACTION_PLUGINS=$ara_location/plugins/actions
+export ANSIBLE_LIBRARY=$ara_location/plugins/modules
+
+export USER=$(whoami)
+
+cd ${base_dir}
+
+# Prepare repo and logs directories
+mkdir -p ${log_dir}
+
+git clone https://github.com/CentOS-PaaS-SIG/contra-env-setup.git
+
+pushd ${PROJECT_REPO}
+
+# Sync the repository
+if [ -z ${PR_NUM} ]; then
+  git fetch origin ${ACTUAL_COMMIT}
+  git checkout FETCH_HEAD
+else
+  # PR was specified so we need to fetch it
+  git fetch origin pull/${PR_NUM}/head:local-testing-branch
+  git checkout local-testing-branch
+fi
+
+popd
+
+# Run the playbook locally with added hook for debugging variables needed for testing
+/usr/bin/ansible-playbook -vv -i "localhost," ${base_dir}/contra-env-setup/playbooks/setup.yml -e user=root \
+                          -e ansible_connection=local -e setup_nested_virt=false -e setup_playbook_hooks=true \
+                          --extra-vars='{"hooks": ["/home/debug_vars.yml"]}'
+
+# Run tests with pytest
+python -m pytest ${base_dir}/test_contra_env_setup.py -v --junitxml=${log_dir}/contra_env_setup_fedora28.xml > ${log_dir}/contra_env_setup_fedora28.log