Skip to content

Commit

Permalink
Add fedora28 testing container, run container tests in parallel
Browse files Browse the repository at this point in the history
This adds the fedora28 testing container to the contra-env-setup CI.
The container contains a fix for a libvirt bug that occurs when first starting the minishift VM, running fix-minishift.sh resolves the issue.
Jenkinsfile has been updated to run tests on both centos7 and fedora28 in parallel.
JUnit archiving has been added, and the test logs have corresponding prefixes for each container.

ed:   test/config/Dockerfiles/contra-env-setup-test-c7/prepare_and_test.sh
  • Loading branch information
dirgim committed Sep 12, 2018
1 parent 0186122 commit dfb15f9
Show file tree
Hide file tree
Showing 14 changed files with 494 additions and 37 deletions.
57 changes: 45 additions & 12 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ timestamps {

// Needed for podTemplate()
env.SLAVE_TAG = env.SLAVE_TAG ?: 'stable'
env.ENVSETUPTESTC7_TAG = env.ENVSETUPTESTC7_TAG ?: 'stable'
env.ENVSETUPTEST_C7_TAG = env.ENVSETUPTEST_C7_TAG ?: 'stable'
env.ENVSETUPTEST_F28_TAG = env.ENVSETUPTEST_F28_TAG ?: 'stable'

env.DOCKER_REPO_URL = env.DOCKER_REPO_URL ?: '172.30.254.79:5000'
env.OPENSHIFT_NAMESPACE = env.OPENSHIFT_NAMESPACE ?: 'continuous-infra'
Expand Down Expand Up @@ -62,7 +63,7 @@ timestamps {
extensions: [
[
$class: 'org.jenkinsci.plugins.ghprb.extensions.status.GhprbSimpleStatus',
commitStatusContext: 'Contra Env Setup Stage Job',
commitStatusContext: 'Contra Env Setup Testing Job',
showMatrixStatus: false,
triggeredStatus: 'Starting job...',
startedStatus: 'Testing...',
Expand Down Expand Up @@ -93,9 +94,12 @@ timestamps {
string(name: 'SLAVE_TAG',
defaultValue: 'stable',
description: 'Tag for slave image'),
string(name: 'ENVSETUPTESTC7_TAG',
string(name: 'ENVSETUPTEST_C7_TAG',
defaultValue: 'stable',
description: 'Tag for contra-env-setup-test-c7 image'),
string(name: 'ENVSETUPTEST_F28_TAG',
defaultValue: 'stable',
description: 'Tag for contra-env-setup-test-f28 image'),
string(name: 'DOCKER_REPO_URL',
defaultValue: '172.30.254.79:5000',
description: 'Docker repo url for Openshift instance'),
Expand Down Expand Up @@ -127,7 +131,14 @@ timestamps {
// This adds the contra-env-setup-test-c7 container to the pod.
containerTemplate(name: 'contra-env-setup-test-c7',
alwaysPullImage: true,
image: DOCKER_REPO_URL + '/' + OPENSHIFT_NAMESPACE + '/contra-env-setup-test-c7:' + ENVSETUPTESTC7_TAG,
image: DOCKER_REPO_URL + '/' + OPENSHIFT_NAMESPACE + '/contra-env-setup-test-c7:' + ENVSETUPTEST_C7_TAG,
ttyEnabled: true,
privileged: true,
workingDir: '/workDir'),
// This adds the contra-env-setup-test-f28 container to the pod.
containerTemplate(name: 'contra-env-setup-test-f28',
alwaysPullImage: true,
image: DOCKER_REPO_URL + '/' + OPENSHIFT_NAMESPACE + '/contra-env-setup-test-f28:' + ENVSETUPTEST_F28_TAG,
ttyEnabled: true,
privileged: true,
workingDir: '/workDir'),
Expand Down Expand Up @@ -176,14 +187,33 @@ timestamps {

currentStage = "test-env-setup"
stage(currentStage) {

envsetupUtils.timedPipelineStep(stepName: currentStage, debug: true) {
// Set stage specific vars
envsetupUtils.setStageEnvVars(currentStage)

// Run contra-env-setup test
pipelineUtils.executeInContainer(currentStage, "contra-env-setup-test-c7", "/home/prepare_and_test.sh")
}
parallel (
'centos7': {
stage("${currentStage}-centos7") {
envsetupUtils.timedPipelineStep(stepName: "${currentStage}-centos7", debug: true) {
// Set stage specific vars
envsetupUtils.setStageEnvVars(currentStage)

// Run contra-env-setup test
pipelineUtils.executeInContainer("${currentStage}-centos7", "contra-env-setup-test-c7", "/home/prepare_and_test.sh")
}
}
},
'fedora28': {
stage("${currentStage}-fedora28") {
envsetupUtils.timedPipelineStep(stepName: "${currentStage}-fedora28", debug: true) {
// Set stage specific vars
envsetupUtils.setStageEnvVars(currentStage)

// Fix for minishift cgrups/cpuset bug
pipelineUtils.executeInContainer("${currentStage}-fedora28", "contra-env-setup-test-f28", "/home/fix-minishift.sh || true")

// Run contra-env-setup test
pipelineUtils.executeInContainer("${currentStage}-fedora28", "contra-env-setup-test-f28", "/home/prepare_and_test.sh")
}
}
}
)
}
} catch (e) {
// Set build result
Expand All @@ -208,6 +238,9 @@ timestamps {
ciMetrics.setMetricTag(jobMeasurement, 'build_result', currentBuild.result)
ciMetrics.setMetricField(jobMeasurement, 'build_time', currentBuild.getDuration())

// Archive JUnit results
step([$class: 'JUnitResultArchiver', testResults: '*.xml'])

}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,25 @@

base_dir=/home

# Set log dir to the WORKSPACE where it can be archived
if [ ! -z "${WORKSPACE}" ]; then
log_dir=${WORKSPACE}
else
log_dir=${base_dir}/logs
fi

# Set environment to use ara with ansible
export ara_location=$(python -c "import os,ara; print(os.path.dirname(ara.__file__))")
export ANSIBLE_CALLBACK_PLUGINS=$ara_location/plugins/callbacks
export ANSIBLE_ACTION_PLUGINS=$ara_location/plugins/actions
export ANSIBLE_LIBRARY=$ara_location/plugins/modules

export USER=$(whoami)

cd ${base_dir}

# Prepare repo and logs directories
mkdir -p ${base_dir}/logs
mkdir -p ${log_dir}

git clone https://github.com/CentOS-PaaS-SIG/contra-env-setup.git

Expand All @@ -30,18 +38,10 @@ fi

popd

# Temporary fix for skipping nested virtualization setup
sed -i 's/- import_tasks: setup_nested_virt.yml/#- import_tasks: setup_nested_virt.yml/g' ${base_dir}/contra-env-setup/playbooks/roles/prereqs/tasks/main.yml

# Run the playbook locally with added hook for debugging variables needed for testing
/usr/bin/ansible-playbook -vv -i "localhost," ${base_dir}/contra-env-setup/playbooks/setup.yml -e user=root \
-e ansible_connection=local -e setup_playbook_hooks=true \
-e ansible_connection=local -e setup_nested_virt=false -e setup_playbook_hooks=true \
--extra-vars='{"hooks": ["/home/debug_vars.yml"]}'

# Run tests with pytest
python -m pytest ${base_dir}/test_contra_env_setup.py -v > ${base_dir}/logs/contra_env_setup.log

# Copy logs to the work directory where they can be archived
if [ ! -z "${WORKSPACE}" ]; then
cp ${base_dir}/logs/* ${WORKSPACE}
fi
python -m pytest ${base_dir}/test_contra_env_setup.py -v --junitxml=${log_dir}/contra_env_setup_centos7.xml > ${log_dir}/contra_env_setup_centos7.log
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ def run_info():
profile = None
minishift_bin = None
oc_bin = None

# Get Playbook ID from ara
playbook_list_cmd = '/usr/bin/ara playbook list -f json'
playbook_list = json.loads(check_output(playbook_list_cmd.split()))
Expand All @@ -33,7 +33,7 @@ def run_info():
if result.get('Name') == 'playbook_hooks : debug-vars':
debug_result_id = result.get('ID')
break

if debug_result_id is not None:
debug_result_cmd = '/usr/bin/ara result show %s --raw -f json' % debug_result_id
debug_result = json.loads(check_output(debug_result_cmd.split()))
Expand All @@ -44,11 +44,11 @@ def run_info():
oc_bin = test_run_info.get('oc_bin')

return {
'playbook_id': playbook_id,
'profile': profile,
'minishift_bin': minishift_bin,
'oc_bin': oc_bin,
}
'playbook_id': playbook_id,
'profile': profile,
'minishift_bin': minishift_bin,
'oc_bin': oc_bin,
}


def test_playbook_success(run_info):
Expand All @@ -70,7 +70,7 @@ def test_playbook_success(run_info):
tasks_ok = stats.get('Ok')
tasks_changed = stats.get('Changed')
tasks_failed = stats.get('Failed')

assert(tasks_failed == 0)
assert(tasks_changed > 0)
assert(tasks_ok > 0)
Expand All @@ -79,7 +79,7 @@ def test_playbook_success(run_info):
def test_binary_locations(run_info):
minishift_bin = run_info.get('minishift_bin')
oc_bin = run_info.get('oc_bin')

assert(minishift_bin is not None)
assert(os.path.isfile(minishift_bin))
assert(oc_bin is not None)
Expand Down Expand Up @@ -113,7 +113,7 @@ def test_buildconfigs(run_info):

assert(oc_result and 'ansible-executor' in oc_result)
assert(oc_result and 'linchpin-executor' in oc_result)
assert(oc_result and 'jenkins-contra-slave' in oc_result)
assert(oc_result and 'jenkins-contra-sample-project-slave' in oc_result)
assert(oc_result and 'jenkins ' in oc_result)


Expand All @@ -134,9 +134,9 @@ def test_builds(run_info):
linchpin_executor_success = True
if 'jenkins-' in line and 'Complete' in line and 'slave' not in line:
jenkins_success = True
if 'jenkins-contra-slave-' in line and 'Complete' in line:
if 'jenkins-contra-sample-project-slave-' in line and 'Complete' in line:
jenkins_contra_slave_success = True


assert(ansible_executor_success)
assert(linchpin_executor_success)
Expand All @@ -154,7 +154,7 @@ def test_imagestreams(run_info):

assert(oc_result and 'ansible-executor' in oc_result)
assert(oc_result and 'linchpin-executor' in oc_result)
assert(oc_result and 'jenkins-contra-slave' in oc_result)
assert(oc_result and 'jenkins-contra-sample-project-slave' in oc_result)
assert(oc_result and 'jenkins' in oc_result)


Expand All @@ -179,7 +179,7 @@ def test_jenkins_master_pod(run_info):
oc_result = check_output(oc_cmd.split())

for line in oc_result.splitlines():
if 'jenkins' in line and 'slave' not in line and 'Running' in line:
if 'jenkins' in line and 'Running' in line:
jenkins_master_pod_running = True

assert(jenkins_master_pod_running)
Expand Down
59 changes: 59 additions & 0 deletions test/config/Dockerfiles/contra-env-setup-test-f28/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
FROM fedora:28
MAINTAINER "Krunoslav Pavic" <[email protected]>

ENV container docker
ENV HOME=/home

WORKDIR $HOME

COPY prepare_and_test.sh $HOME/
COPY test_contra_env_setup.py $HOME/
COPY debug_vars.yml $HOME/

RUN dnf -y install libguestfs libguestfs-tools-c \
git libvirt-client \
PyYAML python2-setuptools virt-install \
python-pip python-devel gcc net-tools \
openssh-clients sshpass \
libselinux-python libffi-devel\
openssl-devel openssh-server libyaml-devel \
python-lxml libvirt libvirt-devel \
libvirt-python ansible pytest \
&& dnf clean all

RUN dnf -y update && dnf clean all
RUN dnf -y install systemd

RUN ssh-keygen -f /root/.ssh/id_rsa -t rsa -N ''
RUN printf "Host *\n StrictHostKeyChecking no" > /root/.ssh/config

RUN dnf -y install libvirt-daemon-driver-* libvirt-daemon libvirt-daemon-kvm qemu-kvm socat && dnf clean all; \
(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \
rm -f /lib/systemd/system/multi-user.target.wants/*;\
rm -f /etc/systemd/system/*.wants/*;\
rm -f /lib/systemd/system/local-fs.target.wants/*; \
rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
rm -f /lib/systemd/system/basic.target.wants/*;\
rm -f /lib/systemd/system/anaconda.target.wants/*; \
systemctl enable libvirtd; \
systemctl enable virtlockd
#systemctl enable debug-shell.service

RUN pip install ara

# The default.xml needs to be have an unique network if there are multiple containers running
COPY default.xml /etc/libvirt/qemu/networks/
COPY docker-machines.xml /etc/libvirt/qemu/networks/

# Edit the service file which includes ExecStartPost to chmod /dev/kvm
RUN sed -i "/Service/a ExecStartPost=\/bin\/chmod 666 /dev/kvm" /usr/lib/systemd/system/libvirtd.service

# Fix for libvirt namespaces
RUN echo "namespaces = []" >> /etc/libvirt/qemu.conf

RUN curl -o /root/minishift.tgz -L https://github.com/minishift/minishift/releases/download/v1.23.0/minishift-1.23.0-linux-amd64.tgz && tar xvzf /root/minishift.tgz -C /root/
RUN curl -o /usr/local/bin/docker-machine-driver-kvm -L https://github.com/dhiltgen/docker-machine-kvm/releases/download/v0.10.0/docker-machine-driver-kvm-centos7 && chmod 755 /usr/local/bin/docker-machine-driver-kvm
COPY fix-minishift.sh $HOME/

CMD ["/usr/sbin/init"]
10 changes: 10 additions & 0 deletions test/config/Dockerfiles/contra-env-setup-test-f28/debug_vars.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
- name: "Set debug vars"
set_fact:
test_run_info:
minishift_bin: "{{ minishift_bin }}"
oc_bin: "{{ oc_bin }}"
profile: "{{ profile }}"

- name: "debug-vars"
debug:
var: test_run_info
12 changes: 12 additions & 0 deletions test/config/Dockerfiles/contra-env-setup-test-f28/default.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<network>
<name>default</name>
<uuid>f487971c-469a-4752-ba15-ff85024834a8</uuid>
<forward mode='nat'/>
<bridge name='virbr4' stp='on' delay='0'/>
<ip address='192.168.126.1' netmask='255.255.255.0'>
<dhcp>
<range start='192.168.126.2' end='192.168.126.254'/>
</dhcp>
</ip>
</network>

Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<network>
<name>docker-machines</name>
<uuid>f487971c-469a-4752-ba15-ff85024834a9</uuid>
<forward mode='nat'/>
<bridge name='virbr1' stp='on' delay='0'/>
<ip address='192.168.43.1' netmask='255.255.255.0'>
<dhcp>
<range start='192.168.43.2' end='192.168.43.254'/>
</dhcp>
</ip>
</network>
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/bin/bash

systemctl start libvirtd || true

/root/minishift-1.23.0-linux-amd64/minishift start -v 5 --profile minishift --disk-size 40gb --memory 8092mb --openshift-version v3.10.0 || /root/minishift-1.23.0-linux-amd64/minishift delete -f

/root/minishift-1.23.0-linux-amd64/minishift delete -f
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/bin/bash

base_dir=/home

# Set log dir to the WORKSPACE where it can be archived
if [ ! -z "${WORKSPACE}" ]; then
log_dir=${WORKSPACE}
else
log_dir=${base_dir}/logs
fi

# Set environment to use ara with ansible
export ara_location=$(python -c "import os,ara; print(os.path.dirname(ara.__file__))")
export ANSIBLE_CALLBACK_PLUGINS=$ara_location/plugins/callbacks
export ANSIBLE_ACTION_PLUGINS=$ara_location/plugins/actions
export ANSIBLE_LIBRARY=$ara_location/plugins/modules

export USER=$(whoami)

cd ${base_dir}

# Prepare repo and logs directories
mkdir -p ${log_dir}

git clone https://github.com/CentOS-PaaS-SIG/contra-env-setup.git

pushd ${PROJECT_REPO}

# Sync the repository
if [ -z ${PR_NUM} ]; then
git fetch origin ${ACTUAL_COMMIT}
git checkout FETCH_HEAD
else
# PR was specified so we need to fetch it
git fetch origin pull/${PR_NUM}/head:local-testing-branch
git checkout local-testing-branch
fi

popd

# Run the playbook locally with added hook for debugging variables needed for testing
/usr/bin/ansible-playbook -vv -i "localhost," ${base_dir}/contra-env-setup/playbooks/setup.yml -e user=root \
-e ansible_connection=local -e setup_nested_virt=false -e setup_playbook_hooks=true \
--extra-vars='{"hooks": ["/home/debug_vars.yml"]}'

# Run tests with pytest
python -m pytest ${base_dir}/test_contra_env_setup.py -v --junitxml=${log_dir}/contra_env_setup_fedora28.xml > ${log_dir}/contra_env_setup_fedora28.log
Loading

0 comments on commit dfb15f9

Please sign in to comment.