From 77170623133423f57d0e214e0b5cbca38827f60e Mon Sep 17 00:00:00 2001 From: Olblak Date: Thu, 2 Apr 2020 15:31:57 +0200 Subject: [PATCH 01/14] Reset env variable compare to upstream --- env/release.mk | 17 ++++++++--------- env/test.mk | 20 +++++++++----------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/env/release.mk b/env/release.mk index e8172139..28c1c83d 100644 --- a/env/release.mk +++ b/env/release.mk @@ -11,18 +11,17 @@ export SSH_OPTS=-p 22 export SCP_OPTS=-P 22 # where to put binary files -export WARDIR=/packages/binary/war${RELEASELINE} -export MSIDIR=/packages/binary/windows${RELEASELINE} +export WARDIR=/srv/releases/jenkins/war${RELEASELINE} +export MSIDIR=/srv/releases/jenkins/windows${RELEASELINE} export OSXDIR=/srv/releases/jenkins/osx${RELEASELINE} -export DEBDIR=/packages/binary/debian${RELEASELINE} -export RPMDIR=/packages/binary/redhat${RELEASELINE} -export SUSEDIR=/packages/binary/opensuse${RELEASELINE} +export DEBDIR=/srv/releases/jenkins/debian${RELEASELINE} +export RPMDIR=/srv/releases/jenkins/redhat${RELEASELINE} +export SUSEDIR=/srv/releases/jenkins/opensuse${RELEASELINE} # where to put repository index and other web contents -export RPM_WEBDIR=/packages/web/redhat${RELEASELINE} -export SUSE_WEBDIR=/packages/web/opensuse${RELEASELINE} -export DEB_WEBDIR=/packages/web/debian${RELEASELINE} -export WAR_WEBDIR=/packages/web/war${RELEASELINE} +export RPM_WEBDIR=/var/www/pkg.jenkins.io.staging/redhat${RELEASELINE} +export SUSE_WEBDIR=/var/www/pkg.jenkins.io.staging/opensuse${RELEASELINE} +export DEB_WEBDIR=/var/www/pkg.jenkins.io.staging/debian${RELEASELINE} # URL to the aforementioned webdir export RPM_URL=https://pkg.jenkins.io/redhat${RELEASELINE} diff --git a/env/test.mk b/env/test.mk index 4d4c4b60..bd9e96ca 100644 --- a/env/test.mk +++ b/env/test.mk @@ -12,19 +12,17 @@ export SCP_OPTS= # where to put binary files export TESTDIR=$(realpath .)/pkg.jenkins.io -export WARDIR=/packages/binary/war${RELEASELINE} -# Concat MSDIR and RELEASELINE in the msi publishing -export MSIDIR=/packages/binary/windows${RELEASELINE} -export OSXDIR=/packages/osx${RELEASELINE} -export DEBDIR=/packages/binary/debian${RELEASELINE} -export RPMDIR=/packages/binary/redhat${RELEASELINE} -export SUSEDIR=/packages/binary/opensuse${RELEASELINE} +export WARDIR=${TESTDIR}/war${RELEASELINE} +export MSIDIR=${TESTDIR}/windows${RELEASELINE} +export OSXDIR=${TESTDIR}/osx${RELEASELINE} +export DEBDIR=${TESTDIR}/debian${RELEASELINE}/binary +export RPMDIR=${TESTDIR}/redhat${RELEASELINE} +export SUSEDIR=${TESTDIR}/opensuse${RELEASELINE} # where to put repository index and other web contents -export RPM_WEBDIR=/packages/web/redhat${RELEASELINE} -export SUSE_WEBDIR=/packages/web/opensuse${RELEASELINE} -export DEB_WEBDIR=/packages/web/debian${RELEASELINE} -export WAR_WEBDIR=/packages/web/war${RELEASELINE} +export RPM_WEBDIR=${TESTDIR}/redhat${RELEASELINE} +export SUSE_WEBDIR=${TESTDIR}/opensuse${RELEASELINE} +export DEB_WEBDIR=${TESTDIR}/debian${RELEASELINE} # URL to the aforementioned webdir. WEBSERVER=test.pkg.jenkins.io:9200 From 3107d602133e8e93c097f628d82b2eb09967faa7 Mon Sep 17 00:00:00 2001 From: Olblak Date: Thu, 2 Apr 2020 15:32:49 +0200 Subject: [PATCH 02/14] Align docker-compose with real production file path --- docker-compose.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 3a7ff015..d5e5c871 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,15 +5,15 @@ services: image: jenkinsciinfra/packaging:latest command: "sleep 99d" environment: - - "BUILDENV='/packaging/env/test.mk'" - - "BRANDING_DIR=/packaging/branding" - - "BRAND=/packaging/jenkins.mk'" - - "CREDENTIAL=credentials/test.mk" - - "GPG_FILE=/packaging/credentials/sandbox.gpg" + - "BUILDENV=/srv/releases/jenkins/env/test.mk" + - "BRANDING_DIR=/srv/releases/jenkins/branding" + - "BRAND=/srv/releases/jenkins/branding/test.mk" + - "CREDENTIAL=/srv/releases/jenkins/credentials/test.mk" + - "GPG_FILE=/srv/releases/jenkins/credentials/sandbox.gpg" - "GPG_KEYNAME=Bogus Test" - "GPG_PASSPHRASE=s3cr3t" - - "GPG_PASSPHRASE_FILE=/packaging/credentials/test.gpg.password.txt" - - "WAR=/packaging/jenkins.war" + - "GPG_PASSPHRASE_FILE=/srv/releases/jenkins/credentials/test.gpg.password.txt" + - "WAR=/srv/releases/jenkins/jenkins.war" volumes: - - .:/packaging - working_dir: "/packaging" + - .:/srv/releases/jenkins + working_dir: "/srv/releases/jenkins" From d12f2581e58f91cf9a8ac61dd3647d725358321b Mon Sep 17 00:00:00 2001 From: Olblak Date: Thu, 2 Apr 2020 15:34:18 +0200 Subject: [PATCH 03/14] Update publishing script to also push artifacts on remote ssh server --- deb/publish/publish.sh | 177 +++++++++++++++++++++++++++------------- rpm/publish/publish.sh | 98 ++++++++++++++-------- suse/publish/publish.sh | 108 +++++++++++++++++------- 3 files changed, 267 insertions(+), 116 deletions(-) diff --git a/deb/publish/publish.sh b/deb/publish/publish.sh index 77a20f67..5583db1e 100755 --- a/deb/publish/publish.sh +++ b/deb/publish/publish.sh @@ -1,63 +1,128 @@ -#!/bin/bash -ex +#!/bin/bash -: "${AGENT_WORKDIR:=/tmp}" -: "${GPG_KEYNAME:?Required valid gpg keyname}" - -bin="$(dirname "$0")" - -## Publish Binary -# -mkdir -p "$DEBDIR" -mkdir -p "$DEB_WEBDIR" +set -euxo pipefail -rsync -avz "$DEB" "$DEBDIR/" +: "${AGENT_WORKDIR:=/tmp}" +: "${GPG_KEYNAME:?Require valid gpg keyname}" +: "${DEB:?Require Debian package}" +: "${DEBDIR:? Require where to put binary files}" +: "${DEB_WEBDIR:? Require where to put repository index and other web contents}" +: "${DEB_URL:? Require Debian repository Url}" # $$ Contains current pid D="$AGENT_WORKDIR/$$" +bin="$(dirname "$0")" + +function clean(){ + rm -rf "$D" +} + # Generate and publish site content -## -mkdir -p "$D/binary" "$D/contents" -cp -R "$bin/contents/." "$D/contents" - -gpg --export -a --output "$D/contents/${ORGANIZATION}.key" "${GPG_KEYNAME}" - -"$BASE/bin/indexGenerator.py" \ - --distribution debian \ - --binaryDir "$DEBDIR" \ - --targetDir "$DEB_WEBDIR" - -"$BASE/bin/branding.py" "$D" - -# build package index -# see http://wiki.debian.org/SecureApt for more details -cp "${DEB}" "$D/binary/" -pushd "$D" - apt-ftparchive packages binary > binary/Packages - apt-ftparchive contents binary > binary/Contents -popd - -apt-ftparchive -c "$bin/release.conf" release "$D/binary" > "$D/binary/Release" - -# sign the release file -rm "$D/binary/Release.gpg" || true - -gpg \ - --batch \ - --pinentry-mode loopback \ - --digest-algo=sha256 \ - -u "$GPG_KEYNAME" \ - -abs \ - -o "$D/binary/Release.gpg" \ - "$D/binary/Release" - -cp \ - "$D"/binary/Packages* \ - "$D"/binary/Release \ - "$D"/binary/Release.gpg \ - "$D"/binary/Contents* \ - "$D"/contents/binary - -rsync -avz "$D/contents/" "$DEB_WEBDIR/" - -rm -rf "$D" +function generateSite(){ + + cp -R "$bin/contents/." "$D/contents" + + gpg --export -a --output "$D/contents/${ORGANIZATION}.key" "${GPG_KEYNAME}" + + "$BASE/bin/indexGenerator.py" \ + --distribution debian \ + --binaryDir "$DEBDIR" \ + --targetDir "$DEB_WEBDIR" + + "$BASE/bin/branding.py" "$D" + + + # build package index + # see http://wiki.debian.org/SecureApt for more details + cp "${DEB}" "$D/binary/" + + pushd "$D" + apt-ftparchive packages binary > binary/Packages + apt-ftparchive contents binary > binary/Contents + popd + + # Remote ftparchive-merge + # https://github.com/kohsuke/apt-ftparchive-merge + pushd $D/binary + mvn org.kohsuke:apt-ftparchive-merge:1.6:merge -Durl="$DEB_URL/binary/" -Dout=../merged + popd + + # Local ftparchive-merge + + cat $D/merged/Packages > $D/binary/Packages + gzip -9c "$D/merged/Packages" > "$D/binary/Packages.gz" + bzip2 "$D/merged/Packages" > "$D/binary/Packages.bz2" + lzma "$D/merged/Packages" > "$D/binary/Packages.lzma" + gzip -9c "$D/merged/Contents" > "$D/binary/Contents.gz" + + apt-ftparchive -c "$bin/release.conf" release "$D/binary" > "$D/binary/Release" + +} + +function init(){ + + mkdir -p "$D/binary" "$D/contents" + + # where to put binary files + mkdir -p "$DEBDIR" # where to put binary files + + # where to put repository index and other web contents + mkdir -p "$DEB_WEBDIR" + ## On remote server + ssh "$SSH_OPTS" "$PKGSERVER" mkdir -p "'$DEBDIR/'" where to put repository index and other web contents +} + +function uploadPackage(){ + # Upload Debian Package + rsync -avz "$DEB" "$DEBDIR/" + rsync -avz -e "ssh $SSH_OPTS" "${DEB}" "$PKGSERVER:${DEBDIR// /\\ }" +} + +function uploadSite(){ + + cp \ + "$D"/binary/Packages* \ + "$D"/binary/Release \ + "$D"/binary/Release.gpg \ + "$D"/binary/Contents* \ + "$D"/contents/binary + + rsync -avz "$D/contents/" "$DEB_WEBDIR/" + rsync -avz -e "ssh $SSH_OPTS" "${DEB}" "$PKGSERVER:${DEBDIR// /\\ }" +} + +function show(){ + echo "Parameters:" + echo "DEB: $DEB" + echo "DEBDIR: $DEBDIR" + echo "DEB_WEBDIR: $DEB_WEBDIR" + echo "SSH_OPTS: $SSH_OPTS" + echo "PKGSERVER: $PKGSERVER" + echo "GPG_KEYNAME: $GPG_KEYNAME" + echo "---" +} + +function signSite(){ + # sign the release file + if [ -f "$D/binary/Release.gpg" ]; then + rm "$D/binary/Release.gpg" + fi + + gpg \ + --batch \ + --pinentry-mode loopback \ + --digest-algo=sha256 \ + -u "$GPG_KEYNAME" \ + -abs \ + -o "$D/binary/Release.gpg" \ + "$D/binary/Release" +} + +show +init +generateSite +signSite +uploadPackage +uploadSite +clean diff --git a/rpm/publish/publish.sh b/rpm/publish/publish.sh index e6c8f0dd..527a695a 100755 --- a/rpm/publish/publish.sh +++ b/rpm/publish/publish.sh @@ -1,28 +1,32 @@ -#!/bin/bash -ex +#!/bin/bash +set -euxo pipefail : "${AGENT_WORKDIR:=/tmp}" -: "${GPG_KEYNAME:?Required valid gpg keyname}" - -mkdir -p "$RPMDIR/" -mkdir -p "$RPM_WEBDIR/" - -rsync -avz "$RPM" "$RPMDIR/" +: "${GPG_KEYNAME:?Require valid gpg keyname}" +: "${RPMDIR:?Require where to put binary files}" +: "${RPM_WEBDIR:?Require where to put index and other web contents}" +: "${RPM_URL:?Require rpm repository url}" +: "${RELEASELINE:?Require rpm release line}" +: "${BASE:? Required base directory}" # $$ Contains current pid D="$AGENT_WORKDIR/$$" -mkdir -p "$D/RPMS/noarch" - -"$BASE/bin/indexGenerator.py" \ - --distribution redhat \ - --binaryDir "$RPMDIR" \ - --targetDir "$RPM_WEBDIR" - -gpg --export -a --output "$D/${ORGANIZATION}.key" "${GPG_KEYNAME}" - -"$BASE/bin/branding.py" "$D" +function clean(){ + rm -rf $D +} -cp "$RPM" "$D/RPMS/noarch" +function generateSite(){ + "$BASE/bin/indexGenerator.py" \ + --distribution redhat \ + --binaryDir "$RPMDIR" \ + --targetDir "$RPM_WEBDIR" + + gpg --export -a --output "$D/${ORGANIZATION}.key" "${GPG_KEYNAME}" + + "$BASE/bin/branding.py" "$D" + + cp "$RPM" "$D/RPMS/noarch" cat > "$D/${ARTIFACTNAME}.repo" << EOF [${ARTIFACTNAME}] @@ -31,20 +35,50 @@ baseurl=${RPM_URL} gpgcheck=1 EOF -pushd "$D" - rsync -avz --exclude RPMS . "$RPM_WEBDIR/" -popd + # generate index + # locally + createrepo --update -o "$RPM_WEBDIR" "$RPMDIR/" + # on the server + ssh "$SSH_OPTS" "$PKGSERVER" createrepo --update -o "'$RPM_WEBDIR'" "'$RPMDIR/'" + +} + +function init(){ + mkdir -p "$D/RPMS/noarch" + + mkdir -p "$RPMDIR/" + # mkdir -p "$RPM_WEBDIR/" # May not be necessary + ssh "$SSH_OPTS" "$PKGSERVER" mkdir -p "'$RPMDIR/'" +} + + +function uploadPackage(){ + rsync -avz "$RPM" "$RPMDIR/" + rsync -avz -e "ssh $SSH_OPTS" "$RPM" "$PKGSERVER:${RPMDIR// /\\ }/" +} -# generate index on the server -createrepo --update -o "$RPM_WEBDIR" "$RPMDIR/" +function show(){ + echo "Parameters:" + echo "RPM: $RPM" + echo "RPMDIR: $RPMDIR" + echo "RPM_WEBDIR: $RPM_WEBDIR" + echo "SSH_OPTS: $SSH_OPTS" + echo "PKGSERVER: $PKGSERVER" + echo "GPG_KEYNAME: $GPG_KEYNAME" + echo "---" +} -gpg \ - --batch \ - --pinentry-mode loopback \ - -u "$GPG_KEYNAME" \ - -a \ - --detach-sign \ - --yes \ - "$RPM_WEBDIR/repodata/repomd.xml" +function uploadSite(){ + pushd "$D" + rsync -avz --exclude RPMS . "$RPM_WEBDIR/" + rsync -avz -e "ssh $SSH_OPTS" --exclude RPMS . "$PKGSERVER:${RPM_WEBDIR// /\\ }" + popd +} -rm -rf $D +show +init +generateSite +signSite +uploadPackage +uploadSite +clean diff --git a/suse/publish/publish.sh b/suse/publish/publish.sh index 61d2ce6c..727ef577 100755 --- a/suse/publish/publish.sh +++ b/suse/publish/publish.sh @@ -1,42 +1,94 @@ -#!/bin/bash -ex +#!/bin/bash -: "${AGENT_WORKDIR:=/tmp}" - -mkdir -p "$SUSEDIR/" -mkdir -p "$SUSE_WEBDIR" +set -euxo pipefail -rsync -avz "$SUSE" "$SUSEDIR/" +: "${AGENT_WORKDIR:=/tmp}" +: "${GPG_KEYNAME:?Required valid gpg keyname}" +: "${BASE:?Require base directory}" +: "${SUSEDIR:? Require where to put binary files}" +: "${SUSE_WEBDIR:? Require where to put repository index and other web contents}" # $$ Contains current pid D="$AGENT_WORKDIR/$$" -mkdir -p $D/RPMS/noarch $D/repodata +function clean(){ + mkdir -rf $D +} + +function generateSite(){ + + "$BASE/bin/indexGenerator.py" \ + --distribution opensuse \ + --binaryDir "$SUSEDIR" \ + --targetDir "$SUSE_WEBDIR" + + gpg --export -a --output "$D/repodata/repomd.xml.key" "${GPG_KEYNAME}" + + "$BASE/bin/branding.py" $D + + cp "$SUSE" $D/RPMS/noarch +} + +function init(){ + # where to put binary files + mkdir -p "$SUSEDIR/" # Local + ssh "$SSH_OPTS" $PKGSERVER mkdir -p "'$SUSEDIR/'" # Remote + + # where to put repository index and other web contents + mkdir -p "$SUSE_WEBDIR" + + mkdir -p $D/RPMS/noarch $D/repodata +} + -"$BASE/bin/indexGenerator.py" \ - --distribution opensuse \ - --binaryDir "$SUSEDIR" \ - --targetDir "$SUSE_WEBDIR" +function show(){ + echo "Parameters:" + echo "SUSE: $SUSE" + echo "SUSEDIR: $SUSEDIR" + echo "SUSE_WEBDIR: $SUSE_WEBDIR" + echo "SSH_OPTS: $SSH_OPTS" + echo "PKGSERVER: $PKGSERVER" + echo "GPG_KEYNAME: $GPG_KEYNAME" + echo "---" +} -gpg --export -a --output "$D/repodata/repomd.xml.key" "${GPG_KEYNAME}" +function uploadPackage(){ + rsync -avz "$SUSE" "$SUSEDIR/" # Local + rsync -avz -e "ssh $SSH_OPTS" "${SUSE}" "$PKGSERVER:${SUSEDIR// /\\ }" # Remote +} -"$BASE/bin/branding.py" $D +function uploadSite(){ -cp "$SUSE" $D/RPMS/noarch + pushd $D + rsync -avz --exclude RPMS . "$SUSE_WEBDIR/" #Local + rsync -avz -e "ssh $SSH_OPTS" --exclude RPMS . "$PKGSERVER:${SUSE_WEBDIR// /\\ }" # Remote + + # generate index on the server + # server needs 'createrepo' pacakge + createrepo --update -o "$SUSE_WEBDIR" "$SUSEDIR/" #Local + ssh "$SSH_OPTS" "$PKGSERVER" createrepo --update -o "'$SUSE_WEBDIR'" "'$SUSEDIR/'" # Remote -pushd $D - rsync -avz --exclude RPMS . "$SUSE_WEBDIR/" + scp "$SCP_OPTS" "$PKGSERVER:${SUSE_WEBDIR// /\\ }/repodata/repomd.xml" repodata/ # Remote + cp "${SUSE_WEBDIR// /\\ }/repodata/repomd.xml" repodata/ # Local - # generate index on the server - # server needs 'createrepo' pacakge -popd + gpg \ + --batch \ + --pinentry-mode loopback \ + -u "$GPG_KEYNAME" \ + -a \ + --detach-sign \ + --yes \ + "$SUSE_WEBDIR/repodata/repomd.xml" -createrepo --update -o "$SUSE_WEBDIR" "$SUSEDIR/" + scp "$SCP_OPTS" repodata/repomd.xml.asc "$PKGSERVER:${SUSE_WEBDIR// /\\ }/repodata/" + cp repodata/repomd.xml.asc "$PKGSERVER:${SUSE_WEBDIR// /\\ }/repodata/" + + popd +} -gpg \ - --batch \ - --pinentry-mode loopback \ - -u "$GPG_KEYNAME" \ - -a \ - --detach-sign \ - --yes \ - "$SUSE_WEBDIR/repodata/repomd.xml" +show +init +generateSite +uploadPackage +uploadSite +clean From b9120b250aa41a8b0bab0de5f8f495eb4d28a11a Mon Sep 17 00:00:00 2001 From: Olblak Date: Thu, 2 Apr 2020 22:10:24 +0200 Subject: [PATCH 04/14] Add fake remote pkg service reachable via ssh --- credentials/ssh/id_rsa | 27 +++++++++++++++++++++++++++ credentials/ssh/id_rsa.pub | 1 + credentials/ssh/known_hosts | 2 ++ docker-compose.yaml | 15 +++++++++++++-- 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 credentials/ssh/id_rsa create mode 100644 credentials/ssh/id_rsa.pub create mode 100644 credentials/ssh/known_hosts diff --git a/credentials/ssh/id_rsa b/credentials/ssh/id_rsa new file mode 100644 index 00000000..268c1b9e --- /dev/null +++ b/credentials/ssh/id_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAw6bQJN8ceL5T7FaNs2QdSEnGgLKy5zvpuORDJLd0Q/wTm5DV +MVxqJzT8hCtL7a4D1NBPaBm6pAvlxCnKMhdZwSi+e2zuzRokOuivlAHyiV7le9Ln +1K/pKkDchAIz8P3YWLt1YZJERDUI8nXW9MC8ZOEt/IudmLTsZQcSr8lXOFIwNZe4 +b/NSr/9sxvSnYQqw5q6j1ZxdjW/KtXPQapU+PObZ/31jzTH9W966LpoF8XYo0xRA +hJvy3f+NxbSVPhjH422wHCeeyfI/+B/xYlVNC3yYR4ONvXbAXsewB0aRhnOfh+sS +kzotOCDMBZHOjKrg27MLs0dfrmjxGffF2Vq+JQIDAQABAoIBAELefZ9Mfg+qhUZu +YqngWr29MVIFQW4UpRIjOeuPo/YkbpMp0iO3wTQ7QN7vaVkHs5mFxM4AlTDCPDpq +SggKwQtqoIfQuGFzQNS9eFzuuXVH8Mj8UW343Ykqd/PKSPRh3hKdp0W81wY01iUA +L4KhaQJVkAETur5Zf74bx8A64UuG+qWbDUGlRGCRk/pl9xGB1z0FQ7Api/6gQd2A +Tnu6ASmQfoKeDsDDOBVy8sHv7HlkU9msowD9TdOj7Gxx9DoPryX6GlAhgi//+jyA +qUCf03kdey5aCVKFVUIOkxpDxYRI0etW0ef0rww+DJDpL7pT0kWMf4sqxgmxPTly +TaOnl+ECgYEA4GpfL7GTPTFvhatxi78TlaATTzjmzPYLrxOF8EHQM3Tz/nDbDknX +x5YtQdq0cv2TNdR2uZ0AyuzZ06j6axyBCJWKgtErN+SO01+Qxa9rcv+Vw7NtdTGs +GUrMa7CU/X4t8jt6UiObIgGRNbvu93ANeEzjIOFn9S9QKQ0vrtfW/W0CgYEA3zAV +/z5rt/AThnajCRPv5c/o9c2TykKy3DFFUdgNTEwnHE04D+xoqH1eoTMNoNXdVbI0 +1C5WTzCpYLuKmbl/aZEwvidi6ssTpYHcviAAz8iqN/TL9Ys5XmJ3iJViNK3IxxHB +TRLMiBC057tS8ZZLRa756weEZ2TUYRydxFntaJkCgYAV/nbbvsSWb7zlVdsn/g8W +T/z0e7grCEY232v2Ew0rrd+n5Tmi2dvbBL3kwWGED5QY53zHTjrgqHvkwZ/hVYbT +54wOrB9XOABDeQ9AQKQAPkpYRsKIhNjAFdOZDlJb0b0BC5E+cZznpU2s/YE7IPFB +BBASjeTZY8ywaUluEltQtQKBgCi2idy000uLdNRbgeQfCez/Hzzvkl0cC6qVJlMG +uW5Imf3UrDxjYLgTnpaDTKIhQS3nwzFNfpsVgmBN9buTFgX44U5euvGft+bCKLVZ ++yvsK/jnI+mXyxBHoAx/S5nWdcCyoXNg0YSkn4uCJWBCjVqZz6crCOEfiIpqgPEX +gnJJAoGBAKvDah87FijEJRBiaroef5buG1jr9pNCBoXIGbvZ1sFwkGWQGgrH6Y3s +EBD083+BBCcIMvzy2leB692axxGhtdyCxfPRN7KiZgT/YC6cCDL1yzhSHtZ8kamb +N8Qqs+wVE4YIdELB+VgKTho1v4gAyzZNuMJMhne6qH+oxNfGUgER +-----END RSA PRIVATE KEY----- diff --git a/credentials/ssh/id_rsa.pub b/credentials/ssh/id_rsa.pub new file mode 100644 index 00000000..3d3e77dd --- /dev/null +++ b/credentials/ssh/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDDptAk3xx4vlPsVo2zZB1IScaAsrLnO+m45EMkt3RD/BObkNUxXGonNPyEK0vtrgPU0E9oGbqkC+XEKcoyF1nBKL57bO7NGiQ66K+UAfKJXuV70ufUr+kqQNyEAjPw/dhYu3VhkkRENQjyddb0wLxk4S38i52YtOxlBxKvyVc4UjA1l7hv81Kv/2zG9KdhCrDmrqPVnF2Nb8q1c9BqlT485tn/fWPNMf1b3roumgXxdijTFECEm/Ld/43FtJU+GMfjbbAcJ57J8j/4H/FiVU0LfJhHg429dsBex7AHRpGGc5+H6xKTOi04IMwFkc6MquDbswuzR1+uaPEZ98XZWr4l olblak@winterfell diff --git a/credentials/ssh/known_hosts b/credentials/ssh/known_hosts new file mode 100644 index 00000000..a352863a --- /dev/null +++ b/credentials/ssh/known_hosts @@ -0,0 +1,2 @@ +|1|3Ao1unSiaoLJcBH+jj4LxlUJvU8=|9VCo6soeOkBokfDfbckMBvnE/6k= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFnJjx5araEbR3BvluFk5ONHqZSVZW1osdn4NuC/UBFPxwcEkkECK0EHR+WTxfTLGybJCTh3H5hTDady7W0EyIs= +|1|Orovxffw11DXksUZda8iwv3XcME=|OIrRb9oqBvY2esPMc+I0K70HLSs= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFnJjx5araEbR3BvluFk5ONHqZSVZW1osdn4NuC/UBFPxwcEkkECK0EHR+WTxfTLGybJCTh3H5hTDady7W0EyIs= diff --git a/docker-compose.yaml b/docker-compose.yaml index d5e5c871..75999dad 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,7 @@ # docker exec -i -t packaging_packaging_1 gpg --import --batch credentials/sandbox.gpg version: '3' +volumes: + sshd: services: packaging: image: jenkinsciinfra/packaging:latest @@ -8,12 +10,21 @@ services: - "BUILDENV=/srv/releases/jenkins/env/test.mk" - "BRANDING_DIR=/srv/releases/jenkins/branding" - "BRAND=/srv/releases/jenkins/branding/test.mk" - - "CREDENTIAL=/srv/releases/jenkins/credentials/test.mk" - "GPG_FILE=/srv/releases/jenkins/credentials/sandbox.gpg" - "GPG_KEYNAME=Bogus Test" - "GPG_PASSPHRASE=s3cr3t" - "GPG_PASSPHRASE_FILE=/srv/releases/jenkins/credentials/test.gpg.password.txt" - "WAR=/srv/releases/jenkins/jenkins.war" + - "RELEASELINE=-experimental" volumes: - - .:/srv/releases/jenkins + - ".:/srv/releases/jenkins" + - "./credentials/ssh:/root/.ssh" working_dir: "/srv/releases/jenkins" + remote: + image: jenkinsciinfra/packaging:latest + command: "/usr/sbin/sshd -D" + ports: + - "2222:22" + volumes: + - "./credentials/ssh/id_rsa.pub:/root/.ssh/authorized_keys:ro" + - sshd:/run/sshd From 3b2cfa3ed015d97720cb5578ad4bf50468e97e2b Mon Sep 17 00:00:00 2001 From: Olblak Date: Thu, 2 Apr 2020 22:13:56 +0200 Subject: [PATCH 05/14] Don't set default value to CREDENTIAL --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f8def310..b42f457b 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,8 @@ BUILDENV ?=./env/test.mk include ${BUILDENV} # refers to whereabouts of code-signing keys -CREDENTIAL ?=./credentials/test.mk +# CREDENTIAL ?=./credentials/test.mk + include ${CREDENTIAL} include ./setup.mk From e90d17574118f3e3281e5adc88cefac9adc158d8 Mon Sep 17 00:00:00 2001 From: Olblak Date: Thu, 2 Apr 2020 22:21:49 +0200 Subject: [PATCH 06/14] Update debian publishing scripts --- deb/publish/publish.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/deb/publish/publish.sh b/deb/publish/publish.sh index 5583db1e..381b9add 100755 --- a/deb/publish/publish.sh +++ b/deb/publish/publish.sh @@ -12,6 +12,9 @@ set -euxo pipefail # $$ Contains current pid D="$AGENT_WORKDIR/$$" +# Convert string to array to correctly escape cli parameter +SSH_OPTS=($SSH_OPTS) + bin="$(dirname "$0")" function clean(){ @@ -69,14 +72,15 @@ function init(){ # where to put repository index and other web contents mkdir -p "$DEB_WEBDIR" - ## On remote server - ssh "$SSH_OPTS" "$PKGSERVER" mkdir -p "'$DEBDIR/'" where to put repository index and other web contents + ## On remote serve + # shellcheck disable=SC2029 + ssh "$PKGSERVER" "${SSH_OPTS[*]}" mkdir -p "$DEBDIR/" } function uploadPackage(){ # Upload Debian Package rsync -avz "$DEB" "$DEBDIR/" - rsync -avz -e "ssh $SSH_OPTS" "${DEB}" "$PKGSERVER:${DEBDIR// /\\ }" + rsync -avz -e \'ssh "${SSH_OPTS[*]}"\' "${DEB}" "$PKGSERVER:${DEBDIR// /\\ }" } function uploadSite(){ @@ -89,7 +93,7 @@ function uploadSite(){ "$D"/contents/binary rsync -avz "$D/contents/" "$DEB_WEBDIR/" - rsync -avz -e "ssh $SSH_OPTS" "${DEB}" "$PKGSERVER:${DEBDIR// /\\ }" + rsync -avz -e "ssh ${SSH_OPTS[*]}" "${DEB}" "$PKGSERVER:${DEBDIR// /\\ }" } function show(){ From fdde09e8821f0083f78722c301fec39964c9982d Mon Sep 17 00:00:00 2001 From: Olblak Date: Thu, 2 Apr 2020 22:22:17 +0200 Subject: [PATCH 07/14] Update rpm publishing scripts --- rpm/publish/publish.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/rpm/publish/publish.sh b/rpm/publish/publish.sh index 527a695a..d89801a4 100755 --- a/rpm/publish/publish.sh +++ b/rpm/publish/publish.sh @@ -6,12 +6,15 @@ set -euxo pipefail : "${RPMDIR:?Require where to put binary files}" : "${RPM_WEBDIR:?Require where to put index and other web contents}" : "${RPM_URL:?Require rpm repository url}" -: "${RELEASELINE:?Require rpm release line}" +: "${RELEASELINE?Require rpm release line}" : "${BASE:? Required base directory}" # $$ Contains current pid D="$AGENT_WORKDIR/$$" +# Convert string to array to correctly escape cli parameter +SSH_OPTS=($SSH_OPTS) + function clean(){ rm -rf $D } @@ -39,7 +42,8 @@ EOF # locally createrepo --update -o "$RPM_WEBDIR" "$RPMDIR/" # on the server - ssh "$SSH_OPTS" "$PKGSERVER" createrepo --update -o "'$RPM_WEBDIR'" "'$RPMDIR/'" + # shellcheck disable=SC2029 + ssh "$PKGSERVER" "${SSH_OPTS[*]}" createrepo --update -o "'$RPM_WEBDIR'" "'$RPMDIR/'" } @@ -48,13 +52,14 @@ function init(){ mkdir -p "$RPMDIR/" # mkdir -p "$RPM_WEBDIR/" # May not be necessary - ssh "$SSH_OPTS" "$PKGSERVER" mkdir -p "'$RPMDIR/'" + # shellcheck disable=SC2029 + ssh "$PKGSERVER" "${SSH_OPTS[*]}" mkdir -p "'$RPMDIR/'" } function uploadPackage(){ rsync -avz "$RPM" "$RPMDIR/" - rsync -avz -e "ssh $SSH_OPTS" "$RPM" "$PKGSERVER:${RPMDIR// /\\ }/" + rsync -avz -e "ssh ${SSH_OPTS[*]}" "$RPM" "$PKGSERVER:${RPMDIR// /\\ }/" } function show(){ @@ -62,7 +67,7 @@ function show(){ echo "RPM: $RPM" echo "RPMDIR: $RPMDIR" echo "RPM_WEBDIR: $RPM_WEBDIR" - echo "SSH_OPTS: $SSH_OPTS" + echo "SSH_OPTS: ${SSH_OPTS[*]}" echo "PKGSERVER: $PKGSERVER" echo "GPG_KEYNAME: $GPG_KEYNAME" echo "---" @@ -71,14 +76,13 @@ function show(){ function uploadSite(){ pushd "$D" rsync -avz --exclude RPMS . "$RPM_WEBDIR/" - rsync -avz -e "ssh $SSH_OPTS" --exclude RPMS . "$PKGSERVER:${RPM_WEBDIR// /\\ }" + rsync -avz -e "ssh ${SSH_OPTS[*]}" --exclude RPMS . "$PKGSERVER:${RPM_WEBDIR// /\\ }" popd } show init generateSite -signSite uploadPackage uploadSite clean From 2e65b20cf9f46ed22559d4e309c44a5c02c84819 Mon Sep 17 00:00:00 2001 From: Olblak Date: Thu, 2 Apr 2020 22:24:55 +0200 Subject: [PATCH 08/14] Update suse publishing scripts --- suse/publish/publish.sh | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/suse/publish/publish.sh b/suse/publish/publish.sh index 727ef577..c8a85171 100755 --- a/suse/publish/publish.sh +++ b/suse/publish/publish.sh @@ -11,8 +11,12 @@ set -euxo pipefail # $$ Contains current pid D="$AGENT_WORKDIR/$$" +# Convert string to array to correctly escape cli parameter +SSH_OPTS=($SSH_OPTS) +SCP_OPTS=($SCP_OPTS) + function clean(){ - mkdir -rf $D + rm -rf $D } function generateSite(){ @@ -32,7 +36,9 @@ function generateSite(){ function init(){ # where to put binary files mkdir -p "$SUSEDIR/" # Local - ssh "$SSH_OPTS" $PKGSERVER mkdir -p "'$SUSEDIR/'" # Remote + + # shellcheck disable=SC2029 + ssh "$PKGSERVER" "${SSH_OPTS[*]}" mkdir -p "'$SUSEDIR/'" # Remote # where to put repository index and other web contents mkdir -p "$SUSE_WEBDIR" @@ -46,7 +52,7 @@ function show(){ echo "SUSE: $SUSE" echo "SUSEDIR: $SUSEDIR" echo "SUSE_WEBDIR: $SUSE_WEBDIR" - echo "SSH_OPTS: $SSH_OPTS" + echo "SSH_OPTS: ${SSH_OPTS[*]}" echo "PKGSERVER: $PKGSERVER" echo "GPG_KEYNAME: $GPG_KEYNAME" echo "---" @@ -54,21 +60,23 @@ function show(){ function uploadPackage(){ rsync -avz "$SUSE" "$SUSEDIR/" # Local - rsync -avz -e "ssh $SSH_OPTS" "${SUSE}" "$PKGSERVER:${SUSEDIR// /\\ }" # Remote + rsync -avz -e "ssh ${SSH_OPTS[*]}" "${SUSE}" "$PKGSERVER:${SUSEDIR// /\\ }" # Remote } function uploadSite(){ pushd $D rsync -avz --exclude RPMS . "$SUSE_WEBDIR/" #Local - rsync -avz -e "ssh $SSH_OPTS" --exclude RPMS . "$PKGSERVER:${SUSE_WEBDIR// /\\ }" # Remote + # shellcheck disable=SC2029 + rsync -avz -e "ssh ${SSH_OPTS[*]}" --exclude RPMS . "$PKGSERVER:${SUSE_WEBDIR// /\\ }" # Remote # generate index on the server # server needs 'createrepo' pacakge createrepo --update -o "$SUSE_WEBDIR" "$SUSEDIR/" #Local - ssh "$SSH_OPTS" "$PKGSERVER" createrepo --update -o "'$SUSE_WEBDIR'" "'$SUSEDIR/'" # Remote + # shellcheck disable=SC2029 + ssh "$PKGSERVER" "${SSH_OPTS[*]}" createrepo --update -o "'$SUSE_WEBDIR'" "'$SUSEDIR/'" # Remote - scp "$SCP_OPTS" "$PKGSERVER:${SUSE_WEBDIR// /\\ }/repodata/repomd.xml" repodata/ # Remote + scp "${SCP_OPTS[*]}" "$PKGSERVER:${SUSE_WEBDIR// /\\ }/repodata/repomd.xml" repodata/ # Remote cp "${SUSE_WEBDIR// /\\ }/repodata/repomd.xml" repodata/ # Local gpg \ @@ -77,11 +85,12 @@ function uploadSite(){ -u "$GPG_KEYNAME" \ -a \ --detach-sign \ + --passphrase-file "$GPG_PASSPHRASE_FILE" \ --yes \ - "$SUSE_WEBDIR/repodata/repomd.xml" + repodata/repomd.xml - scp "$SCP_OPTS" repodata/repomd.xml.asc "$PKGSERVER:${SUSE_WEBDIR// /\\ }/repodata/" - cp repodata/repomd.xml.asc "$PKGSERVER:${SUSE_WEBDIR// /\\ }/repodata/" + scp "${SCP_OPTS[*]}" repodata/repomd.xml.asc "$PKGSERVER:${SUSE_WEBDIR// /\\ }/repodata/" + cp repodata/repomd.xml.asc "${SUSE_WEBDIR// /\\ }/repodata/" popd } From 5677cb31d86634a349f835e11b59af31d3a82ee5 Mon Sep 17 00:00:00 2001 From: Olblak Date: Thu, 2 Apr 2020 22:27:31 +0200 Subject: [PATCH 09/14] Update fake en variables --- env/test.mk | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/env/test.mk b/env/test.mk index bd9e96ca..815315c9 100644 --- a/env/test.mk +++ b/env/test.mk @@ -6,9 +6,12 @@ export JENKINS_URL=https://cloudbees.ci.cloudbees.com/ # the host to publish bits to -export PKGSERVER=${USER}@localhost -export SSH_OPTS= -export SCP_OPTS= +export PKGSERVER=root@remote +# Testing both with and without SSH_OPTS +#export SSH_OPTS=-p 22 +#export SCP_OPTS=-P 22 +export SSH_OPTS=-p 22 +export SCP_OPTS=-P 22 # where to put binary files export TESTDIR=$(realpath .)/pkg.jenkins.io From 4ccda3fdcb69d4b84b94103caf256aea00e92593 Mon Sep 17 00:00:00 2001 From: Olblak Date: Thu, 2 Apr 2020 22:52:31 +0200 Subject: [PATCH 10/14] Debian publishing script correctly compress to stdout --- deb/publish/publish.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deb/publish/publish.sh b/deb/publish/publish.sh index 381b9add..134c55a9 100755 --- a/deb/publish/publish.sh +++ b/deb/publish/publish.sh @@ -55,8 +55,8 @@ function generateSite(){ cat $D/merged/Packages > $D/binary/Packages gzip -9c "$D/merged/Packages" > "$D/binary/Packages.gz" - bzip2 "$D/merged/Packages" > "$D/binary/Packages.bz2" - lzma "$D/merged/Packages" > "$D/binary/Packages.lzma" + bzip2 -c "$D/merged/Packages" > "$D/binary/Packages.bz2" + lzma -c "$D/merged/Packages" > "$D/binary/Packages.lzma" gzip -9c "$D/merged/Contents" > "$D/binary/Contents.gz" apt-ftparchive -c "$bin/release.conf" release "$D/binary" > "$D/binary/Release" From 71cf95a6a73f413dffeae8373a947e495188d7b7 Mon Sep 17 00:00:00 2001 From: Olblak Date: Thu, 2 Apr 2020 22:53:06 +0200 Subject: [PATCH 11/14] Debian publishing script use gpg passphrase file --- deb/publish/publish.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/deb/publish/publish.sh b/deb/publish/publish.sh index 134c55a9..5716ec21 100755 --- a/deb/publish/publish.sh +++ b/deb/publish/publish.sh @@ -118,6 +118,7 @@ function signSite(){ --pinentry-mode loopback \ --digest-algo=sha256 \ -u "$GPG_KEYNAME" \ + --passphrase-file "$GPG_PASSPHRASE_FILE" \ -abs \ -o "$D/binary/Release.gpg" \ "$D/binary/Release" From 9595ca61fc20f5b6d31005817568913e4f929622 Mon Sep 17 00:00:00 2001 From: Olblak Date: Thu, 2 Apr 2020 22:54:58 +0200 Subject: [PATCH 12/14] debian publish.sh correclty escape variable --- deb/publish/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deb/publish/publish.sh b/deb/publish/publish.sh index 5716ec21..1ae7d7a7 100755 --- a/deb/publish/publish.sh +++ b/deb/publish/publish.sh @@ -80,7 +80,7 @@ function init(){ function uploadPackage(){ # Upload Debian Package rsync -avz "$DEB" "$DEBDIR/" - rsync -avz -e \'ssh "${SSH_OPTS[*]}"\' "${DEB}" "$PKGSERVER:${DEBDIR// /\\ }" + rsync -avz -e "ssh ${SSH_OPTS[*]}" "${DEB}" "$PKGSERVER:${DEBDIR// /\\ }" } function uploadSite(){ From d1bb5c19ed911db13d745b468f863727ef6db379 Mon Sep 17 00:00:00 2001 From: Olblak Date: Thu, 2 Apr 2020 22:57:37 +0200 Subject: [PATCH 13/14] Update test variables --- env/test.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env/test.mk b/env/test.mk index 815315c9..59150ca2 100644 --- a/env/test.mk +++ b/env/test.mk @@ -28,7 +28,7 @@ export SUSE_WEBDIR=${TESTDIR}/opensuse${RELEASELINE} export DEB_WEBDIR=${TESTDIR}/debian${RELEASELINE} # URL to the aforementioned webdir. -WEBSERVER=test.pkg.jenkins.io:9200 +WEBSERVER=pkg.jenkins.io export RPM_URL=http://${WEBSERVER}/redhat${RELEASELINE} export SUSE_URL=http://${WEBSERVER}/opensuse${RELEASELINE} export DEB_URL=http://${WEBSERVER}/debian${RELEASELINE} From 813fe4609fe8c76e26f7f8abcc7e3782d8cf9131 Mon Sep 17 00:00:00 2001 From: Olblak Date: Thu, 2 Apr 2020 22:59:46 +0200 Subject: [PATCH 14/14] Switch test variable from http to https --- env/test.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/env/test.mk b/env/test.mk index 59150ca2..35fcdc61 100644 --- a/env/test.mk +++ b/env/test.mk @@ -29,6 +29,6 @@ export DEB_WEBDIR=${TESTDIR}/debian${RELEASELINE} # URL to the aforementioned webdir. WEBSERVER=pkg.jenkins.io -export RPM_URL=http://${WEBSERVER}/redhat${RELEASELINE} -export SUSE_URL=http://${WEBSERVER}/opensuse${RELEASELINE} -export DEB_URL=http://${WEBSERVER}/debian${RELEASELINE} +export RPM_URL=https://${WEBSERVER}/redhat${RELEASELINE} +export SUSE_URL=https://${WEBSERVER}/opensuse${RELEASELINE} +export DEB_URL=https://${WEBSERVER}/debian${RELEASELINE}