Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Tweak rpm signing to make it more robust and work with prod sync #578

Merged
merged 2 commits into from
Jan 26, 2021
Merged

Tweak rpm signing to make it more robust and work with prod sync #578

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
817a0f3
Tweak rpm signing to make it more robust and work with prod sync
peterzhuamazon Jan 25, 2021
a74c85a
Comment out old methods of rpm sync to prod repo
peterzhuamazon Jan 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 58 additions & 54 deletions .github/workflows/prod-sync-rpm.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,59 +20,63 @@ jobs:
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1

- name: Sign And Upload Artifacts
run: |
yum update -y
yum install -y unzip
sleep 5
curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"
unzip awscli-bundle.zip
./awscli-bundle/install -b ~/bin/aws
sleep 5
chmod 755 elasticsearch/linux_distributions/scripts/rpm-addsign.exp
passphrase=${{ secrets.RPM_SIGN_PASSPHRASE }}
/github/home/bin/aws s3 cp s3://opendistro-docs/github-actions/pgp-private-key .
/github/home/bin/aws s3 cp s3://opendistro-docs/github-actions/pgp-public-key .
gpg --import pgp-public-key
gpg --allow-secret-key-import --import pgp-private-key
ls -ltr /github/home/.gnupg/
rpm --import pgp-public-key
rpm -q gpg-pubkey --qf '%{name}-%{version}-%{release} --> %{summary}\n'
echo "%_signature gpg" >> /github/home/.rpmmacros
echo "%_gpg_path /github/home/.gnupg" >> /github/home/.rpmmacros
echo "%_gpg_name OpenDistroForElasticsearch" >> /github/home/.rpmmacros
echo "%_gpg /usr/bin/gpg" >> /github/home/.rpmmacros
- name: (New) Sync artifacts from staging to prod repo
run: rpm-signing.sh passphrase prod-sync

echo "Setup a directory structure on your local machine that mimics the one in S3"
mkdir artifacts-repo
cd artifacts-repo
mkdir yum
mkdir -p downloads/rpms
# - name: (OLD) Sign And Upload Artifacts
# run: |
# yum update -y
# yum install -y unzip
# sleep 5
# curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"
# unzip awscli-bundle.zip
# ./awscli-bundle/install -b ~/bin/aws
# sleep 5
# chmod 755 elasticsearch/linux_distributions/scripts/rpm-addsign.exp
# passphrase=${{ secrets.RPM_SIGN_PASSPHRASE }}
# /github/home/bin/aws s3 cp s3://opendistro-docs/github-actions/pgp-private-key .
# /github/home/bin/aws s3 cp s3://opendistro-docs/github-actions/pgp-public-key .
#
# gpg --import pgp-public-key
# gpg --allow-secret-key-import --import pgp-private-key
#
# ls -ltr /github/home/.gnupg/
#
# rpm --import pgp-public-key
#
# rpm -q gpg-pubkey --qf '%{name}-%{version}-%{release} --> %{summary}\n'
#
# echo "%_signature gpg" >> /github/home/.rpmmacros
# echo "%_gpg_path /github/home/.gnupg" >> /github/home/.rpmmacros
# echo "%_gpg_name OpenDistroForElasticsearch" >> /github/home/.rpmmacros
# echo "%_gpg /usr/bin/gpg" >> /github/home/.rpmmacros
#
# echo "Setup a directory structure on your local machine that mimics the one in S3"
# mkdir artifacts-repo
# cd artifacts-repo
# mkdir yum
# mkdir -p downloads/rpms
#
# echo "Sync the remote yum repo to your local directory. *Before you do this, ensure you export the correct set of AWS credentials.*"
# /github/home/bin/aws s3 sync s3://artifacts.opendistroforelasticsearch.amazon.com/yum/ yum/
# rm -rf yum/staging
# /github/home/bin/aws s3 sync s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/rpms/ downloads/rpms/
#
# echo "Add signatures to the new RPMs and copy them over to the Repo."
# yum install -y expect
# yum install -y rpm-sign
#
# echo "Adding sign to the rpms with the passphrase"
# for VARIABLE in downloads/rpms/*/*.rpm
# do
# ../elasticsearch/linux_distributions/scripts/rpm-addsign.exp $VARIABLE $passphrase
# done
# echo "Verifying the signing"
# find downloads -name *.rpm | xargs -n1 rpm --checksig
# find downloads -name *.rpm | xargs -n1 -I{} cp {} yum/noarch
# yum install -y createrepo
# createrepo -v --update --deltas yum/noarch --max-delta-rpm-size=1000000000
# gpg --detach-sign --armor --batch --yes --passphrase $passphrase yum/noarch/repodata/repomd.xml
# /github/home/bin/aws s3 sync yum/ s3://artifacts.opendistroforelasticsearch.amazon.com/yum
# /github/home/bin/aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/yum/*"

echo "Sync the remote yum repo to your local directory. *Before you do this, ensure you export the correct set of AWS credentials.*"
/github/home/bin/aws s3 sync s3://artifacts.opendistroforelasticsearch.amazon.com/yum/ yum/
rm -rf yum/staging
/github/home/bin/aws s3 sync s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/rpms/ downloads/rpms/
echo "Add signatures to the new RPMs and copy them over to the Repo."
yum install -y expect
yum install -y rpm-sign
echo "Adding sign to the rpms with the passphrase"
for VARIABLE in downloads/rpms/*/*.rpm
do
../elasticsearch/linux_distributions/scripts/rpm-addsign.exp $VARIABLE $passphrase
done
echo "Verifying the signing"
find downloads -name *.rpm | xargs -n1 rpm --checksig
find downloads -name *.rpm | xargs -n1 -I{} cp {} yum/noarch
yum install -y createrepo
createrepo -v --update --deltas yum/noarch --max-delta-rpm-size=1000000000
gpg --detach-sign --armor --batch --yes --passphrase $passphrase yum/noarch/repodata/repomd.xml
/github/home/bin/aws s3 sync yum/ s3://artifacts.opendistroforelasticsearch.amazon.com/yum
/github/home/bin/aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/yum/*"
24 changes: 19 additions & 5 deletions elasticsearch/linux_distributions/rpm-signing.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,20 @@ S3_RELEASE_BUCKET=`echo $S3_RELEASE_BASEURL | awk -F '/' '{print $3}'`
S3_RELEASE_FINAL_BUILD=`yq eval '.urls.ODFE.releases_final_build' $MANIFEST_FILE | sed 's/\///g'`
PLUGIN_PATH=`yq eval '.urls.ODFE.releases' $MANIFEST_FILE | sed "s/^.*$S3_RELEASE_BUCKET\///g"`
PASSPHRASE=$1; if [ -z "$PASSPHRASE" ]; then echo "Please enter passphrase as a parameter"; exit 1; fi
ACTION=$2; if [ ! -z "$ACTION" ]; then echo "About to sync staging to prod repo! Wait for 30 seconds"; sleep 30; fi

REPO_BASEDIR="$ROOT/artifacts-repo"
REPO_YUMDIR="$REPO_BASEDIR/yum"
REPO_RPMSDIR="$REPO_BASEDIR/rpms"

if [ "$ACTION" = "prod-sync" ]
then
aws s3 sync ${S3_PROD_BASEURL}yum/staging/ ${S3_PROD_BASEURL}yum/ --quiet; echo $?
aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/yum/*"
exit 0
fi


if [ -z "$S3_RELEASE_FINAL_BUILD" ]
then
S3_RELEASE_BUILD=`aws s3api list-objects --bucket $S3_RELEASE_BUCKET --prefix "${PLUGIN_PATH}${OD_VERSION}" --query 'Contents[].[Key]' --output text | awk -F '/' '{print $3}' | uniq | tail -n 1`
Expand Down Expand Up @@ -55,15 +64,22 @@ echo "%_gpg /usr/bin/gpg" >> ~/.rpmmacros
# Setup a directory structure on your local machine that mimics the one in S3.
mkdir -p $REPO_YUMDIR/
mkdir -p $REPO_RPMSDIR/
#cd $REPO_BASEDIR/

# Sync the remote yum repo to your local directory. *Before you do this, ensure you export the correct set of AWS credentials.*
echo "Sync yum"
aws s3 sync ${S3_PROD_BASEURL}yum/staging/ $REPO_YUMDIR/ --quiet; echo $?
#aws s3 sync ${S3_PROD_BASEURL}yum/staging-new/ $REPO_YUMDIR/ --quiet; echo $?
echo "Sync rpms"
aws s3 sync ${S3_RELEASE_BASEURL}${OD_VERSION}/${S3_RELEASE_BUILD}/elasticsearch-plugins/ $REPO_RPMSDIR/ --exclude "*" --include "*.rpm" --quiet; echo $?
aws s3 sync ${S3_RELEASE_BASEURL}${OD_VERSION}/${S3_RELEASE_BUILD}/opendistro-libs/ $REPO_RPMSDIR/ --exclude "*" --include "*.rpm" --quiet; echo $?
aws s3 sync ${S3_RELEASE_BASEURL}${OD_VERSION}/odfe/ $REPO_RPMSDIR/ --exclude "*" --include "*.rpm" --quiet; echo $?


# Rename rpms to remove build numbers
for pkg in `ls $REPO_RPMSDIR | grep -i build`
do
mv $REPO_RPMSDIR/$pkg $REPO_RPMSDIR/`echo $pkg | sed 's/-build-[0-9]*//g'`
done

echo $REPO_YUMDIR/; ls -l $REPO_YUMDIR/
echo $REPO_RPMSDIR/; ls -l $REPO_RPMSDIR/
df -h
Expand All @@ -76,7 +92,7 @@ for rpm_package in `ls $REPO_RPMSDIR/`
do
echo "Signing $rpm_package"
./rpm-addsign.exp $REPO_RPMSDIR/$rpm_package $PASSPHRASE
echo "Signing complete"
echo "Signing complete #################################"
done

# Verify the signing
Expand Down Expand Up @@ -104,7 +120,5 @@ gpg --detach-sign --armor --batch --yes --passphrase $PASSPHRASE $REPO_YUMDIR/n
echo "Sync rpms back to the repo"
aws s3 sync $REPO_YUMDIR/ ${S3_PROD_BASEURL}yum/staging/ --quiet; echo $?
aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/yum/staging/*"
#aws s3 sync $REPO_YUMDIR/ ${S3_PROD_BASEURL}yum/staging-new/ --quiet; echo $?
#aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/yum/staging-new/*"