Tweak rpm signing to make it more robust and work with prod sync (#578)

* Tweak rpm signing to make it more robust and work with prod sync * Comment out old methods of rpm sync to prod repo
opendistro-for-elasticsearch · Jan 26, 2021 · 9d839d3 · 9d839d3
1 parent cb4b086
commit 9d839d3
Show file tree

Hide file tree

Showing 2 changed files with 77 additions and 59 deletions.
diff --git a/.github/workflows/prod-sync-rpm.yml b/.github/workflows/prod-sync-rpm.yml
@@ -20,59 +20,63 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
 
-      - name: Sign And Upload Artifacts
-        run: |
-          yum update -y
-          yum install -y unzip
-          sleep 5
-          curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"
-          unzip awscli-bundle.zip
-          ./awscli-bundle/install -b ~/bin/aws
-          sleep 5
-          chmod 755 elasticsearch/linux_distributions/scripts/rpm-addsign.exp
-          passphrase=${{ secrets.RPM_SIGN_PASSPHRASE }}
-          /github/home/bin/aws s3 cp s3://opendistro-docs/github-actions/pgp-private-key .
-          /github/home/bin/aws s3 cp s3://opendistro-docs/github-actions/pgp-public-key .
-                    
-          gpg --import pgp-public-key
-          gpg --allow-secret-key-import --import pgp-private-key
-                    
-          ls -ltr /github/home/.gnupg/
-                    
-          rpm --import pgp-public-key
-                    
-          rpm -q gpg-pubkey --qf '%{name}-%{version}-%{release} --> %{summary}\n'
-                    
-          echo "%_signature gpg" >> /github/home/.rpmmacros
-          echo "%_gpg_path /github/home/.gnupg" >> /github/home/.rpmmacros
-          echo "%_gpg_name OpenDistroForElasticsearch" >> /github/home/.rpmmacros
-          echo "%_gpg /usr/bin/gpg" >> /github/home/.rpmmacros
+      - name: (New) Sync artifacts from staging to prod repo
+        run: rpm-signing.sh passphrase prod-sync
 
-          echo "Setup a directory structure on your local machine that mimics the one in S3"
-          mkdir artifacts-repo
-          cd artifacts-repo
-          mkdir yum
-          mkdir -p downloads/rpms
+#      - name: (OLD) Sign And Upload Artifacts
+#        run: | 
+#          yum update -y
+#          yum install -y unzip
+#          sleep 5
+#          curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"
+#          unzip awscli-bundle.zip
+#          ./awscli-bundle/install -b ~/bin/aws
+#          sleep 5
+#          chmod 755 elasticsearch/linux_distributions/scripts/rpm-addsign.exp
+#          passphrase=${{ secrets.RPM_SIGN_PASSPHRASE }}
+#          /github/home/bin/aws s3 cp s3://opendistro-docs/github-actions/pgp-private-key .
+#          /github/home/bin/aws s3 cp s3://opendistro-docs/github-actions/pgp-public-key .
+#                    
+#          gpg --import pgp-public-key
+#          gpg --allow-secret-key-import --import pgp-private-key
+#                    
+#          ls -ltr /github/home/.gnupg/
+#                    
+#          rpm --import pgp-public-key
+#                    
+#          rpm -q gpg-pubkey --qf '%{name}-%{version}-%{release} --> %{summary}\n'
+#                    
+#          echo "%_signature gpg" >> /github/home/.rpmmacros
+#          echo "%_gpg_path /github/home/.gnupg" >> /github/home/.rpmmacros
+#          echo "%_gpg_name OpenDistroForElasticsearch" >> /github/home/.rpmmacros
+#          echo "%_gpg /usr/bin/gpg" >> /github/home/.rpmmacros
+#
+#          echo "Setup a directory structure on your local machine that mimics the one in S3"
+#          mkdir artifacts-repo
+#          cd artifacts-repo
+#          mkdir yum
+#          mkdir -p downloads/rpms
+#
+#          echo "Sync the remote yum repo to your local directory. *Before you do this, ensure you export the correct set of AWS credentials.*"
+#          /github/home/bin/aws s3 sync s3://artifacts.opendistroforelasticsearch.amazon.com/yum/ yum/
+#          rm -rf yum/staging
+#          /github/home/bin/aws s3 sync s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/rpms/ downloads/rpms/
+#          
+#          echo "Add signatures to the new RPMs and copy them over to the Repo."
+#          yum install -y expect
+#          yum install -y rpm-sign
+#          
+#          echo "Adding sign to the rpms with the passphrase"
+#          for VARIABLE in downloads/rpms/*/*.rpm
+#          do
+#                ../elasticsearch/linux_distributions/scripts/rpm-addsign.exp $VARIABLE $passphrase
+#          done
+#          echo "Verifying the signing"
+#          find downloads -name *.rpm | xargs -n1 rpm --checksig
+#          find downloads -name *.rpm | xargs -n1 -I{} cp {} yum/noarch
+#          yum install -y createrepo
+#          createrepo -v --update --deltas yum/noarch --max-delta-rpm-size=1000000000
+#          gpg --detach-sign --armor --batch --yes  --passphrase $passphrase yum/noarch/repodata/repomd.xml
+#          /github/home/bin/aws s3 sync yum/ s3://artifacts.opendistroforelasticsearch.amazon.com/yum
+#          /github/home/bin/aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/yum/*"
 
-          echo "Sync the remote yum repo to your local directory. *Before you do this, ensure you export the correct set of AWS credentials.*"
-          /github/home/bin/aws s3 sync s3://artifacts.opendistroforelasticsearch.amazon.com/yum/ yum/
-          rm -rf yum/staging
-          /github/home/bin/aws s3 sync s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/rpms/ downloads/rpms/
-          
-          echo "Add signatures to the new RPMs and copy them over to the Repo."
-          yum install -y expect
-          yum install -y rpm-sign
-          
-          echo "Adding sign to the rpms with the passphrase"
-          for VARIABLE in downloads/rpms/*/*.rpm
-          do
-                ../elasticsearch/linux_distributions/scripts/rpm-addsign.exp $VARIABLE $passphrase
-          done
-          echo "Verifying the signing"
-          find downloads -name *.rpm | xargs -n1 rpm --checksig
-          find downloads -name *.rpm | xargs -n1 -I{} cp {} yum/noarch
-          yum install -y createrepo
-          createrepo -v --update --deltas yum/noarch --max-delta-rpm-size=1000000000
-          gpg --detach-sign --armor --batch --yes  --passphrase $passphrase yum/noarch/repodata/repomd.xml
-          /github/home/bin/aws s3 sync yum/ s3://artifacts.opendistroforelasticsearch.amazon.com/yum
-          /github/home/bin/aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/yum/*"
diff --git a/elasticsearch/linux_distributions/rpm-signing.sh b/elasticsearch/linux_distributions/rpm-signing.sh
@@ -14,11 +14,20 @@ S3_RELEASE_BUCKET=`echo $S3_RELEASE_BASEURL | awk -F '/' '{print $3}'`
 S3_RELEASE_FINAL_BUILD=`yq eval '.urls.ODFE.releases_final_build' $MANIFEST_FILE | sed 's/\///g'`
 PLUGIN_PATH=`yq eval '.urls.ODFE.releases' $MANIFEST_FILE | sed "s/^.*$S3_RELEASE_BUCKET\///g"`
 PASSPHRASE=$1; if [ -z "$PASSPHRASE" ]; then echo "Please enter passphrase as a parameter"; exit 1; fi
+ACTION=$2; if [ ! -z "$ACTION" ]; then echo "About to sync staging to prod repo! Wait for 30 seconds"; sleep 30; fi
 
 REPO_BASEDIR="$ROOT/artifacts-repo"
 REPO_YUMDIR="$REPO_BASEDIR/yum"
 REPO_RPMSDIR="$REPO_BASEDIR/rpms"
 
+if [ "$ACTION" = "prod-sync" ]
+then
+  aws s3 sync ${S3_PROD_BASEURL}yum/staging/ ${S3_PROD_BASEURL}yum/ --quiet; echo $?
+  aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/yum/*"
+  exit 0
+fi
+
+
 if [ -z "$S3_RELEASE_FINAL_BUILD" ]
 then
   S3_RELEASE_BUILD=`aws s3api list-objects --bucket $S3_RELEASE_BUCKET --prefix "${PLUGIN_PATH}${OD_VERSION}" --query 'Contents[].[Key]' --output text | awk -F '/' '{print $3}' | uniq | tail -n 1`
@@ -55,15 +64,22 @@ echo "%_gpg /usr/bin/gpg" >> ~/.rpmmacros
 # Setup a directory structure on your local machine that mimics the one in S3.
 mkdir -p $REPO_YUMDIR/
 mkdir -p $REPO_RPMSDIR/
-#cd $REPO_BASEDIR/
 
 # Sync the remote yum repo to your local directory. *Before you do this, ensure you export the correct set of AWS credentials.*
 echo "Sync yum"
 aws s3 sync ${S3_PROD_BASEURL}yum/staging/ $REPO_YUMDIR/ --quiet; echo $?
-#aws s3 sync ${S3_PROD_BASEURL}yum/staging-new/ $REPO_YUMDIR/ --quiet; echo $?
 echo "Sync rpms"
 aws s3 sync ${S3_RELEASE_BASEURL}${OD_VERSION}/${S3_RELEASE_BUILD}/elasticsearch-plugins/ $REPO_RPMSDIR/ --exclude "*" --include "*.rpm"  --quiet; echo $?
+aws s3 sync ${S3_RELEASE_BASEURL}${OD_VERSION}/${S3_RELEASE_BUILD}/opendistro-libs/ $REPO_RPMSDIR/ --exclude "*" --include "*.rpm"  --quiet; echo $?
 aws s3 sync ${S3_RELEASE_BASEURL}${OD_VERSION}/odfe/ $REPO_RPMSDIR/ --exclude "*" --include "*.rpm"  --quiet; echo $?
+
+
+# Rename rpms to remove build numbers
+for pkg in `ls $REPO_RPMSDIR | grep -i build`
+do
+  mv $REPO_RPMSDIR/$pkg $REPO_RPMSDIR/`echo $pkg | sed 's/-build-[0-9]*//g'`
+done
+
 echo $REPO_YUMDIR/; ls -l $REPO_YUMDIR/
 echo $REPO_RPMSDIR/; ls -l $REPO_RPMSDIR/
 df -h
@@ -76,7 +92,7 @@ for rpm_package in `ls $REPO_RPMSDIR/`
 do
   echo "Signing $rpm_package"
   ./rpm-addsign.exp $REPO_RPMSDIR/$rpm_package $PASSPHRASE
-  echo "Signing complete"
+  echo "Signing complete #################################"
 done
 
 # Verify the signing
@@ -104,7 +120,5 @@ gpg --detach-sign --armor --batch --yes  --passphrase $PASSPHRASE $REPO_YUMDIR/n
 echo "Sync rpms back to the repo"
 aws s3 sync $REPO_YUMDIR/ ${S3_PROD_BASEURL}yum/staging/ --quiet; echo $?
 aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/yum/staging/*"
-#aws s3 sync $REPO_YUMDIR/ ${S3_PROD_BASEURL}yum/staging-new/ --quiet; echo $?
-#aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/yum/staging-new/*"