Update remaining libraries

Signed-off-by: Sayali Gaikawad <[email protected]>
opensearch-project · Jun 29, 2022 · 7b73dc9 · 7b73dc9
1 parent 0ab3c0b
commit 7b73dc9
Show file tree

Hide file tree

Showing 10 changed files with 137 additions and 117 deletions.
diff --git a/tests/jenkins/TestCCRPerfTest.groovy b/tests/jenkins/TestCCRPerfTest.groovy
@@ -53,7 +53,7 @@ class TestCCRPerfTest extends BuildPipelineTest {
 
         assertThat(s3DownloadCommands.size(), equalTo(1))
         assertThat(s3DownloadCommands, hasItem(
-            "{file=config.yml, bucket=test_bucket, path=test_config/config-ccr.yml, force=true}".toString()
+            "{file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config-ccr.yml, force=true}".toString()
         ))
     }
 

diff --git a/tests/jenkins/TestPromoteYumRepos.groovy b/tests/jenkins/TestPromoteYumRepos.groovy
@@ -20,9 +20,6 @@ class TestPromoteYumRepos extends BuildPipelineTest {
         super.setUp()
 
         binding.setVariable('PUBLIC_ARTIFACT_URL', 'https://ci.opensearch.org/dbc')
-        binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'artifactPromotionRole')
-        binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'artifactsAccount')
-        binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'prod-bucket-name')
         binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name')
         def signer_client_creds = ["role": "dummy_role",
                        "external_id": "dummy_ID",
@@ -31,6 +28,10 @@ class TestPromoteYumRepos extends BuildPipelineTest {
         binding.setVariable('configs', signer_client_creds)
         helper.registerAllowedMethod("readJSON", [Map.class], {c -> signer_client_creds})
         helper.registerAllowedMethod("git", [Map])
+        helper.registerAllowedMethod("withCredentials", [Map, Closure], { args, closure ->
+            closure.delegate = delegate
+            return helper.callClosure(closure)
+        })
         helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure ->
             closure.delegate = delegate
             return helper.callClosure(closure)

diff --git a/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy b/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy
@@ -56,7 +56,7 @@ class TestRunNonSecurityPerfTestScript extends BuildPipelineTest {
 
         assertThat(s3DownloadCommands.size(), equalTo(1))
         assertThat(s3DownloadCommands, hasItem(
-            "{file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}".toString()
+            "{file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}".toString()
         ))
     }
 

diff --git a/tests/jenkins/TestRunPerfTestScript.groovy b/tests/jenkins/TestRunPerfTestScript.groovy
@@ -56,7 +56,7 @@ class TestRunPerfTestScript extends BuildPipelineTest {
 
         assertThat(s3DownloadCommands.size(), equalTo(2))
         assertThat(s3DownloadCommands, hasItem(
-            "{file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}".toString()
+            "{file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}".toString()
         ))
     }
 

diff --git a/...s/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt b/...s/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt
@@ -42,8 +42,11 @@
         pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0"
         pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9"
     )
-                  runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
-                     runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config-ccr.yml, force=true})
+                  runPerfTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
+                  runPerfTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
+                  runPerfTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
+                     runPerfTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        runPerfTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config-ccr.yml, force=true})
                   runPerfTestScript.usernamePassword({credentialsId=bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
                   runPerfTestScript.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
                      runPerfTestScript.sh(./test.sh perf-test --stack test-single-security-1236-x64-perf-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml     --component cross-cluster-replication)

diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt
@@ -42,8 +42,11 @@
         pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0"
         pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9"
     )
-                  runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
-                     runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true})
+                  runPerfTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
+                  runPerfTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
+                  runPerfTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
+                     runPerfTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        runPerfTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true})
                   runPerfTestScript.usernamePassword({credentialsId=bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
                   runPerfTestScript.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
                      runPerfTestScript.sh(./test.sh perf-test --stack test-single-security-1236-x64-perf-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml  --workload nyc_taxis --test-iters 1 --warmup-iters 1 )
@@ -102,8 +105,11 @@ Performance tests with security for 1236 completed})
         pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0"
         pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9"
     )
-                  runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
-                     runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true})
+                  runPerfTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
+                  runPerfTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
+                  runPerfTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
+                     runPerfTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        runPerfTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true})
                   runPerfTestScript.usernamePassword({credentialsId=bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
                   runPerfTestScript.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
                      runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64-perf-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1 )

diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt
@@ -43,8 +43,11 @@
         pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0"
         pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9"
     )
-                  runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
-                     runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true})
+                  runPerfTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
+                  runPerfTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
+                  runPerfTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
+                     runPerfTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        runPerfTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true})
                   runPerfTestScript.usernamePassword({credentialsId=bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
                   runPerfTestScript.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
                      runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64-perf-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1 )

diff --git a/tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile.txt
@@ -8,47 +8,51 @@
                   promoteYumRepos.library({identifier=jenkins@20211123, retriever=null})
                   promoteYumRepos.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml})
                   InputManifest.asBoolean()
-                  promoteYumRepos.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
-                     promoteYumRepos.println(Pulling Prod Yumrepo)
-                     promoteYumRepos.sh(aws s3 sync s3://prod-bucket-name/releases/bundle/opensearch/1.x/yum/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ --no-progress)
-                  promoteYumRepos.sh(
-        set -e
-        set +x
-        set +x
-
-        echo "Pulling 1.3.0 rpms"
-        cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum
-        curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/rpm/dist/opensearch/opensearch-1.3.0-linux-x64.rpm
-        curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/rpm/dist/opensearch/opensearch-1.3.0-linux-arm64.rpm
-
-        ls -l
-
-        rm -vf repodata/repomd.xml.asc
-
-        echo "Update repo metadata"
-        createrepo --update .
-
-        # Rename .xml to .pom for signing
-        # Please do not add .xml to signer filter
-        # As maven have many .xml and we do not want to sign them
-        # This is an outlier case for yum repo only
-        mv -v repodata/repomd.xml repodata/repomd.pom
-
-        echo "Complete metadata update, awaiting signing repomd.xml"
+                  promoteYumRepos.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
+                  promoteYumRepos.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
+                  promoteYumRepos.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
+                  promoteYumRepos.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
+                     promoteYumRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        promoteYumRepos.println(Pulling Prod Yumrepo)
+                        promoteYumRepos.sh(aws s3 sync s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/yum/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ --no-progress)
+                     promoteYumRepos.sh(
+            set -e
+            set +x
+            set +x
+
+            echo "Pulling 1.3.0 rpms"
+            cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum
+            curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/rpm/dist/opensearch/opensearch-1.3.0-linux-x64.rpm
+            curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/rpm/dist/opensearch/opensearch-1.3.0-linux-arm64.rpm
+
+            ls -l
+
+            rm -vf repodata/repomd.xml.asc
+
+            echo "Update repo metadata"
+            createrepo --update .
+
+            # Rename .xml to .pom for signing
+            # Please do not add .xml to signer filter
+            # As maven have many .xml and we do not want to sign them
+            # This is an outlier case for yum repo only
+            mv -v repodata/repomd.xml repodata/repomd.pom
+
+            echo "Complete metadata update, awaiting signing repomd.xml"
 
         cd -
 
     )
-                  promoteYumRepos.signArtifacts({artifactPath=/tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom, sigtype=.sig, platform=linux})
-                     signArtifacts.echo(PGP Signature Signing)
-                     signArtifacts.fileExists(/tmp/workspace/sign.sh)
-                     signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
-                     signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
-                     signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
-                     signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=signer_client_creds})
-                     signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], signer_client_creds], groovy.lang.Closure)
-                        signArtifacts.readJSON({text=signer_client_creds})
-                        signArtifacts.sh(
+                     promoteYumRepos.signArtifacts({artifactPath=/tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom, sigtype=.sig, platform=linux})
+                        signArtifacts.echo(PGP Signature Signing)
+                        signArtifacts.fileExists(/tmp/workspace/sign.sh)
+                        signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
+                        signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
+                        signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
+                        signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=signer_client_creds})
+                        signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], signer_client_creds], groovy.lang.Closure)
+                           signArtifacts.readJSON({text=signer_client_creds})
+                           signArtifacts.sh(
                 #!/bin/bash
                 set +x
                 export ROLE=dummy_role
@@ -58,27 +62,26 @@
 
                 /tmp/workspace/sign.sh /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom --sigtype=.sig --platform=linux
             )
-                  promoteYumRepos.sh(
-        set -e
-        set +x
+                     promoteYumRepos.sh(
+            set -e
+            set +x
 
-        cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/
+            cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/
 
-        ls -l
+            ls -l
 
-        mv -v repomd.pom repomd.xml
-        mv -v repomd.pom.sig repomd.xml.sig
+            mv -v repomd.pom repomd.xml
+            mv -v repomd.pom.sig repomd.xml.sig
 
-        # This step is required as yum only accept .asc and signing workflow only support .sig
-        cat repomd.xml.sig | gpg --enarmor | sed 's@ARMORED FILE@SIGNATURE@g' > repomd.xml.asc
+            # This step is required as yum only accept .asc and signing workflow only support .sig
+            cat repomd.xml.sig | gpg --enarmor | sed 's@ARMORED FILE@SIGNATURE@g' > repomd.xml.asc
 
-        rm -vf repomd.xml.sig
+            rm -vf repomd.xml.sig
 
-        ls -l
-
-        cd -
+            ls -l
 
+            cd -
     )
-                  promoteYumRepos.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
-                     promoteYumRepos.println(Pushing Prod Yumrepo)
-                     promoteYumRepos.sh(aws s3 sync /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ s3://prod-bucket-name/releases/bundle/opensearch/1.x/yum/ --no-progress)
+                     promoteYumRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        promoteYumRepos.println(Pushing Prod Yumrepo)
+                        promoteYumRepos.sh(aws s3 sync /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/yum/ --no-progress)