Agent load testing (#1467)

* Skeleton of perf Jenkinsfile * load generation wait script * More orch * Working PoC * Script to generate parameters for the load generation pipeline * Add provisioning for agent * Add licenses for load gen scripts * Dynamic Java provisioning for load testing * Script for dynamic Java provisioning for load testing * Fix terrible bug with JDK selection and add more JDKs * Remove TODO * Formatting and lint * Add agent_config param * Add min JDK version and custom config option * Allow concurrent users setting * Disable num_of_runs in rev1 * Use production orchestrator * More documentation and minor changes * Add bare-metal settings * Prep for production * Add metal tag * Add metrics collection * Update .ci/load/README.md Co-authored-by: Manuel de la Peña <[email protected]> * Update .ci/load/README.md Co-authored-by: Manuel de la Peña <[email protected]> * Update .ci/load/README.md Co-authored-by: Manuel de la Peña <[email protected]> * Update .ci/load/Jenkinsfile Co-authored-by: Victor Martinez <[email protected]> * Update .ci/load/Jenkinsfile Co-authored-by: Victor Martinez <[email protected]> * Update .ci/load/Jenkinsfile Co-authored-by: Victor Martinez <[email protected]> * Update .ci/load/Jenkinsfile Co-authored-by: Victor Martinez <[email protected]> * Fixup of review changes * Set interval to fixed time * constant pacing for load gen * Fix bad merge * Bring number of jdks below 250 * Increase timeout * Hide server URL * Switch app to benchmark label Co-authored-by: Manuel de la Peña <[email protected]> Co-authored-by: Victor Martinez <[email protected]>
elastic · Nov 5, 2020 · b0f3854 · b0f3854
1 parent 953f8dc
commit b0f3854
Show file tree

Hide file tree

Showing 10 changed files with 842 additions and 0 deletions.
diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
@@ -0,0 +1,167 @@
+// For documentation on this pipeline, please see the README.md in this directory
+pipeline {
+    agent any
+    environment {
+        REPO = 'apm-agent-java'
+        APP = 'spring-petclinic'
+        APP_BASE_DIR = "src/${APP}"
+        METRICS_BASE_DIR="metrics/"
+        AGENT_BASE_DIR = "agent/"
+        ORCH_URL = 'obs-load-orch.app.elastic.co:8000'
+        // Set below for local development
+        // ORCH_URL='10.0.2.2:8000'
+        DEBUG_MODE = '0' // set to '0' for production
+        LOCUST_RUN_TIME = "${params.duration}"
+        LOCUST_USERS = "${params.concurrent_requests}"
+
+    }
+    options {
+        timeout(time: 72, unit: 'HOURS')
+        buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+        timestamps()
+        ansiColor('xterm')
+        durabilityHint('PERFORMANCE_OPTIMIZED')
+
+    }
+    parameters {
+        // The following snippet is auto-generated. To update it, run the script located in .ci/load/scripts/param_gen and copy in the output
+        choice(choices: ['1.18.1', '1.18.0', '1.18.0.RC1', '1.17.0', '1.16.0', '1.15.0', '1.14.0', '1.13.0', '1.12.0', '1.11.0', '1.10.0', '1.9.0', '1.8.0', '1.7.0', '1.6.1', '1.6.0', '1.5.0', '1.4.0', '1.3.0', '1.2.0', '1.1.0', '1.0.1', '1.0.0', '1.0.0.RC1', '0.7.1', '0.7.0', '0.6.2', '0.6.1', '0.6.0', '0.5.1', '0.1.2', '0.1.1'], name: "apm_version", description: "APM Java Agent version")
+        choice(choices: ['adoptopenjdk-11+28-linux', 'adoptopenjdk-11.0.1+13-linux', 'adoptopenjdk-11.0.2+7-linux', 'adoptopenjdk-11.0.2+9-linux', 'adoptopenjdk-11.0.3+7-linux', 'adoptopenjdk-11.0.4+11-linux', 'adoptopenjdk-11.0.5+10-linux', 'adoptopenjdk-11.0.6+10-linux', 'adoptopenjdk-11.0.7+10-linux', 'adoptopenjdk-11.0.8+10-linux', 'adoptopenjdk-11.0.9+11-linux', 'adoptopenjdk-12+33-linux', 'adoptopenjdk-12.0.1+12-linux', 'adoptopenjdk-12.0.2+10-linux', 'adoptopenjdk-13.0.1+9-linux', 'adoptopenjdk-13.0.2+8-linux', 'adoptopenjdk-14.0.1+7-linux', 'adoptopenjdk-14.0.2+12-linux', 'adoptopenjdk-15+36-linux', 'adoptopenjdk-15.0.1+9-linux', 'openjdk-10+43-linux', 'openjdk-10-linux', 'openjdk-10.0.1-linux', 'openjdk-10.0.2-linux', 'openjdk-11+11-linux', 'openjdk-11+12-linux', 'openjdk-11+13-linux', 'openjdk-11+14-linux', 'openjdk-11+15-linux', 'openjdk-11+16-linux', 'openjdk-11+17-linux', 'openjdk-11+18-linux', 'openjdk-11+19-linux', 'openjdk-11+20-linux', 'openjdk-11+21-linux', 'openjdk-11+22-linux', 'openjdk-11+23-linux', 'openjdk-11+24-linux', 'openjdk-11+25-linux', 'openjdk-11+26-linux', 'openjdk-11+27-linux', 'openjdk-11+28-linux', 'openjdk-11+5-linux', 'openjdk-11-linux', 'openjdk-11.0.1-linux', 'openjdk-11.0.2-linux', 'openjdk-12+23-linux', 'openjdk-12+24-linux', 'openjdk-12+25-linux', 'openjdk-12+27-linux', 'openjdk-12+28-linux', 'openjdk-12+29-linux', 'openjdk-12+30-linux', 'openjdk-12+31-linux', 'openjdk-12+32-linux', 'openjdk-12+33-linux', 'openjdk-12-linux', 'openjdk-12.0.1-linux', 'openjdk-12.0.2-linux', 'openjdk-13+14-linux', 'openjdk-13+15-linux', 'openjdk-13+16-linux', 'openjdk-13+17-linux', 'openjdk-13+18-linux', 'openjdk-13+19-linux', 'openjdk-13+20-linux', 'openjdk-13+21-linux', 'openjdk-13+22-linux', 'openjdk-13+23-linux', 'openjdk-13+24-linux', 'openjdk-13+25-linux', 'openjdk-13+26-linux', 'openjdk-13+27-linux', 'openjdk-13+28-linux', 'openjdk-13+29-linux', 'openjdk-13+30-linux', 'openjdk-13+31-linux', 'openjdk-13+32-linux', 'openjdk-13-linux', 'openjdk-13.0.1-linux', 'openjdk-13.0.2-linux', 'openjdk-14+10-linux', 'openjdk-14+11-linux', 'openjdk-14+12-linux', 'openjdk-14+13-linux', 'openjdk-14+14-linux', 'openjdk-14+15-linux', 'openjdk-14+16-linux', 'openjdk-14+17-linux', 'openjdk-14+25-linux', 'openjdk-14+26-linux', 'openjdk-14+27-linux', 'openjdk-14+28-linux', 'openjdk-14+30-linux', 'openjdk-14+31-linux', 'openjdk-14+32-linux', 'openjdk-14+33-linux', 'openjdk-14+34-linux', 'openjdk-14+9-linux', 'openjdk-14-linux', 'openjdk-14.0.1-linux', 'openjdk-14.0.2+12-linux', 'openjdk-14.0.2-linux', 'openjdk-15+10-linux', 'openjdk-15+11-linux', 'openjdk-15+12-linux', 'openjdk-15+13-linux', 'openjdk-15+14-linux', 'openjdk-15+15-linux', 'openjdk-15+16-linux', 'openjdk-15+17-linux', 'openjdk-15+18-linux', 'openjdk-15+19-linux', 'openjdk-15+20-linux', 'openjdk-15+21-linux', 'openjdk-15+22-linux', 'openjdk-15+23-linux', 'openjdk-15+24-linux', 'openjdk-15+25-linux', 'openjdk-15+26-linux', 'openjdk-15+27-linux', 'openjdk-15+28-linux', 'openjdk-15+29-linux', 'openjdk-15+30-linux', 'openjdk-15+31-linux', 'openjdk-15+32-linux', 'openjdk-15+33-linux', 'openjdk-15+34-linux', 'openjdk-15+36-linux', 'openjdk-15+4-linux', 'openjdk-15+5-linux', 'openjdk-15+6-linux', 'openjdk-15+7-linux', 'openjdk-15+8-linux', 'openjdk-15+9-linux', 'openjdk-15-linux', 'openjdk-15.0.1+9-linux', 'openjdk-9.0.4-linux', 'oracle-10+43-linux', 'oracle-10+46-linux', 'oracle-11+11-linux', 'oracle-11+12-linux', 'oracle-11+13-linux', 'oracle-11+14-linux', 'oracle-11+15-linux', 'oracle-11+16-linux', 'oracle-11+17-linux', 'oracle-11+18-linux', 'oracle-11+19-linux', 'oracle-11+20-linux', 'oracle-11+21-linux', 'oracle-11+22-linux', 'oracle-11+23-linux', 'oracle-11+24-linux', 'oracle-11+25-linux', 'oracle-11+26-linux', 'oracle-11+27-linux', 'oracle-11+28-linux', 'oracle-11+5-linux', 'oracle-11.0.2+7-linux', 'oracle-11.0.2+9-linux', 'oracle-11.0.3+12-linux', 'oracle-11.0.4+10-linux', 'oracle-11.0.5+10-linux', 'oracle-11.0.6+8-linux', 'oracle-12+33-linux', 'oracle-12.0.1+12-linux', 'oracle-12.0.2+10-linux', 'oracle-13+33-linux', 'oracle-13.0.1+9-linux', 'oracle-13.0.2+8-linux', 'oracle-9.0.4+11-linux', 'zulu-10.0.0-linux', 'zulu-10.0.1-linux', 'zulu-10.0.2-linux', 'zulu-11.0.1-linux', 'zulu-11.0.2-linux', 'zulu-11.0.3-linux', 'zulu-11.0.4-linux', 'zulu-11.0.5-linux', 'zulu-11.0.6-linux', 'zulu-11.0.7-linux', 'zulu-11.0.8-linux', 'zulu-11.0.9-linux', 'zulu-12-linux', 'zulu-12.0.0-linux', 'zulu-12.0.1-linux', 'zulu-12.0.2-linux', 'zulu-13-linux', 'zulu-13.0.0-linux', 'zulu-13.0.1-linux', 'zulu-13.0.2-linux', 'zulu-13.0.3-linux', 'zulu-13.0.4-linux', 'zulu-13.0.5-linux', 'zulu-14-linux', 'zulu-14.0.0-linux', 'zulu-14.0.1-linux', 'zulu-14.0.2-linux', 'zulu-15.0.0-linux', 'zulu-15.0.1-linux', 'zulu-7.0.101-linux', 'zulu-7.0.111-linux', 'zulu-7.0.121-linux', 'zulu-7.0.131-linux', 'zulu-7.0.141-linux', 'zulu-7.0.154-linux', 'zulu-7.0.161-linux', 'zulu-7.0.171-linux', 'zulu-7.0.181-linux', 'zulu-7.0.191-linux', 'zulu-7.0.201-linux', 'zulu-7.0.211-linux', 'zulu-7.0.222-linux', 'zulu-7.0.232-linux', 'zulu-7.0.242-linux', 'zulu-7.0.252-linux', 'zulu-7.0.262-linux', 'zulu-7.0.272-linux', 'zulu-7.0.282-linux', 'zulu-7.0.95-linux', 'zulu-8.0.102-linux', 'zulu-8.0.112-linux', 'zulu-8.0.121-linux', 'zulu-8.0.131-linux', 'zulu-8.0.144-linux', 'zulu-8.0.152-linux', 'zulu-8.0.162-linux', 'zulu-8.0.163-linux', 'zulu-8.0.172-linux', 'zulu-8.0.181-linux', 'zulu-8.0.192-linux', 'zulu-8.0.201-linux', 'zulu-8.0.202-linux', 'zulu-8.0.212-linux', 'zulu-8.0.222-linux', 'zulu-8.0.232-linux', 'zulu-8.0.242-linux', 'zulu-8.0.252-linux', 'zulu-8.0.262-linux', 'zulu-8.0.265-linux', 'zulu-8.0.272-linux', 'zulu-8.0.71-linux', 'zulu-8.0.72-linux', 'zulu-8.0.91-linux', 'zulu-8.0.92-linux', 'zulu-9.0.0-linux', 'zulu-9.0.1-linux', 'zulu-9.0.4-linux', 'zulu-9.0.7-linux'], name: "jvm_version", description: "JVM")
+        string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with")
+        string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")
+        // num_of_runs currently unsupported
+        // string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")
+        text(name: "agent_config", "defaultValue": "", description: "Custom APM Agent configuration. (WARNING: May echo to console. Do not supply sensitive data.)")
+        text(name: "locustfile", "defaultValue": "", description: "Locust load-generator plan")
+        booleanParam(name: "local_metrics", description: "Enable local metrics collection?", defaultValue: false)
+        // End script auto-generation
+    }
+
+    stages {
+        stage('Pre-flight'){
+            steps {
+                echo 'Getting authentication information from Vault'
+                withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
+                    setEnvVar('SESSION_TOKEN', sh(script: ".ci/load/scripts/start.sh", returnStdout: true).trim())
+                }
+            }    
+        }
+        stage('Load test') {
+            parallel {
+                stage('Load generation') {
+                    agent { label 'metal' }
+                    steps {
+                            withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
+                                echo 'Preparing load generation..'
+                                whenTrue(Boolean.valueOf(params.locustfile)) {
+                                    echo 'Using user-supplied plan for load-generation with Locust'
+                                    sh script: "echo \"${params.locustfile}\">.ci/load/scripts/locustfile.py"
+                                }
+                                sh(script: ".ci/load/scripts/load_agent.sh")
+                            }
+                    }
+                }
+                stage('Test application') {
+                    agent { label 'benchmark' }
+                    stages{
+                        stage('Provision Java') {
+                            steps {
+                                echo "Provisioning Java version: ${params.jvm_version}"
+                                setEnvVar('JAVA_HOME', sh(script: ".ci/load/scripts/fetch_sdk.sh ${params.jvm_version}", returnStdout: true).trim())
+                                setEnvVar('JAVACMD', "${env.JAVA_HOME}/bin/java")
+                                setEnvVar('PATH', "${env.JAVA_HOME}/bin:$PATH")
+                            }
+                        }
+                        stage ('Provision agent') {
+                            steps {
+                                echo 'Checking out master branch'
+                                dir("${AGENT_BASE_DIR}") {
+                                    gitCheckout(
+                                        basedir: "apm-agent-java",
+                                        branch: 'master',
+                                        repo: "https://github.com/elastic/${REPO}.git",
+                                        credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
+                                        shallow: false
+                                    )
+                                    dir("apm-agent-java"){
+                                        echo 'Switching to requested version'
+                                        sh(script: "git checkout v${params.apm_version}")
+                                        echo 'Building agent'
+                                        sh(script: './mvnw  clean install -DskipTests=true -Dmaven.javadoc.skip=true')
+                                    }
+                                }
+                                whenTrue(Boolean.valueOf(params.agent_config)) {
+                                    echo 'Writing user-supplied agent configuration'
+                                    dir("${AGENT_BASE_DIR}") {
+                                        sh script: "echo \"${params.agent_config}\">custom_config.cfg"
+                                     }
+                                }
+                            }
+                        }
+                        stage('Provision test application') {
+                            steps {
+                                echo 'Checking out test application'
+                                gitCheckout(
+                                    basedir: "${APP_BASE_DIR}",
+                                    branch: 'main',
+                                    repo: "https://github.com/spring-projects/${APP}.git",
+                                    credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
+                                    shallow: false
+                                )
+                            }
+                        }
+                        stage('Provision local metrics collection') {
+                            when {
+                                expression { 
+                                    return params.local_metrics
+                                }
+                            }
+                            steps {
+                                echo 'Enable local metric collection'
+                                gitCheckout(
+                                    basedir: "${METRICS_BASE_DIR}",
+                                    branch: 'master',
+                                    repo: "https://github.com/pstadler/metrics.sh",
+                                    credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
+                                    shallow: false
+                                )
+                                sh(script: "touch metrics.out")
+                                dir("${METRICS_BASE_DIR}"){
+                                    withEnv(["FILE_LOCATION=./metrics.out"]) {
+                                        sh(script: "./metrics.sh -r file &")
+                                    }
+                                }
+                            }
+                        }
+                        stage('Application load') {
+                            steps {
+                                echo 'Starting test application in background..'
+                                dir("${APP_BASE_DIR}"){
+                                    // Launch app in background
+                                    withSecretVault(secret: 'secret/apm-team/ci/apm-load-test-server', user_var_name: 'APM_SERVER_URL', pass_var_name: 'ELASTIC_APM_API_KEY'){
+                                        // Start with packaging things up
+                                        sh(script: "./mvnw package")
+                                        sh(script: "java -jar -javaagent:${WORKSPACE}/${AGENT_BASE_DIR}/apm-agent-java/elastic-apm-agent/target/elastic-apm-agent-${params.apm_version}.jar -Delastic.apm.server_urls=${env.APM_SERVER_URL} -Delastic.apm.secret_token=${env.ELASTIC_APM_API_KEY} -XX:+FlightRecorder -XX:StartFlightRecording=filename=flight.jfr ./target/spring-petclinic-*.jar &")
+                                    }
+                                }
+                                echo 'Starting bandstand client..'
+                                // Foreground the orchestrator script for execution control
+                                withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
+                                    sh(script: ".ci/load/scripts/app.sh")
+                                }
+                            }
+                        }
+                        stage('Collecting results') {
+                            steps {
+                                echo "To view results, JMC is required. Get it here: https://jdk.java.net/jmc/"
+                                archiveArtifacts(allowEmptyArchive: true,
+                                    artifacts: "${APP_BASE_DIR}/**/*.jfr,${METRICS_BASE_DIR}/**/*.out",
+                                    onlyIfSuccessful: false)
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}