Merge remote-tracking branch 'upstream/master' into feature/ci-pipeline-2.0

* upstream/master: (29 commits)
  Add an explicit system test for processes on unix systems (elastic#20320)
  [Autodiscovery] Ignore ErrInputNotFinished errors in autodiscover config checks (elastic#20305)
  [CI] Update README.md with CI references (elastic#20316)
  Add ECK doc links to Heartbeat docs (elastic#20284)
  [Filebeat] Add export tests to x-pack/filebeat (elastic#20156)
  feat(ci): support building docker images for PRs (elastic#20323)
  Update system tests dependency (elastic#20287)
  [Libbeat] Log debug message if the Kibana dashboard can not be imported from the archive (elastic#12211) (elastic#20150)
  [Filebeat][Gsuite] Transform all dates to timestamp with processor (elastic#20308)
  Infer types in Prometheus remote_write (elastic#19944)
  Remove unnecessary restarts of metricsets while using Node autodiscover (elastic#19974)
  docs: update changelog on master branch (elastic#20259)
  feat(ci): support storing artifacts for PRs in separate dirs (elastic#20282)
  [CI] Change upstream reference (elastic#20296)
  [Filebeat] Updates to Suricata module (elastic#20220)
  [docs] Fix Windows download link for agent (elastic#20258)
  [docs] Rename release highlights to what's new (elastic#20255)
  fix: update the display name of the multibranch job (elastic#20265)
  [Elastic Agent] Add basic protocol to control Elastic Agent. (elastic#20146)
  Cisco ASA: Fix message 106100 (elastic#20245)
  ...

Author: v1v

.ci/apm-beats-update.groovy

@@ -27,7 +27,7 @@ pipeline {
   }
   triggers {
     issueCommentTrigger('(?i).*/run\\s+(?:apm-beats-update\\W+)?.*')
-    upstream("Beats/beats-beats-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
+    upstream("Beats/beats/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
   }
   stages {
     stage('Filter build') {
@@ -48,51 +48,53 @@ pipeline {
           }
         }
       }
-      /**
-       Checkout the code and stash it, to use it on other stages.
-      */
-      stage('Checkout') {
-        steps {
-          deleteDir()
-          gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)
-          script {
-            dir("${BEATS_DIR}"){
-              env.GO_VERSION = readFile(".go-version").trim()
-              def regexps =[
-                "^devtools/mage.*",
-                "^libbeat/scripts/Makefile",
-              ]
-              env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps)
-              // Skip all the stages except docs for PR's with asciidoc changes only
-              env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true)
+      stages {
+        /**
+        Checkout the code and stash it, to use it on other stages.
+        */
+        stage('Checkout') {
+          steps {
+            deleteDir()
+            gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)
+            script {
+              dir("${BEATS_DIR}"){
+                env.GO_VERSION = readFile(".go-version").trim()
+                def regexps =[
+                  "^devtools/mage.*",
+                  "^libbeat/scripts/Makefile",
+                ]
+                env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps)
+                // Skip all the stages except docs for PR's with asciidoc changes only
+                env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true)
+              }
             }
           }
         }
-      }
-      /**
-      updates beats updates the framework part and go parts of beats.
-      Then build and test.
-      Finally archive the results.
-      */
-      stage('Update Beats') {
-        options { skipDefaultCheckout() }
-        when {
-          beforeAgent true
-          anyOf {
-            branch 'master'
-            branch "\\d+\\.\\d+"
-            branch "v\\d?"
-            tag "v\\d+\\.\\d+\\.\\d+*"
-            allOf {
-              expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
-              changeRequest()
-            }
+        /**
+        updates beats updates the framework part and go parts of beats.
+        Then build and test.
+        Finally archive the results.
+        */
+        stage('Update Beats') {
+          options { skipDefaultCheckout() }
+          when {
+            beforeAgent true
+            anyOf {
+              branch 'master'
+              branch "\\d+\\.\\d+"
+              branch "v\\d?"
+              tag "v\\d+\\.\\d+\\.\\d+*"
+              allOf {
+                expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
+                changeRequest()
+              }
 
+            }
           }
-        }
-        steps {
-          withGithubNotify(context: 'Check Apm Server Beats Update') {
-            beatsUpdate()
+          steps {
+            withGithubNotify(context: 'Check Apm Server Beats Update') {
+              beatsUpdate()
+            }
           }
         }
       }

.ci/jobs/beats-mbp-2.0.yml

@@ -0,0 +1,59 @@
+---
+- job:
+    name: Beats/beats-mbp-2.0
+    display-name: 'Beats (2.0)'
+    description: 'Beats Main Pipeline 2.0'
+    view: Beats
+    concurrent: true
+    project-type: multibranch
+    prune-dead-branches: true
+    days-to-keep: 30
+    script-path: '.ci/Jenkinsfile'
+    triggers: []
+    wrappers: []
+    scm:
+    - github:
+        branch-discovery: 'no-pr'
+        discover-pr-forks-strategy: 'merge-current'
+        discover-pr-forks-trust: 'permission'
+        discover-pr-origin: 'merge-current'
+        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        discover-tags: true
+        disable-pr-notifications: true
+        notification-context: "beats-ci-2.0"
+        repo: 'beats'
+        repo-owner: 'elastic'
+        credentials-id: github-app-beats-ci
+        ssh-checkout:
+          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        build-strategies:
+        - tags:
+            ignore-tags-older-than: -1
+            ignore-tags-newer-than: 365
+        - change-request:
+            ignore-target-only-changes: true
+        - named-branches:
+            - exact-name:
+                name: 'master'
+                case-sensitive: true
+            - regex-name:
+                regex: '7\.[x789]'
+                case-sensitive: true
+            - regex-name:
+                regex: '8\.\d+'
+                case-sensitive: true
+        clean:
+            after: true
+            before: true
+        prune: true
+        shallow-clone: true
+        depth: 3
+        do-not-fetch-tags: true
+        submodule:
+            disable: false
+            recursive: true
+            parent-credentials: true
+            timeout: 100
+        timeout: '15'
+        use-author: true
+        wipe-workspace: true

.ci/jobs/beats.yml

@@ -1,7 +1,7 @@
 ---
 - job:
     name: Beats/beats
-    display-name: 'Beats (replacement)'
+    display-name: 'Beats'
     description: 'Beats Main Pipeline'
     view: Beats
     concurrent: true

.ci/packaging.groovy

@@ -26,7 +26,7 @@ pipeline {
   triggers {
     issueCommentTrigger('(?i)^\\/packag[ing|e]$')
     // disable upstream trigger on a PR basis
-    upstream("Beats/beats-beats-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
+    upstream("Beats/beats/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
   }
   parameters {
     booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
@@ -188,8 +188,14 @@ def tagAndPush(name){
   if("${env.SNAPSHOT}" == "true"){
     libbetaVer += "-SNAPSHOT"
   }
+
+  def tagName = "${libbetaVer}"
+  if (env.CHANGE_ID?.trim()) {
+    tagName = "pr-${env.CHANGE_ID}"
+  }
+
   def oldName = "${DOCKER_REGISTRY}/beats/${name}:${libbetaVer}"
-  def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${libbetaVer}"
+  def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${tagName}"
   def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}:${env.GIT_BASE_COMMIT}"
   dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
   retry(3){
@@ -222,7 +228,12 @@ def withMacOSEnv(Closure body){
 }
 
 def publishPackages(baseDir){
-  googleStorageUpload(bucket: "gs://${JOB_GCS_BUCKET}/snapshots",
+  def bucketUri = "gs://${JOB_GCS_BUCKET}/snapshots"
+  if (env.CHANGE_ID?.trim()) {
+    bucketUri = "gs://${JOB_GCS_BUCKET}/pull-requests/pr-${env.CHANGE_ID}"
+  }
+
+  googleStorageUpload(bucket: "${bucketUri}",
     credentialsId: "${JOB_GCS_CREDENTIALS}",
     pathPrefix: "${baseDir}/build/distributions/",
     pattern: "${baseDir}/build/distributions/**/*",

.gitignore

@@ -30,6 +30,7 @@ x-pack/dockerlogbeat/temproot.tar
 x-pack/elastic-agent/elastic_agent
 x-pack/elastic-agent/fleet.yml
 x-pack/elastic-agent/pkg/agent/operation/tests/scripts/configurable-1.0-darwin-x86_64/configurable
+x-pack/elastic-agent/pkg/agent/operation/tests/scripts/serviceable-1.0-darwin-x86_64/serviceable
 
 # Editor swap files
 *.swp