Merge remote-tracking branch 'upstream/master' into feature/windows-one-beats

* upstream/master: (35 commits)
  [ci] fix env variable name for xpack filebeats (elastic#19617)
  Cache error responses for cloudfoundry apps metadata (elastic#19181)
  ci: user fixed type of agent (elastic#19625)
  Input v2 cursor testing (elastic#19573)
  Update Jenkinsfile to not inspect removed vendor (elastic#19610)
  Fix ordering and duplicate configs on autodiscover (elastic#19317)
  Prepare input/file for changes in the registrar (elastic#19516)
  Cursor input and manager implementation (elastic#19571)
  [Filebeat] Fix tls mapping in suricata module (elastic#19494)
  [Ingest Manager] Make Agent beta and Constraints experimental (elastic#19586)
  Accept prefix as metric_types for stackdriver metricset in GCP (elastic#19345)
  Implement memlog store operations (elastic#19533)
  introduce journalbeat/pkg in order to provide reusable shared code (elastic#19581)
  Add descriptions to HAProxy fields in Metricbeat (elastic#19561)
  ci: apm-server-update trigered only on upstream, comments, and manual triggered (elastic#19590)
  ci: enable upstream triggering on the packaging job (elastic#19589)
  ci: some jjbb improvements (elastic#19588)
  [MetricBeat] set tags correctly if the dimension value is ARN (elastic#19433)
  [Filebeat] Add default_fields: false to fields.yml in aws module (elastic#19568)
  Add publisher implementation for stateful inputs (elastic#19530)
  ...

Author: v1v

.ci/apm-beats-update.groovy

@@ -2,7 +2,7 @@
 @Library('apm@current') _
 
 pipeline {
-  agent { label 'linux && immutable' }
+  agent none
   environment {
     REPO = 'apm-server'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
@@ -27,63 +27,67 @@ pipeline {
   }
   triggers {
     issueCommentTrigger('(?i).*/run\\s+(?:apm-beats-update\\W+)?.*')
+    upstream("Beats/beats-beats-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
   }
   stages {
-    /**
-     Checkout the code and stash it, to use it on other stages.
-    */
-    stage('Checkout') {
-      options { skipDefaultCheckout() }
-      steps {
-        deleteDir()
-        gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)
-        script {
-          dir("${BEATS_DIR}"){
-            env.GO_VERSION = readFile(".go-version").trim()
-            def regexps =[
-              "^devtools/mage.*",
-              "^libbeat/scripts/Makefile",
-            ]
-            env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps)
-            // Skip all the stages except docs for PR's with asciidoc changes only
-            env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true)
-          }
-        }
-      }
-    }
-    /**
-    updates beats updates the framework part and go parts of beats.
-    Then build and test.
-    Finally archive the results.
-    */
-    stage('Update Beats') {
-      options { skipDefaultCheckout() }
+    stage('Filter build') {
+      agent { label 'ubuntu && immutable' }
       when {
         beforeAgent true
-        anyOf {
-          branch 'master'
-          branch "\\d+\\.\\d+"
-          branch "v\\d?"
-          tag "v\\d+\\.\\d+\\.\\d+*"
-          allOf {
-            expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
-            changeRequest()
+        expression {
+          return isCommentTrigger() || isUserTrigger()
+        }
+      }
+      /**
+       Checkout the code and stash it, to use it on other stages.
+      */
+      stage('Checkout') {
+        steps {
+          deleteDir()
+          gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)
+          script {
+            dir("${BEATS_DIR}"){
+              env.GO_VERSION = readFile(".go-version").trim()
+              def regexps =[
+                "^devtools/mage.*",
+                "^libbeat/scripts/Makefile",
+              ]
+              env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps)
+              // Skip all the stages except docs for PR's with asciidoc changes only
+              env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true)
+            }
           }
-
         }
       }
-      steps {
-        withGithubNotify(context: 'Check Apm Server Beats Update') {
-          beatsUpdate()
+      /**
+      updates beats updates the framework part and go parts of beats.
+      Then build and test.
+      Finally archive the results.
+      */
+      stage('Update Beats') {
+        options { skipDefaultCheckout() }
+        when {
+          beforeAgent true
+          anyOf {
+            branch 'master'
+            branch "\\d+\\.\\d+"
+            branch "v\\d?"
+            tag "v\\d+\\.\\d+\\.\\d+*"
+            allOf {
+              expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
+              changeRequest()
+            }
+
+          }
+        }
+        steps {
+          withGithubNotify(context: 'Check Apm Server Beats Update') {
+            beatsUpdate()
+          }
         }
       }
     }
   }
-  // post {
-  //   cleanup {
-  //     notifyBuildResult()
-  //   }
-  // }
 }
 
 def beatsUpdate() {

.ci/build-docker-images.groovy

@@ -3,7 +3,7 @@
 @Library('apm@current') _
 
 pipeline {
-  agent { label 'linux && immutable' }
+  agent { label 'ubuntu-16 && immutable' }
   environment {
     REPO = 'beats'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"

.ci/jobs/apm-beats-update.yml

@@ -5,7 +5,6 @@
     view: Beats
     concurrent: true
     project-type: multibranch
-    periodic-folder-trigger: 1w
     prune-dead-branches: true
     number-to-keep: 10
     days-to-keep: 30
@@ -19,20 +18,19 @@
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
         discover-tags: true
+        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
         disable-pr-notifications: true
         notification-context: 'apm-beats-update'
-        property-strategies:
-          all-branches:
-          - suppress-scm-triggering: true
         repo: 'beats'
         repo-owner: 'elastic'
         credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
         ssh-checkout:
           credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
         build-strategies:
+        - skip-initial-build: true
         - tags:
             ignore-tags-older-than: -1
-            ignore-tags-newer-than: -1
+            ignore-tags-newer-than: 30
         - named-branches:
             - exact-name:
                 name: 'master'

.ci/jobs/beats.yml

@@ -6,7 +6,6 @@
     view: Beats
     concurrent: true
     project-type: multibranch
-    periodic-folder-trigger: 1w
     prune-dead-branches: true
     number-to-keep: 10
     days-to-keep: 30
@@ -19,6 +18,7 @@
         discover-pr-forks-strategy: 'merge-current'
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
+        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
         discover-tags: true
         notification-context: "beats-ci"
         property-strategies:

.ci/jobs/build-it-docker-images.yml

@@ -16,7 +16,7 @@
         - git:
             url: [email protected]:elastic/beats.git
             refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
-            wipe-workspace: 'True'
+            wipe-workspace: true
             name: origin
             shallow-clone: true
             credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba

.ci/jobs/defaults.yml

@@ -1,4 +1,4 @@
-  
+
 ---
 
 ##### GLOBAL METADATA
@@ -16,5 +16,4 @@
     publishers:
       - email:
           recipients: [email protected]
-    periodic-folder-trigger: 1w
     prune-dead-branches: true

.ci/jobs/packaging.yml

@@ -14,17 +14,19 @@
           discover-pr-forks-trust: 'permission'
           discover-pr-origin: 'merge-current'
           discover-tags: true
-          disable-pr-notifications: false
+          head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
+          disable-pr-notifications: true
           notification-context: 'beats-packaging'
           repo: 'beats'
           repo-owner: 'elastic'
           credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
           ssh-checkout:
             credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
           build-strategies:
+          - skip-initial-build: true
           - tags:
-              ignore-tags-older-than: 30
-              ignore-tags-newer-than: -1
+              ignore-tags-older-than: -1
+              ignore-tags-newer-than: 30
           - named-branches:
               - exact-name:
                   name: 'master'