Migrate semgrep to semgrep-agent (#4446)

.expeditor/export_semgrep_token.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -eou pipefail
+
+SEMGREP_TOKEN=$(vault kv get -field token secret/semgrep)
+SEMGREP_ID=$(vault kv get -field id secret/semgrep)
+
+export SEMGREP_TOKEN
+export SEMGREP_ID

.expeditor/verify.pipeline.yml

@@ -61,37 +61,6 @@ steps:
             - HAB_STUDIO_SUP=false
             - HAB_NONINTERACTIVE=true
 
-  - label: ":semgrep: Custom"
-    expeditor:
-      executor:
-        docker:
-          image: returntocorp/semgrep:0.29.0
-          entrypoint: semgrep
-          command: [
-            "--error",
-            "--exclude", "*.spec.ts",
-            "--config", "/go/src/github.com/chef/automate/semgrep",
-            "/go/src/github.com/chef/automate"
-          ]
-
-  - label: ":semgrep: Published"
-    expeditor:
-      executor:
-        docker:
-          image: returntocorp/semgrep:latest
-          entrypoint: semgrep
-          command: [
-            "--error",
-            "--exclude", "third_party",
-            "--exclude", "*_test.go",
-            "--exclude", "*.pb.go",
-            "--exclude", "*.bindata.go",
-            "--exclude", "*.spec.ts",
-            "--timeout", "120",
-            "--config", "https://semgrep.dev/p/r2c-ci",
-            "/go/src/github.com/chef/automate"
-          ]
-
   #
   # Static & Unit tests
   #

.expeditor/verify_private.pipeline.yml

@@ -101,6 +101,42 @@ steps:
       executor:
         docker:
 
+  - label: ":semgrep: Custom"
+    expeditor:
+      executor:
+        docker:
+          image: returntocorp/semgrep:0.32.0
+          entrypoint: semgrep
+          command: [
+            "--error",
+            "--exclude", "*.spec.ts",
+            "--config", "/go/src/github.com/chef/automate/.semgrep",
+            "/go/src/github.com/chef/automate"
+          ]
+
+  - label: ":semgrep: Published"
+    command:
+    - echo "running in $(pwd)"
+    - export SEMGREP_JOB_URL=$BUILDKITE_BUILD_URL
+    - export SEMGREP_BRANCH=$BUILDKITE_BRANCH
+    - python -m semgrep_agent --publish-token "\$SEMGREP_TOKEN" --publish-deployment \$SEMGREP_ID
+    timeout_in_minutes: 20
+    expeditor:
+      secrets: true
+    plugins:
+      # Temporary workaround per @tduffield; do not propagate this solution too much!
+      - chef/step-hook#v0.1.1:
+          pre-command:
+            - .expeditor/export_semgrep_token.sh
+      - docker#v3.7.0:
+          image: returntocorp/semgrep-agent:v1
+          propogate-environment: true
+          workdir: /go/src/github.com/chef/automate
+          environment:
+            - SEMGREP_TOKEN
+            - SEMGREP_ID
+    soft_fail: true
+
   # Wait for the build to complete before starting anything below this
   # directive. All tests below this wait either require build assets
   # or take a long time.

.semgrep/rxjs-syntax.yaml

@@ -0,0 +1,24 @@
+rules:
+
+  - id: combineLatest-not-combining
+    pattern: combineLatest([$SINGLE_EXPRESSION])
+    fix:  $SINGLE_EXPRESSION
+    message: combineLatest is not needed with a single argument
+    languages: [ts]
+    severity: WARNING
+
+  - id: observable-not-ending-with-dollar-sign
+    patterns:
+    - pattern-either:
+      - pattern: $VAR = $EXPR.select(...);
+      - pattern: $VAR = $EXPR.pipe(...);
+      - pattern: $VAR = observableOf(...)
+      - pattern: $VAR = of(...)
+    - pattern-not: $VAR = d3.select(...);
+    - metavariable-regex:
+        metavariable: '$VAR'
+        regex: '[^$]+(?!\$)$'
+    message: |
+      Observable variable ($VAR) should end with a dollar sign.
+    languages: [ts]
+    severity: ERROR

.semgrepignore

@@ -0,0 +1,32 @@
+# Copied from https://github.com/returntocorp/semgrep-action/blob/develop/src/semgrep_agent/templates/.semgrepignore
+# Note that these are for semgrep-agent ONLY; command line semgrep does NOT use this file.
+
+# Ignore git items
+.gitignore
+.git/
+:include .gitignore
+
+# Common large directories
+node_modules/
+build/
+dist/
+vendor/
+env/
+.env/
+venv/
+.venv/
+*.min.js
+
+# Common test directories
+test/
+tests/
+
+# Semgrep rules folder
+.semgrep
+
+# Chef customizations
+third_party/
+*_test.go
+*.pb.go
+*.bindata.go
+*.spec.ts

Makefile.common_go

@@ -20,6 +20,10 @@ GOLANGCILINTTARBALL:=golangci-lint-$(GOLANGCILINTVERSION)-$(PLATFORM).tar.gz
 
 LINTERARGS?=./...
 
+# Semgrep by default respects .gitignore; these are additive:
+SEMGREP_IGNORE := --exclude third_party --exclude *_test.go --exclude *.pb.go --exclude *.bindata.go
+SEMGREP_CONFIG := https://semgrep.dev/p/r2c-ci
+
 $(REPOROOT)/cache/$(GOLANGCILINTTARBALL):
 	curl --output $(REPOROOT)/cache/$(GOLANGCILINTTARBALL) -L https://github.com/golangci/golangci-lint/releases/download/v$(GOLANGCILINTVERSION)/$(GOLANGCILINTTARBALL)
 
@@ -50,13 +54,13 @@ spell:
 # NB: "third_party" only exists for automate-gateway, but no harm having it for other dirs here.
 semgrep:
 # uncomment if custom rules beyond automate-ui ever get added
-#	semgrep --config $(REPOROOT)/semgrep --exclude third_party --exclude *_test.go --exclude *.pb.go --exclude *.bindata.go
-	semgrep --config https://semgrep.dev/p/r2c-ci --exclude third_party --exclude *_test.go --exclude *.pb.go --exclude *.bindata.go
+# semgrep --config $(REPOROOT)/.semgrep $(SEMGREP_IGNORE)
+	semgrep --config $(SEMGREP_CONFIG) $(SEMGREP_IGNORE)
 
 #: Security validation via semgrep; autofix where possible
 semgrep-and-fix:
 # uncomment if custom rules beyond automate-ui ever get added
-#	semgrep --config $(REPOROOT)/semgrep --exclude third_party --exclude *_test.go --exclude *.pb.go --exclude *.bindata.go --autofix
-	semgrep --config https://semgrep.dev/p/r2c-ci --exclude third_party --exclude *_test.go --exclude *.pb.go --exclude *.bindata.go --autofix
+# semgrep --config $(REPOROOT)/.semgrep $(SEMGREP_IGNORE) --autofix
+	semgrep --config $(SEMGREP_CONFIG) $(SEMGREP_IGNORE) --autofix
 
 .PHONY: lint fmt fmt-check golang_version_check semgrep semgrep-and-fix

components/automate-ui/Makefile

@@ -10,6 +10,11 @@ DEMO_FLAGS := --spec false
 NG_CMD := npm run ng --
 REPOROOT=../..
 
+# Semgrep by default respects .gitignore; these are additive:
+SEMGREP_IGNORE := --exclude *.spec.ts
+SEMGREP_CONFIG := https://semgrep.dev/p/r2c-ci
+
+
 install:
 	npm install
 	npm run install:ui-library
@@ -40,14 +45,18 @@ lint-typescript:
 	npm run lint
 
 #: Security validation via semgrep
-semgrep:
-	semgrep --config $(REPOROOT)/semgrep --exclude *.spec.ts
-	semgrep --config https://semgrep.dev/p/r2c-ci
+semgrep: semgrep-custom semgrep-published
+
+semgrep-custom:
+	semgrep --config $(REPOROOT)/.semgrep $(SEMGREP_IGNORE)
+
+semgrep-published:
+	semgrep --config $(SEMGREP_CONFIG) $(SEMGREP_IGNORE)
 
 #: Security validation via semgrep; autofix where possible
 semgrep-and-fix:
-	semgrep --config $(REPOROOT)/semgrep --exclude *.spec.ts --autofix
-	semgrep --config https://semgrep.dev/p/r2c-ci --autofix
+	semgrep --config $(REPOROOT)/.semgrep $(SEMGREP_IGNORE) --autofix
+	semgrep --config $(SEMGREP_CONFIG) $(SEMGREP_IGNORE) --autofix
 
 pr-ready: unit-all-browsers lint e2e
 
@@ -103,4 +112,4 @@ spell:
 	popd > /dev/null; \
 	exit $$EXIT_CODE
 
-.PHONY: build install test lint lint-html lint-sass lint-typescript semgrep semgrep-and-fix unit e2e license_scout peer-dependencies e2e-aot unit-all-browsers pr-ready start
+.PHONY: build install test lint lint-html lint-sass lint-typescript semgrep semgrep-and-fix semgrep-custom semgrep-published unit e2e license_scout peer-dependencies e2e-aot unit-all-browsers pr-ready start

components/automate-ui/src/app/pages/+compliance/+scanner/containers/nodes-edit/nodes-edit.component.ts

@@ -24,7 +24,7 @@ export class NodesEditComponent implements OnInit {
 
   form: FormGroup;
   backendControl: FormGroup;
-  backendValue: Observable<string>;
+  backendValue$: Observable<string>;
   // Array of secrets available for user to select
   // TODO make ngrx/store selection
   secrets$: Observable<any[]>;
@@ -56,10 +56,10 @@ export class NodesEditComponent implements OnInit {
 
         // Swap fields based on selected "backend" value (ssh, winrm)
         this.backendControl = this.form.get('target_config').get('backend') as FormGroup;
-        this.backendValue = this.backendControl
+        this.backendValue$ = this.backendControl
           .valueChanges
           .pipe(startWith(this.backendControl.value));
-        this.backendValue.subscribe(backend => {
+        this.backendValue$.subscribe(backend => {
           const step = this.form.get('target_config') as FormGroup;
           // step.get('secrets').setValue([]);
           switch (backend) {
@@ -78,7 +78,7 @@ export class NodesEditComponent implements OnInit {
 
         this.secrets$ = combineLatest([
           this.fetchSecrets(),
-          this.backendValue
+          this.backendValue$
         ]).pipe(
           map(([secrets, backend]: [any[], string]) =>
             secrets.filter(s => s.type === backend || s.type === 'sudo'))