From 84333a19a0fc1e78fedc3a9957f1157461c79565 Mon Sep 17 00:00:00 2001 From: Michael Sorens Date: Thu, 19 Nov 2020 17:08:38 -0800 Subject: [PATCH] Migrate semgrep to semgrep-agent (#4446) --- .expeditor/export_semgrep_token.sh | 9 +++++ .expeditor/verify.pipeline.yml | 31 ---------------- .expeditor/verify_private.pipeline.yml | 36 +++++++++++++++++++ {semgrep => .semgrep}/general.yaml | 0 {semgrep => .semgrep}/rxjs-subscription.yaml | 0 .semgrep/rxjs-syntax.yaml | 24 +++++++++++++ .semgrepignore | 32 +++++++++++++++++ Makefile.common_go | 12 ++++--- components/automate-ui/Makefile | 21 +++++++---- .../nodes-edit/nodes-edit.component.ts | 8 ++--- semgrep/rxjs-syntax.yaml | 31 ---------------- 11 files changed, 128 insertions(+), 76 deletions(-) create mode 100755 .expeditor/export_semgrep_token.sh rename {semgrep => .semgrep}/general.yaml (100%) rename {semgrep => .semgrep}/rxjs-subscription.yaml (100%) create mode 100644 .semgrep/rxjs-syntax.yaml create mode 100644 .semgrepignore delete mode 100644 semgrep/rxjs-syntax.yaml diff --git a/.expeditor/export_semgrep_token.sh b/.expeditor/export_semgrep_token.sh new file mode 100755 index 00000000000..1d61686def8 --- /dev/null +++ b/.expeditor/export_semgrep_token.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -eou pipefail + +SEMGREP_TOKEN=$(vault kv get -field token secret/semgrep) +SEMGREP_ID=$(vault kv get -field id secret/semgrep) + +export SEMGREP_TOKEN +export SEMGREP_ID \ No newline at end of file diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index e03fb558702..db2888c3b7a 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -61,37 +61,6 @@ steps: - HAB_STUDIO_SUP=false - HAB_NONINTERACTIVE=true - - label: ":semgrep: Custom" - expeditor: - executor: - docker: - image: returntocorp/semgrep:0.29.0 - entrypoint: semgrep - command: [ - "--error", - "--exclude", "*.spec.ts", - "--config", "/go/src/github.com/chef/automate/semgrep", - "/go/src/github.com/chef/automate" - ] - - - label: ":semgrep: Published" - expeditor: - executor: - docker: - image: returntocorp/semgrep:latest - entrypoint: semgrep - command: [ - "--error", - "--exclude", "third_party", - "--exclude", "*_test.go", - "--exclude", "*.pb.go", - "--exclude", "*.bindata.go", - "--exclude", "*.spec.ts", - "--timeout", "120", - "--config", "https://semgrep.dev/p/r2c-ci", - "/go/src/github.com/chef/automate" - ] - # # Static & Unit tests # diff --git a/.expeditor/verify_private.pipeline.yml b/.expeditor/verify_private.pipeline.yml index 519fb90b64b..eddba64bd28 100644 --- a/.expeditor/verify_private.pipeline.yml +++ b/.expeditor/verify_private.pipeline.yml @@ -101,6 +101,42 @@ steps: executor: docker: + - label: ":semgrep: Custom" + expeditor: + executor: + docker: + image: returntocorp/semgrep:0.32.0 + entrypoint: semgrep + command: [ + "--error", + "--exclude", "*.spec.ts", + "--config", "/go/src/github.com/chef/automate/.semgrep", + "/go/src/github.com/chef/automate" + ] + + - label: ":semgrep: Published" + command: + - echo "running in $(pwd)" + - export SEMGREP_JOB_URL=$BUILDKITE_BUILD_URL + - export SEMGREP_BRANCH=$BUILDKITE_BRANCH + - python -m semgrep_agent --publish-token "\$SEMGREP_TOKEN" --publish-deployment \$SEMGREP_ID + timeout_in_minutes: 20 + expeditor: + secrets: true + plugins: + # Temporary workaround per @tduffield; do not propagate this solution too much! + - chef/step-hook#v0.1.1: + pre-command: + - .expeditor/export_semgrep_token.sh + - docker#v3.7.0: + image: returntocorp/semgrep-agent:v1 + propogate-environment: true + workdir: /go/src/github.com/chef/automate + environment: + - SEMGREP_TOKEN + - SEMGREP_ID + soft_fail: true + # Wait for the build to complete before starting anything below this # directive. All tests below this wait either require build assets # or take a long time. diff --git a/semgrep/general.yaml b/.semgrep/general.yaml similarity index 100% rename from semgrep/general.yaml rename to .semgrep/general.yaml diff --git a/semgrep/rxjs-subscription.yaml b/.semgrep/rxjs-subscription.yaml similarity index 100% rename from semgrep/rxjs-subscription.yaml rename to .semgrep/rxjs-subscription.yaml diff --git a/.semgrep/rxjs-syntax.yaml b/.semgrep/rxjs-syntax.yaml new file mode 100644 index 00000000000..0dcef19c6e4 --- /dev/null +++ b/.semgrep/rxjs-syntax.yaml @@ -0,0 +1,24 @@ +rules: + + - id: combineLatest-not-combining + pattern: combineLatest([$SINGLE_EXPRESSION]) + fix: $SINGLE_EXPRESSION + message: combineLatest is not needed with a single argument + languages: [ts] + severity: WARNING + + - id: observable-not-ending-with-dollar-sign + patterns: + - pattern-either: + - pattern: $VAR = $EXPR.select(...); + - pattern: $VAR = $EXPR.pipe(...); + - pattern: $VAR = observableOf(...) + - pattern: $VAR = of(...) + - pattern-not: $VAR = d3.select(...); + - metavariable-regex: + metavariable: '$VAR' + regex: '[^$]+(?!\$)$' + message: | + Observable variable ($VAR) should end with a dollar sign. + languages: [ts] + severity: ERROR diff --git a/.semgrepignore b/.semgrepignore new file mode 100644 index 00000000000..bb654c7a43b --- /dev/null +++ b/.semgrepignore @@ -0,0 +1,32 @@ +# Copied from https://github.com/returntocorp/semgrep-action/blob/develop/src/semgrep_agent/templates/.semgrepignore +# Note that these are for semgrep-agent ONLY; command line semgrep does NOT use this file. + +# Ignore git items +.gitignore +.git/ +:include .gitignore + +# Common large directories +node_modules/ +build/ +dist/ +vendor/ +env/ +.env/ +venv/ +.venv/ +*.min.js + +# Common test directories +test/ +tests/ + +# Semgrep rules folder +.semgrep + +# Chef customizations +third_party/ +*_test.go +*.pb.go +*.bindata.go +*.spec.ts diff --git a/Makefile.common_go b/Makefile.common_go index 418d6acafac..eaf29aeec46 100644 --- a/Makefile.common_go +++ b/Makefile.common_go @@ -20,6 +20,10 @@ GOLANGCILINTTARBALL:=golangci-lint-$(GOLANGCILINTVERSION)-$(PLATFORM).tar.gz LINTERARGS?=./... +# Semgrep by default respects .gitignore; these are additive: +SEMGREP_IGNORE := --exclude third_party --exclude *_test.go --exclude *.pb.go --exclude *.bindata.go +SEMGREP_CONFIG := https://semgrep.dev/p/r2c-ci + $(REPOROOT)/cache/$(GOLANGCILINTTARBALL): curl --output $(REPOROOT)/cache/$(GOLANGCILINTTARBALL) -L https://github.com/golangci/golangci-lint/releases/download/v$(GOLANGCILINTVERSION)/$(GOLANGCILINTTARBALL) @@ -50,13 +54,13 @@ spell: # NB: "third_party" only exists for automate-gateway, but no harm having it for other dirs here. semgrep: # uncomment if custom rules beyond automate-ui ever get added -# semgrep --config $(REPOROOT)/semgrep --exclude third_party --exclude *_test.go --exclude *.pb.go --exclude *.bindata.go - semgrep --config https://semgrep.dev/p/r2c-ci --exclude third_party --exclude *_test.go --exclude *.pb.go --exclude *.bindata.go +# semgrep --config $(REPOROOT)/.semgrep $(SEMGREP_IGNORE) + semgrep --config $(SEMGREP_CONFIG) $(SEMGREP_IGNORE) #: Security validation via semgrep; autofix where possible semgrep-and-fix: # uncomment if custom rules beyond automate-ui ever get added -# semgrep --config $(REPOROOT)/semgrep --exclude third_party --exclude *_test.go --exclude *.pb.go --exclude *.bindata.go --autofix - semgrep --config https://semgrep.dev/p/r2c-ci --exclude third_party --exclude *_test.go --exclude *.pb.go --exclude *.bindata.go --autofix +# semgrep --config $(REPOROOT)/.semgrep $(SEMGREP_IGNORE) --autofix + semgrep --config $(SEMGREP_CONFIG) $(SEMGREP_IGNORE) --autofix .PHONY: lint fmt fmt-check golang_version_check semgrep semgrep-and-fix diff --git a/components/automate-ui/Makefile b/components/automate-ui/Makefile index 233b5b8172d..28e906376d2 100644 --- a/components/automate-ui/Makefile +++ b/components/automate-ui/Makefile @@ -10,6 +10,11 @@ DEMO_FLAGS := --spec false NG_CMD := npm run ng -- REPOROOT=../.. +# Semgrep by default respects .gitignore; these are additive: +SEMGREP_IGNORE := --exclude *.spec.ts +SEMGREP_CONFIG := https://semgrep.dev/p/r2c-ci + + install: npm install npm run install:ui-library @@ -40,14 +45,18 @@ lint-typescript: npm run lint #: Security validation via semgrep -semgrep: - semgrep --config $(REPOROOT)/semgrep --exclude *.spec.ts - semgrep --config https://semgrep.dev/p/r2c-ci +semgrep: semgrep-custom semgrep-published + +semgrep-custom: + semgrep --config $(REPOROOT)/.semgrep $(SEMGREP_IGNORE) + +semgrep-published: + semgrep --config $(SEMGREP_CONFIG) $(SEMGREP_IGNORE) #: Security validation via semgrep; autofix where possible semgrep-and-fix: - semgrep --config $(REPOROOT)/semgrep --exclude *.spec.ts --autofix - semgrep --config https://semgrep.dev/p/r2c-ci --autofix + semgrep --config $(REPOROOT)/.semgrep $(SEMGREP_IGNORE) --autofix + semgrep --config $(SEMGREP_CONFIG) $(SEMGREP_IGNORE) --autofix pr-ready: unit-all-browsers lint e2e @@ -103,4 +112,4 @@ spell: popd > /dev/null; \ exit $$EXIT_CODE -.PHONY: build install test lint lint-html lint-sass lint-typescript semgrep semgrep-and-fix unit e2e license_scout peer-dependencies e2e-aot unit-all-browsers pr-ready start +.PHONY: build install test lint lint-html lint-sass lint-typescript semgrep semgrep-and-fix semgrep-custom semgrep-published unit e2e license_scout peer-dependencies e2e-aot unit-all-browsers pr-ready start diff --git a/components/automate-ui/src/app/pages/+compliance/+scanner/containers/nodes-edit/nodes-edit.component.ts b/components/automate-ui/src/app/pages/+compliance/+scanner/containers/nodes-edit/nodes-edit.component.ts index 965c72fdec3..a608ed917cb 100644 --- a/components/automate-ui/src/app/pages/+compliance/+scanner/containers/nodes-edit/nodes-edit.component.ts +++ b/components/automate-ui/src/app/pages/+compliance/+scanner/containers/nodes-edit/nodes-edit.component.ts @@ -24,7 +24,7 @@ export class NodesEditComponent implements OnInit { form: FormGroup; backendControl: FormGroup; - backendValue: Observable; + backendValue$: Observable; // Array of secrets available for user to select // TODO make ngrx/store selection secrets$: Observable; @@ -56,10 +56,10 @@ export class NodesEditComponent implements OnInit { // Swap fields based on selected "backend" value (ssh, winrm) this.backendControl = this.form.get('target_config').get('backend') as FormGroup; - this.backendValue = this.backendControl + this.backendValue$ = this.backendControl .valueChanges .pipe(startWith(this.backendControl.value)); - this.backendValue.subscribe(backend => { + this.backendValue$.subscribe(backend => { const step = this.form.get('target_config') as FormGroup; // step.get('secrets').setValue([]); switch (backend) { @@ -78,7 +78,7 @@ export class NodesEditComponent implements OnInit { this.secrets$ = combineLatest([ this.fetchSecrets(), - this.backendValue + this.backendValue$ ]).pipe( map(([secrets, backend]: [any[], string]) => secrets.filter(s => s.type === backend || s.type === 'sudo')) diff --git a/semgrep/rxjs-syntax.yaml b/semgrep/rxjs-syntax.yaml deleted file mode 100644 index a90877d9b12..00000000000 --- a/semgrep/rxjs-syntax.yaml +++ /dev/null @@ -1,31 +0,0 @@ -rules: - - - id: combineLatest-not-combining - pattern: combineLatest([$SINGLE_EXPRESSION]) - fix: $SINGLE_EXPRESSION - message: combineLatest is not needed with a single argument - languages: [ts] - severity: WARNING - - - id: observables-from-select-or-pipe-not-ending-with-dollar-sign - patterns: - - pattern-regex: '\w+\s*(:.+?)?=.*?(select|pipe)' - - pattern-either: - - pattern-inside: $VAR = $EXPR.select(...); - - pattern-inside: $VAR = $EXPR.pipe(...); - - pattern-not-inside: $VAR = d3.select(...); - message: | - Observable variable should end with a dollar sign. - languages: [ts] - severity: ERROR - - - id: observables-from-values-not-ending-with-dollar-sign - patterns: - - pattern-regex: '\w+\s*(:.+?)?=\s*(observableOf|of)' - - pattern-either: - - pattern-inside: $VAR = observableOf(...) - - pattern-inside: $VAR = of(...) - message: | - Observable variable should end with a dollar sign. - languages: [ts] - severity: ERROR