From 9346de69ad335d892aaf28a2d42fd79c655fb550 Mon Sep 17 00:00:00 2001 From: Edwin Joassart Date: Thu, 10 Oct 2024 16:29:52 +0200 Subject: [PATCH] minor: scan PR and all artifacts with trufflehogs --- .github/workflows/flowzone.yml | 125 +++++++++++++++++++++++++++++++++ README.md | 5 ++ flowzone.yml | 87 +++++++++++++++++++++++ 3 files changed, 217 insertions(+) diff --git a/.github/workflows/flowzone.yml b/.github/workflows/flowzone.yml index 80a8e94b3..f59cccee9 100644 --- a/.github/workflows/flowzone.yml +++ b/.github/workflows/flowzone.yml @@ -271,6 +271,11 @@ on: type: boolean required: false default: true + scan_secrets: + description: Scan for secrets in the PR and artifacts using TruffleHog OSS + type: boolean + required: false + default: true outputs: cloudflare_deployment_url: description: Cloudflare Deployment URL @@ -607,6 +612,42 @@ jobs: -f ref="${REF}" \ -f sha="${SHA}" \ --include + scan_secrets: + name: Scan Commits for Secrets + runs-on: ubuntu-latest + needs: + - versioned_source + if: inputs.scan_secrets == 'true' + steps: + - name: Generate GitHub App installation token + uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a + continue-on-error: true + id: gh_app_token + with: + app_id: ${{ inputs.app_id }} + installation_retrieval_mode: id + installation_retrieval_payload: ${{ inputs.installation_id }} + private_key: ${{ secrets.GH_APP_PRIVATE_KEY }} + permissions: |- + { + "contents": "read", + "metadata": "read" + } + - name: Checkout versioned commit + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 + with: + fetch-depth: ${{ needs.versioned_source.outputs.depth || 0 }} + fetch-tags: true + submodules: recursive + ref: ${{ needs.versioned_source.outputs.sha || '¯ (ツ)_/¯' }} + token: ${{ steps.gh_app_token.outputs.token || secrets.FLOWZONE_TOKEN }} + persist-credentials: false + - name: Scan + uses: trufflesecurity/trufflehog@v3.82.7 + with: + base: ${{ github.event.pull_request.base.sha }} + head: ${{ github.event.pull_request.head.ref }} + extra_args: --only-verified release_notes: name: Generate release notes runs-on: ${{ fromJSON(inputs.runs_on) }} @@ -970,6 +1011,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets defaults: run: working-directory: ${{ inputs.working_directory }} @@ -1085,6 +1127,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets defaults: run: working-directory: ${{ inputs.working_directory }} @@ -1313,6 +1356,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets defaults: run: working-directory: ${{ inputs.working_directory }} @@ -1415,6 +1459,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets if: inputs.cargo_targets != '' defaults: run: @@ -1498,6 +1543,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets if: inputs.balena_slugs != '' defaults: run: @@ -1571,6 +1617,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets defaults: run: working-directory: . @@ -1757,6 +1804,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets if: inputs.cloudflare_website != '' defaults: run: @@ -1805,6 +1853,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets if: inputs.cloudformation_templates != '' defaults: run: @@ -2016,6 +2065,18 @@ jobs: # FIXME: workaround when `npm pack` for npm 6.x dumps tarball into the current directory because it has no `--pack-destination` flag [[ "$(npm --version)" =~ ^6\..* ]] && find . -maxdepth 1 -name '*.tgz' -exec mv {} ${{ runner.temp }}/npm-pack \; || true + - name: Install trufflehog + if: inputs.scan_secrets == 'true' + run: | + # install trufflesecurity/trufflehog + curl https://github.com/trufflesecurity/trufflehog/releases/download/v3.82.7/trufflehog_3.82.7_linux_amd64.tar.gz -o trufflehog.tar.gz && \ + tar -xzf trufflehog.tar.gz && \ + chmod +x trufflehog + - name: Scan artifacts + if: inputs.scan_secrets == 'true' + run: ./trufflehog filesystem ${{env.scanPath}} --no-update --only-verified --fail + env: + scanPath: ${{ runner.temp }}/npm-pack - name: Upload artifact if: needs.is_npm.outputs.npm_private != 'true' && steps.node_versions.outputs.max == matrix.node_version uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 @@ -2027,6 +2088,11 @@ jobs: if: needs.is_npm.outputs.npm_docs == 'true' shell: bash run: npm run doc + - name: Scan artifacts + if: inputs.scan_secrets == 'true' + run: ./trufflehog filesystem ${{env.scanPath}} --no-update --only-verified --fail + env: + scanPath: ./docs - name: Compress docs if: needs.is_npm.outputs.npm_docs == 'true' && steps.node_versions.outputs.max == matrix.node_version run: tar --auto-compress -cvf ${{ runner.temp }}/docs.tar.zst ./docs @@ -2548,6 +2614,17 @@ jobs: run: | docker save ${{ join(fromJSON(steps.test_meta.outputs.json).tags,' ') }} -o ${DOCKER_TAR} zstd -v ${DOCKER_TAR} + - name: Install trufflehog + if: inputs.scan_secrets == 'true' + run: | + # install trufflesecurity/trufflehog + curl https://github.com/trufflesecurity/trufflehog/releases/download/v3.82.7/trufflehog_3.82.7_linux_amd64.tar.gz -o trufflehog.tar.gz && \ + tar -xzf trufflehog.tar.gz && \ + chmod +x trufflehog + - name: Scan for secrets + if: inputs.scan_secrets == 'true' + run: | + trufflehog docker --image file://${DOCKER_TAR} --only-verified --no-update --fail - name: Enable KVM group perms if: contains(matrix.runs_on,'ubuntu-latest') || contains(matrix.runs_on,'ubuntu-22.04') continue-on-error: true @@ -3419,6 +3496,18 @@ jobs: - name: Run poetry install run: | poetry install + - name: Install trufflehog + if: inputs.scan_secrets == 'true' + run: | + # install trufflesecurity/trufflehog + curl https://github.com/trufflesecurity/trufflehog/releases/download/v3.82.7/trufflehog_3.82.7_linux_amd64.tar.gz -o trufflehog.tar.gz && \ + tar -xzf trufflehog.tar.gz && \ + chmod +x trufflehog + - name: Scan artifacts + if: inputs.scan_secrets == 'true' + run: ./trufflehog filesystem ${{env.scanPath}} --no-update --only-verified --fail + env: + scanPath: ./ - name: Publish draft release env: PYPI_TOKEN: ${{ secrets.PYPI_TEST_TOKEN }} @@ -3551,6 +3640,18 @@ jobs: if: | inputs.docusaurus_website == false run: npm run deploy-docs --if-present + - name: Install trufflehog + if: inputs.scan_secrets == 'true' + run: | + # install trufflesecurity/trufflehog + curl https://github.com/trufflesecurity/trufflehog/releases/download/v3.82.7/trufflehog_3.82.7_linux_amd64.tar.gz -o trufflehog.tar.gz && \ + tar -xzf trufflehog.tar.gz && \ + chmod +x trufflehog + - name: Scan artifacts + if: inputs.scan_secrets == 'true' + run: ./trufflehog filesystem ${{env.scanPath}} --no-update --only-verified --fail + env: + scanPath: build/ - name: Update deploy branch for merged PRs if: github.event.pull_request.state != 'open' run: | @@ -3689,6 +3790,18 @@ jobs: with: path: ${{ runner.temp }} name: release-notes + - name: Install trufflehog + if: inputs.scan_secrets == 'true' + run: | + # install trufflesecurity/trufflehog + curl https://github.com/trufflesecurity/trufflehog/releases/download/v3.82.7/trufflehog_3.82.7_linux_amd64.tar.gz -o trufflehog.tar.gz && \ + tar -xzf trufflehog.tar.gz && \ + chmod +x trufflehog + - name: Scan artifacts + if: inputs.scan_secrets == 'true' + run: ./trufflehog filesystem ${{env.scanPath}} --no-update --only-verified --fail + env: + scanPath: ${{ runner.temp }} - name: Publish draft release uses: softprops/action-gh-release@01570a1f39cb168c169c802c3bceb9e93fb10974 with: @@ -4010,6 +4123,18 @@ jobs: - name: Compress run: | tar --auto-compress -cvf ${{ needs.cargo_test.outputs.package }}-${{ matrix.target }}.tar.gz -C target/${{ matrix.target }}/release ${{ needs.cargo_test.outputs.package }} + - name: Install trufflehog + if: inputs.scan_secrets == 'true' + run: | + # install trufflesecurity/trufflehog + curl https://github.com/trufflesecurity/trufflehog/releases/download/v3.82.7/trufflehog_3.82.7_linux_amd64.tar.gz -o trufflehog.tar.gz && \ + tar -xzf trufflehog.tar.gz && \ + chmod +x trufflehog + - name: Scan artifacts + if: inputs.scan_secrets == 'true' + run: ./trufflehog filesystem ${{env.scanPath}} --no-update --only-verified --fail + env: + scanPath: ${{ needs.cargo_test.outputs.package }}-${{ matrix.target }}.tar.gz - name: Upload artifact uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 with: diff --git a/README.md b/README.md index 80dfe97dd..a3039a31e 100644 --- a/README.md +++ b/README.md @@ -374,6 +374,11 @@ jobs: # Required: false generate_sbom: true + # Scan for secrets in the PR and artifacts using TruffleHog OSS + # Type: boolean + # Required: false + scan_secrets: true + ``` diff --git a/flowzone.yml b/flowzone.yml index f7d510fdc..b9ef1b123 100644 --- a/flowzone.yml +++ b/flowzone.yml @@ -693,6 +693,20 @@ "bom": "'"$(base64 -w 0 "${{ env.BOM_FILE }}")"'" }' + - &setupTruffleHog + name: Install trufflehog + if: inputs.scan_secrets == 'true' + run: | + # install trufflesecurity/trufflehog + curl https://github.com/trufflesecurity/trufflehog/releases/download/v3.82.7/trufflehog_3.82.7_linux_amd64.tar.gz -o trufflehog.tar.gz && \ + tar -xzf trufflehog.tar.gz && \ + chmod +x trufflehog + + - &scanArtifacts + name: Scan artifacts + if: inputs.scan_secrets == 'true' + run: ./trufflehog filesystem ${{env.scanPath}} --no-update --only-verified --fail + - &convenienceFunctions name: Convenience functions id: functions @@ -1011,6 +1025,11 @@ on: type: boolean required: false default: true + scan_secrets: + description: "Scan for secrets in the PR and artifacts using TruffleHog OSS" + type: boolean + required: false + default: true outputs: cloudflare_deployment_url: description: "Cloudflare Deployment URL" @@ -1319,6 +1338,25 @@ jobs: REF: "refs/tags/${{ steps.versionist.outputs.tag }}" SHA: ${{ steps.create_tag.outputs.sha }} + # Scan the PR for unintentional secrets leakage + # Note that this scans the commits themselfs but not the artifacts it produces (see later steps for that) + scan_secrets: + name: Scan Commits for Secrets + runs-on: ubuntu-latest + needs: + - versioned_source + if: inputs.scan_secrets == 'true' + steps: + - *getGitHubAppToken + - *checkoutVersionedSha + + - name: Scan + uses: trufflesecurity/trufflehog@v3.82.7 + with: + base: ${{ github.event.pull_request.base.sha }} + head: ${{ github.event.pull_request.head.ref }} + extra_args: "--only-verified" + release_notes: name: Generate release notes runs-on: ${{ fromJSON(inputs.runs_on) }} @@ -1639,6 +1677,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets <<: *customWorkingDirectory @@ -1748,6 +1787,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets <<: *customWorkingDirectory @@ -1959,6 +1999,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets <<: *customWorkingDirectory @@ -2044,6 +2085,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets if: inputs.cargo_targets != '' <<: *customWorkingDirectory @@ -2093,6 +2135,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets if: inputs.balena_slugs != '' <<: *customWorkingDirectory @@ -2131,6 +2174,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets <<: *rootWorkingDirectory @@ -2243,6 +2287,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets if: inputs.cloudflare_website != '' <<: *customWorkingDirectory @@ -2274,6 +2319,7 @@ jobs: timeout-minutes: ${{ fromJSON(inputs.jobs_timeout_minutes) }} needs: - versioned_source + - scan_secrets if: inputs.cloudformation_templates != '' <<: *customWorkingDirectory @@ -2458,6 +2504,12 @@ jobs: # FIXME: workaround when `npm pack` for npm 6.x dumps tarball into the current directory because it has no `--pack-destination` flag [[ "$(npm --version)" =~ ^6\..* ]] && find . -maxdepth 1 -name '*.tgz' -exec mv {} ${{ runner.temp }}/npm-pack \; || true + # Scan artifacts for secrets + - *setupTruffleHog + - <<: *scanArtifacts + env: + scanPath: ${{ runner.temp }}/npm-pack + - name: Upload artifact if: needs.is_npm.outputs.npm_private != 'true' && steps.node_versions.outputs.max == matrix.node_version uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 @@ -2471,6 +2523,10 @@ jobs: shell: bash run: npm run doc + - <<: *scanArtifacts + env: + scanPath: ./docs + - name: Compress docs if: needs.is_npm.outputs.npm_docs == 'true' && steps.node_versions.outputs.max == matrix.node_version run: tar --auto-compress -cvf ${{ runner.temp }}/docs.tar.zst ./docs @@ -2833,6 +2889,13 @@ jobs: docker save ${{ join(fromJSON(steps.test_meta.outputs.json).tags,' ') }} -o ${DOCKER_TAR} zstd -v ${DOCKER_TAR} + # scan image for secrets + - *setupTruffleHog + - name: Scan for secrets + if: inputs.scan_secrets == 'true' + run: | + trufflehog docker --image file://${DOCKER_TAR} --only-verified --no-update --fail + # https://github.com/actions/runner-images/discussions/7191#discussioncomment-8351370 # https://github.com/reactivecircus/android-emulator-runner?tab=readme-ov-file#running-hardware-accelerated-emulators-on-linux-runners # https://github.com/ankidroid/Anki-Android/commit/3a5ecaa9837691817022d11b0dbe383b8e82d9fe @@ -3301,6 +3364,12 @@ jobs: run: | poetry install + # Scan artifacts for secrets + - *setupTruffleHog + - <<: *scanArtifacts + env: + scanPath: ./ + - name: Publish draft release env: PYPI_TOKEN: ${{ secrets.PYPI_TEST_TOKEN }} @@ -3403,6 +3472,12 @@ jobs: inputs.docusaurus_website == false run: npm run deploy-docs --if-present + # Scan artifacts for secrets + - *setupTruffleHog + - <<: *scanArtifacts + env: + scanPath: build/ + - name: Update deploy branch for merged PRs if: github.event.pull_request.state != 'open' run: | @@ -3527,6 +3602,12 @@ jobs: path: ${{ runner.temp }} name: release-notes + # Scan artifacts for secrets + - *setupTruffleHog + - <<: *scanArtifacts + env: + scanPath: ${{ runner.temp }} + # https://github.com/softprops/action-gh-release - name: Publish draft release uses: softprops/action-gh-release@01570a1f39cb168c169c802c3bceb9e93fb10974 # v2.1.0 @@ -3784,6 +3865,12 @@ jobs: run: | tar --auto-compress -cvf ${{ needs.cargo_test.outputs.package }}-${{ matrix.target }}.tar.gz -C target/${{ matrix.target }}/release ${{ needs.cargo_test.outputs.package }} + # Scan artifacts for secrets + - *setupTruffleHog + - <<: *scanArtifacts + env: + scanPath: ${{ needs.cargo_test.outputs.package }}-${{ matrix.target }}.tar.gz + - name: Upload artifact uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: