From d298402594da8a1ea50c5cbb65313206fa900800 Mon Sep 17 00:00:00 2001 From: Miroslav Shubernetskiy Date: Mon, 24 Oct 2022 14:47:58 -0400 Subject: [PATCH 1/8] adding test-org CI + --noServer flag --- .github/workflows/test.yml | 35 +++++++++++++++++++++++++++++++++++ Dockerfile | 8 ++++++-- cmd/github-analyzer/main.go | 5 ++--- docker-compose.yml | 8 +++----- pkg/config/config.go | 2 +- 5 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..c223192 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,35 @@ +name: test + +on: + push: + branches: + - main + pull_request: + +jobs: + test-org: + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Generate org-level access token for test-org + id: org-token + uses: getsentry/action-github-app-token@v1 + with: + app_id: ${{ secrets.TEST_GITHUB_APP_ID }} + private_key: ${{ secrets.TEST_GITHUB_APP_PRIVATE_KEY }} + + - name: Scan test-org + env: + GH_SECURITY_AUDITOR_TOKEN: ${{ steps.org-token.outputs.token }} + run: | + docker-compose run --rm co-github-analyzer \ + --organization ${{ secrets.TEST_GITHUB_ORG }} \ + --userPermissionStats \ + --noServer + + - name: Run tests on output data + run: | + true # TODO diff --git a/Dockerfile b/Dockerfile index c2d150f..873bd84 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,13 +2,17 @@ FROM golang:1.19-alpine +RUN apk add --no-cache make + WORKDIR /ghanalyzer -ADD . /ghanalyzer +ADD go.* /ghanalyzer/ RUN go mod download RUN go env -w GO111MODULE=on -RUN mkdir -p bin && go generate && go build -v -o bin/github-analyzer cmd/github-analyzer/main.go +ADD . /ghanalyzer/ + +RUN make all ENTRYPOINT [ "/ghanalyzer/bin/github-analyzer" ] diff --git a/cmd/github-analyzer/main.go b/cmd/github-analyzer/main.go index fd9556b..1f6a738 100644 --- a/cmd/github-analyzer/main.go +++ b/cmd/github-analyzer/main.go @@ -175,9 +175,6 @@ func NewRootCommand() *cobra.Command { rootCmd.Flags(). BoolVarP(&config.ViperEnv.UserPermissionStats, "userPermissionStats", "", false, "enable user permission statistics (might be slow in large orgs due to throttling limits)") - rootCmd.Flags(). - BoolVarP(&config.ViperEnv.DisableServer, "disableServer", "", false, "do not spin up an HTTP server, and only emit data in the designated output folder") - rootCmd.Flags(). BoolVarP(&config.ViperEnv.EnableScraping, "enableScraping", "", false, "enable experimental checks that rely on screen scraping") rootCmd.Flags(). @@ -189,6 +186,8 @@ func NewRootCommand() *cobra.Command { rootCmd.Flags(). IntVarP(&config.ViperEnv.Port, "port", "", 3000, "port for local http server used to display HTML with summary of findings (if you are using docker you will need to override the default port appropriately)") + rootCmd.Flags(). + BoolVarP(&config.ViperEnv.DisableServer, "disableServer", "", false, "do not spin up an HTTP server, and only emit data in the designated output folder") return rootCmd } diff --git a/docker-compose.yml b/docker-compose.yml index 45b9208..e0280d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,15 +2,13 @@ version: "3.8" services: co-github-analyzer: - # image allows to cache all deps hence to speed up CI - # CI image: ghcr.io/crashappsec/github-analyzer:latest - build: . # CI + build: . container_name: github-analyzer - working_dir: $PWD ports: - 3000:3000 + working_dir: $PWD volumes: - - $PWD:$PWD + - $PWD:$PWD # this allows to share ./output/ environment: GH_SECURITY_AUDITOR_TOKEN: ${GH_SECURITY_AUDITOR_TOKEN:-} GH_SECURITY_AUDITOR_USERNAME: ${GH_SECURITY_AUDITOR_USERNAME:-} diff --git a/pkg/config/config.go b/pkg/config/config.go index d85a74c..7e914c3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -9,7 +9,6 @@ const ( type ViperEnvVars struct { CfgFile string `mapstructure:"CFG_FILE"` EnableScraping bool `mapstructure:"ENABLE_SCRAPING"` - DisableServer bool `mapstructure:"DISABLE_SERVER"` UserPermissionStats bool `mapstructure:"USER_PERMISSION_STATS"` Version bool `mapstructure:"VERSION"` Organization string `mapstructure:"ORGANIZATION"` @@ -17,6 +16,7 @@ type ViperEnvVars struct { OutputDir string `mapstructure:"OUTPUT_DIR"` Password string `mapstructure:"PASSWORD"` Port int `mapstructure:"PORT"` + DisableServer bool `mapstructure:"DISABLE_SERVER"` ScmURL string `mapstructure:"SCM_URL"` Token string `mapstructure:"TOKEN"` Username string `mapstructure:"USERNAME"` From 29767ffbfa3f8e0dd7188841aea013cbd9533f14 Mon Sep 17 00:00:00 2001 From: Miroslav Shubernetskiy Date: Mon, 24 Oct 2022 14:59:59 -0400 Subject: [PATCH 2/8] saving output as an artifact for local inspection --- .github/workflows/test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c223192..ff4916d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,6 +30,13 @@ jobs: --userPermissionStats \ --noServer + - name: "Upload Artifact" + uses: actions/upload-artifact@v3 + with: + name: output + path: output + retention-days: 7 + - name: Run tests on output data run: | true # TODO From 79cd93adb8710b4b8e2f3f99cfd8a8c1247af22e Mon Sep 17 00:00:00 2001 From: Miroslav Shubernetskiy Date: Mon, 24 Oct 2022 16:18:59 -0400 Subject: [PATCH 3/8] running tests in CI --- .github/workflows/test.yml | 25 +++++++++++++++++++++---- Dockerfile | 12 ++++++++---- README.md | 2 +- docker-compose.yml | 16 ++++++++++++++-- pkg/github/auditor/auditor_test.go | 10 ++++++++-- 5 files changed, 52 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ff4916d..0e32b60 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ on: pull_request: jobs: - test-org: + analyzer: runs-on: ubuntu-latest steps: @@ -25,10 +25,10 @@ jobs: env: GH_SECURITY_AUDITOR_TOKEN: ${{ steps.org-token.outputs.token }} run: | - docker-compose run --rm co-github-analyzer \ + docker-compose run --rm github-analyzer \ --organization ${{ secrets.TEST_GITHUB_ORG }} \ --userPermissionStats \ - --noServer + --disableServer - name: "Upload Artifact" uses: actions/upload-artifact@v3 @@ -37,6 +37,23 @@ jobs: path: output retention-days: 7 + asserts: + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Generate org-level access token for test-org + id: org-token + uses: getsentry/action-github-app-token@v1 + with: + app_id: ${{ secrets.TEST_GITHUB_APP_ID }} + private_key: ${{ secrets.TEST_GITHUB_APP_PRIVATE_KEY }} + - name: Run tests on output data + env: + GH_SECURITY_AUDITOR_TOKEN: ${{ steps.org-token.outputs.token }} + GH_SECURITY_AUDITOR_ORGANIZATION: ${{ secrets.TEST_GITHUB_ORG }} run: | - true # TODO + docker-compose run --rm tests diff --git a/Dockerfile b/Dockerfile index 873bd84..06640c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,4 @@ -# syntax=docker/dockerfile:1 - -FROM golang:1.19-alpine +FROM golang:1.19-alpine as build RUN apk add --no-cache make @@ -15,4 +13,10 @@ ADD . /ghanalyzer/ RUN make all -ENTRYPOINT [ "/ghanalyzer/bin/github-analyzer" ] +# ---------------------------------------------------------------------------- + +FROM alpine + +COPY --from=build /ghanalyzer/bin/github-analyzer /bin/github-analyzer + +ENTRYPOINT [ "/bin/github-analyzer" ] diff --git a/README.md b/README.md index 1868efb..207026f 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ You can see available options via the `--help` flag. ```sh docker compose run \ --rm --service-ports \ - co-github-analyzer \ + github-analyzer \ --organization \ --output output \ --token "$GH_SECURITY_AUDITOR_TOKEN" diff --git a/docker-compose.yml b/docker-compose.yml index e0280d1..9065875 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,8 @@ version: "3.8" services: - co-github-analyzer: + github-analyzer: build: . - container_name: github-analyzer ports: - 3000:3000 working_dir: $PWD @@ -14,3 +13,16 @@ services: GH_SECURITY_AUDITOR_USERNAME: ${GH_SECURITY_AUDITOR_USERNAME:-} GH_SECURITY_AUDITOR_PASSWORD: ${GH_SECURITY_AUDITOR_PASSWORD:-} GH_SECURITY_AUDITOR_OTP_SEED: ${GH_SECURITY_AUDITOR_OTP_SEED:-} + + tests: + image: golang:1.19 + command: make test + init: true + working_dir: $PWD + volumes: + - $PWD:$PWD # this allows to share ./output/ + environment: + GH_SECURITY_AUDITOR_TOKEN: ${GH_SECURITY_AUDITOR_TOKEN:-} + GH_SECURITY_AUDITOR_USERNAME: ${GH_SECURITY_AUDITOR_USERNAME:-} + GH_SECURITY_AUDITOR_PASSWORD: ${GH_SECURITY_AUDITOR_PASSWORD:-} + GH_SECURITY_AUDITOR_OTP_SEED: ${GH_SECURITY_AUDITOR_OTP_SEED:-} diff --git a/pkg/github/auditor/auditor_test.go b/pkg/github/auditor/auditor_test.go index d70aee5..7e54a91 100644 --- a/pkg/github/auditor/auditor_test.go +++ b/pkg/github/auditor/auditor_test.go @@ -42,12 +42,18 @@ func TestSampleOrg(t *testing.T) { Max: 3 * time.Minute, Jitter: true, } - name := "github-security-auditor-test-org" + + name := os.Getenv("GH_SECURITY_AUDITOR_ORGANIZATION") + if name == "" { + name = "github-security-auditor-test-org" + } + org, err := org.NewOrganization(ctx, auditor.client, back, name) + assert.Nil(t, err, "Could not create organization") assert.NotNil(t, org.CoreStats, "Could not fetch core stats") assert.Equal(t, name, *org.CoreStats.Login) - assert.GreaterOrEqual(t, 1, org.CoreStats.TotalPrivateRepos) + assert.GreaterOrEqual(t, 1, *org.CoreStats.TotalPrivateRepos) assert.NotNil( t, org.CoreStats.TwoFactorRequirementEnabled, From ea401cb04dd12d75c8e3208705fd3a40e056e12e Mon Sep 17 00:00:00 2001 From: Miroslav Shubernetskiy Date: Mon, 24 Oct 2022 18:07:04 -0400 Subject: [PATCH 4/8] generating version on each build --- .gitignore | 1 + Dockerfile | 2 +- Makefile | 15 ++++++++++----- cmd/github-analyzer/main.go | 2 +- cmd/github-analyzer/version.sh | 4 +++- cmd/github-analyzer/version.txt | 1 - 6 files changed, 16 insertions(+), 9 deletions(-) delete mode 100644 cmd/github-analyzer/version.txt diff --git a/.gitignore b/.gitignore index e4ad979..52c8503 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ _testmain.go tags wiki *.envrc* +version.txt /VERSION.cache bin/ diff --git a/Dockerfile b/Dockerfile index 06640c2..cc98d36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM golang:1.19-alpine as build -RUN apk add --no-cache make +RUN apk add --no-cache git make WORKDIR /ghanalyzer diff --git a/Makefile b/Makefile index b4a76ad..739a731 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,14 @@ .PHONY: all -all: ## compile auditor - mkdir -p bin - go generate +all: bin generate ## compile auditor go build -v -o bin/github-analyzer cmd/github-analyzer/main.go +bin: + mkdir -p bin + +.PHONY: generate +generate: + go generate cmd/github-analyzer/main.go + .PHONY: lint lint: ## lint everything with pre-commit pre-commit run --all-files --show-diff-on-failure @@ -22,11 +27,11 @@ fmt: ## go format gofmt -w ./$* .PHONY: vet -vet: ## go vet +vet: generate ## go vet go vet ./... .PHONY: test -test: ## run go tests (requires GitHub to be reachable via the network) +test: generate ## run go tests (requires GitHub to be reachable via the network) go test -v -race -coverprofile coverage.txt ./... .PHONY: help diff --git a/cmd/github-analyzer/main.go b/cmd/github-analyzer/main.go index 1f6a738..2457206 100644 --- a/cmd/github-analyzer/main.go +++ b/cmd/github-analyzer/main.go @@ -21,7 +21,7 @@ import ( "github.com/spf13/viper" ) -//go:generate sh version.sh +//go:generate sh version.sh version.txt //go:embed version.txt var version string diff --git a/cmd/github-analyzer/version.sh b/cmd/github-analyzer/version.sh index 319982b..6fb5e6d 100644 --- a/cmd/github-analyzer/version.sh +++ b/cmd/github-analyzer/version.sh @@ -1,2 +1,4 @@ #!/bin/sh -git describe --tags --long > version.txt + +destination=${1:-version.txt} +git describe --tags --long | tee $destination diff --git a/cmd/github-analyzer/version.txt b/cmd/github-analyzer/version.txt deleted file mode 100644 index 45b3c63..0000000 --- a/cmd/github-analyzer/version.txt +++ /dev/null @@ -1 +0,0 @@ -v0.1.4-pre-alpha-5-g1722c63 From f089c95c45226f9932388bdf59bfc5fba46401b5 Mon Sep 17 00:00:00 2001 From: Miroslav Shubernetskiy Date: Mon, 24 Oct 2022 18:42:39 -0400 Subject: [PATCH 5/8] using ldflags to set version to allow "go install" --- Makefile | 13 +++++++++---- cmd/github-analyzer/main.go | 4 +--- cmd/github-analyzer/version.sh | 4 ---- 3 files changed, 10 insertions(+), 11 deletions(-) delete mode 100644 cmd/github-analyzer/version.sh diff --git a/Makefile b/Makefile index 739a731..9e197f6 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,18 @@ +BIN=$(notdir $(wildcard cmd/*)) +VERSION=$(shell git describe --tags --long) + .PHONY: all -all: bin generate ## compile auditor - go build -v -o bin/github-analyzer cmd/github-analyzer/main.go +all: $(addprefix bin/,$(BIN)) ## compile auditor + +bin/%: bin generate + go build -v -ldflags "-X main.version=$(VERSION)" -o $@ cmd/$*/main.go bin: mkdir -p bin .PHONY: generate -generate: - go generate cmd/github-analyzer/main.go +generate: ## generate go:generate files + go generate ./... .PHONY: lint lint: ## lint everything with pre-commit diff --git a/cmd/github-analyzer/main.go b/cmd/github-analyzer/main.go index 2457206..192bcf6 100644 --- a/cmd/github-analyzer/main.go +++ b/cmd/github-analyzer/main.go @@ -21,9 +21,7 @@ import ( "github.com/spf13/viper" ) -//go:generate sh version.sh version.txt -//go:embed version.txt -var version string +var version = "(devel)" func main() { if err := NewRootCommand().Execute(); err != nil { diff --git a/cmd/github-analyzer/version.sh b/cmd/github-analyzer/version.sh deleted file mode 100644 index 6fb5e6d..0000000 --- a/cmd/github-analyzer/version.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -destination=${1:-version.txt} -git describe --tags --long | tee $destination From 6dcfd30284c0e7593f98b14d303936321cabb7aa Mon Sep 17 00:00:00 2001 From: Miroslav Shubernetskiy Date: Mon, 24 Oct 2022 18:51:43 -0400 Subject: [PATCH 6/8] reading version info from BuildInfo --- Makefile | 6 +++++- cmd/github-analyzer/main.go | 20 ++++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9e197f6..1179e3e 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,11 @@ VERSION=$(shell git describe --tags --long) all: $(addprefix bin/,$(BIN)) ## compile auditor bin/%: bin generate - go build -v -ldflags "-X main.version=$(VERSION)" -o $@ cmd/$*/main.go + go build \ + -v \ + -ldflags "-X main.version=$(VERSION)" \ + -o $@ \ + cmd/$*/main.go bin: mkdir -p bin diff --git a/cmd/github-analyzer/main.go b/cmd/github-analyzer/main.go index 192bcf6..c91304b 100644 --- a/cmd/github-analyzer/main.go +++ b/cmd/github-analyzer/main.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "os" + "runtime/debug" "strings" _ "embed" @@ -23,6 +24,21 @@ import ( var version = "(devel)" +func getVersion() (response string) { + // inspired from + // https://github.com/mvdan/sh/blob/6ba49e2c622e3f56330f4de6238a390f395db2d8/cmd/shfmt/main.go#L181-L192 + if info, ok := debug.ReadBuildInfo(); ok && version == "(devel)" { + mod := &info.Main + if mod.Replace != nil { + mod = mod.Replace + } + if mod.Version != "" { + version = mod.Version + } + } + return version +} + func main() { if err := NewRootCommand().Execute(); err != nil { log.Logger.Errorf("Scan completed with errors", err) @@ -135,7 +151,7 @@ func NewRootCommand() *cobra.Command { rootCmd := &cobra.Command{ Use: fmt.Sprintf( "github-analyzer (%s)", - strings.TrimSuffix(version, "\n"), + strings.TrimSuffix(getVersion(), "\n"), ), Short: "A tool to collect statistics and highlight potential security issues within a GitHub org", Long: "A tool to collect statistics and highlight potential security issues within a GitHub org", @@ -146,7 +162,7 @@ func NewRootCommand() *cobra.Command { PreRun: func(cmd *cobra.Command, args []string) { onlyPrintVersion, _ := cmd.Flags().GetBool("version") if onlyPrintVersion { - fmt.Println(version) + fmt.Println(getVersion()) os.Exit(0) } cmd.MarkFlagRequired("organization") From 73079a3fba003e067331bc4d909779c1e84b7d6e Mon Sep 17 00:00:00 2001 From: Miroslav Shubernetskiy Date: Mon, 24 Oct 2022 19:30:24 -0400 Subject: [PATCH 7/8] ignoring dist folder --- .gitignore | 2 ++ Makefile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 52c8503..1bce003 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,5 @@ bin/ # For example, set up .git/info/exclude or use a global .gitignore. githubsecurity.json + +dist/ diff --git a/Makefile b/Makefile index 1179e3e..43c0185 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ bin: mkdir -p bin .PHONY: generate -generate: ## generate go:generate files +generate: ## process go:generate files go generate ./... .PHONY: lint From 06d55816af32fea6c7642adbe8ebca63276541a3 Mon Sep 17 00:00:00 2001 From: Miroslav Shubernetskiy Date: Mon, 24 Oct 2022 19:49:37 -0400 Subject: [PATCH 8/8] adding goreleaser --- .github/workflows/release.yml | 34 +++++++++++++++++++++++++++++ .goreleaser.yaml | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 .github/workflows/release.yml create mode 100644 .goreleaser.yaml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ecd30e7 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,34 @@ +name: release + +permissions: + contents: write + +on: + push: + tags: + - "*" + +jobs: + goreleaser: + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Fetch git tags + run: git fetch --force --tags + + - name: Setup go + uses: actions/setup-go@v3 + with: + go-version: ">=1.19.2" + cache: true + + - uses: goreleaser/goreleaser-action@v2 + with: + distribution: goreleaser + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ github.token }} diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..827a30c --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,41 @@ +# https://goreleaser.com/customization/build/ + +before: + hooks: + - go mod tidy + - go generate ./... + +builds: + - main: cmd/github-analyzer/main.go + binary: github-analyzer + env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin + goarch: + - amd64 + - arm + - arm64 + +archives: + - replacements: + darwin: Darwin + linux: Linux + windows: Windows + 386: i386 + amd64: x86_64 + +checksum: + name_template: "checksums.txt" + +snapshot: + name_template: "{{ incpatch .Version }}-next" + +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:"