feat: (CCIE-1585) Add more detailed ownership tags to fogg components (…

…#918) * skeleton for obtaining tags from config * add tags to template * stub stamps * feat: fill in github repository for tags * golden files * CCIE-1806 * deadcode * feat: compute file path tag (#930) * chore: first change * feat: add filepath of template being executed * tag name change * fix: add func for git origin (#931) * fix: add func for git origin * move test; deadcode * timestamp * update golden files * fix git remote url test * stamp cleanup and fix git remote test * remove user from template * update golden files * progress * update golden files * ci update golden files * update actions * token * commit from ci -- updated golden files * fix: populate git related tags with templated terraform (#934) * progress * handle git log errors in script * fix json string formatting * add git data source to template * git helper tags * branch tag * cleanup * add git authors to tags * commit from ci -- updated golden files * git author query formatting * commit from ci -- updated golden files --------- Co-authored-by: Jake Heath <[email protected]> Co-authored-by: Jake Heath <[email protected]> Co-authored-by: jakeyheath <[email protected]> Co-authored-by: dtsai-czi <[email protected]>
chanzuckerberg · Sep 18, 2023 · 6832494 · 6832494
1 parent bf8d37e
commit 6832494
Show file tree

Hide file tree

Showing 100 changed files with 2,628 additions and 246 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,23 +1,49 @@
 on: pull_request
 
 jobs:
+  golden:
+    runs-on: [self-hosted, ARM64, linux]
+    steps:
+      - name: Generate token
+        id: generate_token
+        uses: chanzuckerberg/[email protected]
+        with:
+          app_id: ${{ secrets.CZI_RELEASE_PLEASE_APP_ID }}
+          private_key: ${{ secrets.CZI_RELEASE_PLEASE_PK }}
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ steps.generate_token.outputs.token }}
+          ref: ${{ github.event.pull_request.head.ref }}
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Run tests
+        run: make update-golden-files
+      - name: Commit updated mod tidy
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: -A
+          message: commit from ci -- updated golden files
   test:
-    runs-on: self-hosted
+    runs-on: [self-hosted, ARM64, linux]
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
+          cache: true
       - name: Run tests
         run: make test-ci
 
   lint:
-    runs-on: self-hosted
+    runs-on: [self-hosted, ARM64, linux]
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
+          cache: true
       - uses: actions/setup-node@v3
         with:
           node-version: 18

diff --git a/.github/workflows/conventional_commits_title.yml b/.github/workflows/conventional_commits_title.yml
@@ -9,6 +9,6 @@ on:
 
 jobs:
   conventional_commit_title:
-    runs-on: self-hosted
+    runs-on: [self-hosted, ARM64, linux]
     steps:
       - uses: chanzuckerberg/github-actions/.github/actions/conventional-commits@main
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -6,7 +6,7 @@ on:
 name: release-please
 jobs:
   release-please:
-    runs-on: self-hosted
+    runs-on: [self-hosted, ARM64, linux]
     steps:
       - uses: actions/github-script@v5
         id: configure-changelog
@@ -38,16 +38,17 @@ jobs:
           changelog-types: ${{ steps.configure-changelog.outputs.result }}
           token: ${{ steps.generate_token.outputs.token }}
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         # we need to fetch all history and tags
         # so we build the proper version
         with:
           fetch-depth: 0
         if: ${{ steps.release.outputs.release_created }}
 
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
+          cache: true
         if: ${{ steps.release.outputs.release_created }}
 
       - name: Run GoReleaser

diff --git a/apply/apply.go b/apply/apply.go
@@ -8,10 +8,12 @@ import (
 	"io/fs"
 	"net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"regexp"
 	"sort"
 	"strings"
+	"text/template"
 
 	"github.com/pkg/errors"
 	"golang.org/x/exp/slices"
@@ -521,10 +523,24 @@ func removeExtension(path string) string {
 	return strings.TrimSuffix(path, filepath.Ext(path))
 }
 
+func getGitRemoteOriginURL(cwd ...string) string {
+	dir := "."
+	if len(cwd) > 0 {
+		dir = cwd[0]
+	}
+	cmd := exec.Command("git", "remote", "get-url", "--push", "origin")
+	cmd.Dir = dir
+	out, err := cmd.Output()
+	if err != nil {
+		logrus.Warnf("unable to get git output: %s", err)
+		return ""
+	}
+	return strings.TrimSpace(string(out))
+}
+
 func applyTemplate(sourceFile io.Reader, commonTemplates fs.FS, dest afero.Fs, path string, overrides interface{}) error {
-	dir, _ := filepath.Split(path)
-	ospath := filepath.FromSlash(dir)
-	err := dest.MkdirAll(ospath, 0775)
+	dir := filepath.Dir(path)
+	err := dest.MkdirAll(dir, 0775)
 	if err != nil {
 		return errs.WrapUserf(err, "couldn't create %s directory", dir)
 	}
@@ -534,7 +550,14 @@ func applyTemplate(sourceFile io.Reader, commonTemplates fs.FS, dest afero.Fs, p
 	if err != nil {
 		return errs.WrapUser(err, "unable to open file")
 	}
-	t, e := util.OpenTemplate(path, sourceFile, commonTemplates)
+	t, e := util.OpenTemplate(path, sourceFile, commonTemplates, []template.FuncMap{
+		{
+			"cwd": func() string {
+				return filepath.Dir(writer.Name())
+			},
+			"git_origin": getGitRemoteOriginURL,
+		},
+	}...)
 	if e != nil {
 		return e
 	}

diff --git a/apply/apply_test.go b/apply/apply_test.go
@@ -29,6 +29,25 @@ func init() {
 	logrus.SetFormatter(formatter)
 }
 
+type testGitRemote struct {
+	path            string
+	expectedRemotes []string
+}
+
+func TestGetGitRemoteOriginURL(t *testing.T) {
+	r := require.New(t)
+	sshRemote := "[email protected]:chanzuckerberg/fogg"
+	httpsRemote := "https://github.com/chanzuckerberg/fogg"
+	acceptableRemotes := []string{sshRemote, httpsRemote}
+	tests := []testGitRemote{
+		{path: ".", expectedRemotes: acceptableRemotes},
+	}
+	for _, test := range tests {
+		remote := getGitRemoteOriginURL(test.path)
+		r.Contains(test.expectedRemotes, remote)
+	}
+}
+
 func randomString(n int) string {
 	var letter = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
 

diff --git a/plan/plan.go b/plan/plan.go
@@ -46,7 +46,7 @@ type ComponentCommon struct {
 	Accounts              map[string]*json.Number    `yaml:"all_accounts"`
 	Backend               Backend                    `yaml:"backend"`
 	ComponentBackends     map[string]Backend         `yaml:"comonent_backends"`
-	Env                   string                     ` yaml:"env"`
+	Env                   string                     `yaml:"env"`
 	ExtraVars             map[string]string          `yaml:"extra_vars"`
 	Name                  string                     `yaml:"name"`
 	Owner                 string                     `yaml:"owner"`

diff --git a/templates/templates/common/make_git_helpers.tmpl b/templates/templates/common/make_git_helpers.tmpl
@@ -0,0 +1,13 @@
+{{ define "make_git_helpers" }}
+SHELL := bash
+soft_git_log:
+	@git log --pretty=format:"{ \"sha\": \"%H\" }" -1 HEAD || echo "{ \"sha\": \"unknown\" }"
+soft_git_user:
+	@echo "{\"name\": \"$$(git config --global --get user.name)\"}" || echo "{ \"user\": \"unknown\" }"
+soft_git_email:
+	@echo "{\"email\": \"$$(git config --global --get user.email)\"}" || echo "{ \"email\": \"unknown\" }"
+soft_git_branch:
+	@echo "{\"branch\": \"$$(git rev-parse --abbrev-ref HEAD)\"}" || echo "{ \"branch\": \"unknown\" }"
+soft_git_authors:
+	@echo "{\"authors\": \"$$(git log --format=\"%an\" -- . | grep -v "[bot]" | head -10)\"}" || echo "{ \"authors\": \"unknown\" }"
+{{ end }}
diff --git a/templates/templates/component/terraform/Makefile.tmpl b/templates/templates/component/terraform/Makefile.tmpl
@@ -1,3 +1,4 @@
 {{ template "fogg_header" }}
 {{ template "make_vars" . }}
 {{ template "make_help" }}
+{{ template "make_git_helpers" }}
diff --git a/templates/templates/component/terraform/fogg.tf.tmpl b/templates/templates/component/terraform/fogg.tf.tmpl
@@ -119,15 +119,58 @@ variable "owner" {
   default = "{{ .Owner }}"
 }
 
+data "external" "git_sha" {
+  program = [
+    "make",
+    "soft_git_log",
+  ]
+}
+
+data "external" "git_user" {
+  program = [
+    "make",
+    "soft_git_user"
+  ]
+}
+
+data "external" "git_email" {
+  program = [
+    "make",
+    "soft_git_email"
+  ]
+}
+
+data "external" "git_branch" {
+  program = [
+    "make",
+    "soft_git_branch"
+  ]
+}
+
+data "external" "git_authors" {
+  program = [
+    "make",
+    "soft_git_authors"
+  ]
+}
+
 # tflint-ignore: terraform_unused_declarations
 variable "tags" {
   type =  object({project: string, env: string, service: string, owner: string, managedBy: string})
   default = {
-    project   = "{{ .Project }}"
-    env       = "{{ .Env }}"
-    service   = "{{ .Name }}"
-    owner     = "{{ .Owner }}"
-    managedBy = "terraform"
+    project               = "{{ .Project }}"
+    env                   = "{{ .Env }}"
+    service               = "{{ .Name }}"
+    owner                 = "{{ .Owner }}"
+    terraformLastApplyTime = timestamp()
+    terraformWorkspaceDir = "{{ cwd }}"
+    gitRepository         = "{{ git_origin }}"
+    gitSHA                = data.external.git_sha.result.sha
+    gitUser               = data.external.git_user.result.name
+    gitEmail              = data.external.git_email.result.email
+    gitBranch             = data.external.git_branch.result.branch
+    gitAuthors            = data.external.git_authors.result.authors
+    managedBy             = "terraform"
   }
 }
 

diff --git a/testdata/auth0_provider_yaml/terraform/accounts/foo/Makefile b/testdata/auth0_provider_yaml/terraform/accounts/foo/Makefile
diff --git a/testdata/auth0_provider_yaml/terraform/accounts/foo/fogg.tf b/testdata/auth0_provider_yaml/terraform/accounts/foo/fogg.tf
diff --git a/testdata/auth0_provider_yaml/terraform/envs/bar/bam/Makefile b/testdata/auth0_provider_yaml/terraform/envs/bar/bam/Makefile