From 90236ef5f9099de7c556ea4991d11de56ddae31a Mon Sep 17 00:00:00 2001 From: sonalmahajan15 Date: Wed, 24 Apr 2024 11:46:20 -0700 Subject: [PATCH 1/5] add arg --- tools/cmd/golden-test/main.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/cmd/golden-test/main.go b/tools/cmd/golden-test/main.go index 9c255adf..fd513faf 100644 --- a/tools/cmd/golden-test/main.go +++ b/tools/cmd/golden-test/main.go @@ -55,7 +55,7 @@ type BranchResult struct { // Run runs the golden tests on the base branch and the test branch and writes the summary and // diff to the writer. -func Run(writer io.Writer, baseBranch, testBranch string) error { +func Run(writer io.Writer, baseBranch, testBranch, groupingFlag string) error { // First verify that the git repository is clean. out, err := exec.Command("git", "status", "--porcelain=v1").CombinedOutput() if err != nil { @@ -134,7 +134,7 @@ func Run(writer io.Writer, baseBranch, testBranch string) error { // Run the built NilAway binary on the stdlib and parse the diagnostics. var buf bytes.Buffer - cmd := exec.Command("bin/nilaway", "-include-errors-in-files", "/", "-json", "-pretty-print=false", "-group-error-messages=true", "std") + cmd := exec.Command("bin/nilaway", "-include-errors-in-files", "/", "-json", "-pretty-print=false", "-group-error-messages="+groupingFlag, "std") cmd.Stdout = &buf // Inherit env vars such that users can control the resource usages via GOMEMLIMIT, GOGC // etc. env vars. @@ -281,6 +281,7 @@ func main() { baseBranch := fset.String("base-branch", "main", "the base branch to compare against") testBranch := fset.String("test-branch", "", "the test branch to run golden tests (default current branch)") resultFile := fset.String("result-file", "", "the file to write the diff to, default stdout") + groupingFlag := fset.Bool("group-error-messages", true, "group similar error messages (default true)") if err := fset.Parse(os.Args[1:]); err != nil { log.Printf("failed to parse flags: %v\n", err) flag.PrintDefaults() @@ -296,7 +297,7 @@ func main() { writer = w } - if err := Run(writer, *baseBranch, *testBranch); err != nil { + if err := Run(writer, *baseBranch, *testBranch, fmt.Sprintf("%t", *groupingFlag)); err != nil { log.Printf("failed to run golden test: %v", err) var e *exec.ExitError if errors.As(err, &e) { From b65436ee75e436641ff3dde691e4a9717af021e5 Mon Sep 17 00:00:00 2001 From: sonalmahajan15 Date: Thu, 25 Apr 2024 14:15:56 -0700 Subject: [PATCH 2/5] updated CI job --- .github/workflows/golden-test-no-grouping.yml | 81 +++++++++++++++++++ .github/workflows/golden-test.yml | 10 +-- tools/cmd/golden-test/main.go | 10 ++- tools/cmd/golden-test/main_test.go | 4 +- 4 files changed, 95 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/golden-test-no-grouping.yml diff --git a/.github/workflows/golden-test-no-grouping.yml b/.github/workflows/golden-test-no-grouping.yml new file mode 100644 index 00000000..7c0713ed --- /dev/null +++ b/.github/workflows/golden-test-no-grouping.yml @@ -0,0 +1,81 @@ +name: Golden Test w/o grouping + +# NilAway output may change due to introduction of new feature or bug fixes. Since NilAway is still +# at early stage of development, constantly updating / maintaining the golden test output will be +# a burden. Therefore, we run this as a separate CI job and post the differences as a PR comment +# for manual reviews. +on: + pull_request: + +jobs: + golden-test-no-grouping: + name: Golden Test w/o grouping + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + name: Check out repository + + - name: Fetch base branch (${{ github.event.pull_request.base.ref }}) locally + run: git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }} + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.22.x + cache: false + + - name: Golden Test w/o grouping + id: golden_test_no_grouping + # Run golden test by comparing HEAD and the base branch (the target branch of the PR). + # GitHub Actions terminates the job if it hits the resource limits. Here we limit the + # memory usage to 8GiB to avoid that. + run: | + make golden-test GOMEMLIMIT=8192MiB ARGS="-base-branch ${{ github.event.pull_request.base.ref }} -result-file ${{ runner.temp }}/golden-test-no-grouping-result.md -group-error-messages=false" + + - uses: actions/github-script@v7 + with: + script: | + const fsp = require('fs').promises; + + const issueNumber = context.issue.number; + const owner = context.repo.owner; + const repo = context.repo.repo; + const rawData = await fsp.readFile(`${{ runner.temp }}/golden-test-no-grouping-result.md`, 'utf8'); + + // GitHub API has a limit of 65536 bytes for a comment body, so here we shrink the + // diff part (anything between
and
) to 10,000 characters if it + // is too long. + const pattern = /(
)([\s\S]*?)(<\/details>)/; + + const body = rawData.replace(pattern, function(match, p1, p2, p3) { + if (p2.length > 10000) { + return p1 + p2.substring(0, 5000) + '\n\n ...(truncated)...\n\n' + p2.substring(p2.length - 5000) + p3; + } + // No need to change anything if it is not too long. + return match; + }); + + // First find the comments made by the bot. + const comments = await github.rest.issues.listComments({ + owner: owner, + repo: repo, + issue_number: issueNumber + }); + const botComment = comments.data.find(comment => comment.user.login === 'github-actions[bot]' && comment.body.startsWith('## Golden Test') && comment.body.endsWith('w/o grouping')); + + // Update or create the PR comment. + if (botComment) { + await github.rest.issues.updateComment({ + owner: owner, + repo: repo, + comment_id: botComment.id, + body: body + }); + } else { + await github.rest.issues.createComment({ + owner: owner, + repo: repo, + issue_number: issueNumber, + body: body + }); + } diff --git a/.github/workflows/golden-test.yml b/.github/workflows/golden-test.yml index 7fd98863..ad1bfe9b 100644 --- a/.github/workflows/golden-test.yml +++ b/.github/workflows/golden-test.yml @@ -1,4 +1,4 @@ -name: Golden Test +name: Golden Test w/ grouping # NilAway output may change due to introduction of new feature or bug fixes. Since NilAway is still # at early stage of development, constantly updating / maintaining the golden test output will be @@ -9,7 +9,7 @@ on: jobs: golden-test: - name: Golden Test + name: Golden Test w/ grouping runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -24,13 +24,13 @@ jobs: go-version: 1.22.x cache: false - - name: Golden Test + - name: Golden Test w/ grouping id: golden_test # Run golden test by comparing HEAD and the base branch (the target branch of the PR). # GitHub Actions terminates the job if it hits the resource limits. Here we limit the # memory usage to 8GiB to avoid that. run: | - make golden-test GOMEMLIMIT=8192MiB ARGS="-base-branch ${{ github.event.pull_request.base.ref }} -result-file ${{ runner.temp }}/golden-test-result.md" + make golden-test GOMEMLIMIT=8192MiB ARGS="-base-branch ${{ github.event.pull_request.base.ref }} -result-file ${{ runner.temp }}/golden-test-result.md -group-error-messages=true" - uses: actions/github-script@v7 with: @@ -61,7 +61,7 @@ jobs: repo: repo, issue_number: issueNumber }); - const botComment = comments.data.find(comment => comment.user.login === 'github-actions[bot]' && comment.body.startsWith('## Golden Test')); + const botComment = comments.data.find(comment => comment.user.login === 'github-actions[bot]' && comment.body.startsWith('## Golden Test') && comment.body.endsWith('w/ grouping')); // Update or create the PR comment. if (botComment) { diff --git a/tools/cmd/golden-test/main.go b/tools/cmd/golden-test/main.go index fd513faf..25d97e0d 100644 --- a/tools/cmd/golden-test/main.go +++ b/tools/cmd/golden-test/main.go @@ -149,7 +149,7 @@ func Run(writer io.Writer, baseBranch, testBranch, groupingFlag string) error { branch.Result = diagnostics } - WriteDiff(writer, branches) + WriteDiff(writer, branches, groupingFlag) return nil } @@ -178,12 +178,16 @@ func ParseDiagnostics(reader io.Reader) (map[Diagnostic]bool, error) { // WriteDiff writes the summary and the diff (if the base and test are different) between the base // and test diagnostics to the writer. If the writer is os.Stdout, it will write the diff in color. -func WriteDiff(writer io.Writer, branches [2]*BranchResult) { +func WriteDiff(writer io.Writer, branches [2]*BranchResult, groupingFlag string) { // Compute the diagnostic differences between base and test branches. minuses, pluses := Diff(branches[0].Result, branches[1].Result), Diff(branches[1].Result, branches[0].Result) // Write the summary lines first. - MustFprint(fmt.Fprintf(writer, "## Golden Test\n\n")) + if groupingFlag == "true" { + MustFprint(fmt.Fprintf(writer, "## Golden Test w/ grouping\n\n")) + } else { + MustFprint(fmt.Fprintf(writer, "## Golden Test w/o grouping\n\n")) + } if len(pluses) == 0 && len(minuses) == 0 { MustFprint(fmt.Fprint(writer, "> [!NOTE] \n")) MustFprint(fmt.Fprintf(writer, "> ✅ NilAway errors reported on standard libraries are **identical**.\n")) diff --git a/tools/cmd/golden-test/main_test.go b/tools/cmd/golden-test/main_test.go index 63f932b6..28aef71d 100644 --- a/tools/cmd/golden-test/main_test.go +++ b/tools/cmd/golden-test/main_test.go @@ -64,7 +64,7 @@ func TestWriteDiff(t *testing.T) { {Name: "base", ShortSHA: "123456", Result: base}, {Name: "test", ShortSHA: "456789", Result: test}, } - WriteDiff(&buf, branches) + WriteDiff(&buf, branches, "true") require.Contains(t, buf.String(), "## Golden Test") // Must contain the title. require.Contains(t, buf.String(), "are **identical**") @@ -72,7 +72,7 @@ func TestWriteDiff(t *testing.T) { base[Diagnostic{Posn: "src/file2:10:2", Message: "nil pointer dereference"}] = true test[Diagnostic{Posn: "src/file4:10:2", Message: "bar error"}] = true buf.Reset() - WriteDiff(&buf, branches) + WriteDiff(&buf, branches, "true") s := buf.String() require.Contains(t, buf.String(), "## Golden Test") // Must contain the title. require.Contains(t, s, "are **different**") From 7bea03f879e3e00659e24687be3b1a6fe808f9a2 Mon Sep 17 00:00:00 2001 From: sonalmahajan15 Date: Thu, 25 Apr 2024 21:19:39 -0700 Subject: [PATCH 3/5] consolidated CI job --- .github/workflows/golden-test-no-grouping.yml | 81 ------------------- .github/workflows/golden-test.yml | 22 +++-- 2 files changed, 16 insertions(+), 87 deletions(-) delete mode 100644 .github/workflows/golden-test-no-grouping.yml diff --git a/.github/workflows/golden-test-no-grouping.yml b/.github/workflows/golden-test-no-grouping.yml deleted file mode 100644 index 7c0713ed..00000000 --- a/.github/workflows/golden-test-no-grouping.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Golden Test w/o grouping - -# NilAway output may change due to introduction of new feature or bug fixes. Since NilAway is still -# at early stage of development, constantly updating / maintaining the golden test output will be -# a burden. Therefore, we run this as a separate CI job and post the differences as a PR comment -# for manual reviews. -on: - pull_request: - -jobs: - golden-test-no-grouping: - name: Golden Test w/o grouping - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - name: Check out repository - - - name: Fetch base branch (${{ github.event.pull_request.base.ref }}) locally - run: git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }} - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: 1.22.x - cache: false - - - name: Golden Test w/o grouping - id: golden_test_no_grouping - # Run golden test by comparing HEAD and the base branch (the target branch of the PR). - # GitHub Actions terminates the job if it hits the resource limits. Here we limit the - # memory usage to 8GiB to avoid that. - run: | - make golden-test GOMEMLIMIT=8192MiB ARGS="-base-branch ${{ github.event.pull_request.base.ref }} -result-file ${{ runner.temp }}/golden-test-no-grouping-result.md -group-error-messages=false" - - - uses: actions/github-script@v7 - with: - script: | - const fsp = require('fs').promises; - - const issueNumber = context.issue.number; - const owner = context.repo.owner; - const repo = context.repo.repo; - const rawData = await fsp.readFile(`${{ runner.temp }}/golden-test-no-grouping-result.md`, 'utf8'); - - // GitHub API has a limit of 65536 bytes for a comment body, so here we shrink the - // diff part (anything between
and
) to 10,000 characters if it - // is too long. - const pattern = /(
)([\s\S]*?)(<\/details>)/; - - const body = rawData.replace(pattern, function(match, p1, p2, p3) { - if (p2.length > 10000) { - return p1 + p2.substring(0, 5000) + '\n\n ...(truncated)...\n\n' + p2.substring(p2.length - 5000) + p3; - } - // No need to change anything if it is not too long. - return match; - }); - - // First find the comments made by the bot. - const comments = await github.rest.issues.listComments({ - owner: owner, - repo: repo, - issue_number: issueNumber - }); - const botComment = comments.data.find(comment => comment.user.login === 'github-actions[bot]' && comment.body.startsWith('## Golden Test') && comment.body.endsWith('w/o grouping')); - - // Update or create the PR comment. - if (botComment) { - await github.rest.issues.updateComment({ - owner: owner, - repo: repo, - comment_id: botComment.id, - body: body - }); - } else { - await github.rest.issues.createComment({ - owner: owner, - repo: repo, - issue_number: issueNumber, - body: body - }); - } diff --git a/.github/workflows/golden-test.yml b/.github/workflows/golden-test.yml index ad1bfe9b..ceb1f0be 100644 --- a/.github/workflows/golden-test.yml +++ b/.github/workflows/golden-test.yml @@ -1,4 +1,4 @@ -name: Golden Test w/ grouping +name: Golden Test # NilAway output may change due to introduction of new feature or bug fixes. Since NilAway is still # at early stage of development, constantly updating / maintaining the golden test output will be @@ -9,8 +9,18 @@ on: jobs: golden-test: - name: Golden Test w/ grouping + name: Golden Test ${{ matrix.grouping }} runs-on: ubuntu-latest + strategy: + matrix: + include: + - grouping: 'w/ grouping' + group_errors: 'true' + test_id: 'golden_test_with_grouping' + - grouping: 'w/o grouping' + group_errors: 'false' + test_id: 'golden_test_without_grouping' + steps: - uses: actions/checkout@v4 name: Check out repository @@ -24,13 +34,13 @@ jobs: go-version: 1.22.x cache: false - - name: Golden Test w/ grouping - id: golden_test + - name: Golden Test ${{ matrix.grouping }} + id: ${{ matrix.test_id }} # Run golden test by comparing HEAD and the base branch (the target branch of the PR). # GitHub Actions terminates the job if it hits the resource limits. Here we limit the # memory usage to 8GiB to avoid that. run: | - make golden-test GOMEMLIMIT=8192MiB ARGS="-base-branch ${{ github.event.pull_request.base.ref }} -result-file ${{ runner.temp }}/golden-test-result.md -group-error-messages=true" + make golden-test GOMEMLIMIT=8192MiB ARGS="-base-branch ${{ github.event.pull_request.base.ref }} -result-file ${{ runner.temp }}/golden-test-result.md -group-error-messages=${{ matrix.group_errors }}" - uses: actions/github-script@v7 with: @@ -61,7 +71,7 @@ jobs: repo: repo, issue_number: issueNumber }); - const botComment = comments.data.find(comment => comment.user.login === 'github-actions[bot]' && comment.body.startsWith('## Golden Test') && comment.body.endsWith('w/ grouping')); + const botComment = comments.data.find(comment => comment.user.login === 'github-actions[bot]' && comment.body.startsWith('## Golden Test') && comment.body.endsWith(${{ matrix.grouping }})); // Update or create the PR comment. if (botComment) { From 1d1b14f56e8c391f5d8f9ea5951a14c4665f515c Mon Sep 17 00:00:00 2001 From: sonalmahajan15 Date: Thu, 25 Apr 2024 21:33:38 -0700 Subject: [PATCH 4/5] fix consolidated CI job --- .github/workflows/golden-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/golden-test.yml b/.github/workflows/golden-test.yml index ceb1f0be..311580a3 100644 --- a/.github/workflows/golden-test.yml +++ b/.github/workflows/golden-test.yml @@ -35,7 +35,7 @@ jobs: cache: false - name: Golden Test ${{ matrix.grouping }} - id: ${{ matrix.test_id }} + id: golden_test # Run golden test by comparing HEAD and the base branch (the target branch of the PR). # GitHub Actions terminates the job if it hits the resource limits. Here we limit the # memory usage to 8GiB to avoid that. @@ -71,7 +71,7 @@ jobs: repo: repo, issue_number: issueNumber }); - const botComment = comments.data.find(comment => comment.user.login === 'github-actions[bot]' && comment.body.startsWith('## Golden Test') && comment.body.endsWith(${{ matrix.grouping }})); + const botComment = comments.data.find(comment => comment.user.login === 'github-actions[bot]' && comment.body.startsWith('## Golden Test ${{ matrix.grouping }}')); // Update or create the PR comment. if (botComment) { From fc95ef9060dcebdb8bb5a52123850af03210a9fb Mon Sep 17 00:00:00 2001 From: sonalmahajan15 Date: Fri, 26 Apr 2024 15:30:33 -0700 Subject: [PATCH 5/5] remove unused field --- .github/workflows/golden-test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/golden-test.yml b/.github/workflows/golden-test.yml index 311580a3..3d320189 100644 --- a/.github/workflows/golden-test.yml +++ b/.github/workflows/golden-test.yml @@ -16,10 +16,8 @@ jobs: include: - grouping: 'w/ grouping' group_errors: 'true' - test_id: 'golden_test_with_grouping' - grouping: 'w/o grouping' group_errors: 'false' - test_id: 'golden_test_without_grouping' steps: - uses: actions/checkout@v4