ci: Retain complete a3p-integration directories and resulting slogfil…

…es (#10446)

## Description
Extracted from #10165 and dependent upon Agoric/agoric-3-proposals#193 .
This expands `test-docker-build` CI job artifacts from just core eval scripts to their containing a3p-integration directory plus the job's slogfile.

### Security Considerations
n/a

### Scaling Considerations
The final slogfile from #10165 was about 67 MB, which is smaller than the 161 MB deployment-test-results artifact. But regardless, I've limited retention to ~~4 days on success/10 days on failure~~ 10 days, and we could also compress before upload if even that is too much.

### Documentation Considerations
n/a

### Testing Considerations
n/a

### Upgrade Considerations
n/a

.github/actions/post-test/action.yml

@@ -30,7 +30,7 @@ runs:
         DATADOG_SITE: ${{ inputs.datadog-site }}
       continue-on-error: true
       run: |
-        ./scripts/ci-collect-testruns.sh
+        ./scripts/ci/collect-testruns.sh
         if [ "${{ inputs.datadog-token }}" != "" ]; then
           export DATADOG_API_KEY="${{ inputs.datadog-token }}"
           export DD_ENV="ci"

.github/workflows/integration.yml

@@ -229,6 +229,14 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
+      - name: update job environment
+        run: |
+          if [ ":$HOME" = : ]; then
+            # Keep the first two path components, e.g. /home/runner.
+            HOME="(pwd | sed -E 's#^(/[^/]+/[^/]+).*#\1#')"
+            echo "HOME=$HOME" >> "$GITHUB_ENV"
+          fi
+          echo "SLOGFILE=$HOME/test.slog" >> "$GITHUB_ENV"
       - name: free up disk space
         run: |
           # Workaround to provide additional free space for testing.
@@ -279,30 +287,28 @@ jobs:
       - name: run proposals tests
         run: yarn test
         working-directory: a3p-integration
-      - name: collect all core eval scripts
-        # Core eval scripts will be copied under /tmp/core_eval_scripts directory
-        # colons in the parent directory name will be replaced with dashes
+      - name: copy a3p-integration
+        if: (success() || failure())
         run: |
-          find . -type d -path "./a3p-integration/proposals/*/submission" | while read -r dir; do
-            # Get the parent directory name
-            parent_dir=$(basename "$(dirname "$dir")")
-            # Replace colons with dashes in the parent directory name
-            sanitized_parent_dir=${parent_dir//:/-}
-            # Create the destination directory under /tmp/core_eval_scripts if it doesn't exist
-            destination_dir="/tmp/core_eval_scripts/$sanitized_parent_dir"
-            mkdir -p "$destination_dir"
-            # Copy everything from the $dir to the destination directory
-            cp -r "$dir"/* "$destination_dir/"
-            echo "Copied contents of $dir to $destination_dir"
-          done
-      - name: archive core eval scripts
-        # The core eval scripts can be found at the bottom of `Summary` page of
-        # `Integration tests` workflow once the workflow is completed.
-        # Ref: https://github.com/actions/upload-artifact?tab=readme-ov-file#where-does-the-upload-go
+          dir='/tmp/export/a3p-integration'
+          rm -rf "$dir"
+          scripts/ci/export-a3p.sh a3p-integration "$dir"
+      # Artifacts can be found at the bottom of `Summary` page of
+      # `Integration tests` workflow once the workflow is completed.
+      # Ref: https://github.com/actions/upload-artifact?tab=readme-ov-file#where-does-the-upload-go
+      - name: archive a3p-integration
+        if: (success() || failure())
+        uses: actions/upload-artifact@v4
+        with:
+          name: a3p-integration
+          path: /tmp/export/a3p-integration
+      - name: archive slogfile
+        if: (success() || failure())
         uses: actions/upload-artifact@v4
         with:
-          name: core-eval-scripts
-          path: /tmp/core_eval_scripts
+          name: slogfile
+          path: ${{ env.SLOGFILE }}
+          retention-days: 10
       - name: notify on failure
         if: failure() && github.event_name != 'pull_request'
         uses: ./.github/actions/notify-status

a3p-integration/package.json

@@ -13,7 +13,7 @@
     "doctor": "yarn synthetic-chain doctor"
   },
   "dependencies": {
-    "@agoric/synthetic-chain": "^0.3.0",
+    "@agoric/synthetic-chain": "^0.4.0",
     "@types/better-sqlite3": "^7.6.11"
   },
   "packageManager": "[email protected]",

a3p-integration/yarn.lock

@@ -5,9 +5,9 @@ __metadata:
   version: 8
   cacheKey: 10c0
 
-"@agoric/synthetic-chain@npm:^0.3.0":
-  version: 0.3.0
-  resolution: "@agoric/synthetic-chain@npm:0.3.0"
+"@agoric/synthetic-chain@npm:^0.4.0":
+  version: 0.4.0
+  resolution: "@agoric/synthetic-chain@npm:0.4.0"
   dependencies:
     "@endo/zip": "npm:^1.0.7"
     better-sqlite3: "npm:^9.6.0"
@@ -16,7 +16,7 @@ __metadata:
     execa: "npm:^9.3.1"
   bin:
     synthetic-chain: dist/cli/cli.js
-  checksum: 10c0/17c6241bdc48b8a2a7608c9d4d7c0a0c76fb10d4ee44a31a1150104a792bcd1133f4b1a7e8ab26673a07450b3ceabccd9911999117568221b49221b6ee4306a1
+  checksum: 10c0/3cb31bba30f4c64ac459da39aed639a3ab0405aa1d867b4897bbad21cedd40f2142785be7f2e55b7198c58c6a31674fb534e115f7e20ebe20f4de80bfb55e7c8
   languageName: node
   linkType: hard
 
@@ -1034,7 +1034,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "root-workspace-0b6124@workspace:."
   dependencies:
-    "@agoric/synthetic-chain": "npm:^0.3.0"
+    "@agoric/synthetic-chain": "npm:^0.4.0"
     "@types/better-sqlite3": "npm:^7.6.11"
   languageName: unknown
   linkType: soft

scripts/ci/export-a3p.sh

@@ -0,0 +1,50 @@
+#! /usr/bin/env bash
+set -ueo pipefail
+
+# Usage: export-a3p.sh <a3p_dir> <dest_dir>
+# Copy an agoric-3-proposals directory, omitting `agoric-sdk`, `node_modules`,
+# and `.yarn`, and replacing colons in proposal directory names (children of
+# $a3p_dir/proposals) with dashes.
+# https://github.com/Agoric/agoric-3-proposals
+#
+# The result is ready to be uploaded as an artifact from GitHub Actions.
+# https://github.com/actions/upload-artifact
+
+a3p_dir="${1-}"
+dest_dir="${2-}"
+if ! [ "$#" -eq 2 -a -n "$a3p_dir" -a -n "$dest_dir" ]; then
+  echo "Usage: $0 <a3p_dir> <dest_dir>" >&2
+  exit 64
+fi
+if [ -e "$dest_dir" ]; then
+  echo "Destination already exists: $dest_dir"
+  exit 1
+fi
+
+# Because of the exclusions, copying applies to disjoint subtrees of $a3p_dir:
+# * children except `agoric-sdk` and `proposals`
+# * file children of `proposals`, i.e. `proposals/*`
+# * grandchildren of `proposals`, i.e. `proposals/*/*`
+#
+# We accomplish this by consuming (subdir?, depth, `find` filter?) tuples.
+SUBSEP="$(printf '\x1C')"
+printf "    $SUBSEP 1 $SUBSEP -not -name agoric-sdk -not -name proposals
+  proposals $SUBSEP 1 $SUBSEP -type f
+  proposals $SUBSEP 2
+" \
+  | sed -E 's/^ *|#.*//g; /^[[:space:]]*$/d;' \
+  | while IFS="$SUBSEP" read subdir depth filter; do
+    find "$a3p_dir/"$subdir -mindepth ${depth:-1} -maxdepth ${depth:-1} \
+      -name node_modules -prune -o -name .yarn -prune -o $filter -print \
+      | while read -r path; do
+        relpath="${path#"$a3p_dir/"}"
+        reldir="$(dirname -- "$relpath")"
+        dest_subdir="$dest_dir/$reldir"
+        [ "$reldir" = . ] && dest_subdir="$dest_dir" # remove trailing `/.`
+        sanitized="${dest_subdir//:/-}"              # replace each `:` with `-`
+        mkdir -p "$sanitized"
+        cp -r -- "$path" "$sanitized/"
+        echo "Copied $(dirname -- "$path") to $sanitized"
+      done \
+      | uniq
+  done