[docker] Check that version matches tag before publishing release images

We currently dont check that the tag and the aptos-node cargo.toml version match This ensures that before publish we have to check the tag and version match only for release images Test Plan: wrote unittests and added a testing workflow made the script possibel to unittest
aptos-labs · Jan 23, 2024 · 5d5f6d4 · 5d5f6d4
1 parent 5ae2542
commit 5d5f6d4
Show file tree

Hide file tree

Showing 6 changed files with 2,514 additions and 126 deletions.
diff --git a/.github/workflows/test-copy-images-to-dockerhub.yaml b/.github/workflows/test-copy-images-to-dockerhub.yaml
@@ -0,0 +1,28 @@
+name: Test Release Images
+on:
+  pull_request:
+    paths:
+      - "docker/release-images.mjs"
+      - "docker/__tests__/**"
+  push:
+    branches:
+      - main
+    paths:
+      - "docker/release-images.mjs"
+      - "docker/__tests__/**"
+
+permissions:
+  contents: read
+
+jobs:
+  test-copy-images:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version-file: .node-version
+      - uses: pnpm/action-setup@v2
+      - run: pnpm install
+      - name: Test Release Images
+        run: ./docker/test.sh
diff --git a/docker/__tests__/release-images.test.js b/docker/__tests__/release-images.test.js
@@ -0,0 +1,30 @@
+#!/usr/bin/env -S pnpm test release-images.test.js
+
+import { assertTagMatchesSourceVersion, getImageReleaseGroupByImageTagPrefix, isReleaseImage } from '../release-images.mjs';
+
+describe('releaseImages', () => {
+    it('gets aptos-node as the default image group', () => {
+        const prefix = 'image-banana';
+        const releaseGroup = getImageReleaseGroupByImageTagPrefix(prefix);
+        expect(releaseGroup).toEqual('aptos-node');
+    });
+    it('gets indexer image group', () => {
+        const prefix = 'aptos-indexer-grpc-vX.Y.Z';
+        const releaseGroup = getImageReleaseGroupByImageTagPrefix(prefix);
+        expect(releaseGroup).toEqual('aptos-indexer-grpc');
+    });
+    it('gets aptos-node as the node image group', () => {
+        const prefix = 'aptos-node-vX.Y.Z';
+        const releaseGroup = getImageReleaseGroupByImageTagPrefix(prefix);
+        expect(releaseGroup).toEqual('aptos-node');
+    });
+    it('determines image is a release image', () => {
+        expect(isReleaseImage("nightly-banana")).toEqual(false);
+        expect(isReleaseImage("aptos-node-v1.2.3")).toEqual(true);
+    });
+    it('asserts version match', () => {
+        // toThrow apparently matches a prefix, so this works but it does actually test against the real config version
+        // Which... hilariously means this would fail if the version was ever 0.0.0
+        expect(() => assertTagMatchesSourceVersion("aptos-node-v0.0.0")).toThrow("image tag does not match cargo version: aptos-node-v0.0.0");
+    });
+});
diff --git a/docker/release-images.mjs b/docker/release-images.mjs
@@ -31,6 +31,7 @@
 // You can also run this script locally with the DRY_RUN flag to test it out:
 // IMAGE_TAG_PREFIX=devnet AWS_ACCOUNT_ID=bla GCP_DOCKER_ARTIFACT_REPO_US=bla GCP_DOCKER_ARTIFACT_REPO=bla GIT_SHA=bla ./docker/release-images.mjs --wait-for-image-seconds=3600 --dry-run
 //
+// You can also run unittests by running docker/__tests__/release-images.test.js
 
 // When we release aptos-node, we also want to release related images for tooling, testing, etc. Similarly, other images have other related images
 // that we can release together, ie in a release group.
@@ -98,118 +99,183 @@ import { execSync } from "node:child_process";
 import { dirname } from "node:path";
 import { chdir } from "node:process";
 import { promisify } from "node:util";
+import fs from "node:fs";
+
 const sleep = promisify(setTimeout);
 
-chdir(dirname(process.argv[1]) + "/.."); // change workdir to the root of the repo
-// install repo pnpm dependencies
-execSync("pnpm install --frozen-lockfile", { stdio: "inherit" });
-await import("zx/globals");
+// These are lazy globals
+let toml;
+let core;
+
+function pnpmInstall() {
+  // change workdir to the root of the repo
+  chdir(dirname(process.argv[1]) + "/..");
+  // install repo pnpm dependencies
+  execSync("pnpm install --frozen-lockfile", { stdio: "inherit" });
+}
 
-const REQUIRED_ARGS = ["GIT_SHA", "GCP_DOCKER_ARTIFACT_REPO", "GCP_DOCKER_ARTIFACT_REPO_US", "AWS_ACCOUNT_ID", "IMAGE_TAG_PREFIX"];
-const OPTIONAL_ARGS = ["WAIT_FOR_IMAGE_SECONDS", "DRY_RUN"];
+// This is kinda gross but means we can just run the script directly and it will
+// work without running pnpm install before hand...
+async function lazyImports() {
+  await import("zx/globals");
+  toml = await import("toml");
+  core = await import("@actions/core");
+}
 
-const parsedArgs = {};
+const Environment = {
+  CI: "ci",
+  LOCAL: "local",
+  TEST: "test",
+};
 
-for (const arg of REQUIRED_ARGS) {
-  const argValue = argv[arg.toLowerCase().replaceAll("_", "-")] ?? process.env[arg];
-  if (!argValue) {
-    console.error(chalk.red(`ERROR: Missing required argument or environment variable: ${arg}`));
-    process.exit(1);
+
+function getEnvironment() {
+  if (process.env.CI === "true") {
+    return Environment.CI;
+  } else if (import.meta.jest !== undefined) {
+    return Environment.TEST;
+  } else {
+    return Environment.LOCAL;
   }
-  parsedArgs[arg] = argValue;
 }
 
-for (const arg of OPTIONAL_ARGS) {
-  const argValue = argv[arg.toLowerCase().replaceAll("_", "-")] ?? process.env[arg];
-  parsedArgs[arg] = argValue;
+function reportError(message, opts={throwOnFailure: false}) {
+  if (getEnvironment() === Environment.CI) {
+    core.setFailed(message);
+  } else if (getEnvironment() === Environment.LOCAL) {
+    console.error(message);
+  } else if (getEnvironment() === Environment.TEST) {
+    // Errors in tests are expected and mess up formatting
+    console.log(message);
+  }
+  if (opts.throwOnFailure) {
+    throw new Error(message);
+  }
 }
 
-let crane;
+async function main() {
+  const REQUIRED_ARGS = ["GIT_SHA", "GCP_DOCKER_ARTIFACT_REPO", "GCP_DOCKER_ARTIFACT_REPO_US", "AWS_ACCOUNT_ID", "IMAGE_TAG_PREFIX"];
+  const OPTIONAL_ARGS = ["WAIT_FOR_IMAGE_SECONDS", "DRY_RUN"];
 
-if (process.env.CI === "true") {
-  console.log("installing crane automatically in CI");
-  await $`curl -sL https://github.com/google/go-containerregistry/releases/download/v0.15.1/go-containerregistry_Linux_x86_64.tar.gz > crane.tar.gz`;
-  const sha = (await $`shasum -a 256 ./crane.tar.gz | awk '{ print $1 }'`).toString().trim();
-  if (sha !== "d4710014a3bd135eb1d4a9142f509cfd61d2be242e5f5785788e404448a4f3f2") {
-    console.error(chalk.red(`ERROR: sha256 mismatch for crane.tar.gz got: ${sha}`));
-    process.exit(1);
+  const parsedArgs = {};
+
+  await assertExecutingInRepoRoot();
+
+  for (const arg of REQUIRED_ARGS) {
+    const argValue = argv[arg.toLowerCase().replaceAll("_", "-")] ?? process.env[arg];
+    if (!argValue) {
+      console.error(chalk.red(`ERROR: Missing required argument or environment variable: ${arg}`));
+      process.exit(1);
+    }
+    parsedArgs[arg] = argValue;
   }
-  await $`tar -xf crane.tar.gz`;
-  crane = "./crane";
-} else {
-  if ((await $`command -v crane`.exitCode) !== 0) {
-    console.log(
-      chalk.red(
-        "ERROR: could not find crane binary in PATH - follow https://github.com/google/go-containerregistry/tree/main/cmd/crane#installation to install",
-      ),
-    );
-    process.exit(1);
+
+  for (const arg of OPTIONAL_ARGS) {
+    const argValue = argv[arg.toLowerCase().replaceAll("_", "-")] ?? process.env[arg];
+    parsedArgs[arg] = argValue;
   }
-  crane = "crane";
-}
 
-const AWS_ECR = `${parsedArgs.AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/aptos`;
-const GCP_ARTIFACT_REPO = parsedArgs.GCP_DOCKER_ARTIFACT_REPO;
-const GCP_ARTIFACT_REPO_US = parsedArgs.GCP_DOCKER_ARTIFACT_REPO_US;
-const DOCKERHUB = "docker.io/aptoslabs";
+  let crane;
 
-const INTERNAL_TARGET_REGISTRIES = [
-  GCP_ARTIFACT_REPO,
-  GCP_ARTIFACT_REPO_US,
-  AWS_ECR,
-];
+  if (getEnvironment() === Environment.CI) {
+    console.log("installing crane automatically in CI");
+    await $`curl -sL https://github.com/google/go-containerregistry/releases/download/v0.15.1/go-containerregistry_Linux_x86_64.tar.gz > crane.tar.gz`;
+    const sha = (await $`shasum -a 256 ./crane.tar.gz | awk '{ print $1 }'`).toString().trim();
+    if (sha !== "d4710014a3bd135eb1d4a9142f509cfd61d2be242e5f5785788e404448a4f3f2") {
+      console.error(chalk.red(`ERROR: sha256 mismatch for crane.tar.gz got: ${sha}`));
+      process.exit(1);
+    }
+    await $`tar -xf crane.tar.gz`;
+    crane = "./crane";
+  } else {
+    if ((await $`command -v crane`.exitCode) !== 0) {
+      console.log(
+        chalk.red(
+          "ERROR: could not find crane binary in PATH - follow https://github.com/google/go-containerregistry/tree/main/cmd/crane#installation to install",
+        ),
+      );
+      process.exit(1);
+    }
+    crane = "crane";
+  }
 
-const ALL_TARGET_REGISTRIES = [
-  ...INTERNAL_TARGET_REGISTRIES,
-  DOCKERHUB,
-];
+  const AWS_ECR = `${parsedArgs.AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/aptos`;
+  const GCP_ARTIFACT_REPO = parsedArgs.GCP_DOCKER_ARTIFACT_REPO;
+  const GCP_ARTIFACT_REPO_US = parsedArgs.GCP_DOCKER_ARTIFACT_REPO_US;
+  const DOCKERHUB = "docker.io/aptoslabs";
 
-// default 10 seconds
-parsedArgs.WAIT_FOR_IMAGE_SECONDS = parseInt(parsedArgs.WAIT_FOR_IMAGE_SECONDS ?? 10, 10);
+  const INTERNAL_TARGET_REGISTRIES = [
+    GCP_ARTIFACT_REPO,
+    GCP_ARTIFACT_REPO_US,
+    AWS_ECR,
+  ];
 
-// dry run
-console.log(`INFO: dry run: ${parsedArgs.DRY_RUN}`);
+  const ALL_TARGET_REGISTRIES = [
+    ...INTERNAL_TARGET_REGISTRIES,
+    DOCKERHUB,
+  ];
 
-// get the appropriate release group based on the image tag prefix
-const imageReleaseGroup = getImageReleaseGroupByImageTagPrefix(parsedArgs.IMAGE_TAG_PREFIX);
-console.log(`INFO: image release group: ${imageReleaseGroup}`);
+  // default 10 seconds
+  parsedArgs.WAIT_FOR_IMAGE_SECONDS = parseInt(parsedArgs.WAIT_FOR_IMAGE_SECONDS ?? 10, 10);
 
-// only release the images that are part of the release group
-const imageNamesToRelease = IMAGES_TO_RELEASE_BY_RELEASE_GROUP[imageReleaseGroup];
-console.log(`INFO: image names to release: ${JSON.stringify(imageNamesToRelease)}`);
+  // dry run
+  console.log(`INFO: dry run: ${parsedArgs.DRY_RUN}`);
 
-// iterate over all images to release, including their release configurations
-const imagesToRelease = {};
-for (const imageName of imageNamesToRelease) {
-  imagesToRelease[imageName] = IMAGES_TO_RELEASE[imageName];
-}
-for (const [image, imageConfig] of Object.entries(imagesToRelease)) {
-  for (const [profile, features] of Object.entries(imageConfig)) {
-    // build profiles that are not the default "release" will have a separate prefix
-    const profilePrefix = profile === "release" ? "" : profile;
-    for (const feature of features) {
-      const featureSuffix = feature === Features.Default ? "" : feature;
-      const targetRegistries = IMAGES_TO_RELEASE_ONLY_INTERNAL.includes(image) ? INTERNAL_TARGET_REGISTRIES : ALL_TARGET_REGISTRIES;
-
-      for (const targetRegistry of targetRegistries) {
-        const imageSource = `${parsedArgs.GCP_DOCKER_ARTIFACT_REPO}/${image}:${joinTagSegments(
-          profilePrefix,
-          featureSuffix,
-          parsedArgs.GIT_SHA,
-        )}`;
-        const imageTarget = `${targetRegistry}/${image}:${joinTagSegments(parsedArgs.IMAGE_TAG_PREFIX, profilePrefix, featureSuffix)}`;
-        console.info(chalk.green(`INFO: copying ${imageSource} to ${imageTarget}`));
-        if (parsedArgs.DRY_RUN) {
-          continue;
+  // Assert if we have a release version that the version matches the cargo version
+  if (isReleaseImage(parsedArgs.IMAGE_TAG_PREFIX)) {
+    assertTagMatchesSourceVersion(parsedArgs.IMAGE_TAG_PREFIX);
+  }
+
+  // get the appropriate release group based on the image tag prefix
+  const imageReleaseGroup = getImageReleaseGroupByImageTagPrefix(parsedArgs.IMAGE_TAG_PREFIX);
+  console.log(`INFO: image release group: ${imageReleaseGroup}`);
+
+  // only release the images that are part of the release group
+  const imageNamesToRelease = IMAGES_TO_RELEASE_BY_RELEASE_GROUP[imageReleaseGroup];
+  console.log(`INFO: image names to release: ${JSON.stringify(imageNamesToRelease)}`);
+
+  // iterate over all images to release, including their release configurations
+  const imagesToRelease = {};
+  for (const imageName of imageNamesToRelease) {
+    imagesToRelease[imageName] = IMAGES_TO_RELEASE[imageName];
+  }
+  for (const [image, imageConfig] of Object.entries(imagesToRelease)) {
+    for (const [profile, features] of Object.entries(imageConfig)) {
+      // build profiles that are not the default "release" will have a separate prefix
+      const profilePrefix = profile === "release" ? "" : profile;
+      for (const feature of features) {
+        const featureSuffix = feature === Features.Default ? "" : feature;
+        const targetRegistries = IMAGES_TO_RELEASE_ONLY_INTERNAL.includes(image) ? INTERNAL_TARGET_REGISTRIES : ALL_TARGET_REGISTRIES;
+
+        for (const targetRegistry of targetRegistries) {
+          const imageSource = `${parsedArgs.GCP_DOCKER_ARTIFACT_REPO}/${image}:${joinTagSegments(
+            profilePrefix,
+            featureSuffix,
+            parsedArgs.GIT_SHA,
+          )}`;
+          const imageTarget = `${targetRegistry}/${image}:${joinTagSegments(parsedArgs.IMAGE_TAG_PREFIX, profilePrefix, featureSuffix)}`;
+          console.info(chalk.green(`INFO: copying ${imageSource} to ${imageTarget}`));
+          if (parsedArgs.DRY_RUN) {
+            continue;
+          }
+          await waitForImageToBecomeAvailable(imageSource, parsedArgs.WAIT_FOR_IMAGE_SECONDS);
+          await $`${crane} copy ${imageSource} ${imageTarget}`;
+          await $`${crane} copy ${imageSource} ${joinTagSegments(imageTarget, parsedArgs.GIT_SHA)}`;
         }
-        await waitForImageToBecomeAvailable(imageSource, parsedArgs.WAIT_FOR_IMAGE_SECONDS);
-        await $`${crane} copy ${imageSource} ${imageTarget}`;
-        await $`${crane} copy ${imageSource} ${joinTagSegments(imageTarget, parsedArgs.GIT_SHA)}`;
       }
     }
   }
 }
 
+async function assertExecutingInRepoRoot() {
+  const gitRoot = (await $`git rev-parse --show-toplevel`).toString().trim();
+  const currentDir = process.cwd();
+  if (gitRoot !== currentDir) {
+    console.error(chalk.red(`ERROR: must execute this script from the root of the repo: ${gitRoot}`));
+    process.exit(1);
+  }
+}
+
 // joinTagSegments joins tag segments with a dash, but only if the segment is not empty
 function joinTagSegments(...segments) {
   return segments.filter((s) => s).join("_");
@@ -218,7 +284,7 @@ function joinTagSegments(...segments) {
 // The image tag prefix is used to determine the release group. Examples:
 // * tag a release as "aptos-node-vX.Y.Z"
 // * tag a release as "aptos-indexer-grpc-vX.Y.Z"
-function getImageReleaseGroupByImageTagPrefix(prefix) {
+export function getImageReleaseGroupByImageTagPrefix(prefix) {
   // iterate over the keys in IMAGES_TO_RELEASE_BY_RELEASE_GROUP
   // if the prefix includes the release group, then return the release group
   for (const [imageReleaseGroup, imagesToRelease] of Object.entries(IMAGES_TO_RELEASE_BY_RELEASE_GROUP)) {
@@ -230,6 +296,28 @@ function getImageReleaseGroupByImageTagPrefix(prefix) {
   return "aptos-node";
 }
 
+const APTOS_RELEASE_REGEX = /aptos-node-v(\d+\.\d+\.\d+)/;
+
+export function assertTagMatchesSourceVersion(imageTag) {
+  const config = toml.parse(fs.readFileSync("aptos-node/Cargo.toml"));
+  const configVersion = config.package.version;
+  if (!doesTagMatchConfig(imageTag, configVersion)) {
+    reportError(`image tag does not match cargo version: ${imageTag} !== ${configVersion}`, {throwOnFailure: true});
+  }
+}
+
+export function doesTagMatchConfig(imageTag, configVersion) {
+  if (!APTOS_RELEASE_REGEX.test(imageTag)) {
+    reportError(`image tag does not match cargo version: ${imageTag} !== ${configVersion}`, {throwOnFailure: true});
+  }
+  const version = imageTag.match(APTOS_RELEASE_REGEX)[1];
+  return version === configVersion;
+}
+
+export function isReleaseImage(imageTag) {
+  return APTOS_RELEASE_REGEX.test(imageTag);
+}
+
 async function waitForImageToBecomeAvailable(imageToWaitFor, waitForImageSeconds) {
   const WAIT_TIME_IN_BETWEEN_ATTEMPTS = 10000; // 10 seconds in ms
   const startTimeMs = Date.now();
@@ -263,3 +351,14 @@ async function waitForImageToBecomeAvailable(imageToWaitFor, waitForImageSeconds
   );
   process.exit(1);
 }
+
+// This prevents tests from executing main
+if (import.meta.jest === undefined) {
+  pnpmInstall();
+  await lazyImports();
+  await main()
+} else {
+  // Because we do this weird import in order to test we also have to resolve imports
+  // However we force the caller to actually install pnpm first here
+  await lazyImports();
+}
diff --git a/docker/test.sh b/docker/test.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# cd to repo root
+cd "$(git rev-parse --show-toplevel)"
+
+pnpm install
+pnpm test docker/__tests__