chore(codegen): generate and run SSDK protocol tests during CI (#2558)

This sets up a server-only mode for generate-clients, and a set of yarn targets that generate and run server protocol tests. The CodeBuild CI for clients uses Node 10, while SSDKs require Node 14, so this sets up an additional GitHub action for running the SSDK protocol tests and excludes SSDK protocol tests from the client targets. When the client is updated to support 14 as a minimum version, this distinction can be removed.
aws · Jul 14, 2021 · c2e88a4 · c2e88a4
1 parent dff5cd4
commit c2e88a4
Show file tree

Hide file tree

Showing 7 changed files with 142 additions and 6 deletions.
diff --git a/.github/workflows/server-protocol-tests.yml b/.github/workflows/server-protocol-tests.yml
@@ -0,0 +1,49 @@
+name: server-protocol-tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-java@v1
+        with:
+          java-version: '11'
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          cache: 'yarn'
+
+      - name: build and publish smithy-typescript
+        run: |
+          git clone --depth 1 https://github.com/awslabs/smithy-typescript.git
+          cd smithy-typescript
+          ./gradlew clean build publishToMavenLocal
+          cd ..
+
+      - name: install dependencies
+        run: |
+          yarn install
+
+      - name: generate protocol tests
+        run: |
+          yarn generate-clients --server-artifacts
+
+      # there are new dependencies as a result of generating server protocol tests
+      - name: install dependencies again
+        run: |
+          yarn install
+
+      - name: run protocol tests
+        run: |
+          yarn test:server-protocols
+
+
diff --git a/codegen/protocol-test-codegen/smithy-build.json b/codegen/protocol-test-codegen/smithy-build.json
@@ -118,6 +118,11 @@
             "license": "Apache-2.0"
           },
           "private": true
+        },
+        "typescript-ssdk-codegen": {
+          "package": "@aws-sdk/aws-restjson-server",
+          "packageVersion": "1.0.0-alpha.1",
+          "disableDefaultValidation": true
         }
       }
     },

diff --git a/...codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AwsProtocolUtils.java b/...codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AwsProtocolUtils.java
@@ -293,6 +293,16 @@ private static boolean filterProtocolTests(
         if (testCase.getId().equals("QueryCustomizedError")) {
             return true;
         }
+        if (testCase.getId().equals("RestJsonStreamingTraitsRequireLengthWithBlob") && settings.generateServerSdk()) {
+            return true;
+        }
+        //TODO: enable with Smithy 1.10
+        if ((testCase.getId().equals("RestJsonAllQueryStringTypes")
+                || testCase.getId().equals("RestJsonQueryStringEscaping"))
+                && settings.generateServerSdk()) {
+            return true;
+        }
+
         return false;
     }
 }
diff --git a/package.json b/package.json
@@ -13,15 +13,17 @@
     "clear-build-info": "rimraf ./packages/**/*.tsbuildinfo ./clients/**/*.tsbuildinfo  ./lib/**/*.tsbuildinfo",
     "remove-documentation": "rimraf ./docs",
     "build:crypto-dependencies": "lerna run --scope '@aws-sdk/{types,util-utf8-browser,util-locate-window,hash-node}' --include-dependencies build",
-    "build:protocols": "yarn build:crypto-dependencies && lerna run --scope '@aws-sdk/aws-*' --include-dependencies build",
+    "build:protocols": "yarn build:crypto-dependencies && lerna run --scope '@aws-sdk/aws-*' --ignore '@aws-sdk/*-server' --include-dependencies build",
+    "build:server-protocols": "yarn build:crypto-dependencies && lerna run --scope '@aws-sdk/*-server' --include-dependencies build",
     "build:all": "yarn build:crypto-dependencies && lerna run build",
     "build-documentation": "yarn remove-documentation && typedoc",
     "pretest:all": "yarn build:all",
     "test:all": "jest --coverage --passWithNoTests && lerna run test --scope '@aws-sdk/{fetch-http-handler,hash-blob-browser}'",
     "test:functional": "jest --config tests/functional/jest.config.js --passWithNoTests",
     "test:integration-legacy": "cucumber-js --fail-fast",
     "test:integration": "jest --config jest.config.integ.js --passWithNoTests",
-    "test:protocols": "yarn build:protocols && lerna run test --scope '@aws-sdk/aws-*'",
+    "test:protocols": "yarn build:protocols && lerna run test --scope '@aws-sdk/aws-*' --ignore '@aws-sdk/*-server'",
+    "test:server-protocols": "yarn build:server-protocols && lerna run test --scope '@aws-sdk/*-server'",
     "pretest:e2e": "yarn build:crypto-dependencies && lerna run --scope '@aws-sdk/{client-cloudformation,karma-credential-loader}' --include-dependencies build",
     "test:e2e": "node ./tests/e2e/index.js",
     "local-publish": "node ./scripts/verdaccio-publish/index.js"
@@ -118,4 +120,4 @@
     ],
     "**/*.{ts,js,md,json}": "prettier --write"
   }
-}
+}
diff --git a/scripts/generate-clients/code-prettify.js b/scripts/generate-clients/code-prettify.js
@@ -2,11 +2,16 @@
 const { spawnProcess } = require("../utils/spawn-process");
 const path = require("path");
 
-const prettifyCode = async (dir) =>
-  spawnProcess(path.join(__dirname, "..", "..", "node_modules", ".bin", "pprettier"), [
+const prettifyCode = async (dir) => {
+  await spawnProcess(path.join(__dirname, "..", "..", "node_modules", ".bin", "pprettier"), [
     "--write",
     `${dir}/*/typescript-codegen/**/*.{ts,js,md,json}`,
   ]);
+  await spawnProcess(path.join(__dirname, "..", "..", "node_modules", ".bin", "pprettier"), [
+    "--write",
+    `${dir}/*/typescript-ssdk-codegen/**/*.{ts,js,md,json}`,
+  ]);
+};
 
 module.exports = {
   prettifyCode,

diff --git a/scripts/generate-clients/copy-to-clients.js b/scripts/generate-clients/copy-to-clients.js
@@ -151,6 +151,53 @@ const copyToClients = async (sourceDir, destinationDir) => {
   }
 };
 
+const copyServerTests = async (sourceDir, destinationDir) => {
+  for (const modelName of readdirSync(sourceDir)) {
+    if (modelName === "source") continue;
+
+    const artifactPath = join(sourceDir, modelName, "typescript-ssdk-codegen");
+    const packageManifestPath = join(artifactPath, "package.json");
+    if (!existsSync(packageManifestPath)) {
+      console.error(`${modelName} generates empty server, skip.`);
+      continue;
+    }
+
+    const packageManifest = JSON.parse(readFileSync(packageManifestPath).toString());
+    const packageName = packageManifest.name;
+    const testName = packageName.replace("@aws-sdk/", "");
+
+    console.log(`copying ${packageName} from ${artifactPath} to ${destinationDir}`);
+    const destPath = join(destinationDir, testName);
+    const overwritablePredicate = getOverwritablePredicate(packageName);
+
+    for (const packageSub of readdirSync(artifactPath)) {
+      const packageSubPath = join(artifactPath, packageSub);
+      const destSubPath = join(destPath, packageSub);
+
+      if (packageSub === "package.json") {
+        //copy manifest file
+        const destManifest = existsSync(destSubPath) ? JSON.parse(readFileSync(destSubPath).toString()) : {};
+        const mergedManifest = {
+          ...mergeManifest(packageManifest, destManifest),
+          homepage: `https://github.com/aws/aws-sdk-js-v3/tree/main/protocol_tests/${testName}`,
+          repository: {
+            type: "git",
+            url: "https://github.com/aws/aws-sdk-js-v3.git",
+            directory: `protocol_tests/${testName}`,
+          },
+        };
+        writeFileSync(destSubPath, JSON.stringify(mergedManifest, null, 2).concat(`\n`));
+      } else if (overwritablePredicate(packageSub) || !existsSync(destSubPath)) {
+        if (lstatSync(packageSubPath).isDirectory()) removeSync(destSubPath);
+        copySync(packageSubPath, destSubPath, {
+          overwrite: true,
+        });
+      }
+    }
+  }
+};
+
 module.exports = {
   copyToClients,
+  copyServerTests,
 };
diff --git a/scripts/generate-clients/index.js b/scripts/generate-clients/index.js
@@ -3,7 +3,7 @@ const yargs = require("yargs");
 const path = require("path");
 const { emptyDirSync, rmdirSync } = require("fs-extra");
 const { generateClients, generateProtocolTests } = require("./code-gen");
-const { copyToClients } = require("./copy-to-clients");
+const { copyToClients, copyServerTests } = require("./copy-to-clients");
 const {
   CODE_GEN_SDK_OUTPUT_DIR,
   CODE_GEN_PROTOCOL_TESTS_OUTPUT_DIR,
@@ -19,6 +19,7 @@ const {
   globs,
   output: clientsDir,
   noProtocolTest,
+  s: serverOnly,
 } = yargs
   .alias("m", "models")
   .string("m")
@@ -34,10 +35,27 @@ const {
   .alias("n", "noProtocolTest")
   .boolean("n")
   .describe("n", "Disable generating protocol test files")
+  .alias("s", "server-artifacts")
+  .boolean("s")
+  .describe("s", "Generate server artifacts instead of client ones")
+  .default("s", false)
+  .conflicts("s", ["m", "g", "n"])
   .help().argv;
 
 (async () => {
   try {
+    if (serverOnly) {
+      await generateProtocolTests();
+      await prettifyCode(CODE_GEN_PROTOCOL_TESTS_OUTPUT_DIR);
+      await copyServerTests(CODE_GEN_PROTOCOL_TESTS_OUTPUT_DIR, PROTOCOL_TESTS_CLIENTS_DIR);
+
+      emptyDirSync(CODE_GEN_PROTOCOL_TESTS_OUTPUT_DIR);
+      emptyDirSync(TEMP_CODE_GEN_INPUT_DIR);
+
+      rmdirSync(TEMP_CODE_GEN_INPUT_DIR);
+      return;
+    }
+
     await generateClients(models || globs);
     if (!noProtocolTest) await generateProtocolTests();