fix: process input object, union and interface metadata in model intr…

…ospection schema codegen (#795)

* fix: add input object metadata in model introspection schema

* fix: use one place for types to skip

* add e2e tests

* fix CI failure

* fix cleanup script

* feat: add union and interface type in model intro schema

* fix test description

* fix e2e

* rm unused code

* fix: fine grain argument field type & better error msg for getType

* fix: remove new types in field type

* rename arguments to attributes

* rm union and interface in root level schema

* add unit test cases for list of scalar, enum and input

.codebuild/e2e_workflow.yml

@@ -145,24 +145,25 @@ batch:
       depend-on:
         - publish_to_local_registry
     - identifier: >-
-        l_build_app_ts_uninitialized_project_codegen_js_uninitialized_project_modelgen_android_uninitialized_project_modelgen_flutter
+        l_build_app_ts_push_codegen_admin_modelgen_uninitialized_project_codegen_js_uninitialized_project_modelgen_android
       buildspec: .codebuild/run_e2e_tests.yml
       env:
         compute-type: BUILD_GENERAL1_LARGE
         variables:
           TEST_SUITE: >-
-            src/__tests__/build-app-ts.test.ts|src/__tests__/uninitialized-project-codegen-js.test.ts|src/__tests__/uninitialized-project-modelgen-android.test.ts|src/__tests__/uninitialized-project-modelgen-flutter.test.ts
+            src/__tests__/build-app-ts.test.ts|src/__tests__/push-codegen-admin-modelgen.test.ts|src/__tests__/uninitialized-project-codegen-js.test.ts|src/__tests__/uninitialized-project-modelgen-android.test.ts
           CLI_REGION: ap-southeast-1
           DISABLE_ESLINT_PLUGIN: true
       depend-on:
         - publish_to_local_registry
-    - identifier: l_uninitialized_project_modelgen_ios_uninitialized_project_modelgen_js
+    - identifier: >-
+        l_uninitialized_project_modelgen_flutter_uninitialized_project_modelgen_ios_uninitialized_project_modelgen_js
       buildspec: .codebuild/run_e2e_tests.yml
       env:
         compute-type: BUILD_GENERAL1_LARGE
         variables:
           TEST_SUITE: >-
-            src/__tests__/uninitialized-project-modelgen-ios.test.ts|src/__tests__/uninitialized-project-modelgen-js.test.ts
+            src/__tests__/uninitialized-project-modelgen-flutter.test.ts|src/__tests__/uninitialized-project-modelgen-ios.test.ts|src/__tests__/uninitialized-project-modelgen-js.test.ts
           CLI_REGION: ap-southeast-2
       depend-on:
         - publish_to_local_registry
@@ -251,29 +252,30 @@ batch:
         - publish_to_local_registry
         - build_windows
     - identifier: >-
-        w_build_app_ts_uninitialized_project_codegen_js_uninitialized_project_modelgen_android_uninitialized_project_modelgen_flutter
+        w_build_app_ts_push_codegen_admin_modelgen_uninitialized_project_codegen_js_uninitialized_project_modelgen_android
       buildspec: .codebuild/run_e2e_tests_windows.yml
       env:
         compute-type: BUILD_GENERAL1_LARGE
         image: $WINDOWS_IMAGE_2019
         type: WINDOWS_SERVER_2019_CONTAINER
         variables:
           TEST_SUITE: >-
-            src/__tests__/build-app-ts.test.ts|src/__tests__/uninitialized-project-codegen-js.test.ts|src/__tests__/uninitialized-project-modelgen-android.test.ts|src/__tests__/uninitialized-project-modelgen-flutter.test.ts
+            src/__tests__/build-app-ts.test.ts|src/__tests__/push-codegen-admin-modelgen.test.ts|src/__tests__/uninitialized-project-codegen-js.test.ts|src/__tests__/uninitialized-project-modelgen-android.test.ts
           CLI_REGION: us-east-1
           DISABLE_ESLINT_PLUGIN: true
       depend-on:
         - publish_to_local_registry
         - build_windows
-    - identifier: w_uninitialized_project_modelgen_ios_uninitialized_project_modelgen_js
+    - identifier: >-
+        w_uninitialized_project_modelgen_flutter_uninitialized_project_modelgen_ios_uninitialized_project_modelgen_js
       buildspec: .codebuild/run_e2e_tests_windows.yml
       env:
         compute-type: BUILD_GENERAL1_LARGE
         image: $WINDOWS_IMAGE_2019
         type: WINDOWS_SERVER_2019_CONTAINER
         variables:
           TEST_SUITE: >-
-            src/__tests__/uninitialized-project-modelgen-ios.test.ts|src/__tests__/uninitialized-project-modelgen-js.test.ts
+            src/__tests__/uninitialized-project-modelgen-flutter.test.ts|src/__tests__/uninitialized-project-modelgen-ios.test.ts|src/__tests__/uninitialized-project-modelgen-js.test.ts
           CLI_REGION: us-east-1
       depend-on:
         - publish_to_local_registry

packages/amplify-codegen-e2e-core/src/utils/sdk-calls.ts

@@ -14,6 +14,7 @@ import {
   AmplifyBackend,
 } from 'aws-sdk';
 import _ from 'lodash';
+import { getProjectMeta } from './projectMeta';
 
 export const getDDBTable = async (tableName: string, region: string) => {
   const service = new DynamoDB({ region });
@@ -42,6 +43,19 @@ export const bucketNotExists = async (bucket: string) => {
   }
 };
 
+export const getDeploymentBucketObject = async (projectRoot: string, objectKey: string) => {
+  const meta = getProjectMeta(projectRoot);
+  const deploymentBucket = meta.providers.awscloudformation.DeploymentBucketName;
+  const s3 = new S3();
+  const result = await s3
+    .getObject({
+      Bucket: deploymentBucket,
+      Key: objectKey,
+    })
+    .promise();
+  return result.Body.toLocaleString();
+};
+
 export const deleteS3Bucket = async (bucket: string, providedS3Client: S3 | undefined = undefined) => {
   const s3 = providedS3Client ? providedS3Client : new S3();
   let continuationToken: Required<Pick<S3.ListObjectVersionsOutput, 'KeyMarker' | 'VersionIdMarker'>> = undefined;

packages/amplify-codegen-e2e-tests/schemas/admin-modelgen.graphql

@@ -0,0 +1,53 @@
+input AMPLIFY { globalAuthRule: AuthRule = { allow: public } } # FOR TESTING ONLY!
+
+type Todo @model {
+  id: ID!
+  name: String!
+  description: String
+  phone: Phone
+}
+type Phone {
+  number: String
+}
+enum BillingSource {
+  CLIENT
+  PROJECT
+}
+input CustomInput {
+  customField1: String!
+  customField2: BillingSource
+  customField3: NestedInput!
+}
+input NestedInput {
+  content: String! = "hello"
+}
+interface ICustom {
+  firstName: String!
+  lastName: String
+  birthdays: [INestedCustom!]!
+}
+interface INestedCustom {
+  birthDay: AWSDate!
+}
+# The member types of a Union type must all be Object base types.
+union CustomUnion = Todo | Phone
+
+type Query {
+  getAllTodo(msg: String, input: CustomInput): String @function(name: "echofunction-${env}")
+  echo(msg: String!): String
+  echo2(todoId: ID!): Todo
+  echo3: [Todo!]!
+  echo4(number: String): Phone
+  echo5: [CustomUnion!]!
+  echo6(customInput: CustomInput): String!
+  echo7: [ICustom]!
+  echo8(msg: [Float], msg2: [Int!], enumType: BillingSource, enumList: [BillingSource], inputType: [CustomInput]): [String]
+  echo9(msg: [Float]!, msg2: [Int!]!, enumType: BillingSource!, enumList: [BillingSource!]!, inputType: [CustomInput!]!): [String!]!    
+
+}
+type Mutation {
+  mutate(msg: [String!]!): Todo
+}
+type Subscription {
+  onMutate(msg: String): [Todo!]
+}

packages/amplify-codegen-e2e-tests/src/__tests__/push-codegen-admin-modelgen.test.ts

@@ -0,0 +1,19 @@
+import { DEFAULT_JS_CONFIG, createNewProjectDir } from "@aws-amplify/amplify-codegen-e2e-core";
+import { deleteAmplifyProject, testPushAdminModelgen, testPushCodegen } from "../codegen-tests-base";
+
+const schema = 'admin-modelgen.graphql';
+
+describe('Amplify push with codegen tests - admin modelgen', () => {
+  let projectRoot: string;
+  beforeEach(async () => {
+    projectRoot = await createNewProjectDir('pushCodegenAdminModelgen');
+  });
+
+  afterEach(async () => {
+    await deleteAmplifyProject(projectRoot);
+  });
+
+  it(`should not throw error for executing the admin modelgen step required by studio CMS usage post push given the schema with input, union and interface types`, async () => {
+    await testPushAdminModelgen(DEFAULT_JS_CONFIG, projectRoot, schema);
+  });
+});

packages/amplify-codegen-e2e-tests/src/cleanup-e2e-resources.ts

@@ -45,6 +45,7 @@ type AmplifyAppInfo = {
 
 type S3BucketInfo = {
   name: string;
+  region: string;
   jobId?: string;
   cbInfo?: CodeBuild.Build;
 };
@@ -118,7 +119,16 @@ const getOrphanS3TestBuckets = async (account: AWSAccountInfo): Promise<S3Bucket
   const s3Client = new aws.S3(getAWSConfig(account));
   const listBucketResponse = await s3Client.listBuckets().promise();
   const staleBuckets = listBucketResponse.Buckets.filter(testBucketStalenessFilter);
-  return staleBuckets.map(it => ({ name: it.Name }));
+  const bucketInfos = await Promise.all(
+    staleBuckets.map(async (staleBucket): Promise<S3BucketInfo> => {
+      const region = await getBucketRegion(account, staleBucket.Name);
+      return {
+        name: staleBucket.Name,
+        region,
+      };
+    }),
+  );
+  return bucketInfos;
 };
 
 /**
@@ -276,27 +286,52 @@ const getJobCodeBuildDetails = async (jobIds: string[]): Promise<CodeBuild.Build
   }
 };
 
+const getBucketRegion = async (account: AWSAccountInfo, bucketName: string): Promise<string> => {
+  const awsConfig = getAWSConfig(account);
+  const s3Client = new aws.S3(awsConfig);
+  const location = await s3Client.getBucketLocation({ Bucket: bucketName }).promise();
+  const region = location.LocationConstraint ?? 'us-east-1';
+  return region;
+};
+
 const getS3Buckets = async (account: AWSAccountInfo): Promise<S3BucketInfo[]> => {
-  const s3Client = new aws.S3(getAWSConfig(account));
+  const awsConfig = getAWSConfig(account);
+  const s3Client = new aws.S3(awsConfig);
   const buckets = await s3Client.listBuckets().promise();
   const result: S3BucketInfo[] = [];
   for (const bucket of buckets.Buckets) {
+    let region: string | undefined;
     try {
-      const bucketDetails = await s3Client.getBucketTagging({ Bucket: bucket.Name }).promise();
+      region = await getBucketRegion(account, bucket.Name);
+      // Operations on buckets created in opt-in regions appear to require region-specific clients
+      const regionalizedClient = new aws.S3({
+        region,
+        ...(awsConfig as object),
+      });
+      const bucketDetails = await regionalizedClient.getBucketTagging({ Bucket: bucket.Name }).promise();
       const jobId = getJobId(bucketDetails.TagSet);
       if (jobId) {
         result.push({
           name: bucket.Name,
+          region,
           jobId
         });
       }
     } catch (e) {
-      if (e.code !== 'NoSuchTagSet' && e.code !== 'NoSuchBucket') {
+      // TODO: Why do we process the bucket even with these particular errors?
+      if (e.code === 'NoSuchTagSet' || e.code === 'NoSuchBucket') {
+        result.push({
+          name: bucket.Name,
+          region: region ?? 'us-east-1',
+        });
+      } else if (e.code === 'InvalidToken') {
+        // We see some buckets in some accounts that were somehow created in an opt-in region different from the one to which the account is
+        // actually opted in. We don't quite know how this happened, but for now, we'll make a note of the inconsistency and continue
+        // processing the rest of the buckets.
+        console.error(`Skipping processing ${account.accountId}, bucket ${bucket.Name}`, e);
+      } else {
         throw e;
       }
-      result.push({
-        name: bucket.Name,
-      });
     }
   }
   return result;
@@ -516,8 +551,12 @@ const deleteBucket = async (account: AWSAccountInfo, accountIndex: number, bucke
   const { name } = bucket;
   try {
     console.log(`${generateAccountInfo(account, accountIndex)} Deleting S3 Bucket ${name}`);
-    const s3 = new aws.S3(getAWSConfig(account));
-    await deleteS3Bucket(name, s3);
+    const awsConfig = getAWSConfig(account);
+    const regionalizedS3Client = new aws.S3({
+      region: bucket.region,
+      ...(awsConfig as object),
+    });
+    await deleteS3Bucket(name, regionalizedS3Client);
   } catch (e) {
     console.log(`${generateAccountInfo(account, accountIndex)} Deleting bucket ${name} failed with error ${e.message}`);
     if (e.code === 'ExpiredTokenException') {

packages/amplify-codegen-e2e-tests/src/codegen-tests-base/push-codegen.ts

@@ -6,7 +6,10 @@ import {
     amplifyPushWithCodegenAdd,
     AmplifyFrontendConfig,
     amplifyPushWithCodegenUpdate,
-    updateAPIWithResolutionStrategyWithModels
+    updateAPIWithResolutionStrategyWithModels,
+    getProjectMeta,
+    getDeploymentBucketObject,
+    amplifyPush
 } from "@aws-amplify/amplify-codegen-e2e-core";
 import { existsSync } from "fs";
 import path from 'path';
@@ -38,3 +41,38 @@ export async function testPushCodegen(config: AmplifyFrontendConfig, projectRoot
     expect(existsSync(userSourceCodePath)).toBe(true);
     expect(isNotEmptyDir(path.join(projectRoot, config.modelgenDir))).toBe(true);
 }
+
+export async function testPushAdminModelgen(config: AmplifyFrontendConfig, projectRoot: string, schema: string) {
+    // init project and add API category
+    await initProjectWithProfile(projectRoot, { ...config, disableAmplifyAppCreation: false, });
+    const {
+      DeploymentBucketName: bucketName,
+      Region: region,
+      AmplifyAppId: appId,
+    } = getProjectMeta(projectRoot).providers.awscloudformation;
+
+    expect(bucketName).toBeDefined()
+    expect(region).toBeDefined();
+    expect(appId).toBeDefined();
+
+    const projectName = createRandomName();
+    await addApiWithoutSchema(projectRoot, { apiName: projectName });
+    await updateApiSchema(projectRoot, projectName, schema);
+    // add codegen succeeds
+    await amplifyPush(projectRoot);
+
+    /**
+     * Source code from
+     * https://github.com/aws-amplify/amplify-cli/blob/1da5de70c57b15a76f02c92364af4889d1585229/packages/amplify-provider-awscloudformation/src/admin-modelgen.ts#L85-L93
+     */
+    const s3ApiModelsPrefix = `models/${projectName}/`;
+    const cmsArtifactLocalToS3Keys = [
+      `${s3ApiModelsPrefix}schema.graphql`,
+      `${s3ApiModelsPrefix}schema.js`,
+      `${s3ApiModelsPrefix}modelIntrospection.json`,
+    ];
+    // expect CMS assets to be present in S3
+    cmsArtifactLocalToS3Keys.forEach(async (key) => {
+      await expect(getDeploymentBucketObject(projectRoot, key)).resolves.not.toThrow();
+    });
+}

packages/appsync-modelgen-plugin/API.md

@@ -28,7 +28,7 @@ export interface AppSyncModelPluginConfig extends RawDocumentsConfig {
 // @public (undocumented)
 export type Argument = {
     name: string;
-    type: FieldType;
+    type: InputFieldType;
     isArray: boolean;
     isRequired: boolean;
     isArrayNullable?: boolean;
@@ -94,14 +94,27 @@ export type FieldAttribute = ModelAttribute;
 export type Fields = Record<string, Field>;
 
 // @public (undocumented)
-export type FieldType = 'ID' | 'String' | 'Int' | 'Float' | 'AWSDate' | 'AWSTime' | 'AWSDateTime' | 'AWSTimestamp' | 'AWSEmail' | 'AWSURL' | 'AWSIPAddress' | 'Boolean' | 'AWSJSON' | 'AWSPhone' | {
+export type FieldType = ScalarType | {
     enum: string;
 } | {
     model: string;
 } | {
     nonModel: string;
 };
 
+// @public (undocumented)
+export type Input = {
+    name: string;
+    attributes: Arguments;
+};
+
+// @public (undocumented)
+export type InputFieldType = ScalarType | {
+    enum: string;
+} | {
+    input: string;
+};
+
 // @public (undocumented)
 export type ModelAttribute = {
     type: string;
@@ -119,6 +132,7 @@ export type ModelIntrospectionSchema = {
     queries?: SchemaQueries;
     mutations?: SchemaMutations;
     subscriptions?: SchemaSubscriptions;
+    inputs?: SchemaInputs;
 };
 
 // Warning: (ae-forgotten-export) The symbol "RawAppSyncModelConfig" needs to be exported by the entry point index.d.ts
@@ -136,6 +150,9 @@ export type PrimaryKeyInfo = {
     sortKeyFieldNames: string[];
 };
 
+// @public (undocumented)
+export type ScalarType = 'ID' | 'String' | 'Int' | 'Float' | 'AWSDate' | 'AWSTime' | 'AWSDateTime' | 'AWSTimestamp' | 'AWSEmail' | 'AWSURL' | 'AWSIPAddress' | 'Boolean' | 'AWSJSON' | 'AWSPhone';
+
 // @public (undocumented)
 export type SchemaEnum = {
     name: string;
@@ -145,6 +162,9 @@ export type SchemaEnum = {
 // @public (undocumented)
 export type SchemaEnums = Record<string, SchemaEnum>;
 
+// @public (undocumented)
+export type SchemaInputs = Record<string, Input>;
+
 // @public (undocumented)
 export type SchemaModel = {
     name: string;