Support Modular Auto Generation

tools/js-sdk-release-tools/package.json

@@ -1,12 +1,16 @@
 {
   "name": "@azure-tools/js-sdk-release-tools",
-  "version": "2.7.11",
+  "version": "2.7.12",
   "description": "",
+  "files": [
+    "dist"
+  ],
   "scripts": {
     "dev": "tsx watch src/changelogToolCli.ts",
     "start": "node dist/changelogToolCli.js",
     "debug": "node --inspect-brk dist/changelogToolCli.js",
-    "build": "rimraf dist && tsc -p .",
+    "build": "rimraf dist && tsc -p . && npm run copy-files",
+    "copy-files": "copyfiles -f src/common/ciYamlTemplates/*.template.yml dist/common/ciYamlTemplates/",
     "prepack": "npm run build",
     "test": "vitest --run",
     "test:watch": "vitest"
@@ -22,17 +26,18 @@
   "license": "MIT",
   "dependencies": {
     "@azure-tools/openapi-tools-common": "^1.2.2",
+    "@npmcli/package-json": "^5.2.0",
     "@ts-common/azure-js-dev-tools": "^21.1.0",
     "colors": "1.4.0",
     "command-line-args": "^5.1.1",
     "comment-json": "^4.1.0",
+    "copyfiles": "^2.4.1",
     "fs-extra": "^11.2.0",
     "glob": "^11.0.0",
     "js-yaml": "^4.1.0",
     "parse-ts-to-ast": "^0.1.1",
     "semver": "^7.3.5",
     "shelljs": "^0.8.4",
-    "shx": "^0.3.4",
     "simple-git": "^3.5.0",
     "ts-morph": "^23.0.0",
     "tslib": "^1.9.3",
@@ -42,7 +47,9 @@
     "yaml": "^1.10.2"
   },
   "devDependencies": {
+    "@types/fs-extra": "^11.0.4",
     "@types/node": "^20.12.12",
+    "@types/npmcli__package-json": "^4.0.4",
     "@types/shelljs": "^0.8.15",
     "@types/unixify": "^1.0.2",
     "rimraf": "^3.0.2",

tools/js-sdk-release-tools/packages/typescript-codegen-breaking-change-detector/src/azure/detect-breaking-changes.ts

@@ -145,7 +145,7 @@ async function detectBreakingChangesCore(projectContext: ProjectContext): Promis
     );
     return breakingChangeResults;
   } catch (err) {
-    logger.error(`Failed to detect breaking changes due to ${(err as Error).stack ?? err}`);
+    logger.error(`Failed to detect breaking changes due to ${(err as Error)?.stack ?? err}`);
     return undefined;
   }
 }

tools/js-sdk-release-tools/src/autoGenerateInPipeline.ts

@@ -1,88 +1,112 @@
 #!/usr/bin/env node
 
 import * as path from 'path';
-import { generateMgmt } from "./hlc/generateMgmt";
+import { generateMgmt } from './hlc/generateMgmt';
 import { backupNodeModules, restoreNodeModules } from './utils/backupNodeModules';
-import { logger } from "./utils/logger";
-import { generateRLCInPipeline } from "./llc/generateRLCInPipeline/generateRLCInPipeline";
-import { RunningEnvironment } from "./utils/runningEnvironment";
+import { logger } from './utils/logger';
+import { generateRLCInPipeline } from './llc/generateRLCInPipeline/generateRLCInPipeline';
+import { ModularClientPackageOptions, SDKType } from './common/types';
+import { generateAzureSDKPackage } from './mlc/clientGenerator/modularClientPackageGenerator';
+import { parseInputJson } from './utils/generateInputUtils';
 
 const shell = require('shelljs');
 const fs = require('fs');
 
-async function automationGenerateInPipeline(inputJsonPath: string, outputJsonPath: string, use: string | undefined, typespecEmitter: string | undefined, sdkGenerationType: string | undefined) {
+async function automationGenerateInPipeline(
+    inputJsonPath: string,
+    outputJsonPath: string,
+    use: string | undefined,
+    typespecEmitter: string | undefined,
+    sdkGenerationType: string | undefined,
+    local: boolean
+) {
     const inputJson = JSON.parse(fs.readFileSync(inputJsonPath, { encoding: 'utf-8' }));
-    const specFolder: string = inputJson['specFolder'];
-    const readmeFiles: string[] | string | undefined = inputJson['relatedReadmeMdFiles'] ? inputJson['relatedReadmeMdFiles'] : inputJson['relatedReadmeMdFile'];
-    const typespecProjectFolder: string[] | string | undefined = inputJson['relatedTypeSpecProjectFolder'];
-    const gitCommitId: string = inputJson['headSha'];
-    const repoHttpsUrl: string = inputJson['repoHttpsUrl'];
-    const autorestConfig: string | undefined = inputJson['autorestConfig'];
-    const downloadUrlPrefix: string | undefined = inputJson.installInstructionInput?.downloadUrlPrefix;
-    const skipGeneration: boolean | undefined = inputJson['skipGeneration'];
+    const {
+        sdkType,
+        specFolder,
+        readmeMd,
+        gitCommitId,
+        outputJson,
+        repoHttpsUrl,
+        downloadUrlPrefix,
+        skipGeneration,
+        runningEnvironment,
+        typespecProject,
+        autorestConfig
+    } = await parseInputJson(inputJson);
 
-    if (!readmeFiles && !typespecProjectFolder) {
-        throw new Error(`readme files and typespec project info are both undefined`);
-    }
-
-    if (readmeFiles && (typeof readmeFiles !== 'string') && readmeFiles.length !== 1) {
-        throw new Error(`get ${readmeFiles.length} readme files`);
-    }
-
-    if (typespecProjectFolder && (typeof typespecProjectFolder !== 'string') && typespecProjectFolder.length !== 1) {
-        throw new Error(`get ${typespecProjectFolder.length} typespec project`);
-    }
-
-    const isTypeSpecProject = !!typespecProjectFolder;
-
-    const packages: any[] = [];
-    const outputJson = {
-        packages: packages
-    };
-    const readmeMd = isTypeSpecProject ? undefined : typeof readmeFiles === 'string' ? readmeFiles : readmeFiles![0];
-    const typespecProject = isTypeSpecProject ? typeof typespecProjectFolder === 'string' ? typespecProjectFolder : typespecProjectFolder![0] : undefined;
-    const isMgmt = isTypeSpecProject ? false : readmeMd!.includes('resource-manager');
-    const runningEnvironment = typeof readmeFiles === 'string' || typeof typespecProjectFolder === 'string' ? RunningEnvironment.SdkGeneration : RunningEnvironment.SwaggerSdkAutomation;
     try {
-        await backupNodeModules(String(shell.pwd()));
-        if (isMgmt) {
-            await generateMgmt({
-                sdkRepo: String(shell.pwd()),
-                swaggerRepo: specFolder,
-                readmeMd: readmeMd!,
-                gitCommitId: gitCommitId,
-                use: use,
-                outputJson: outputJson,
-                swaggerRepoUrl: repoHttpsUrl,
-                downloadUrlPrefix: downloadUrlPrefix,
-                skipGeneration: skipGeneration,
-                runningEnvironment: runningEnvironment
-            });
-        } else {
-            await generateRLCInPipeline({
-                sdkRepo: String(shell.pwd()),
-                swaggerRepo: path.isAbsolute(specFolder) ? specFolder : path.join(String(shell.pwd()), specFolder),
-                readmeMd: readmeMd,
-                typespecProject: typespecProject,
-                autorestConfig,
-                use: use,
-                typespecEmitter: !!typespecEmitter ? typespecEmitter : `@azure-tools/typespec-ts`,
-                outputJson: outputJson,
-                skipGeneration: skipGeneration,
-                sdkGenerationType: (sdkGenerationType === "command") ? "command" : "script",
-                runningEnvironment: runningEnvironment,
-                swaggerRepoUrl: repoHttpsUrl,
-                gitCommitId: gitCommitId,
-            })
+        if (!local) {
+            await backupNodeModules(String(shell.pwd()));
+        }
+        switch (sdkType) {
+            case SDKType.HighLevelClient:
+                await generateMgmt({
+                    sdkRepo: String(shell.pwd()),
+                    swaggerRepo: specFolder,
+                    readmeMd: readmeMd!,
+                    gitCommitId: gitCommitId,
+                    use: use,
+                    outputJson: outputJson,
+                    swaggerRepoUrl: repoHttpsUrl,
+                    downloadUrlPrefix: downloadUrlPrefix,
+                    skipGeneration: skipGeneration,
+                    runningEnvironment: runningEnvironment
+                });
+                break;
+            case SDKType.RestLevelClient:
+                await generateRLCInPipeline({
+                    sdkRepo: String(shell.pwd()),
+                    swaggerRepo: path.isAbsolute(specFolder) ? specFolder : path.join(String(shell.pwd()), specFolder),
+                    readmeMd: readmeMd,
+                    typespecProject: typespecProject,
+                    autorestConfig,
+                    use: use,
+                    typespecEmitter: !!typespecEmitter ? typespecEmitter : `@azure-tools/typespec-ts`,
+                    outputJson: outputJson,
+                    skipGeneration: skipGeneration,
+                    sdkGenerationType: sdkGenerationType === 'command' ? 'command' : 'script',
+                    runningEnvironment: runningEnvironment,
+                    swaggerRepoUrl: repoHttpsUrl,
+                    gitCommitId: gitCommitId
+                });
+                break;
+
+            case SDKType.ModularClient: {
+                const typeSpecDirectory = path.posix.join(specFolder, typespecProject!);
+                const sdkRepoRoot = String(shell.pwd());
+                const skip = skipGeneration ?? false;
+                const repoUrl = repoHttpsUrl;
+                const options: ModularClientPackageOptions = {
+                    sdkRepoRoot,
+                    specRepoRoot: specFolder,
+                    typeSpecDirectory,
+                    gitCommitId,
+                    skip,
+                    repoUrl,
+                    local,
+                    // support MPG for now
+                    versionPolicyName: 'management'
+                };
+                const packageResult = await generateAzureSDKPackage(options);
+                outputJson.packages = [packageResult];
+                break;
+            }
+            default:
+                break;
         }
     } catch (e) {
-        const packageName = outputJson.packages?.[0].packageName;
-        logger.error(`Failed to generate SDK for package ${"'" + packageName + "'" ?? ''} due to ${(e as Error)?.stack ?? e}.`);
+        const packageNameStr = `'${outputJson.packages?.[0]?.packageName}' `;
+        logger.error(`Failed to generate SDK for package ${packageNameStr ?? ''}due to ${(e as Error)?.stack ?? e}.`);
         logger.error(`Please review the detail errors for potential fixes.`);
-        logger.error(`If the issue persists, contact the support channel at https://aka.ms/azsdk/js-teams-channel and include this spec pull request.`)
+        logger.error(
+            `If the issue persists, contact the support channel at https://aka.ms/azsdk/js-teams-channel and include this spec pull request.`
+        );
         throw e;
     } finally {
-        await restoreNodeModules(String(shell.pwd()));
+        if (!local) {
+            await restoreNodeModules(String(shell.pwd()));
+        }
         fs.writeFileSync(outputJsonPath, JSON.stringify(outputJson, null, '  '), { encoding: 'utf-8' });
     }
 }
@@ -93,10 +117,13 @@ const optionDefinitions = [
     { name: 'sdkGenerationType', type: String },
     { name: 'inputJsonPath', type: String },
     { name: 'outputJsonPath', type: String },
+    // this option should be only used in local run, it will skip backup node modules, etc.
+    // do NOT set to true in sdk automation pipeline 
+    { name: 'local', type: Boolean, defaultValue: false }
 ];
 const commandLineArgs = require('command-line-args');
 const options = commandLineArgs(optionDefinitions);
-automationGenerateInPipeline(options.inputJsonPath, options.outputJsonPath, options.use, options.typespecEmitter, options.sdkGenerationType).catch(e => {
+automationGenerateInPipeline(options.inputJsonPath, options.outputJsonPath, options.use, options.typespecEmitter, options.sdkGenerationType, options.local ?? false).catch(e => {
     logger.error(e.message);
     process.exit(1);
 });

tools/js-sdk-release-tools/src/common/ciYamlTemplates/ci.mgmt.template.yml

@@ -0,0 +1,33 @@
+# NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file.
+
+trigger:
+  branches:
+    include:
+      - main
+      - feature/*
+      - release/*
+      - hotfix/*
+    exclude:
+      - feature/v4
+  paths:
+    include: null
+
+pr:
+  branches:
+    include:
+      - main
+      - feature/*
+      - release/*
+      - hotfix/*
+    exclude:
+      - feature/v4
+  paths:
+    include: null
+
+extends:
+  template: /eng/pipelines/templates/stages/archetype-sdk-client.yml
+  parameters:
+    ServiceDirectory: null
+    Artifacts:
+      - name: null
+        safeName: null