feat(amazonq): /doc: add support for infrastructure diagrams

packages/amazonq/.changes/next-release/Feature-088c8b44-eb51-4495-84fb-2e02878ff4bc.json

@@ -0,0 +1,4 @@
+{
+	"type": "Feature",
+	"description": "Amazon Q /doc: Add support for infrastructure diagrams"
+}

packages/core/src/amazonqDoc/controllers/chat/controller.ts

@@ -35,7 +35,7 @@ import { BaseChatSessionStorage } from '../../../amazonq/commons/baseChatStorage
 import { DocMessenger } from '../../messenger'
 import { AuthController } from '../../../amazonq/auth/controller'
 import { openUrl } from '../../../shared/utilities/vsCodeUtils'
-import { openDeletedDiff, openDiff } from '../../../amazonq/commons/diff'
+import { createAmazonQUri, openDeletedDiff, openDiff } from '../../../amazonq/commons/diff'
 import {
     getWorkspaceFoldersByPrefixes,
     getWorkspaceRelativePath,
@@ -203,7 +203,14 @@ export class DocController {
                     uploadId = session?.state?.uploadHistory[codeGenerationId].uploadId
                 }
                 const rightPath = path.join(uploadId, zipFilePath)
-                await openDiff(pathInfos.absolutePath, rightPath, tabId, this.scheme)
+
+                if (rightPath.toLowerCase().endsWith('.svg')) {
+                    // use open instead of diff for svg
+                    const rightPathUri = createAmazonQUri(rightPath, tabId, this.scheme);
+                    await vscode.commands.executeCommand('vscode.open', rightPathUri);
+                } else {
+                    await openDiff(pathInfos.absolutePath, rightPath, tabId, this.scheme);
+                }
             }
         }
     }

packages/core/src/amazonqDoc/session/sessionState.ts

@@ -49,6 +49,7 @@ import {
     WorkspaceEmptyError,
 } from '../errors'
 import { DocMessenger } from '../messenger'
+import { ZipStream } from '../../shared'
 
 abstract class CodeGenBase {
     private pollCount = 360
@@ -363,7 +364,9 @@ export class PrepareCodeGenState implements SessionState {
                 this.config.workspaceRoots,
                 this.config.workspaceFolders,
                 action.telemetry,
-                span
+                span,
+                new ZipStream(),
+                true
             )
             const uploadId = randomUUID()
             const { uploadUrl, kmsKeyArn } = await this.config.proxyClient.createUploadUrl(

packages/core/src/amazonqFeatureDev/util/files.ts

@@ -5,7 +5,11 @@
 
 import * as vscode from 'vscode'
 import * as path from 'path'
-import { collectFiles } from '../../shared/utilities/workspaceUtils'
+import {
+    collectFiles2,
+    CollectFilesFileFilter,
+    DEFAULT_EXTRA_EXCLUDE_PATTERNS,
+} from '../../shared/utilities/workspaceUtils'
 
 import { ContentLengthError, PrepareRepoFailedError } from '../errors'
 import { getLogger } from '../../shared/logger/logger'
@@ -26,6 +30,15 @@ export async function checkForDevFile(root: string) {
     return hasDevFile
 }
 
+function isInfraDiagramFile(relativePath: string) {
+    return (
+        relativePath.toLowerCase() == 'infra.dot' ||
+        relativePath.toLowerCase().endsWith('/infra.dot') ||
+        relativePath.toLowerCase() == 'infra.svg' ||
+        relativePath.toLowerCase().endsWith('/infra.svg')
+    )
+}
+
 /**
  * given the root path of the repo it zips its files in memory and generates a checksum for it.
  */
@@ -34,13 +47,42 @@ export async function prepareRepoData(
     workspaceFolders: CurrentWsFolders,
     telemetry: TelemetryHelper,
     span: Span<AmazonqCreateUpload>,
-    zip: ZipStream = new ZipStream()
+    zip: ZipStream = new ZipStream(),
+    isIncludeInfraDiagram: boolean = false
 ) {
     try {
         const autoBuildSetting = CodeWhispererSettings.instance.getAutoBuildSetting()
         const useAutoBuildFeature = autoBuildSetting[repoRootPaths[0]] ?? false
+        const extraExcludeFilePatterns:string[] = [];
+        let extraFileFilterFn: CollectFilesFileFilter | undefined = undefined;
+
         // We only respect gitignore file rules if useAutoBuildFeature is on, this is to avoid dropping necessary files for building the code (e.g. png files imported in js code)
-        const files = await collectFiles(repoRootPaths, workspaceFolders, true, maxRepoSizeBytes, !useAutoBuildFeature)
+        if (!useAutoBuildFeature) {
+            if (isIncludeInfraDiagram) {
+                // ensure svg is not filtered out by files search
+                extraExcludeFilePatterns.push(...(DEFAULT_EXTRA_EXCLUDE_PATTERNS.filter(p=>!p.endsWith('.svg'))));
+                // ensure only infra diagram is included from all png files
+                extraFileFilterFn = (relativePath: string) => {
+                    if (!relativePath.toLowerCase().endsWith(".svg")) {
+                        return false;
+                    }
+                    return !isInfraDiagramFile(relativePath);
+                };
+            } else {
+                extraExcludeFilePatterns.push(...DEFAULT_EXTRA_EXCLUDE_PATTERNS)
+            }
+        }
+
+        const files = await collectFiles2(
+            repoRootPaths,
+            workspaceFolders,
+            {
+                maxSizeLimitInBytes: maxRepoSizeBytes,
+                useGitIgnoreFileAsFilter: true,
+                extraExcludeFilePatterns,
+                extraFileFilterFn
+            }
+        );
 
         let totalBytes = 0
         const ignoredExtensionMap = new Map<string, number>()
@@ -58,9 +100,16 @@ export async function prepareRepoData(
             }
             const isCodeFile_ = isCodeFile(file.relativeFilePath)
             const isDevFile = file.relativeFilePath === 'devfile.yaml'
-            // When useAutoBuildFeature is on, only respect the gitignore rules filtered earlier and apply the size limit, otherwise, exclude all non code files and gitignore files
-            const isNonCodeFileAndIgnored = useAutoBuildFeature ? false : !isCodeFile_ || isDevFile
-            if (fileSize >= maxFileSizeBytes || isNonCodeFileAndIgnored) {
+            const isInfraDiagramFileExt = isInfraDiagramFile(file.relativeFilePath);
+
+            let isExcludeFile = fileSize >= maxFileSizeBytes;
+            // When useAutoBuildFeature is on, only respect the gitignore rules filtered earlier and apply the size limit
+            if (!isExcludeFile && !useAutoBuildFeature) {
+                isExcludeFile = isDevFile
+                    || (!isCodeFile && (!isIncludeInfraDiagram || !isInfraDiagramFileExt))
+            }
+
+            if (isExcludeFile) {
                 if (!isCodeFile_) {
                     const re = /(?:\.([^.]+))?$/
                     const extensionArray = re.exec(file.relativeFilePath)
@@ -73,6 +122,7 @@ export async function prepareRepoData(
                 }
                 continue
             }
+
             totalBytes += fileSize
             // Paths in zip should be POSIX compliant regardless of OS
             // Reference: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT

packages/core/src/shared/utilities/workspaceUtils.ts

@@ -19,6 +19,7 @@ import * as parser from '@gerhobbelt/gitignore-parser'
 import fs from '../fs/fs'
 import { ChildProcess } from './processUtils'
 import { isWin } from '../vscode/env'
+import { maxRepoSizeBytes } from '../../amazonqFeatureDev/constants'
 
 type GitIgnoreRelativeAcceptor = {
     folderPath: string
@@ -268,32 +269,48 @@ export function checkUnsavedChanges(): boolean {
     return vscode.workspace.textDocuments.some((doc) => doc.isDirty)
 }
 
+export const DEFAULT_EXTRA_EXCLUDE_PATTERNS = [
+    '**/package-lock.json',
+    '**/yarn.lock',
+    '**/*.zip',
+    '**/*.tar.gz',
+    '**/*.bin',
+    '**/*.png',
+    '**/*.jpg',
+    '**/*.svg',
+    '**/*.pyc',
+    '**/*.pdf',
+    '**/*.ttf',
+    '**/*.ico',
+    '**/license.txt',
+    '**/License.txt',
+    '**/LICENSE.txt',
+    '**/license.md',
+    '**/License.md',
+    '**/LICENSE.md',
+];
+
 export function getExcludePattern(defaultExcludePatterns: boolean = true) {
-    const globAlwaysExcludedDirs = getGlobDirExcludedPatterns().map((pattern) => `**/${pattern}/*`)
-    const extraPatterns = [
-        '**/package-lock.json',
-        '**/yarn.lock',
-        '**/*.zip',
-        '**/*.tar.gz',
-        '**/*.bin',
-        '**/*.png',
-        '**/*.jpg',
-        '**/*.svg',
-        '**/*.pyc',
-        '**/*.pdf',
-        '**/*.ttf',
-        '**/*.ico',
-        '**/license.txt',
-        '**/License.txt',
-        '**/LICENSE.txt',
-        '**/license.md',
-        '**/License.md',
-        '**/LICENSE.md',
-    ]
-    const allPatterns = [...globAlwaysExcludedDirs, ...(defaultExcludePatterns ? extraPatterns : [])]
-    return `{${allPatterns.join(',')}}`
+    const globAlwaysExcludedDirs = getGlobalExcludePatterns();
+    const allPatterns = [...globAlwaysExcludedDirs];
+
+    if (defaultExcludePatterns) {
+        allPatterns.push(...DEFAULT_EXTRA_EXCLUDE_PATTERNS)
+    }
+
+    return excludePatternsAsString(allPatterns);
 }
 
+export function getGlobalExcludePatterns() {
+    return getGlobDirExcludedPatterns()
+        .map((pattern) => `**/${pattern}/*`)
+}
+
+export function excludePatternsAsString(patterns: string[]): string {
+    return `{${patterns.join(',')}}`
+}
+
+
 /**
  * @param rootPath root folder to look for .gitignore files
  * @param addExtraPatterns whether to add extra exclude patterns even if not in gitignore
@@ -313,30 +330,26 @@ async function filterOutGitignoredFiles(
     return gitIgnoreFilter.filterFiles(files)
 }
 
-/**
- * collects all files that are marked as source
- * @param sourcePaths the paths where collection starts
- * @param workspaceFolders the current workspace folders opened
- * @param respectGitIgnore whether to respect gitignore file
- * @param addExtraIgnorePatterns whether to add extra exclude patterns even if not in gitignore
- * @returns all matched files
- */
-export async function collectFiles(
+
+export type CollectFilesResultItem = {
+    workspaceFolder: vscode.WorkspaceFolder;
+    relativeFilePath: string;
+    fileUri: vscode.Uri;
+    fileContent: string;
+    zipFilePath: string;
+}
+export type CollectFilesFileFilter = (relativePath: string)=>boolean; // returns true if file should be filtered out
+
+export async function collectFiles2(
     sourcePaths: string[],
     workspaceFolders: CurrentWsFolders,
-    respectGitIgnore: boolean = true,
-    maxSize = 200 * 1024 * 1024, // 200 MB
-    defaultExcludePatterns: boolean = true
-): Promise<
-    {
-        workspaceFolder: vscode.WorkspaceFolder
-        relativeFilePath: string
-        fileUri: vscode.Uri
-        fileContent: string
-        zipFilePath: string
-    }[]
-> {
-    const storage: Awaited<ReturnType<typeof collectFiles>> = []
+    options: {
+        maxSizeLimitInBytes?: number; // 200 MB default
+        useGitIgnoreFileAsFilter: boolean;
+        extraExcludeFilePatterns: string[];
+        extraFileFilterFn?: CollectFilesFileFilter
+    }): Promise<CollectFilesResultItem[]> {
+    const storage: Awaited<CollectFilesResultItem[]> = []
 
     const workspaceFoldersMapping = getWorkspaceFoldersByPrefixes(workspaceFolders)
     const workspaceToPrefix = new Map<vscode.WorkspaceFolder, string>(
@@ -360,14 +373,20 @@ export async function collectFiles(
     }
 
     let totalSizeBytes = 0
+    const excludePatterns = [...getGlobalExcludePatterns()];
+    if (options.extraExcludeFilePatterns?.length) {
+        excludePatterns.push(...options.extraExcludeFilePatterns);
+    }
+    const excludePatternFilter = excludePatternsAsString(excludePatterns);
+
     for (const rootPath of sourcePaths) {
         const allFiles = await vscode.workspace.findFiles(
             new vscode.RelativePattern(rootPath, '**'),
-            getExcludePattern(defaultExcludePatterns)
+            excludePatternFilter
         )
 
-        const files = respectGitIgnore
-            ? await filterOutGitignoredFiles(rootPath, allFiles, defaultExcludePatterns)
+        const files = options.useGitIgnoreFileAsFilter
+            ? await filterOutGitignoredFiles(rootPath, allFiles, false)
             : allFiles
 
         for (const file of files) {
@@ -376,8 +395,12 @@ export async function collectFiles(
                 continue
             }
 
+            if (options.extraFileFilterFn && options.extraFileFilterFn(relativePath.relativePath)) {
+                continue;
+            }
+
             const fileStat = await fs.stat(file)
-            if (totalSizeBytes + fileStat.size > maxSize) {
+            if (totalSizeBytes + fileStat.size > (options.maxSizeLimitInBytes ?? maxRepoSizeBytes)) {
                 throw new ToolkitError(
                     'The project you have selected for source code is too large to use as context. Please select a different folder to use',
                     { code: 'ContentLengthError' }
@@ -404,6 +427,30 @@ export async function collectFiles(
     return storage
 }
 
+/**
+ * collects all files that are marked as source
+ * @param sourcePaths the paths where collection starts
+ * @param workspaceFolders the current workspace folders opened
+ * @param respectGitIgnore whether to respect gitignore file
+ * @param addExtraIgnorePatterns whether to add extra exclude patterns even if not in gitignore
+ * @returns all matched files
+ */
+export async function collectFiles(
+    sourcePaths: string[],
+    workspaceFolders: CurrentWsFolders,
+    respectGitIgnore: boolean = true,
+    maxSize = 200 * 1024 * 1024, // 200 MB
+    defaultExcludePatterns: boolean = true
+): Promise<CollectFilesResultItem[]> {
+    const extraExcludeFilePatterns: string[] = defaultExcludePatterns ? DEFAULT_EXTRA_EXCLUDE_PATTERNS : [];
+     return collectFiles2(sourcePaths, workspaceFolders, {
+        maxSizeLimitInBytes: maxSize,
+        useGitIgnoreFileAsFilter: respectGitIgnore,
+        extraExcludeFilePatterns
+    })
+
+}
+
 const readFile = async (file: vscode.Uri) => {
     try {
         const fileContent = await fs.readFileText(file, new TextDecoder('utf8', { fatal: false }))