diff --git a/packages/amazonq/.changes/next-release/Feature-088c8b44-eb51-4495-84fb-2e02878ff4bc.json b/packages/amazonq/.changes/next-release/Feature-088c8b44-eb51-4495-84fb-2e02878ff4bc.json new file mode 100644 index 00000000000..da1c7b4de35 --- /dev/null +++ b/packages/amazonq/.changes/next-release/Feature-088c8b44-eb51-4495-84fb-2e02878ff4bc.json @@ -0,0 +1,4 @@ +{ + "type": "Feature", + "description": "Amazon Q /doc: Add support for infrastructure diagrams" +} diff --git a/packages/amazonq/test/unit/amazonqFeatureDev/util/files.test.ts b/packages/amazonq/test/unit/amazonqFeatureDev/util/files.test.ts index 4351e7832e6..e05cdee976c 100644 --- a/packages/amazonq/test/unit/amazonqFeatureDev/util/files.test.ts +++ b/packages/amazonq/test/unit/amazonqFeatureDev/util/files.test.ts @@ -6,6 +6,7 @@ import * as vscode from 'vscode' import assert from 'assert' import { prepareRepoData, + PrepareRepoDataOptions, TelemetryHelper, ContentLengthError, maxRepoSizeBytes, @@ -46,23 +47,23 @@ const testDevfilePrepareRepo = async (devfileEnabled: boolean) => { .stub(CodeWhispererSettings.instance, 'getAutoBuildSetting') .returns(devfileEnabled ? { [workspace.uri.fsPath]: true } : {}) - await testPrepareRepoData([workspace], expectedFiles) + await testPrepareRepoData([workspace], expectedFiles, { telemetry: new TelemetryHelper() }) } const testPrepareRepoData = async ( workspaces: vscode.WorkspaceFolder[], expectedFiles: string[], + prepareRepoDataOptions: PrepareRepoDataOptions, expectedTelemetryMetrics?: Array<{ metricName: MetricName; value: any }> ) => { expectedFiles.sort((a, b) => a.localeCompare(b)) - const telemetry = new TelemetryHelper() const result = await prepareRepoData( workspaces.map((ws) => ws.uri.fsPath), workspaces as CurrentWsFolders, - telemetry, { record: () => {}, - } as unknown as Span + } as unknown as Span, + prepareRepoDataOptions ) assert.strictEqual(Buffer.isBuffer(result.zipFileBuffer), true) @@ -84,6 +85,8 @@ const testPrepareRepoData = async ( describe('file utils', () => { describe('prepareRepoData', function () { + const defaultPrepareRepoDataOptions: PrepareRepoDataOptions = { telemetry: new TelemetryHelper() } + afterEach(() => { sinon.restore() }) @@ -92,9 +95,23 @@ describe('file utils', () => { const folder = await TestFolder.create() await folder.write('file1.md', 'test content') await folder.write('file2.md', 'test content') + await folder.write('docs/infra.svg', 'test content') const workspace = getWorkspaceFolder(folder.path) - await testPrepareRepoData([workspace], ['file1.md', 'file2.md']) + await testPrepareRepoData([workspace], ['file1.md', 'file2.md'], defaultPrepareRepoDataOptions) + }) + + it('infrastructure diagram is included', async function () { + const folder = await TestFolder.create() + await folder.write('file1.md', 'test content') + await folder.write('file2.svg', 'test content') + await folder.write('docs/infra.svg', 'test content') + const workspace = getWorkspaceFolder(folder.path) + + await testPrepareRepoData([workspace], ['file1.md', 'docs/infra.svg'], { + telemetry: new TelemetryHelper(), + isIncludeInfraDiagram: true, + }) }) it('prepareRepoData ignores denied file extensions', async function () { @@ -102,11 +119,9 @@ describe('file utils', () => { await folder.write('file.mp4', 'test content') const workspace = getWorkspaceFolder(folder.path) - await testPrepareRepoData( - [workspace], - [], - [{ metricName: 'amazonq_bundleExtensionIgnored', value: { filenameExt: 'mp4', count: 1 } }] - ) + await testPrepareRepoData([workspace], [], defaultPrepareRepoDataOptions, [ + { metricName: 'amazonq_bundleExtensionIgnored', value: { filenameExt: 'mp4', count: 1 } }, + ]) }) it('should ignore devfile.yaml when setting is disabled', async function () { @@ -122,14 +137,18 @@ describe('file utils', () => { const folder = await TestFolder.create() await folder.write('file.md', 'test content') const workspace = getWorkspaceFolder(folder.path) - const telemetry = new TelemetryHelper() sinon.stub(fs, 'stat').resolves({ size: 2 * maxRepoSizeBytes } as vscode.FileStat) await assert.rejects( () => - prepareRepoData([workspace.uri.fsPath], [workspace], telemetry, { - record: () => {}, - } as unknown as Span), + prepareRepoData( + [workspace.uri.fsPath], + [workspace], + { + record: () => {}, + } as unknown as Span, + defaultPrepareRepoDataOptions + ), ContentLengthError ) }) @@ -144,7 +163,11 @@ describe('file utils', () => { const workspace2 = getWorkspaceFolder(folder.path + '/innerFolder') const folderName = path.basename(folder.path) - await testPrepareRepoData([workspace1, workspace2], [`${folderName}_${workspace1.name}/${testFilePath}`]) + await testPrepareRepoData( + [workspace1, workspace2], + [`${folderName}_${workspace1.name}/${testFilePath}`], + defaultPrepareRepoDataOptions + ) }) }) }) diff --git a/packages/core/src/amazonq/session/sessionState.ts b/packages/core/src/amazonq/session/sessionState.ts index 21652b3dd5d..d2d81ae6670 100644 --- a/packages/core/src/amazonq/session/sessionState.ts +++ b/packages/core/src/amazonq/session/sessionState.ts @@ -7,7 +7,7 @@ import * as vscode from 'vscode' import { ToolkitError } from '../../shared/errors' import globals from '../../shared/extensionGlobals' import { getLogger } from '../../shared/logger/logger' -import { telemetry } from '../../shared/telemetry/telemetry' +import { AmazonqCreateUpload, Span, telemetry } from '../../shared/telemetry/telemetry' import { VirtualFileSystem } from '../../shared/virtualFilesystem' import { CodeReference, UploadHistory } from '../webview/ui/connector' import { AuthUtil } from '../../codewhisperer/util/authUtil' @@ -25,7 +25,7 @@ import { SessionStateInteraction, SessionStatePhase, } from '../commons/types' -import { prepareRepoData, getDeletedFileInfos, registerNewFiles } from '../util/files' +import { prepareRepoData, getDeletedFileInfos, registerNewFiles, PrepareRepoDataOptions } from '../util/files' import { uploadCode } from '../util/upload' export const EmptyCodeGenID = 'EMPTY_CURRENT_CODE_GENERATION_ID' @@ -227,11 +227,11 @@ export abstract class BasePrepareCodeGenState implements SessionState { amazonqConversationId: this.config.conversationId, credentialStartUrl: AuthUtil.instance.startUrl, }) - const { zipFileBuffer, zipFileChecksum } = await prepareRepoData( + const { zipFileBuffer, zipFileChecksum } = await this.prepareProjectZip( this.config.workspaceRoots, this.config.workspaceFolders, - action.telemetry, - span + span, + { telemetry: action.telemetry } ) const uploadId = randomUUID() const { uploadUrl, kmsKeyArn } = await this.config.proxyClient.createUploadUrl( @@ -251,6 +251,15 @@ export abstract class BasePrepareCodeGenState implements SessionState { const nextState = this.createNextState({ ...this.config, uploadId }) return nextState.interact(action) } + + protected async prepareProjectZip( + workspaceRoots: string[], + workspaceFolders: CurrentWsFolders, + span: Span, + options: PrepareRepoDataOptions + ) { + return await prepareRepoData(workspaceRoots, workspaceFolders, span, options) + } } export interface CodeGenerationParams { diff --git a/packages/core/src/amazonq/util/files.ts b/packages/core/src/amazonq/util/files.ts index d97b706596f..497e0076dcb 100644 --- a/packages/core/src/amazonq/util/files.ts +++ b/packages/core/src/amazonq/util/files.ts @@ -5,7 +5,12 @@ import * as vscode from 'vscode' import * as path from 'path' -import { collectFiles, getWorkspaceFoldersByPrefixes } from '../../shared/utilities/workspaceUtils' +import { + collectFiles, + CollectFilesFileFilter, + DefaultExtraExcludePatterns, + getWorkspaceFoldersByPrefixes, +} from '../../shared/utilities/workspaceUtils' import { ContentLengthError, PrepareRepoFailedError } from '../../amazonqFeatureDev/errors' import { getLogger } from '../../shared/logger/logger' @@ -24,27 +29,71 @@ import { isPresent } from '../../shared/utilities/collectionUtils' import { AuthUtil } from '../../codewhisperer/util/authUtil' import { TelemetryHelper } from '../util/telemetryHelper' +export const SvgFileExtension = '.svg' + export async function checkForDevFile(root: string) { const devFilePath = root + '/devfile.yaml' const hasDevFile = await fs.existsFile(devFilePath) return hasDevFile } +function isInfraDiagramFile(relativePath: string) { + return ( + relativePath.toLowerCase().endsWith(path.join('docs', 'infra.dot')) || + relativePath.toLowerCase().endsWith(path.join('docs', 'infra.svg')) + ) +} + +export type PrepareRepoDataOptions = { + telemetry?: TelemetryHelper + zip?: ZipStream + isIncludeInfraDiagram?: boolean +} + /** * given the root path of the repo it zips its files in memory and generates a checksum for it. */ export async function prepareRepoData( repoRootPaths: string[], workspaceFolders: CurrentWsFolders, - telemetry: TelemetryHelper, span: Span, - zip: ZipStream = new ZipStream() + options?: PrepareRepoDataOptions ) { try { + const telemetry = options?.telemetry + const isIncludeInfraDiagram = options?.isIncludeInfraDiagram ?? false + const zip = options?.zip ?? new ZipStream() + const autoBuildSetting = CodeWhispererSettings.instance.getAutoBuildSetting() const useAutoBuildFeature = autoBuildSetting[repoRootPaths[0]] ?? false + const extraExcludeFilePatterns: string[] = [] + let extraFileFilterFn: CollectFilesFileFilter | undefined = undefined + // We only respect gitignore file rules if useAutoBuildFeature is on, this is to avoid dropping necessary files for building the code (e.g. png files imported in js code) - const files = await collectFiles(repoRootPaths, workspaceFolders, true, maxRepoSizeBytes, !useAutoBuildFeature) + if (!useAutoBuildFeature) { + if (isIncludeInfraDiagram) { + // ensure svg is not filtered out by files search + extraExcludeFilePatterns.push( + ...DefaultExtraExcludePatterns.filter((p) => !p.endsWith(SvgFileExtension)) + ) + // ensure only infra diagram is included from all svg files + extraFileFilterFn = (relativePath: string) => { + if (!relativePath.toLowerCase().endsWith(SvgFileExtension)) { + return false + } + return !isInfraDiagramFile(relativePath) + } + } else { + extraExcludeFilePatterns.push(...DefaultExtraExcludePatterns) + } + } + + const files = await collectFiles(repoRootPaths, workspaceFolders, { + maxSizeLimitInBytes: maxRepoSizeBytes, + useGitIgnoreFileAsFilter: true, + extraExcludeFilePatterns, + extraFileFilterFn, + }) let totalBytes = 0 const ignoredExtensionMap = new Map() @@ -68,9 +117,15 @@ export async function prepareRepoData( } const isCodeFile_ = isCodeFile(file.relativeFilePath) const isDevFile = file.relativeFilePath === 'devfile.yaml' - // When useAutoBuildFeature is on, only respect the gitignore rules filtered earlier and apply the size limit, otherwise, exclude all non code files and gitignore files - const isNonCodeFileAndIgnored = useAutoBuildFeature ? false : !isCodeFile_ || isDevFile - if (fileSize >= maxFileSizeBytes || isNonCodeFileAndIgnored) { + const isInfraDiagramFileExt = isInfraDiagramFile(file.relativeFilePath) + + let isExcludeFile = fileSize >= maxFileSizeBytes + // When useAutoBuildFeature is on, only respect the gitignore rules filtered earlier and apply the size limit + if (!isExcludeFile && !useAutoBuildFeature) { + isExcludeFile = isDevFile || (!isCodeFile_ && (!isIncludeInfraDiagram || !isInfraDiagramFileExt)) + } + + if (isExcludeFile) { if (!isCodeFile_) { const re = /(?:\.([^.]+))?$/ const extensionArray = re.exec(file.relativeFilePath) @@ -83,6 +138,7 @@ export async function prepareRepoData( } continue } + totalBytes += fileSize // Paths in zip should be POSIX compliant regardless of OS // Reference: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT @@ -117,7 +173,10 @@ export async function prepareRepoData( } } - telemetry.setRepositorySize(totalBytes) + if (telemetry) { + telemetry.setRepositorySize(totalBytes) + } + span.record({ amazonqRepositorySize: totalBytes }) const zipResult = await zip.finalize() diff --git a/packages/core/src/amazonqDoc/controllers/chat/controller.ts b/packages/core/src/amazonqDoc/controllers/chat/controller.ts index 8c52ab9187f..1b1fb09c364 100644 --- a/packages/core/src/amazonqDoc/controllers/chat/controller.ts +++ b/packages/core/src/amazonqDoc/controllers/chat/controller.ts @@ -35,13 +35,13 @@ import { BaseChatSessionStorage } from '../../../amazonq/commons/baseChatStorage import { DocMessenger } from '../../messenger' import { AuthController } from '../../../amazonq/auth/controller' import { openUrl } from '../../../shared/utilities/vsCodeUtils' -import { openDeletedDiff, openDiff } from '../../../amazonq/commons/diff' +import { createAmazonQUri, openDeletedDiff, openDiff } from '../../../amazonq/commons/diff' import { getWorkspaceFoldersByPrefixes, getWorkspaceRelativePath, isMultiRootWorkspace, } from '../../../shared/utilities/workspaceUtils' -import { getPathsFromZipFilePath } from '../../../amazonq/util/files' +import { getPathsFromZipFilePath, SvgFileExtension } from '../../../amazonq/util/files' import { FollowUpTypes } from '../../../amazonq/commons/types' import { DocGenerationTask } from '../docGenerationTask' import { DevPhase } from '../../types' @@ -204,7 +204,13 @@ export class DocController { uploadId = session?.state?.uploadHistory[codeGenerationId].uploadId } const rightPath = path.join(uploadId, zipFilePath) - await openDiff(pathInfos.absolutePath, rightPath, tabId, this.scheme) + if (rightPath.toLowerCase().endsWith(SvgFileExtension)) { + const rightPathUri = createAmazonQUri(rightPath, tabId, this.scheme) + const infraDiagramContent = await vscode.workspace.openTextDocument(rightPathUri) + await vscode.window.showTextDocument(infraDiagramContent) + } else { + await openDiff(pathInfos.absolutePath, rightPath, tabId, this.scheme) + } } } } diff --git a/packages/core/src/amazonqDoc/session/sessionState.ts b/packages/core/src/amazonqDoc/session/sessionState.ts index 71650dd2b78..d03051170a0 100644 --- a/packages/core/src/amazonqDoc/session/sessionState.ts +++ b/packages/core/src/amazonqDoc/session/sessionState.ts @@ -8,7 +8,7 @@ import { DocGenerationStep, docScheme, getFileSummaryPercentage, Mode } from '.. import { i18n } from '../../shared/i18n-helper' -import { NewFileInfo, SessionState, SessionStateAction, SessionStateConfig } from '../types' +import { CurrentWsFolders, NewFileInfo, SessionState, SessionStateAction, SessionStateConfig } from '../types' import { ContentLengthError, DocServiceError, @@ -23,6 +23,8 @@ import { import { DocMessenger } from '../messenger' import { BaseCodeGenState, BasePrepareCodeGenState, CreateNextStateParams } from '../../amazonq/session/sessionState' import { Intent } from '../../amazonq/commons/types' +import { AmazonqCreateUpload, Span } from '../../shared/telemetry/telemetry' +import { prepareRepoData, PrepareRepoDataOptions } from '../../amazonq/util/files' export class DocCodeGenState extends BaseCodeGenState { protected handleProgress(messenger: DocMessenger, action: SessionStateAction, detail?: string): void { @@ -132,4 +134,16 @@ export class DocPrepareCodeGenState extends BasePrepareCodeGenState { protected override createNextState(config: SessionStateConfig): SessionState { return super.createNextState(config, DocCodeGenState) } + + protected override async prepareProjectZip( + workspaceRoots: string[], + workspaceFolders: CurrentWsFolders, + span: Span, + options: PrepareRepoDataOptions + ) { + return await prepareRepoData(workspaceRoots, workspaceFolders, span, { + ...options, + isIncludeInfraDiagram: true, + }) + } } diff --git a/packages/core/src/amazonqFeatureDev/index.ts b/packages/core/src/amazonqFeatureDev/index.ts index b3c17319b67..55114de0a06 100644 --- a/packages/core/src/amazonqFeatureDev/index.ts +++ b/packages/core/src/amazonqFeatureDev/index.ts @@ -11,5 +11,5 @@ export { Session } from './session/session' export { FeatureDevClient } from './client/featureDev' export { FeatureDevChatSessionStorage } from './storages/chatSession' export { TelemetryHelper } from '../amazonq/util/telemetryHelper' -export { prepareRepoData } from '../amazonq/util/files' +export { prepareRepoData, PrepareRepoDataOptions } from '../amazonq/util/files' export { ChatControllerEventEmitters, FeatureDevController } from './controllers/chat/controller' diff --git a/packages/core/src/amazonqFeatureDev/session/sessionState.ts b/packages/core/src/amazonqFeatureDev/session/sessionState.ts index 63e1cbdf2b3..ca8ffa8510c 100644 --- a/packages/core/src/amazonqFeatureDev/session/sessionState.ts +++ b/packages/core/src/amazonqFeatureDev/session/sessionState.ts @@ -77,7 +77,9 @@ export class MockCodeGenState implements SessionState { const files = await collectFiles( this.config.workspaceFolders.map((f) => path.join(f.uri.fsPath, './mock-data')), this.config.workspaceFolders, - false + { + useGitIgnoreFileAsFilter: false, + } ) const newFileContents = files.map((f) => ({ zipFilePath: f.zipFilePath, diff --git a/packages/core/src/codewhisperer/util/zipUtil.ts b/packages/core/src/codewhisperer/util/zipUtil.ts index 18379e051ad..3e4199d8f9f 100644 --- a/packages/core/src/codewhisperer/util/zipUtil.ts +++ b/packages/core/src/codewhisperer/util/zipUtil.ts @@ -410,12 +410,9 @@ export class ZipUtil { return } - const sourceFiles = await collectFiles( - projectPaths, - vscode.workspace.workspaceFolders as CurrentWsFolders, - true, - this.getProjectScanPayloadSizeLimitInBytes() - ) + const sourceFiles = await collectFiles(projectPaths, vscode.workspace.workspaceFolders as CurrentWsFolders, { + maxSizeLimitInBytes: this.getProjectScanPayloadSizeLimitInBytes(), + }) for (const file of sourceFiles) { const projectName = path.basename(file.workspaceFolder.uri.fsPath) const zipEntryPath = this.getZipEntryPath(projectName, file.relativeFilePath) diff --git a/packages/core/src/shared/utilities/workspaceUtils.ts b/packages/core/src/shared/utilities/workspaceUtils.ts index f624a3ad2e1..22228e3a73c 100644 --- a/packages/core/src/shared/utilities/workspaceUtils.ts +++ b/packages/core/src/shared/utilities/workspaceUtils.ts @@ -19,6 +19,7 @@ import * as parser from '@gerhobbelt/gitignore-parser' import fs from '../fs/fs' import { ChildProcess } from './processUtils' import { isWin } from '../vscode/env' +import { maxRepoSizeBytes } from '../../amazonqFeatureDev/constants' type GitIgnoreRelativeAcceptor = { folderPath: string @@ -268,30 +269,44 @@ export function checkUnsavedChanges(): boolean { return vscode.workspace.textDocuments.some((doc) => doc.isDirty) } +export const DefaultExtraExcludePatterns = [ + '**/package-lock.json', + '**/yarn.lock', + '**/*.zip', + '**/*.tar.gz', + '**/*.bin', + '**/*.png', + '**/*.jpg', + '**/*.svg', + '**/*.pyc', + '**/*.pdf', + '**/*.ttf', + '**/*.ico', + '**/license.txt', + '**/License.txt', + '**/LICENSE.txt', + '**/license.md', + '**/License.md', + '**/LICENSE.md', +] + export function getExcludePattern(defaultExcludePatterns: boolean = true) { - const globAlwaysExcludedDirs = getGlobDirExcludedPatterns().map((pattern) => `**/${pattern}/*`) - const extraPatterns = [ - '**/package-lock.json', - '**/yarn.lock', - '**/*.zip', - '**/*.tar.gz', - '**/*.bin', - '**/*.png', - '**/*.jpg', - '**/*.svg', - '**/*.pyc', - '**/*.pdf', - '**/*.ttf', - '**/*.ico', - '**/license.txt', - '**/License.txt', - '**/LICENSE.txt', - '**/license.md', - '**/License.md', - '**/LICENSE.md', - ] - const allPatterns = [...globAlwaysExcludedDirs, ...(defaultExcludePatterns ? extraPatterns : [])] - return `{${allPatterns.join(',')}}` + const globAlwaysExcludedDirs = getGlobalExcludePatterns() + const allPatterns = [...globAlwaysExcludedDirs] + + if (defaultExcludePatterns) { + allPatterns.push(...DefaultExtraExcludePatterns) + } + + return excludePatternsAsString(allPatterns) +} + +export function getGlobalExcludePatterns() { + return getGlobDirExcludedPatterns().map((pattern) => `**/${pattern}/*`) +} + +export function excludePatternsAsString(patterns: string[]): string { + return `{${patterns.join(',')}}` } /** @@ -313,30 +328,33 @@ async function filterOutGitignoredFiles( return gitIgnoreFilter.filterFiles(files) } +export type CollectFilesResultItem = { + workspaceFolder: vscode.WorkspaceFolder + relativeFilePath: string + fileUri: vscode.Uri + fileContent: string + zipFilePath: string +} +export type CollectFilesFileFilter = (relativePath: string) => boolean // returns true if file should be filtered out + /** - * collects all files that are marked as source + * search and collect source files * @param sourcePaths the paths where collection starts * @param workspaceFolders the current workspace folders opened - * @param respectGitIgnore whether to respect gitignore file - * @param addExtraIgnorePatterns whether to add extra exclude patterns even if not in gitignore + * @param options - filtering options * @returns all matched files */ export async function collectFiles( sourcePaths: string[], workspaceFolders: CurrentWsFolders, - respectGitIgnore: boolean = true, - maxSize = 200 * 1024 * 1024, // 200 MB - defaultExcludePatterns: boolean = true -): Promise< - { - workspaceFolder: vscode.WorkspaceFolder - relativeFilePath: string - fileUri: vscode.Uri - fileContent: string - zipFilePath: string - }[] -> { - const storage: Awaited> = [] + options?: { + maxSizeLimitInBytes?: number // 200 MB default + useGitIgnoreFileAsFilter?: boolean // default true + extraExcludeFilePatterns?: string[] // default DefaultExtraExcludePatterns + extraFileFilterFn?: CollectFilesFileFilter + } +): Promise { + const storage: Awaited = [] const workspaceFoldersMapping = getWorkspaceFoldersByPrefixes(workspaceFolders) const workspaceToPrefix = new Map( @@ -360,15 +378,24 @@ export async function collectFiles( } let totalSizeBytes = 0 + + const useGitIgnoreFileAsFilter = options?.useGitIgnoreFileAsFilter ?? true + const extraExcludeFilePatterns = options?.extraExcludeFilePatterns ?? DefaultExtraExcludePatterns + const maxSizeLimitInBytes = options?.maxSizeLimitInBytes ?? maxRepoSizeBytes + + const excludePatterns = [...getGlobalExcludePatterns()] + if (extraExcludeFilePatterns.length) { + excludePatterns.push(...extraExcludeFilePatterns) + } + const excludePatternFilter = excludePatternsAsString(excludePatterns) + for (const rootPath of sourcePaths) { const allFiles = await vscode.workspace.findFiles( new vscode.RelativePattern(rootPath, '**'), - getExcludePattern(defaultExcludePatterns) + excludePatternFilter ) - const files = respectGitIgnore - ? await filterOutGitignoredFiles(rootPath, allFiles, defaultExcludePatterns) - : allFiles + const files = useGitIgnoreFileAsFilter ? await filterOutGitignoredFiles(rootPath, allFiles, false) : allFiles for (const file of files) { const relativePath = getWorkspaceRelativePath(file.fsPath, { workspaceFolders }) @@ -376,8 +403,12 @@ export async function collectFiles( continue } + if (options?.extraFileFilterFn && options.extraFileFilterFn(relativePath.relativePath)) { + continue + } + const fileStat = await fs.stat(file) - if (totalSizeBytes + fileStat.size > maxSize) { + if (totalSizeBytes + fileStat.size > maxSizeLimitInBytes) { throw new ToolkitError( 'The project you have selected for source code is too large to use as context. Please select a different folder to use', { code: 'ContentLengthError' } diff --git a/packages/core/src/testInteg/perf/collectFiles.test.ts b/packages/core/src/testInteg/perf/collectFiles.test.ts index d2bc705f29a..506d9b5da7c 100644 --- a/packages/core/src/testInteg/perf/collectFiles.test.ts +++ b/packages/core/src/testInteg/perf/collectFiles.test.ts @@ -42,7 +42,7 @@ function performanceTestWrapper(totalFiles: number) { }, execute: async ({ workspace }: { workspace: vscode.WorkspaceFolder }) => { return { - result: await collectFiles([workspace.uri.fsPath], [workspace], true), + result: await collectFiles([workspace.uri.fsPath], [workspace]), } }, verify: ( diff --git a/packages/core/src/testInteg/perf/prepareRepoData.test.ts b/packages/core/src/testInteg/perf/prepareRepoData.test.ts index eb11f183ae8..c1ba1df1223 100644 --- a/packages/core/src/testInteg/perf/prepareRepoData.test.ts +++ b/packages/core/src/testInteg/perf/prepareRepoData.test.ts @@ -46,9 +46,14 @@ function performanceTestWrapper(numFiles: number, fileSize: number) { return { workspace, fsSpy, numFiles, fileSize } }, execute: async (setup: setupResult) => { - return await prepareRepoData([setup.workspace.uri.fsPath], [setup.workspace], telemetry, { - record: () => {}, - } as unknown as Span) + return await prepareRepoData( + [setup.workspace.uri.fsPath], + [setup.workspace], + { + record: () => {}, + } as unknown as Span, + { telemetry } + ) }, verify: async (setup: setupResult, result: resultType) => { verifyResult(setup, result, telemetry, numFiles * fileSize) diff --git a/packages/core/src/testInteg/shared/utilities/workspaceUtils.test.ts b/packages/core/src/testInteg/shared/utilities/workspaceUtils.test.ts index d64d3a81d6c..025ce502bf0 100644 --- a/packages/core/src/testInteg/shared/utilities/workspaceUtils.test.ts +++ b/packages/core/src/testInteg/shared/utilities/workspaceUtils.test.ts @@ -196,7 +196,7 @@ describe('workspaceUtils', () => { const result = await collectFiles( workspaces.map((ws) => ws.uri.fsPath), workspaces, - false + { useGitIgnoreFileAsFilter: false } ) assert.strictEqual( result.length, @@ -256,7 +256,7 @@ describe('workspaceUtils', () => { await writeFile(['src', 'folder3', 'negate_test1'], fileContent) await writeFile(['src', 'folder3', 'negate_test6'], fileContent) - const result = (await collectFiles([workspaceFolder.uri.fsPath], [workspaceFolder], true)) + const result = (await collectFiles([workspaceFolder.uri.fsPath], [workspaceFolder])) // for some reason, uri created inline differ in subfields, so skipping them from assertion .map(({ fileUri, zipFilePath, ...r }) => ({ ...r })) @@ -331,7 +331,7 @@ describe('workspaceUtils', () => { // add a non license file too, to make sure it is returned await toFile(fileContent, path.join(workspace.uri.fsPath, 'non-license.md')) - const result = await collectFiles([workspace.uri.fsPath], [workspace], true) + const result = await collectFiles([workspace.uri.fsPath], [workspace]) assert.deepStrictEqual(1, result.length) assert.deepStrictEqual('non-license.md', result[0].relativeFilePath)