From bbb56530d6ecc6050604e687d38d57b4c5f92e74 Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Thu, 9 Jan 2025 16:17:22 +0100 Subject: [PATCH] Watch changes on directories --- .../test/trace-provider.test.ts | 22 +-- .../grammar-workspace-manager.ts | 9 +- .../src/grammar/internal-grammar-util.ts | 2 +- .../src/lsp/document-update-handler.ts | 28 ++-- .../src/node/node-file-system-provider.ts | 18 +++ packages/langium/src/service-registry.ts | 9 -- packages/langium/src/test/index.ts | 1 + .../langium/src/test/virtual-file-system.ts | 62 +++++++ packages/langium/src/utils/uri-utils.ts | 151 ++++++++++++++++++ .../langium/src/workspace/document-builder.ts | 48 +++++- packages/langium/src/workspace/documents.ts | 42 +++-- .../src/workspace/file-system-provider.ts | 10 ++ .../src/workspace/workspace-manager.ts | 46 ++++-- .../parser/langium-parser-builder.test.ts | 2 +- .../langium/test/references/linker.test.ts | 2 +- .../test/serializer/json-serializer.test.ts | 24 +-- .../langium/test/service-registry.test.ts | 2 +- packages/langium/test/utils/caching.test.ts | 21 ++- packages/langium/test/utils/uri-trie.test.ts | 101 ++++++++++++ packages/langium/test/utils/uri-utils.test.ts | 39 +++++ .../test/workspace/document-builder.test.ts | 57 ++++++- 21 files changed, 601 insertions(+), 95 deletions(-) create mode 100644 packages/langium/src/test/virtual-file-system.ts create mode 100644 packages/langium/test/utils/uri-trie.test.ts diff --git a/packages/langium-sprotty/test/trace-provider.test.ts b/packages/langium-sprotty/test/trace-provider.test.ts index d1667a4d7..342cfee63 100644 --- a/packages/langium-sprotty/test/trace-provider.test.ts +++ b/packages/langium-sprotty/test/trace-provider.test.ts @@ -38,7 +38,7 @@ describe('DefaultTraceProvider', async () => { node a { node b {} } - `, { documentUri: 'test://test.model' }); + `, { documentUri: 'test:/test.txt' }); const model = document.parseResult.value; const source = model.nodes[0].nodes[0]; expect(source).toBeDefined(); @@ -47,7 +47,7 @@ describe('DefaultTraceProvider', async () => { id: 'node0' }; services.diagram.TraceProvider.trace(target, source); - expect(target.trace).toBe('test://test.model?2%3A16-2%3A25#%2Fnodes%400%2Fnodes%400'); + expect(target.trace).toBe('test:/test.txt?2%3A16-2%3A25#%2Fnodes%400%2Fnodes%400'); }); test('finds source node', async () => { @@ -55,12 +55,12 @@ describe('DefaultTraceProvider', async () => { node a { node b {} } - `, { documentUri: 'test://test.model' }); + `, { documentUri: 'test:/test.txt' }); const model = document.parseResult.value; const target: TracedModelElement = { type: 'node', id: 'node0', - trace: 'test://test.model?2%3A16-2%3A25#%2Fnodes%400%2Fnodes%400' + trace: 'test:/test.txt?2%3A16-2%3A25#%2Fnodes%400%2Fnodes%400' }; const source = services.diagram.TraceProvider.getSource(target); expect(source).toBeDefined(); @@ -72,7 +72,7 @@ describe('DefaultTraceProvider', async () => { node a { node b {} } - `, { documentUri: 'test://test.model' }); + `, { documentUri: 'test:/test.txt' }); const model = document.parseResult.value; const source = model.nodes[0].nodes[0]; expect(source).toBeDefined(); @@ -87,7 +87,7 @@ describe('DefaultTraceProvider', async () => { { type: 'node', id: 'node1', - trace: 'test://test.model?2%3A16-2%3A25#%2Fnodes%400%2Fnodes%400' + trace: 'test:/test.txt?2%3A16-2%3A25#%2Fnodes%400%2Fnodes%400' } ] }; @@ -101,7 +101,7 @@ describe('DefaultTraceProvider', async () => { node a { node b {} } - `, { documentUri: 'test://test.model' }); + `, { documentUri: 'test:/test.txt' }); const model = document.parseResult.value; const source = model.nodes[0].nodes[0]; expect(source).toBeDefined(); @@ -116,7 +116,7 @@ describe('DefaultTraceProvider', async () => { { type: 'node', id: 'node0', - trace: 'test://test.model?1%3A12-3%3A13#%2Fnodes%400' + trace: 'test:/test.txt?1%3A12-3%3A13#%2Fnodes%400' } ] }; @@ -130,7 +130,7 @@ describe('DefaultTraceProvider', async () => { node a { node b {} } - `, { documentUri: 'test://test.model' }); + `, { documentUri: 'test:/test.txt' }); const model = document.parseResult.value; const source = model.nodes[0].nodes[0]; expect(source).toBeDefined(); @@ -141,12 +141,12 @@ describe('DefaultTraceProvider', async () => { { type: 'node', id: 'node0', - trace: 'test://test.model?1%3A12-3%3A13#%2Fnodes%400' + trace: 'test:/test.txt?1%3A12-3%3A13#%2Fnodes%400' }, { type: 'node', id: 'node1', - trace: 'test://test.model?2%3A16-2%3A25#%2Fnodes%400%2Fnodes%400' + trace: 'test:/test.txt?2%3A16-2%3A25#%2Fnodes%400%2Fnodes%400' } ] }; diff --git a/packages/langium-vscode/src/language-server/grammar-workspace-manager.ts b/packages/langium-vscode/src/language-server/grammar-workspace-manager.ts index 531b2ce49..63747cfdc 100644 --- a/packages/langium-vscode/src/language-server/grammar-workspace-manager.ts +++ b/packages/langium-vscode/src/language-server/grammar-workspace-manager.ts @@ -52,14 +52,15 @@ export class LangiumGrammarWorkspaceManager extends DefaultWorkspaceManager { return super.initializeWorkspace(folders, cancelToken); } - protected override includeEntry(workspaceFolder: WorkspaceFolder, entry: FileSystemNode, fileExtensions: string[]): boolean { - if (this.matcher) { + override includeEntry(entry: FileSystemNode): boolean { + const workspaceFolder = this.workspaceFolders?.find(folder => UriUtils.contains(folder.uri, entry.uri)); + if (this.matcher && workspaceFolder) { // create path relative to workspace folder root: /user/foo/workspace/entry.txt -> entry.txt const relPath = path.relative(URI.parse(workspaceFolder.uri).path, entry.uri.path); const ignored = this.matcher.ignores(relPath); - return !ignored && (entry.isDirectory || (entry.isFile && fileExtensions.includes(UriUtils.extname(entry.uri)))); + return !ignored && (entry.isDirectory || (entry.isFile && this.fileExtensions.has(UriUtils.extname(entry.uri)))); } - return super.includeEntry(workspaceFolder, entry, fileExtensions); + return super.includeEntry(entry); } } diff --git a/packages/langium/src/grammar/internal-grammar-util.ts b/packages/langium/src/grammar/internal-grammar-util.ts index a784bc9a0..17fb5f2b4 100644 --- a/packages/langium/src/grammar/internal-grammar-util.ts +++ b/packages/langium/src/grammar/internal-grammar-util.ts @@ -181,7 +181,7 @@ export async function createServicesForGrammar language.LanguageMetaData.fileExtensions) - .map(ext => ext.startsWith('.') ? ext.substring(1) : ext) - .distinct() - .toArray(); - if (fileExtensions.length > 0) { - const connection = services.lsp.Connection; - const options: DidChangeWatchedFilesRegistrationOptions = { - watchers: [{ - globPattern: fileExtensions.length === 1 - ? `**/*.${fileExtensions[0]}` - : `**/*.{${fileExtensions.join(',')}}` - }] - }; - connection?.client.register(DidChangeWatchedFilesNotification.type, options); - } + const connection = services.lsp.Connection; + const options: DidChangeWatchedFilesRegistrationOptions = { + watchers: [{ + // We need to watch all file changes in the workspace + // Otherwise we miss changes to directories + globPattern: '**/*' + }] + }; + connection?.client.register(DidChangeWatchedFilesNotification.type, options); } protected fireDocumentUpdate(changed: URI[], deleted: URI[]): void { - // Filter out URIs that do not have a service in the registry - // Running the document builder update will fail for those URIs - changed = changed.filter(uri => this.serviceRegistry.hasServices(uri)); // Only fire the document update when the workspace manager is ready // Otherwise, we might miss the initial indexing of the workspace this.workspaceManager.ready.then(() => { diff --git a/packages/langium/src/node/node-file-system-provider.ts b/packages/langium/src/node/node-file-system-provider.ts index 740929a23..06242bebb 100644 --- a/packages/langium/src/node/node-file-system-provider.ts +++ b/packages/langium/src/node/node-file-system-provider.ts @@ -15,6 +15,24 @@ export class NodeFileSystemProvider implements FileSystemProvider { encoding: NodeTextEncoding = 'utf-8'; + async stat(uri: URI): Promise { + const stat = await fs.promises.stat(uri.fsPath); + return { + isFile: stat.isFile(), + isDirectory: stat.isDirectory(), + uri + }; + } + + statSync(uri: URI): FileSystemNode { + const stat = fs.statSync(uri.fsPath); + return { + isFile: stat.isFile(), + isDirectory: stat.isDirectory(), + uri + }; + } + readFile(uri: URI): Promise { return fs.promises.readFile(uri.fsPath, this.encoding); } diff --git a/packages/langium/src/service-registry.ts b/packages/langium/src/service-registry.ts index bbf55bfd7..9cd3aefe0 100644 --- a/packages/langium/src/service-registry.ts +++ b/packages/langium/src/service-registry.ts @@ -41,7 +41,6 @@ export interface ServiceRegistry { */ export class DefaultServiceRegistry implements ServiceRegistry { - protected singleton?: LangiumCoreServices; protected readonly languageIdMap = new Map(); protected readonly fileExtensionMap = new Map(); @@ -67,17 +66,9 @@ export class DefaultServiceRegistry implements ServiceRegistry { this.fileExtensionMap.set(ext, language); } this.languageIdMap.set(data.languageId, language); - if (this.languageIdMap.size === 1) { - this.singleton = language; - } else { - this.singleton = undefined; - } } getServices(uri: URI): LangiumCoreServices { - if (this.singleton !== undefined) { - return this.singleton; - } if (this.languageIdMap.size === 0) { throw new Error('The service registry is empty. Use `register` to register the services of a language.'); } diff --git a/packages/langium/src/test/index.ts b/packages/langium/src/test/index.ts index c8b4ae10a..6456ec04c 100644 --- a/packages/langium/src/test/index.ts +++ b/packages/langium/src/test/index.ts @@ -7,3 +7,4 @@ */ export * from './langium-test.js'; +export * from './virtual-file-system.js'; diff --git a/packages/langium/src/test/virtual-file-system.ts b/packages/langium/src/test/virtual-file-system.ts new file mode 100644 index 000000000..25d21b01f --- /dev/null +++ b/packages/langium/src/test/virtual-file-system.ts @@ -0,0 +1,62 @@ +/****************************************************************************** + * Copyright 2024 TypeFox GmbH + * This program and the accompanying materials are made available under the + * terms of the MIT License, which is available in the project root. + ******************************************************************************/ + +import { URI } from 'vscode-uri'; +import type { FileSystemNode, FileSystemProvider } from '../workspace/file-system-provider.js'; +import { UriTrie } from '../utils/uri-utils.js'; + +export class VirtualFileSystemProvider implements FileSystemProvider { + + private readonly trie = new UriTrie(); + + insert(uri: URI | string, content: string): void { + this.trie.insert(uri, content); + } + + delete(uri: URI | string): void { + this.trie.delete(uri); + } + + stat(uri: URI): Promise { + return Promise.resolve(this.statSync(uri)); + } + + statSync(uri: URI): FileSystemNode { + const node = this.trie.findNode(uri); + if (node) { + return { + isDirectory: node.element === undefined, + isFile: node.element !== undefined, + uri + }; + } else { + throw new Error('File not found'); + } + } + + readFile(uri: URI): Promise { + const data = this.trie.find(uri); + if (typeof data === 'string') { + return Promise.resolve(data); + } else { + throw new Error('File not found'); + } + } + + readDirectory(uri: URI): Promise { + const node = this.trie.findNode(uri); + if (!node) { + throw new Error('Directory not found'); + } + const children = this.trie.findChildren(uri); + return Promise.resolve(children.map(child => ({ + isDirectory: child.element === undefined, + isFile: child.element !== undefined, + uri: URI.parse(child.uri) + }))); + } + +} diff --git a/packages/langium/src/utils/uri-utils.ts b/packages/langium/src/utils/uri-utils.ts index 81fb80d43..9f927905a 100644 --- a/packages/langium/src/utils/uri-utils.ts +++ b/packages/langium/src/utils/uri-utils.ts @@ -40,4 +40,155 @@ export namespace UriUtils { return URI.parse(uri.toString()).toString(); } + export function contains(parent: URI | string, child: URI | string): boolean { + let parentPath = typeof parent === 'string' ? parent : parent.path; + let childPath = typeof child === 'string' ? child : child.path; + // Trim trailing slashes + if (childPath.charAt(childPath.length - 1) === '/') { + childPath = childPath.slice(0, -1); + } + if (parentPath.charAt(parentPath.length - 1) === '/') { + parentPath = parentPath.slice(0, -1); + } + // If the paths are equal, simply return true + if (childPath === parentPath) { + return true; + } + // If the child path is shorter than the parent path, it can't be a child + if (childPath.length < parentPath.length) { + return false; + } + // If the path does not feature a slash after the parent path, it can't be a child + if (childPath.charAt(parentPath.length) !== '/') { + return false; + } + // Check if the child path starts with the parent path + return childPath.startsWith(parentPath); + } + +} + +interface InternalUriTrieNode { + name: string, + children: Map>; + parent?: InternalUriTrieNode; + // If this element is set, the node represents a leaf in the trie + element?: T; +} + +export interface UriTrieNode { + name: string; + uri: string; + element?: T; +} + +export class UriTrie { + + private readonly root: InternalUriTrieNode = { name: '', children: new Map() }; + + clear(): void { + this.root.children.clear(); + } + + insert(uri: URI | string, element: T): void { + const node = this.getNode(UriUtils.normalize(uri), true); + node.element = element; + } + + delete(uri: URI | string): void { + const nodeToDelete = this.getNode(UriUtils.normalize(uri), false); + if (nodeToDelete?.parent) { + nodeToDelete.parent.children.delete(nodeToDelete.name); + } + } + + has(uri: URI | string): boolean { + return this.getNode(UriUtils.normalize(uri), false)?.element !== undefined; + } + + hasNode(uri: URI | string): boolean { + return this.getNode(UriUtils.normalize(uri), false) !== undefined; + } + + find(uri: URI | string): T | undefined { + return this.getNode(UriUtils.normalize(uri), false)?.element; + } + + findNode(uri: URI | string): UriTrieNode | undefined { + const uriString = UriUtils.normalize(uri); + const node = this.getNode(uriString, false); + if (!node) { + return undefined; + } + return { + name: node.name, + uri: UriUtils.joinPath(URI.parse(uriString), node.name).toString(), + element: node.element + }; + } + + findChildren(uri: URI | string): Array> { + const uriString = UriUtils.normalize(uri); + const node = this.getNode(uriString, false); + if (!node) { + return []; + } + return Array.from(node.children.values()).map(child => ({ + name: child.name, + uri: UriUtils.joinPath(URI.parse(uriString), child.name).toString(), + element: child.element + })); + } + + all(): T[] { + return this.collectDocuments(this.root); + } + + findAll(prefix: URI | string): T[] { + const node = this.getNode(UriUtils.normalize(prefix), false); + if (!node) { + return []; + } + return this.collectDocuments(node); + } + + private getNode(uri: string, create: true): InternalUriTrieNode; + private getNode(uri: string, create: false): InternalUriTrieNode | undefined; + private getNode(uri: string, create: boolean): InternalUriTrieNode | undefined { + const parts = uri.split('/'); + if (uri.charAt(uri.length - 1) === '/') { + // Remove the last part if the URI ends with a slash + parts.pop(); + } + let current = this.root; + for (const part of parts) { + let child = current.children.get(part); + if (!child) { + if (create) { + child = { + name: part, + children: new Map(), + parent: current + }; + current.children.set(part, child); + } else { + return undefined; + } + } + current = child; + } + return current; + } + + private collectDocuments(node: InternalUriTrieNode): T[] { + const result: T[] = []; + if (node.element) { + result.push(node.element); + } + for (const child of node.children.values()) { + result.push(...this.collectDocuments(child)); + } + return result; + } + } diff --git a/packages/langium/src/workspace/document-builder.ts b/packages/langium/src/workspace/document-builder.ts index fedd687a4..1bfca75b6 100644 --- a/packages/langium/src/workspace/document-builder.ts +++ b/packages/langium/src/workspace/document-builder.ts @@ -20,6 +20,8 @@ import { stream } from '../utils/stream.js'; import type { URI } from '../utils/uri-utils.js'; import { ValidationCategory } from '../validation/validation-registry.js'; import { DocumentState } from './documents.js'; +import type { FileSystemProvider } from './file-system-provider.js'; +import type { WorkspaceManager } from './workspace-manager.js'; export interface BuildOptions { /** @@ -132,6 +134,8 @@ export class DefaultDocumentBuilder implements DocumentBuilder { protected readonly textDocuments: TextDocumentProvider | undefined; protected readonly indexManager: IndexManager; protected readonly serviceRegistry: ServiceRegistry; + protected readonly fileSystemProvider: FileSystemProvider; + protected readonly workspaceManager: () => WorkspaceManager; protected readonly updateListeners: DocumentUpdateListener[] = []; protected readonly buildPhaseListeners = new MultiMap(); protected readonly documentPhaseListeners = new MultiMap(); @@ -144,6 +148,8 @@ export class DefaultDocumentBuilder implements DocumentBuilder { this.langiumDocumentFactory = services.workspace.LangiumDocumentFactory; this.textDocuments = services.workspace.TextDocuments; this.indexManager = services.workspace.IndexManager; + this.fileSystemProvider = services.workspace.FileSystemProvider; + this.workspaceManager = () => services.workspace.WorkspaceManager; this.serviceRegistry = services.ServiceRegistry; } @@ -192,13 +198,20 @@ export class DefaultDocumentBuilder implements DocumentBuilder { async update(changed: URI[], deleted: URI[], cancelToken = CancellationToken.None): Promise { this.currentState = DocumentState.Changed; // Remove all metadata of documents that are reported as deleted + const deletedUris: URI[] = []; for (const deletedUri of deleted) { - this.langiumDocuments.deleteDocument(deletedUri); - this.buildState.delete(deletedUri.toString()); - this.indexManager.remove(deletedUri); + // Since the deleted URI might point to a directory, we delete all documents within + const deletedDocs = this.langiumDocuments.deleteDocuments(deletedUri); + for (const doc of deletedDocs) { + deletedUris.push(doc.uri); + this.buildState.delete(doc.uri.toString()); + this.indexManager.remove(doc.uri); + } } + // Since the changed URI might point to a directory, we need to check all (nested) documents in that directory + const changedUris = (await Promise.all(changed.map(uri => this.findChangedUris(uri)))).flat(); // Set the state of all changed documents to `Changed` so they are completely rebuilt - for (const changedUri of changed) { + for (const changedUri of changedUris) { const invalidated = this.langiumDocuments.invalidateDocument(changedUri); if (!invalidated) { // We create an unparsed, invalid document. @@ -211,7 +224,7 @@ export class DefaultDocumentBuilder implements DocumentBuilder { this.buildState.delete(changedUri.toString()); } // Set the state of all documents that should be relinked to `ComputedScopes` (if not already lower) - const allChangedUris = stream(changed).concat(deleted).map(uri => uri.toString()).toSet(); + const allChangedUris = stream(changedUris).concat(deletedUris).map(uri => uri.toString()).toSet(); this.langiumDocuments.all .filter(doc => !allChangedUris.has(doc.uri.toString()) && this.shouldRelink(doc, allChangedUris)) .forEach(doc => { @@ -221,7 +234,7 @@ export class DefaultDocumentBuilder implements DocumentBuilder { doc.diagnostics = undefined; }); // Notify listeners of the update - await this.emitUpdate(changed, deleted); + await this.emitUpdate(changedUris, deletedUris); // Only allow interrupting the execution after all state changes are done await interruptAndCheck(cancelToken); @@ -239,6 +252,29 @@ export class DefaultDocumentBuilder implements DocumentBuilder { await this.buildDocuments(rebuildDocuments, this.updateBuildOptions, cancelToken); } + protected async findChangedUris(changed: URI): Promise { + // Most common case is that the document/textDocument at the specified URI has changed + const document = this.langiumDocuments.getDocument(changed) ?? this.textDocuments?.get(changed); + if (document) { + return [changed]; + } + // If the document doesn't exist yet, we need to check what kind of file has changed + try { + const stat = await this.fileSystemProvider.stat(changed); + if (stat.isDirectory) { + // If a directory has changed, we need to check all documents in that directory + const uris = await this.workspaceManager().searchDirectory(changed); + return uris; + } else if (this.workspaceManager().includeEntry(stat)) { + // Return the changed URI if it's a file that we can handle + return [changed]; + } + } catch { + // If we can't determine the file type, we discard the change + } + return []; + } + protected async emitUpdate(changed: URI[], deleted: URI[]): Promise { await Promise.all(this.updateListeners.map(listener => listener(changed, deleted))); } diff --git a/packages/langium/src/workspace/documents.ts b/packages/langium/src/workspace/documents.ts index a94dc3563..0995e911b 100644 --- a/packages/langium/src/workspace/documents.ts +++ b/packages/langium/src/workspace/documents.ts @@ -24,7 +24,7 @@ import type { Stream } from '../utils/stream.js'; import { TextDocument } from './documents.js'; import { CancellationToken } from '../utils/cancellation.js'; import { stream } from '../utils/stream.js'; -import { URI } from '../utils/uri-utils.js'; +import { URI, UriTrie } from '../utils/uri-utils.js'; /** * A Langium document holds the parse result (AST and CST) and any additional state that is derived @@ -387,6 +387,13 @@ export interface LangiumDocuments { * @returns the affected {@link LangiumDocument} if existing for convenience */ deleteDocument(uri: URI): LangiumDocument | undefined; + /** + * If the given URI is a directory, remove all documents within this directory. + * If it is a file, just remove that single document from the documents. + * + * @returns the affected {@link LangiumDocument}s if existing for convenience + */ + deleteDocuments(uri: URI): LangiumDocument[]; } export class DefaultLangiumDocuments implements LangiumDocuments { @@ -394,7 +401,7 @@ export class DefaultLangiumDocuments implements LangiumDocuments { protected readonly langiumDocumentFactory: LangiumDocumentFactory; protected readonly serviceRegistry: ServiceRegistry; - protected readonly documentMap: Map = new Map(); + protected readonly documentTrie = new UriTrie(); constructor(services: LangiumSharedCoreServices) { this.langiumDocumentFactory = services.workspace.LangiumDocumentFactory; @@ -402,20 +409,25 @@ export class DefaultLangiumDocuments implements LangiumDocuments { } get all(): Stream { - return stream(this.documentMap.values()); + return stream(this.documentTrie.all()); } addDocument(document: LangiumDocument): void { const uriString = document.uri.toString(); - if (this.documentMap.has(uriString)) { + if (this.documentTrie.has(uriString)) { throw new Error(`A document with the URI '${uriString}' is already present.`); } - this.documentMap.set(uriString, document); + this.documentTrie.insert(uriString, document); } getDocument(uri: URI): LangiumDocument | undefined { const uriString = uri.toString(); - return this.documentMap.get(uriString); + return this.documentTrie.find(uriString); + } + + getDocuments(folder: URI): LangiumDocument[] { + const uriString = folder.toString(); + return this.documentTrie.findAll(uriString); } async getOrCreateDocument(uri: URI, cancellationToken?: CancellationToken): Promise { @@ -444,12 +456,12 @@ export class DefaultLangiumDocuments implements LangiumDocuments { } hasDocument(uri: URI): boolean { - return this.documentMap.has(uri.toString()); + return this.documentTrie.has(uri.toString()); } invalidateDocument(uri: URI): LangiumDocument | undefined { const uriString = uri.toString(); - const langiumDoc = this.documentMap.get(uriString); + const langiumDoc = this.documentTrie.find(uriString); if (langiumDoc) { const linker = this.serviceRegistry.getServices(uri).references.Linker; linker.unlink(langiumDoc); @@ -462,11 +474,21 @@ export class DefaultLangiumDocuments implements LangiumDocuments { deleteDocument(uri: URI): LangiumDocument | undefined { const uriString = uri.toString(); - const langiumDoc = this.documentMap.get(uriString); + const langiumDoc = this.documentTrie.find(uriString); if (langiumDoc) { langiumDoc.state = DocumentState.Changed; - this.documentMap.delete(uriString); + this.documentTrie.delete(uriString); } return langiumDoc; } + + deleteDocuments(folder: URI): LangiumDocument[] { + const uriString = folder.toString(); + const langiumDocs = this.documentTrie.findAll(uriString); + for (const langiumDoc of langiumDocs) { + langiumDoc.state = DocumentState.Changed; + } + this.documentTrie.delete(uriString); + return langiumDocs; + } } diff --git a/packages/langium/src/workspace/file-system-provider.ts b/packages/langium/src/workspace/file-system-provider.ts index 0c7a58d6c..84d128ec3 100644 --- a/packages/langium/src/workspace/file-system-provider.ts +++ b/packages/langium/src/workspace/file-system-provider.ts @@ -18,6 +18,8 @@ export type FileSystemFilter = (node: FileSystemNode) => boolean; * Provides methods to interact with an abstract file system. The default implementation is based on the node.js `fs` API. */ export interface FileSystemProvider { + stat(uri: URI): Promise; + statSync(uri: URI): FileSystemNode; /** * Reads a document asynchronously from a given URI. * @returns The string content of the file with the specified URI. @@ -32,6 +34,14 @@ export interface FileSystemProvider { export class EmptyFileSystemProvider implements FileSystemProvider { + stat(_uri: URI): Promise { + throw new Error('No file system is available.'); + } + + statSync(_uri: URI): FileSystemNode { + throw new Error('No file system is available.'); + } + readFile(): Promise { throw new Error('No file system is available.'); } diff --git a/packages/langium/src/workspace/workspace-manager.ts b/packages/langium/src/workspace/workspace-manager.ts index cdc9c7c4a..a3adbaff2 100644 --- a/packages/langium/src/workspace/workspace-manager.ts +++ b/packages/langium/src/workspace/workspace-manager.ts @@ -65,6 +65,21 @@ export interface WorkspaceManager { */ initializeWorkspace(folders: WorkspaceFolder[], cancelToken?: CancellationToken): Promise; + /** + * Searches for workspace files in the given directory and its subdirectories. + * Note that this method does not create documents for the found files. + * @param uri The URI of the directory to search in. + * @returns A promise that resolves to an array of URIs of the found files. + */ + searchDirectory(uri: URI): Promise; + + /** + * Determine whether the given file system node shall be included in the workspace. + * @param entry The file system node to check. + * @returns `true` if the entry shall be included, `false` otherwise. + */ + includeEntry(entry: FileSystemNode): boolean; + } export class DefaultWorkspaceManager implements WorkspaceManager { @@ -78,6 +93,7 @@ export class DefaultWorkspaceManager implements WorkspaceManager { protected readonly mutex: WorkspaceLock; protected readonly _ready = new Deferred(); protected folders?: WorkspaceFolder[]; + protected fileExtensions = new Set(); constructor(services: LangiumSharedCoreServices) { this.serviceRegistry = services.ServiceRegistry; @@ -85,6 +101,7 @@ export class DefaultWorkspaceManager implements WorkspaceManager { this.documentBuilder = services.workspace.DocumentBuilder; this.fileSystemProvider = services.workspace.FileSystemProvider; this.mutex = services.workspace.WorkspaceLock; + this.fileExtensions = new Set(this.serviceRegistry.all.flatMap(e => e.LanguageMetaData.fileExtensions)); } get ready(): Promise { @@ -118,7 +135,6 @@ export class DefaultWorkspaceManager implements WorkspaceManager { * This methods loads all documents in the workspace and other documents and returns them. */ protected async performStartup(folders: WorkspaceFolder[]): Promise { - const fileExtensions = this.serviceRegistry.all.flatMap(e => e.LanguageMetaData.fileExtensions); const documents: LangiumDocument[] = []; const collector = (document: LangiumDocument) => { documents.push(document); @@ -130,10 +146,15 @@ export class DefaultWorkspaceManager implements WorkspaceManager { // we can still assume that all library documents and file documents are loaded by the time we start building documents. // The mutex prevents anything from performing a workspace build until we check the cancellation token await this.loadAdditionalDocuments(folders, collector); + const uris: URI[] = []; await Promise.all( - folders.map(wf => [wf, this.getRootFolder(wf)] as [WorkspaceFolder, URI]) - .map(async entry => this.traverseFolder(...entry, fileExtensions, collector)) + folders.map(wf => this.getRootFolder(wf)) + .map(async entry => this.traverseFolder(entry, uris)) ); + await Promise.all(uris.map(async uri => { + const document = await this.langiumDocuments.getOrCreateDocument(uri); + collector(document); + })); this._ready.resolve(); return documents; } @@ -160,24 +181,29 @@ export class DefaultWorkspaceManager implements WorkspaceManager { * Traverse the file system folder identified by the given URI and its subfolders. All * contained files that match the file extensions are added to the collector. */ - protected async traverseFolder(workspaceFolder: WorkspaceFolder, folderPath: URI, fileExtensions: string[], collector: (document: LangiumDocument) => void): Promise { + protected async traverseFolder(folderPath: URI, uris: URI[]): Promise { const content = await this.fileSystemProvider.readDirectory(folderPath); await Promise.all(content.map(async entry => { - if (this.includeEntry(workspaceFolder, entry, fileExtensions)) { + if (this.includeEntry(entry)) { if (entry.isDirectory) { - await this.traverseFolder(workspaceFolder, entry.uri, fileExtensions, collector); + await this.traverseFolder(entry.uri, uris); } else if (entry.isFile) { - const document = await this.langiumDocuments.getOrCreateDocument(entry.uri); - collector(document); + uris.push(entry.uri); } } })); } + async searchDirectory(uri: URI): Promise { + const uris: URI[] = []; + await this.traverseFolder(uri, uris); + return uris; + } + /** * Determine whether the given folder entry shall be included while indexing the workspace. */ - protected includeEntry(_workspaceFolder: WorkspaceFolder, entry: FileSystemNode, fileExtensions: string[]): boolean { + includeEntry(entry: FileSystemNode): boolean { const name = UriUtils.basename(entry.uri); if (name.startsWith('.')) { return false; @@ -186,7 +212,7 @@ export class DefaultWorkspaceManager implements WorkspaceManager { return name !== 'node_modules' && name !== 'out'; } else if (entry.isFile) { const extname = UriUtils.extname(entry.uri); - return fileExtensions.includes(extname); + return this.fileExtensions.has(extname); } return false; } diff --git a/packages/langium/test/parser/langium-parser-builder.test.ts b/packages/langium/test/parser/langium-parser-builder.test.ts index 0ac83c330..4549cde22 100644 --- a/packages/langium/test/parser/langium-parser-builder.test.ts +++ b/packages/langium/test/parser/langium-parser-builder.test.ts @@ -760,7 +760,7 @@ describe('Fragment rules', () => { } } }); - const document = services.shared.workspace.LangiumDocumentFactory.fromString('def a def b a[].b', URI.parse('file:///test')); + const document = services.shared.workspace.LangiumDocumentFactory.fromString('def a def b a[].b', URI.parse('file:///test.txt')); await services.shared.workspace.DocumentBuilder.build([document]); expect(document.parseResult.lexerErrors).toHaveLength(0); expect(document.parseResult.parserErrors).toHaveLength(0); diff --git a/packages/langium/test/references/linker.test.ts b/packages/langium/test/references/linker.test.ts index 96935b82d..cda971c65 100644 --- a/packages/langium/test/references/linker.test.ts +++ b/packages/langium/test/references/linker.test.ts @@ -46,7 +46,7 @@ describe('DefaultLinker', async () => { const document = await cyclicParser(` node a referrer a - `, { documentUri: 'test://test.model' }); + `, { documentUri: 'test:/test.txt' }); const model = document.parseResult.value; expect(model.referrers[0]?.node?.error).toBeDefined(); expect(model.referrers[0].node.error?.message).toBe( diff --git a/packages/langium/test/serializer/json-serializer.test.ts b/packages/langium/test/serializer/json-serializer.test.ts index 62b28e847..94c89c122 100644 --- a/packages/langium/test/serializer/json-serializer.test.ts +++ b/packages/langium/test/serializer/json-serializer.test.ts @@ -62,12 +62,12 @@ describe('JsonSerializer', async () => { const document1 = await parse(` element a `, { - documentUri: 'file:///test1.langium' + documentUri: 'file:///test1.txt' }); const document2 = await parse(` element b refers a `, { - documentUri: 'file:///test2.langium' + documentUri: 'file:///test2.txt' }); await services.shared.workspace.DocumentBuilder.build([document1, document2]); const json = serializer.serialize(document2.parseResult.value, { space: 4 }); @@ -79,7 +79,7 @@ describe('JsonSerializer', async () => { "$type": "Element", "name": "b", "other": { - "$ref": "file:///test1.langium#/elements@0" + "$ref": "file:///test1.txt#/elements@0" } } ] @@ -91,12 +91,12 @@ describe('JsonSerializer', async () => { const document1 = await parse(` element a `, { - documentUri: 'file:///test1.langium' + documentUri: 'file:///test1.txt' }); const document2 = await parse(` element b refers a `, { - documentUri: 'file:///test2.langium' + documentUri: 'file:///test2.txt' }); await services.shared.workspace.DocumentBuilder.build([document1, document2]); const json = serializer.serialize(document2.parseResult.value, { @@ -111,7 +111,7 @@ describe('JsonSerializer', async () => { "$type": "Element", "name": "b", "other": { - "$ref": "file:///foo/test1.langium#/elements@0" + "$ref": "file:///foo/test1.txt#/elements@0" } } ] @@ -147,7 +147,7 @@ describe('JsonSerializer', async () => { const document1 = await parse(` element a `, { - documentUri: 'file:///test1.langium' + documentUri: 'file:///test1.txt' }); await services.shared.workspace.DocumentBuilder.build([document1]); const json = expandToStringLF` @@ -158,7 +158,7 @@ describe('JsonSerializer', async () => { "$type": "Element", "name": "b", "other": { - "$ref": "file:///test1.langium#/elements@0" + "$ref": "file:///test1.txt#/elements@0" } } ] @@ -173,7 +173,7 @@ describe('JsonSerializer', async () => { const document1 = await parse(` element a `, { - documentUri: 'file:///test1.langium' + documentUri: 'file:///test1.txt' }); await services.shared.workspace.DocumentBuilder.build([document1]); const json = expandToStringLF` @@ -184,7 +184,7 @@ describe('JsonSerializer', async () => { "$type": "Element", "name": "b", "other": { - "$ref": "file:///foo/test1.langium#/elements@0" + "$ref": "file:///foo/test1.txt#/elements@0" } } ] @@ -209,7 +209,7 @@ describe('JsonSerializer', async () => { "$type": "Element", "name": "b", "other": { - "$ref": "file:///does-not-exist.langium#/elements@0" + "$ref": "file:///does-not-exist.txt#/elements@0" } } ] @@ -217,7 +217,7 @@ describe('JsonSerializer', async () => { `; const model = serializer.deserialize(json); expect(model.elements).toHaveLength(1); - expect(model.elements[0].other?.error?.message).toEqual('Could not find document for URI: file:///does-not-exist.langium#/elements@0'); + expect(model.elements[0].other?.error?.message).toEqual('Could not find document for URI: file:///does-not-exist.txt#/elements@0'); }); }); diff --git a/packages/langium/test/service-registry.test.ts b/packages/langium/test/service-registry.test.ts index 2bb7d26ce..06ab460ac 100644 --- a/packages/langium/test/service-registry.test.ts +++ b/packages/langium/test/service-registry.test.ts @@ -13,7 +13,7 @@ import { DefaultServiceRegistry, EmptyFileSystem, URI, createDefaultSharedCoreMo describe('DefaultServiceRegistry', () => { test('should work with a single language', () => { - const language: LangiumCoreServices = { LanguageMetaData: { fileExtensions: ['.foo'], languageId: 'foo' } } as any; + const language: LangiumCoreServices = { LanguageMetaData: { fileExtensions: ['.bar'], languageId: 'bar' } } as any; const registry = new DefaultServiceRegistry(createSharedCoreServices()); registry.register(language); expect(registry.getServices(URI.parse('file:/foo.bar'))).toBe(language); diff --git a/packages/langium/test/utils/caching.test.ts b/packages/langium/test/utils/caching.test.ts index ba16ee72b..0242ce53d 100644 --- a/packages/langium/test/utils/caching.test.ts +++ b/packages/langium/test/utils/caching.test.ts @@ -6,7 +6,7 @@ /* eslint-disable dot-notation */ -import { beforeEach, describe, expect, test } from 'vitest'; +import { afterEach, beforeEach, describe, expect, test } from 'vitest'; import type { DefaultDocumentBuilder} from 'langium'; import { DocumentCache, DocumentState, EmptyFileSystem, URI, WorkspaceCache } from 'langium'; import { createLangiumGrammarServices } from 'langium/grammar'; @@ -17,8 +17,18 @@ const document1 = workspace.LangiumDocumentFactory.fromString('', URI.file('/doc workspace.TextDocuments.set(document1.textDocument); const document2 = workspace.LangiumDocumentFactory.fromString('', URI.file('/document2.langium')); workspace.TextDocuments.set(document2.textDocument); -workspace.LangiumDocuments.addDocument(document1); -workspace.LangiumDocuments.addDocument(document2); + +beforeEach(async () => { + // Rebuild both documents to ensure that the following build calls don't pick up on other documents + workspace.LangiumDocuments.addDocument(document1); + workspace.LangiumDocuments.addDocument(document2); + await workspace.DocumentBuilder.build([document1, document2]); +}); + +afterEach(async () => { + workspace.LangiumDocuments.deleteDocument(document1.uri); + workspace.LangiumDocuments.deleteDocument(document2.uri); +}); describe('Document Cache', () => { @@ -123,11 +133,6 @@ describe('Document Cache', () => { describe('Workspace Cache', () => { - beforeEach(async () => { - // Rebuild both documents to ensure that the following build calls don't pick up on other documents - await workspace.DocumentBuilder.build([document1, document2]); - }); - test('Should get and set on the whole workspace', () => { const cache = new WorkspaceCache(services.shared); expect(cache.has('key')).toBe(false); diff --git a/packages/langium/test/utils/uri-trie.test.ts b/packages/langium/test/utils/uri-trie.test.ts new file mode 100644 index 000000000..8ce811cad --- /dev/null +++ b/packages/langium/test/utils/uri-trie.test.ts @@ -0,0 +1,101 @@ +/****************************************************************************** + * Copyright 2024 TypeFox GmbH + * This program and the accompanying materials are made available under the + * terms of the MIT License, which is available in the project root. + ******************************************************************************/ + +import { describe, expect, test, beforeEach } from 'vitest'; +import { UriTrie } from 'langium'; + +describe('UriTrie', () => { + let trie: UriTrie; + + beforeEach(() => { + trie = new UriTrie(); + }); + + test('inserts and finds documents', () => { + const uri = 'file:///test.txt'; + expect(trie.find(uri)).toBeUndefined(); + trie.insert(uri, uri); + expect(trie.find(uri)).toBe(uri); + }); + + test('inserts and has documents', () => { + const uri = 'file:///test.txt'; + expect(trie.has(uri)).toBeFalsy(); + trie.insert(uri, uri); + expect(trie.has(uri)).toBeTruthy(); + }); + + test('returns false for has on parents', () => { + const uri = 'file:///parent/test.txt'; + trie.insert(uri, uri); + expect(trie.has(uri)).toBeTruthy(); + expect(trie.has('file:///parent')).toBeFalsy(); + }); + + test('deletes documents', () => { + const uri = 'file:///test.txt'; + trie.insert(uri, uri); + expect(trie.find(uri)).toBe(uri); + trie.delete(uri); + expect(trie.find(uri)).toBeUndefined(); + }); + + test('deletes directories', () => { + const document0 = 'file:///test.txt'; + const document1 = 'file:///parent/test1.txt'; + const document2 = 'file:///parent/test2.txt'; + trie.insert(document0, document0); + trie.insert(document1, document1); + trie.insert(document2, document2); + expect(trie.find(document1)).toBe(document1); + expect(trie.find(document2)).toBe(document2); + trie.delete('file:///parent'); + expect(trie.find(document0)).toBe(document0); + expect(trie.find(document1)).toBeUndefined(); + expect(trie.find(document2)).toBeUndefined(); + }); + + test('deletes directories with trailing slash', () => { + const document1 = 'file:///parent/test1.txt'; + const document2 = 'file:///parent/test2.txt'; + trie.insert(document1, document1); + trie.insert(document2, document2); + expect(trie.find(document1)).toBe(document1); + expect(trie.find(document2)).toBe(document2); + trie.delete('file:///parent/'); + expect(trie.find(document1)).toBeUndefined(); + expect(trie.find(document2)).toBeUndefined(); + }); + + test('finds all documents', () => { + const document1 = 'file:///test.txt'; + const document2 = 'file:///parent/test.txt'; + trie.insert(document1, document1); + trie.insert(document2, document2); + expect(trie.all()).toEqual([document1, document2]); + }); + + test('finds documents by prefix', () => { + const document0 = 'file:///test.txt'; + const document1 = 'file:///parent/test1.txt'; + const document2 = 'file:///parent/test2.txt'; + trie.insert(document0, document0); + trie.insert(document1, document1); + trie.insert(document2, document2); + expect(trie.findAll('file:///')).toEqual([document0, document1, document2]); + expect(trie.findAll('file:///parent')).toEqual([document1, document2]); + // Ensure that the trailing slash does not affect the result + expect(trie.findAll('file:///parent/')).toEqual([document1, document2]); + }); + + test('returns undefined for non-existing documents', () => { + expect(trie.find('file:///test.txt')).toBeUndefined(); + }); + + test('returns empty array for non-existing prefixes', () => { + expect(trie.findAll('file:///test')).toEqual([]); + }); +}); \ No newline at end of file diff --git a/packages/langium/test/utils/uri-utils.test.ts b/packages/langium/test/utils/uri-utils.test.ts index b4686d238..7eae53c11 100644 --- a/packages/langium/test/utils/uri-utils.test.ts +++ b/packages/langium/test/utils/uri-utils.test.ts @@ -72,3 +72,42 @@ describe('URIUtils#normalize', () => { }); }); + +describe('URIUtils#contains', () => { + + test('Should return true for equal URIs', () => { + const parent = 'file:///path/to/file'; + expect(UriUtils.contains(parent, parent)).toBeTruthy(); + }); + + test('Should return true for equal URIs with trailing slashes', () => { + const parent = 'file:///path/to/file'; + expect(UriUtils.contains(parent + '/', parent)).toBeTruthy(); + expect(UriUtils.contains(parent, parent + '/')).toBeTruthy(); + }); + + test('Should return true for child URIs', () => { + const parent = 'file:///path/to'; + const child = 'file:///path/to/file'; + expect(UriUtils.contains(parent, child)).toBeTruthy(); + }); + + test('Should return true for child URIs with trailing slashes', () => { + const parent = 'file:///path/to/'; + const child = 'file:///path/to/file'; + expect(UriUtils.contains(parent, child)).toBeTruthy(); + }); + + test('Should return false for parent URIs', () => { + const parent = 'file:///path/to/file'; + const child = 'file:///path/to'; + expect(UriUtils.contains(parent, child)).toBeFalsy(); + }); + + test('Should return false for unrelated URIs', () => { + const parent = 'file:///path/to/directory'; + const unrelated = 'file:///path/to/other'; + expect(UriUtils.contains(parent, unrelated)).toBeFalsy(); + }); + +}); diff --git a/packages/langium/test/workspace/document-builder.test.ts b/packages/langium/test/workspace/document-builder.test.ts index 1241bb44e..ca5f0b7b8 100644 --- a/packages/langium/test/workspace/document-builder.test.ts +++ b/packages/langium/test/workspace/document-builder.test.ts @@ -4,13 +4,14 @@ * terms of the MIT License, which is available in the project root. ******************************************************************************/ -import type { AstNode, DocumentBuilder, FileSystemProvider, LangiumDocument, LangiumDocumentFactory, LangiumDocuments, Module, Reference, ValidationChecks } from 'langium'; -import { AstUtils, DocumentState, TextDocument, URI, isOperationCancelled, startCancelableOperation } from 'langium'; +import type { AstNode, DocumentBuilder, FileSystemNode, FileSystemProvider, LangiumDocument, LangiumDocumentFactory, LangiumDocuments, Module, Reference, ValidationChecks } from 'langium'; +import { AstUtils, DocumentState, TextDocument, URI, UriUtils, isOperationCancelled, startCancelableOperation } from 'langium'; import { createServicesForGrammar } from 'langium/grammar'; import { describe, expect, test, vi, beforeEach, afterEach } from 'vitest'; import { CancellationToken } from 'vscode-languageserver'; import { fail } from 'assert'; import type { LangiumServices, LangiumSharedServices, TextDocuments } from 'langium/lsp'; +import { VirtualFileSystemProvider } from 'langium/test'; describe('DefaultDocumentBuilder', () => { async function createServices(shared?: Module) { @@ -87,6 +88,42 @@ describe('DefaultDocumentBuilder', () => { expect(called).toBe(true); }); + test('emits `onUpdate` on `update` call in a directory', async () => { + const virtualFileSystem = new VirtualFileSystemProvider(); + const services = await createServices({ + workspace: { + FileSystemProvider: () => virtualFileSystem + } + }); + const workspace = services.shared.workspace; + const documentFactory = workspace.LangiumDocumentFactory; + const documents = workspace.LangiumDocuments; + const uri1 = URI.parse('file:/dir1/test.txt'); + const uri2 = URI.parse('file:/dir2/test.txt'); + virtualFileSystem.insert(uri1, ''); + virtualFileSystem.insert(uri2, ''); + const document1 = await documentFactory.fromUri(uri1); + documents.addDocument(document1); + const document2 = await documentFactory.fromUri(uri2); + documents.addDocument(document2); + + const builder = workspace.DocumentBuilder; + await builder.build([document1, document2], {}); + let deleted = false; + let updated = false; + builder.onUpdate((changedUris, deletedUris) => { + if (UriUtils.equals(changedUris[0], uri1)) { + updated = true; + } + if (UriUtils.equals(deletedUris[0], uri2)) { + deleted = true; + } + }); + await builder.update([URI.parse('file:/dir1')], [URI.parse('file:/dir2')]); + expect(deleted).toBe(true); + expect(updated).toBe(true); + }); + test('Check all onBuidPhase callbacks', async () => { const services = await createServices(); const documentFactory = services.shared.workspace.LangiumDocumentFactory; @@ -703,6 +740,22 @@ describe('DefaultDocumentBuilder', () => { class MockFileSystemProvider implements FileSystemProvider { isMockFileSystemProvider = true; + async stat(uri: URI): Promise { + return { + isDirectory: false, + isFile: true, + uri + }; + } + + statSync(uri: URI): FileSystemNode { + return { + isDirectory: false, + isFile: true, + uri + }; + } + // Return an empty string for any file readFile(_uri: URI): Promise{ return Promise.resolve('');