diff --git a/package.json b/package.json index b706286303..e718521e72 100644 --- a/package.json +++ b/package.json @@ -516,6 +516,11 @@ "default": true, "description": "Whether to automatic updating import path when rename or move a file" }, + "vetur.languageFeatures.semanticTokens": { + "type": "boolean", + "default": true, + "description": "Whether to enable semantic highlighting. Currently only works for typescript" + }, "vetur.trace.server": { "type": "string", "enum": [ @@ -555,7 +560,14 @@ "description": "Enable template interpolation service that offers hover / definition / references in Vue interpolations." } } - } + }, + "semanticTokenScopes": [ + { + "scopes": { + "property.refValue": ["entity.name.function"] + } + } + ] }, "devDependencies": { "@rollup/plugin-commonjs": "^17.1.0", diff --git a/server/src/config.ts b/server/src/config.ts index 521edaf163..283a66e1aa 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -65,6 +65,7 @@ export interface VLSConfig { languageFeatures: { codeActions: boolean; updateImportOnFileMove: boolean; + semanticTokens: boolean; }; trace: { server: 'off' | 'messages' | 'verbose'; @@ -128,7 +129,8 @@ export function getDefaultVLSConfig(): VLSFullConfig { }, languageFeatures: { codeActions: true, - updateImportOnFileMove: true + updateImportOnFileMove: true, + semanticTokens: true }, trace: { server: 'off' diff --git a/server/src/embeddedSupport/languageModes.ts b/server/src/embeddedSupport/languageModes.ts index 47ea62d792..089fe08d09 100644 --- a/server/src/embeddedSupport/languageModes.ts +++ b/server/src/embeddedSupport/languageModes.ts @@ -31,7 +31,7 @@ import { getCSSMode, getSCSSMode, getLESSMode, getPostCSSMode } from '../modes/s import { getJavascriptMode } from '../modes/script/javascript'; import { VueHTMLMode } from '../modes/template'; import { getStylusMode } from '../modes/style/stylus'; -import { DocumentContext } from '../types'; +import { DocumentContext, SemanticTokenData } from '../types'; import { VueInfoService } from '../services/vueInfoService'; import { DependencyService } from '../services/dependencyService'; import { nullMode } from '../modes/nullMode'; @@ -74,6 +74,7 @@ export interface LanguageMode { getColorPresentations?(document: TextDocument, color: Color, range: Range): ColorPresentation[]; getFoldingRanges?(document: TextDocument): FoldingRange[]; getRenameFileEdit?(renames: FileRename): TextDocumentEdit[]; + getSemanticTokens?(document: TextDocument, range?: Range): SemanticTokenData[]; onDocumentChanged?(filePath: string): void; onDocumentRemoved(document: TextDocument): void; diff --git a/server/src/modes/script/javascript.ts b/server/src/modes/script/javascript.ts index f5032f69ea..dc676203d6 100644 --- a/server/src/modes/script/javascript.ts +++ b/server/src/modes/script/javascript.ts @@ -44,7 +44,13 @@ import { BasicComponentInfo, VLSFormatConfig } from '../../config'; import { VueInfoService } from '../../services/vueInfoService'; import { getComponentInfo } from './componentInfo'; import { DependencyService, RuntimeLibrary } from '../../services/dependencyService'; -import { CodeActionData, CodeActionDataKind, OrganizeImportsActionData, RefactorActionData } from '../../types'; +import { + CodeActionData, + CodeActionDataKind, + OrganizeImportsActionData, + RefactorActionData, + SemanticTokenOffsetData +} from '../../types'; import { IServiceHost } from '../../services/typescriptService/serviceHost'; import { isVirtualVueTemplateFile, @@ -57,11 +63,18 @@ import { isVCancellationRequested, VCancellationToken } from '../../utils/cancel import { EnvironmentService } from '../../services/EnvironmentService'; import { getCodeActionKind } from './CodeActionKindConverter'; import { FileRename } from 'vscode-languageserver'; +import { + addCompositionApiRefTokens, + getTokenModifierFromClassification, + getTokenTypeFromClassification +} from './semanticToken'; // Todo: After upgrading to LS server 4.0, use CompletionContext for filtering trigger chars // https://microsoft.github.io/language-server-protocol/specification#completion-request-leftwards_arrow_with_hook const NON_SCRIPT_TRIGGERS = ['<', '*', ':']; +const SEMANTIC_TOKEN_CONTENT_LENGTH_LIMIT = 80000; + export async function getJavascriptMode( serviceHost: IServiceHost, env: EnvironmentService, @@ -788,6 +801,64 @@ export async function getJavascriptMode( return textDocumentEdit; }, + getSemanticTokens(doc: TextDocument, range?: Range) { + const { scriptDoc, service } = updateCurrentVueTextDocument(doc); + const scriptText = scriptDoc.getText(); + if (scriptText.trim().length > SEMANTIC_TOKEN_CONTENT_LENGTH_LIMIT) { + return []; + } + + const fileFsPath = getFileFsPath(doc.uri); + const textSpan = range + ? convertTextSpan(range, scriptDoc) + : { + start: 0, + length: scriptText.length + }; + const { spans } = service.getEncodedSemanticClassifications( + fileFsPath, + textSpan, + tsModule.SemanticClassificationFormat.TwentyTwenty + ); + + const data: SemanticTokenOffsetData[] = []; + let index = 0; + + while (index < spans.length) { + // [start, length, encodedClassification, start2, length2, encodedClassification2] + const start = spans[index++]; + const length = spans[index++]; + const encodedClassification = spans[index++]; + const classificationType = getTokenTypeFromClassification(encodedClassification); + if (classificationType < 0) { + continue; + } + + const modifierSet = getTokenModifierFromClassification(encodedClassification); + + data.push({ + start, + length, + classificationType, + modifierSet + }); + } + + const program = service.getProgram(); + if (program) { + addCompositionApiRefTokens(tsModule, program, fileFsPath, data); + } + + return data.map(({ start, ...rest }) => { + const startPosition = scriptDoc.positionAt(start); + + return { + ...rest, + line: startPosition.line, + character: startPosition.character + }; + }); + }, dispose() { jsDocuments.dispose(); } @@ -1114,3 +1185,13 @@ function getFoldingRangeKind(span: ts.OutliningSpan): FoldingRangeKind | undefin return undefined; } } + +function convertTextSpan(range: Range, doc: TextDocument): ts.TextSpan { + const start = doc.offsetAt(range.start); + const end = doc.offsetAt(range.end); + + return { + start, + length: end - start + }; +} diff --git a/server/src/modes/script/semanticToken.ts b/server/src/modes/script/semanticToken.ts new file mode 100644 index 0000000000..56fd20dc0c --- /dev/null +++ b/server/src/modes/script/semanticToken.ts @@ -0,0 +1,149 @@ +import ts from 'typescript'; +import { SemanticTokensLegend, SemanticTokenModifiers, SemanticTokenTypes } from 'vscode-languageserver'; +import { RuntimeLibrary } from '../../services/dependencyService'; +import { SemanticTokenOffsetData } from '../../types'; + +/* tslint:disable:max-line-length */ +/** + * extended from https://github.com/microsoft/TypeScript/blob/35c8df04ad959224fad9037e340c1e50f0540a49/src/services/classifier2020.ts#L9 + * so that we don't have to map it into our own legend + */ +export const enum TokenType { + class, + enum, + interface, + namespace, + typeParameter, + type, + parameter, + variable, + enumMember, + property, + function, + member +} + +/* tslint:disable:max-line-length */ +/** + * adopted from https://github.com/microsoft/TypeScript/blob/35c8df04ad959224fad9037e340c1e50f0540a49/src/services/classifier2020.ts#L13 + * so that we don't have to map it into our own legend + */ +export const enum TokenModifier { + declaration, + static, + async, + readonly, + defaultLibrary, + local, + + // vue composition api + refValue +} + +export function getSemanticTokenLegends(): SemanticTokensLegend { + const tokenModifiers: string[] = []; + + ([ + [TokenModifier.declaration, SemanticTokenModifiers.declaration], + [TokenModifier.static, SemanticTokenModifiers.static], + [TokenModifier.async, SemanticTokenModifiers.async], + [TokenModifier.readonly, SemanticTokenModifiers.readonly], + [TokenModifier.defaultLibrary, SemanticTokenModifiers.defaultLibrary], + [TokenModifier.local, 'local'], + + // vue + [TokenModifier.refValue, 'refValue'] + ] as const).forEach(([tsModifier, legend]) => (tokenModifiers[tsModifier] = legend)); + + const tokenTypes: string[] = []; + + ([ + [TokenType.class, SemanticTokenTypes.class], + [TokenType.enum, SemanticTokenTypes.enum], + [TokenType.interface, SemanticTokenTypes.interface], + [TokenType.namespace, SemanticTokenTypes.namespace], + [TokenType.typeParameter, SemanticTokenTypes.typeParameter], + [TokenType.type, SemanticTokenTypes.type], + [TokenType.parameter, SemanticTokenTypes.parameter], + [TokenType.variable, SemanticTokenTypes.variable], + [TokenType.enumMember, SemanticTokenTypes.enumMember], + [TokenType.property, SemanticTokenTypes.property], + [TokenType.function, SemanticTokenTypes.function], + + // member is renamed to method in vscode codebase to match LSP default + [TokenType.member, SemanticTokenTypes.method] + ] as const).forEach(([tokenType, legend]) => (tokenTypes[tokenType] = legend)); + + return { + tokenModifiers, + tokenTypes + }; +} + +export function getTokenTypeFromClassification(tsClassification: number): number { + return (tsClassification >> TokenEncodingConsts.typeOffset) - 1; +} + +export function getTokenModifierFromClassification(tsClassification: number) { + return tsClassification & TokenEncodingConsts.modifierMask; +} + +const enum TokenEncodingConsts { + typeOffset = 8, + modifierMask = (1 << typeOffset) - 1 +} + +export function addCompositionApiRefTokens( + tsModule: RuntimeLibrary['typescript'], + program: ts.Program, + fileFsPath: string, + exists: SemanticTokenOffsetData[] +): void { + const sourceFile = program.getSourceFile(fileFsPath); + + if (!sourceFile) { + return; + } + + const typeChecker = program.getTypeChecker(); + + walk(sourceFile, node => { + if (!ts.isIdentifier(node) || node.text !== 'value' || !ts.isPropertyAccessExpression(node.parent)) { + return; + } + const propertyAccess = node.parent; + + let parentSymbol = typeChecker.getTypeAtLocation(propertyAccess.expression).symbol; + + if (parentSymbol.flags & tsModule.SymbolFlags.Alias) { + parentSymbol = typeChecker.getAliasedSymbol(parentSymbol); + } + + if (parentSymbol.name !== 'Ref') { + return; + } + + const start = node.getStart(); + const length = node.getWidth(); + const exist = exists.find(token => token.start === start && token.length === length); + const encodedModifier = 1 << TokenModifier.refValue; + + if (exist) { + exist.modifierSet |= encodedModifier; + } else { + exists.push({ + classificationType: TokenType.property, + length: node.getEnd() - node.getStart(), + modifierSet: encodedModifier, + start: node.getStart() + }); + } + }); +} + +function walk(node: ts.Node, callback: (node: ts.Node) => void) { + node.forEachChild(child => { + callback(child); + walk(child, callback); + }); +} diff --git a/server/src/services/projectService.ts b/server/src/services/projectService.ts index 647f26ca05..32f5f0c79d 100644 --- a/server/src/services/projectService.ts +++ b/server/src/services/projectService.ts @@ -22,6 +22,10 @@ import { FoldingRangeParams, Hover, Location, + SemanticTokens, + SemanticTokensBuilder, + SemanticTokensParams, + SemanticTokensRangeParams, SignatureHelp, SymbolInformation, TextDocumentEdit, @@ -33,7 +37,7 @@ import { URI } from 'vscode-uri'; import { LanguageId } from '../embeddedSupport/embeddedSupport'; import { LanguageMode, LanguageModes } from '../embeddedSupport/languageModes'; import { NULL_COMPLETION, NULL_HOVER, NULL_SIGNATURE } from '../modes/nullMode'; -import { DocumentContext, CodeActionData } from '../types'; +import { DocumentContext, CodeActionData, SemanticTokenData } from '../types'; import { VCancellationToken } from '../utils/cancellationToken'; import { getFileFsPath } from '../utils/paths'; import { DependencyService } from './dependencyService'; @@ -60,6 +64,7 @@ export interface ProjectService { onCodeAction(params: CodeActionParams): Promise; onCodeActionResolve(action: CodeAction): Promise; onWillRenameFile(fileRename: FileRename): Promise; + onSemanticTokens(params: SemanticTokensParams | SemanticTokensRangeParams): Promise; doValidate(doc: TextDocument, cancellationToken?: VCancellationToken): Promise; dispose(): Promise; } @@ -336,6 +341,35 @@ export async function createProjectService( return textDocumentEdit ?? []; }, + async onSemanticTokens(params: SemanticTokensParams | SemanticTokensRangeParams) { + if (!env.getConfig().vetur.languageFeatures.semanticTokens) { + return { + data: [] + }; + } + + const { textDocument } = params; + const range = 'range' in params ? params.range : undefined; + const doc = documentService.getDocument(textDocument.uri)!; + const modes = languageModes.getAllLanguageModeRangesInDocument(doc); + const data: SemanticTokenData[] = []; + + for (const mode of modes) { + const tokenData = mode.mode.getSemanticTokens?.(doc, range); + + data.push(...(tokenData ?? [])); + } + + const builder = new SemanticTokensBuilder(); + const sorted = data.sort((a, b) => { + return a.line - b.line || a.character - b.character; + }); + sorted.forEach(token => + builder.push(token.line, token.character, token.length, token.classificationType, token.modifierSet) + ); + + return builder.build(); + }, async doValidate(doc: TextDocument, cancellationToken?: VCancellationToken) { const diagnostics: Diagnostic[] = []; if (doc.languageId === 'vue') { diff --git a/server/src/services/vls.ts b/server/src/services/vls.ts index ea58cc099d..194632fa21 100644 --- a/server/src/services/vls.ts +++ b/server/src/services/vls.ts @@ -26,7 +26,12 @@ import { CompletionParams, ExecuteCommandParams, FoldingRangeParams, - RenameFilesParams + RenameFilesParams, + SemanticTokensParams, + SemanticTokens, + SemanticTokensRangeParams, + SemanticTokensRequest, + SemanticTokensRangeRequest } from 'vscode-languageserver'; import { ColorInformation, @@ -44,9 +49,10 @@ import { FoldingRange, DocumentUri, CodeAction, - CodeActionKind + CodeActionKind, + TextDocumentIdentifier } from 'vscode-languageserver-types'; -import type { TextDocument } from 'vscode-languageserver-textdocument'; +import type { Range, TextDocument } from 'vscode-languageserver-textdocument'; import { NULL_COMPLETION, NULL_HOVER, NULL_SIGNATURE } from '../modes/nullMode'; import { createDependencyService, createNodeModulesPaths } from './dependencyService'; @@ -63,6 +69,7 @@ import { getVueVersionKey } from '../utils/vueVersion'; import { accessSync, constants, existsSync } from 'fs'; import { sleep } from '../utils/sleep'; import { URI } from 'vscode-uri'; +import { getSemanticTokenLegends } from '../modes/script/semanticToken'; interface ProjectConfig { vlsFullConfig: VLSFullConfig; @@ -384,6 +391,8 @@ export class VLS { this.lspConnection.onCodeAction(this.onCodeAction.bind(this)); this.lspConnection.onCodeActionResolve(this.onCodeActionResolve.bind(this)); this.lspConnection.workspace.onWillRenameFiles(this.onWillRenameFiles.bind(this)); + this.lspConnection.languages.semanticTokens.on(this.onSemanticToken.bind(this)); + this.lspConnection.languages.semanticTokens.onRange(this.onSemanticToken.bind(this)); this.lspConnection.onDocumentColor(this.onDocumentColors.bind(this)); this.lspConnection.onColorPresentation(this.onColorPresentations.bind(this)); @@ -620,6 +629,12 @@ export class VLS { }; } + async onSemanticToken(params: SemanticTokensParams | SemanticTokensRangeParams): Promise { + const project = await this.getProjectService(params.textDocument.uri); + + return project?.onSemanticTokens(params) ?? { data: [] as number[] }; + } + private triggerValidation(textDocument: TextDocument): void { if (textDocument.uri.includes('node_modules')) { return; @@ -713,7 +728,12 @@ export class VLS { executeCommandProvider: { commands: [] }, - foldingRangeProvider: true + foldingRangeProvider: true, + semanticTokensProvider: { + range: true, + full: true, + legend: getSemanticTokenLegends() + } }; } } diff --git a/server/src/types.ts b/server/src/types.ts index aee303b498..6299d0826a 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -35,3 +35,19 @@ export interface OrganizeImportsActionData extends BaseCodeActionData { } export type CodeActionData = RefactorActionData | CombinedFixActionData | OrganizeImportsActionData; + +interface SemanticTokenClassification { + classificationType: number; + modifierSet: number; +} + +export interface SemanticTokenData extends SemanticTokenClassification { + line: number; + character: number; + length: number; +} + +export interface SemanticTokenOffsetData extends SemanticTokenClassification { + start: number; + length: number; +} diff --git a/test/codeTestRunner.ts b/test/codeTestRunner.ts index 14cbf82773..4d0a797510 100644 --- a/test/codeTestRunner.ts +++ b/test/codeTestRunner.ts @@ -25,7 +25,7 @@ async function run(execPath: string, testWorkspaceRelativePath: string, mochaArg return await runTests({ vscodeExecutablePath: execPath, - version: '1.52.1', + version: '1.53.2', extensionDevelopmentPath: EXT_ROOT, extensionTestsPath: extTestPath, extensionTestsEnv: mochaArgs, @@ -81,7 +81,7 @@ function installMissingDependencies(fixturePath: string) { } async function go() { - const execPath = await downloadAndUnzipVSCode('1.52.1'); + const execPath = await downloadAndUnzipVSCode('1.53.2'); await runAllTests(execPath); } diff --git a/test/lsp/features/semanticTokens/basic.test.ts b/test/lsp/features/semanticTokens/basic.test.ts new file mode 100644 index 0000000000..f0885da7e8 --- /dev/null +++ b/test/lsp/features/semanticTokens/basic.test.ts @@ -0,0 +1,73 @@ +import { SemanticTokenModifiers, SemanticTokenTypes } from 'vscode-languageserver-protocol'; +import { getTokenRange, testSemanticTokens } from '../../../semanticTokenHelper'; +import { getDocUri } from '../../path'; + +describe('semantic tokens', () => { + const docPath = 'semanticTokens/Basic.vue'; + const docUri = getDocUri(docPath); + + it('provide semantic tokens', async () => { + await testSemanticTokens(docUri, [ + { + type: SemanticTokenTypes.variable, + range: getTokenRange(7, 6, 'aConst'), + modifiers: [SemanticTokenModifiers.declaration, SemanticTokenModifiers.readonly] + }, + { + type: SemanticTokenTypes.class, + range: getTokenRange(9, 15, 'Vue'), + modifiers: [] + }, + { + type: SemanticTokenTypes.method, + range: getTokenRange(9, 19, 'extend'), + modifiers: [] + }, + { + type: SemanticTokenTypes.property, + range: getTokenRange(10, 2, 'methods'), + modifiers: [SemanticTokenModifiers.declaration] + }, + { + type: SemanticTokenTypes.method, + range: getTokenRange(11, 4, 'abc'), + modifiers: [SemanticTokenModifiers.declaration] + }, + { + type: SemanticTokenTypes.method, + range: getTokenRange(12, 11, 'log'), + modifiers: [] + }, + { + type: SemanticTokenTypes.variable, + range: getTokenRange(12, 15, 'aConst'), + modifiers: [SemanticTokenModifiers.readonly] + }, + { + type: SemanticTokenTypes.method, + range: getTokenRange(14, 4, 'log'), + modifiers: [SemanticTokenModifiers.declaration] + }, + { + type: SemanticTokenTypes.parameter, + range: getTokenRange(14, 8, 'str'), + modifiers: [SemanticTokenModifiers.declaration] + }, + { + type: SemanticTokenTypes.variable, + range: getTokenRange(15, 6, 'console'), + modifiers: [SemanticTokenModifiers.defaultLibrary] + }, + { + type: SemanticTokenTypes.method, + range: getTokenRange(15, 14, 'log'), + modifiers: [SemanticTokenModifiers.defaultLibrary] + }, + { + type: SemanticTokenTypes.parameter, + range: getTokenRange(15, 18, 'str'), + modifiers: [] + } + ]); + }); +}); diff --git a/test/lsp/fixture/semanticTokens/Basic.vue b/test/lsp/fixture/semanticTokens/Basic.vue new file mode 100644 index 0000000000..0540a211be --- /dev/null +++ b/test/lsp/fixture/semanticTokens/Basic.vue @@ -0,0 +1,20 @@ + + + \ No newline at end of file diff --git a/test/semanticTokenHelper.ts b/test/semanticTokenHelper.ts new file mode 100644 index 0000000000..1dbe1aa37b --- /dev/null +++ b/test/semanticTokenHelper.ts @@ -0,0 +1,64 @@ +import assert from 'assert'; +import vscode from 'vscode'; +import { showFile } from './editorHelper'; +import { sameLineRange } from './util'; + +/** + * group result by tokens to better distinguish + */ +export function assertResult(actual: Uint32Array, expected: number[]) { + const actualGrouped = group(actual); + const expectedGrouped = group(expected); + + assert.deepStrictEqual(actualGrouped, expectedGrouped); +} + +function group(tokens: Uint32Array | number[]) { + const result: number[][] = []; + + let index = 0; + while (index < tokens.length) { + result.push(Array.from(tokens.slice(index, (index += 5)))); + } + + return result; +} + +export interface UnEncodedSemanticTokenData { + range: vscode.Range; + type: string; + modifiers: string[]; +} + +function encodeExpected(legend: vscode.SemanticTokensLegend, tokens: UnEncodedSemanticTokenData[]) { + const builder = new vscode.SemanticTokensBuilder(legend); + + for (const token of tokens) { + builder.push(token.range, token.type, token.modifiers); + } + + return Array.from(builder.build().data); +} + +async function getLegend(uri: vscode.Uri): Promise { + const res = await vscode.commands.executeCommand( + 'vscode.provideDocumentSemanticTokensLegend', + uri + ); + + return res!; +} + +export async function testSemanticTokens(docUri: vscode.Uri, expected: UnEncodedSemanticTokenData[]) { + await showFile(docUri); + + const result = await vscode.commands.executeCommand( + 'vscode.provideDocumentSemanticTokens', + docUri + ); + assertResult(result!.data, encodeExpected(await getLegend(docUri), expected)); +} + +export function getTokenRange(line: number, startChar: number, identifier: string) { + return sameLineRange(line, startChar, startChar + identifier.length); +} diff --git a/test/vue3/features/semanticTokens/basic.test.ts b/test/vue3/features/semanticTokens/basic.test.ts new file mode 100644 index 0000000000..033b3f32b9 --- /dev/null +++ b/test/vue3/features/semanticTokens/basic.test.ts @@ -0,0 +1,38 @@ +import { SemanticTokenModifiers, SemanticTokenTypes } from 'vscode-languageserver-protocol'; +import { getTokenRange, testSemanticTokens } from '../../../semanticTokenHelper'; +import { getDocUri } from '../../path'; + +describe('semantic tokens', () => { + const docPath = 'semanticTokens/Basic.vue'; + const docUri = getDocUri(docPath); + + it('provide semantic tokens', async () => { + await testSemanticTokens(docUri, [ + { + type: SemanticTokenTypes.method, + range: getTokenRange(7, 2, 'setup'), + modifiers: [SemanticTokenModifiers.declaration] + }, + { + type: SemanticTokenTypes.variable, + range: getTokenRange(8, 10, 'a'), + modifiers: [SemanticTokenModifiers.readonly, SemanticTokenModifiers.declaration, 'local'] + }, + { + type: SemanticTokenTypes.function, + range: getTokenRange(8, 14, 'ref'), + modifiers: [] + }, + { + type: SemanticTokenTypes.variable, + range: getTokenRange(10, 4, 'a'), + modifiers: [SemanticTokenModifiers.readonly, 'local'] + }, + { + type: SemanticTokenTypes.property, + range: getTokenRange(10, 6, 'value'), + modifiers: ['refValue'] + } + ]); + }); +}); diff --git a/test/vue3/fixture/semanticTokens/Basic.vue b/test/vue3/fixture/semanticTokens/Basic.vue new file mode 100644 index 0000000000..cdc3006150 --- /dev/null +++ b/test/vue3/fixture/semanticTokens/Basic.vue @@ -0,0 +1,14 @@ + + + \ No newline at end of file