From db5d52916dcb97e33d95d6d9476a0f7b86a6e68c Mon Sep 17 00:00:00 2001 From: Ivan Goncharov Date: Tue, 1 Oct 2019 17:28:16 +0300 Subject: [PATCH] Switch Lexer to be a proper class, remove 'createLexer' --- src/index.js | 3 +- src/language/__tests__/lexer-test.js | 14 ++-- src/language/index.js | 3 +- src/language/lexer.js | 79 ++++++++----------- src/language/parser.js | 4 +- .../__tests__/stripIgnoredCharacters-test.js | 4 +- src/utilities/stripIgnoredCharacters.js | 4 +- tstypes/index.d.ts | 3 +- tstypes/language/index.d.ts | 2 +- tstypes/language/lexer.d.ts | 9 +-- 10 files changed, 53 insertions(+), 72 deletions(-) diff --git a/src/index.js b/src/index.js index 46133b23db..4c1945ec56 100644 --- a/src/index.js +++ b/src/index.js @@ -178,7 +178,7 @@ export { printLocation, printSourceLocation, // Lex - createLexer, + Lexer, TokenKind, // Parse parse, @@ -207,7 +207,6 @@ export { } from './language'; export type { - Lexer, ParseOptions, SourceLocation, Location, diff --git a/src/language/__tests__/lexer-test.js b/src/language/__tests__/lexer-test.js index bfec7cab77..e60d4f69d7 100644 --- a/src/language/__tests__/lexer-test.js +++ b/src/language/__tests__/lexer-test.js @@ -12,15 +12,15 @@ import { GraphQLError } from '../../error/GraphQLError'; import { Source } from '../source'; import { TokenKind } from '../tokenKind'; -import { createLexer, isPunctuatorTokenKind } from '../lexer'; +import { Lexer, isPunctuatorTokenKind } from '../lexer'; function lexOne(str) { - const lexer = createLexer(new Source(str)); + const lexer = new Lexer(new Source(str)); return lexer.advance(); } function lexSecond(str) { - const lexer = createLexer(new Source(str)); + const lexer = new Lexer(new Source(str)); lexer.advance(); return lexer.advance(); } @@ -187,7 +187,7 @@ describe('Lexer', () => { try { const str = ['', '', ' ?', ''].join('\n'); const source = new Source(str, 'foo.js', { line: 11, column: 12 }); - createLexer(source).advance(); + new Lexer(source).advance(); } catch (error) { caughtError = error; } @@ -206,7 +206,7 @@ describe('Lexer', () => { let caughtError; try { const source = new Source('?', 'foo.js', { line: 1, column: 5 }); - createLexer(source).advance(); + new Lexer(source).advance(); } catch (error) { caughtError = error; } @@ -847,7 +847,7 @@ describe('Lexer', () => { it('lex reports useful information for dashes in names', () => { const source = new Source('a-b'); - const lexer = createLexer(source); + const lexer = new Lexer(source); const firstToken = lexer.advance(); expect(firstToken).to.contain({ kind: TokenKind.NAME, @@ -872,7 +872,7 @@ describe('Lexer', () => { } `); - const lexer = createLexer(source); + const lexer = new Lexer(source); const startToken = lexer.token; let endToken; do { diff --git a/src/language/index.js b/src/language/index.js index 3d8a821703..a23644a532 100644 --- a/src/language/index.js +++ b/src/language/index.js @@ -13,8 +13,7 @@ export type { KindEnum } from './kinds'; export { TokenKind } from './tokenKind'; export type { TokenKindEnum } from './tokenKind'; -export { createLexer } from './lexer'; -export type { Lexer } from './lexer'; +export { Lexer } from './lexer'; export { parse, parseValue, parseType } from './parser'; export type { ParseOptions } from './parser'; diff --git a/src/language/lexer.js b/src/language/lexer.js index 3fe1ad469f..43c279879a 100644 --- a/src/language/lexer.js +++ b/src/language/lexer.js @@ -10,83 +10,70 @@ import { dedentBlockStringValue } from './blockString'; import { type TokenKindEnum, TokenKind } from './tokenKind'; /** - * Given a Source object, this returns a Lexer for that source. + * Given a Source object, creates a Lexer for that source. * A Lexer is a stateful stream generator in that every time * it is advanced, it returns the next token in the Source. Assuming the * source lexes, the final Token emitted by the lexer will be of kind * EOF, after which the lexer will repeatedly return the same EOF token * whenever called. */ -export function createLexer(source: Source): Lexer { - const startOfFileToken = new Tok(TokenKind.SOF, 0, 0, 0, 0, null); - const lexer: Lexer = { - source, - lastToken: startOfFileToken, - token: startOfFileToken, - line: 1, - lineStart: 0, - advance: advanceLexer, - lookahead, - }; - return lexer; -} - -function advanceLexer() { - this.lastToken = this.token; - const token = (this.token = this.lookahead()); - return token; -} - -function lookahead() { - let token = this.token; - if (token.kind !== TokenKind.EOF) { - do { - // Note: next is only mutable during parsing, so we cast to allow this. - token = token.next || ((token: any).next = readToken(this, token)); - } while (token.kind === TokenKind.COMMENT); - } - return token; -} - -/** - * The return type of createLexer. - */ -export type Lexer = { - source: Source, +export class Lexer { + source: Source; /** * The previously focused non-ignored token. */ - lastToken: Token, + lastToken: Token; /** * The currently focused non-ignored token. */ - token: Token, + token: Token; /** * The (1-indexed) line containing the current token. */ - line: number, + line: number; /** * The character offset at which the current line begins. */ - lineStart: number, + lineStart: number; + + constructor(source: Source) { + const startOfFileToken = new Tok(TokenKind.SOF, 0, 0, 0, 0, null); + + this.source = source; + this.lastToken = startOfFileToken; + this.token = startOfFileToken; + this.line = 1; + this.lineStart = 0; + } /** * Advances the token stream to the next non-ignored token. */ - advance(): Token, + advance(): Token { + this.lastToken = this.token; + const token = (this.token = this.lookahead()); + return token; + } /** * Looks ahead and returns the next non-ignored token, but does not change * the Lexer's state. */ - lookahead(): Token, - - ... -}; + lookahead(): Token { + let token = this.token; + if (token.kind !== TokenKind.EOF) { + do { + // Note: next is only mutable during parsing, so we cast to allow this. + token = token.next || ((token: any).next = readToken(this, token)); + } while (token.kind === TokenKind.COMMENT); + } + return token; + } +} /** * @internal diff --git a/src/language/parser.js b/src/language/parser.js index ce38a0f773..8455be389c 100644 --- a/src/language/parser.js +++ b/src/language/parser.js @@ -11,7 +11,7 @@ import { Kind } from './kinds'; import { Source } from './source'; import { DirectiveLocation } from './directiveLocation'; import { type TokenKindEnum, TokenKind } from './tokenKind'; -import { type Lexer, createLexer, isPunctuatorTokenKind } from './lexer'; +import { Lexer, isPunctuatorTokenKind } from './lexer'; import { type Location, type Token, @@ -177,7 +177,7 @@ class Parser { `Must provide Source. Received: ${inspect(sourceObj)}`, ); - this._lexer = createLexer(sourceObj); + this._lexer = new Lexer(sourceObj); this._options = options || {}; } diff --git a/src/utilities/__tests__/stripIgnoredCharacters-test.js b/src/utilities/__tests__/stripIgnoredCharacters-test.js index 95093d993d..2c24e0c4be 100644 --- a/src/utilities/__tests__/stripIgnoredCharacters-test.js +++ b/src/utilities/__tests__/stripIgnoredCharacters-test.js @@ -8,7 +8,7 @@ import invariant from '../../jsutils/invariant'; import { parse } from '../../language/parser'; import { Source } from '../../language/source'; -import { createLexer } from '../../language/lexer'; +import { Lexer } from '../../language/lexer'; import { stripIgnoredCharacters } from '../stripIgnoredCharacters'; @@ -59,7 +59,7 @@ const nonPunctuatorTokens = [ ]; function lexValue(str) { - const lexer = createLexer(new Source(str)); + const lexer = new Lexer(new Source(str)); const value = lexer.advance().value; invariant(lexer.advance().kind === '', 'Expected EOF'); diff --git a/src/utilities/stripIgnoredCharacters.js b/src/utilities/stripIgnoredCharacters.js index 8eae59b8c8..e6ad305a1b 100644 --- a/src/utilities/stripIgnoredCharacters.js +++ b/src/utilities/stripIgnoredCharacters.js @@ -4,7 +4,7 @@ import inspect from '../jsutils/inspect'; import { Source } from '../language/source'; import { TokenKind } from '../language/tokenKind'; -import { createLexer, isPunctuatorTokenKind } from '../language/lexer'; +import { Lexer, isPunctuatorTokenKind } from '../language/lexer'; import { dedentBlockStringValue, getBlockStringIndentation, @@ -71,7 +71,7 @@ export function stripIgnoredCharacters(source: string | Source): string { } const body = sourceObj.body; - const lexer = createLexer(sourceObj); + const lexer = new Lexer(sourceObj); let strippedBody = ''; let wasLastAddedTokenNonPunctuator = false; diff --git a/tstypes/index.d.ts b/tstypes/index.d.ts index b5c614bcda..17701c5cd5 100644 --- a/tstypes/index.d.ts +++ b/tstypes/index.d.ts @@ -177,7 +177,7 @@ export { printLocation, printSourceLocation, // Lex - createLexer, + Lexer, TokenKind, // Parse parse, @@ -206,7 +206,6 @@ export { } from './language'; export { - Lexer, ParseOptions, SourceLocation, Location, diff --git a/tstypes/language/index.d.ts b/tstypes/language/index.d.ts index 7cd95901c9..9f789853e9 100644 --- a/tstypes/language/index.d.ts +++ b/tstypes/language/index.d.ts @@ -5,7 +5,7 @@ export { printLocation, printSourceLocation } from './printLocation'; export { Kind, KindEnum } from './kinds'; export { TokenKind, TokenKindEnum } from './tokenKind'; -export { createLexer, Lexer } from './lexer'; +export { Lexer } from './lexer'; export { parse, parseValue, parseType, ParseOptions } from './parser'; export { print } from './printer'; export { diff --git a/tstypes/language/lexer.d.ts b/tstypes/language/lexer.d.ts index f1a7257f54..fb6ec700ec 100644 --- a/tstypes/language/lexer.d.ts +++ b/tstypes/language/lexer.d.ts @@ -10,12 +10,7 @@ import { Source } from './source'; * EOF, after which the lexer will repeatedly return the same EOF token * whenever called. */ -export function createLexer(source: Source): Lexer; - -/** - * The return type of createLexer. - */ -export interface Lexer { +export class Lexer { source: Source; /** @@ -38,6 +33,8 @@ export interface Lexer { */ lineStart: number; + constructor(source: Source); + /** * Advances the token stream to the next non-ignored token. */