Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Switch Lexer to be a proper class, remove 'createLexer' #2210

Merged
merged 1 commit into from
Oct 14, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions src/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ export {
printLocation,
printSourceLocation,
// Lex
createLexer,
Lexer,
TokenKind,
// Parse
parse,
Expand Down Expand Up @@ -207,7 +207,6 @@ export {
} from './language';

export type {
Lexer,
ParseOptions,
SourceLocation,
Location,
Expand Down
14 changes: 7 additions & 7 deletions src/language/__tests__/lexer-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,15 @@ import { GraphQLError } from '../../error/GraphQLError';

import { Source } from '../source';
import { TokenKind } from '../tokenKind';
import { createLexer, isPunctuatorTokenKind } from '../lexer';
import { Lexer, isPunctuatorTokenKind } from '../lexer';

function lexOne(str) {
const lexer = createLexer(new Source(str));
const lexer = new Lexer(new Source(str));
return lexer.advance();
}

function lexSecond(str) {
const lexer = createLexer(new Source(str));
const lexer = new Lexer(new Source(str));
lexer.advance();
return lexer.advance();
}
Expand Down Expand Up @@ -187,7 +187,7 @@ describe('Lexer', () => {
try {
const str = ['', '', ' ?', ''].join('\n');
const source = new Source(str, 'foo.js', { line: 11, column: 12 });
createLexer(source).advance();
new Lexer(source).advance();
} catch (error) {
caughtError = error;
}
Expand All @@ -206,7 +206,7 @@ describe('Lexer', () => {
let caughtError;
try {
const source = new Source('?', 'foo.js', { line: 1, column: 5 });
createLexer(source).advance();
new Lexer(source).advance();
} catch (error) {
caughtError = error;
}
Expand Down Expand Up @@ -847,7 +847,7 @@ describe('Lexer', () => {

it('lex reports useful information for dashes in names', () => {
const source = new Source('a-b');
const lexer = createLexer(source);
const lexer = new Lexer(source);
const firstToken = lexer.advance();
expect(firstToken).to.contain({
kind: TokenKind.NAME,
Expand All @@ -872,7 +872,7 @@ describe('Lexer', () => {
}
`);

const lexer = createLexer(source);
const lexer = new Lexer(source);
const startToken = lexer.token;
let endToken;
do {
Expand Down
3 changes: 1 addition & 2 deletions src/language/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,7 @@ export type { KindEnum } from './kinds';
export { TokenKind } from './tokenKind';
export type { TokenKindEnum } from './tokenKind';

export { createLexer } from './lexer';
export type { Lexer } from './lexer';
export { Lexer } from './lexer';

export { parse, parseValue, parseType } from './parser';
export type { ParseOptions } from './parser';
Expand Down
79 changes: 33 additions & 46 deletions src/language/lexer.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,83 +10,70 @@ import { dedentBlockStringValue } from './blockString';
import { type TokenKindEnum, TokenKind } from './tokenKind';

/**
* Given a Source object, this returns a Lexer for that source.
* Given a Source object, creates a Lexer for that source.
* A Lexer is a stateful stream generator in that every time
* it is advanced, it returns the next token in the Source. Assuming the
* source lexes, the final Token emitted by the lexer will be of kind
* EOF, after which the lexer will repeatedly return the same EOF token
* whenever called.
*/
export function createLexer(source: Source): Lexer {
const startOfFileToken = new Tok(TokenKind.SOF, 0, 0, 0, 0, null);
const lexer: Lexer = {
source,
lastToken: startOfFileToken,
token: startOfFileToken,
line: 1,
lineStart: 0,
advance: advanceLexer,
lookahead,
};
return lexer;
}

function advanceLexer() {
this.lastToken = this.token;
const token = (this.token = this.lookahead());
return token;
}

function lookahead() {
let token = this.token;
if (token.kind !== TokenKind.EOF) {
do {
// Note: next is only mutable during parsing, so we cast to allow this.
token = token.next || ((token: any).next = readToken(this, token));
} while (token.kind === TokenKind.COMMENT);
}
return token;
}

/**
* The return type of createLexer.
*/
export type Lexer = {
source: Source,
export class Lexer {
source: Source;

/**
* The previously focused non-ignored token.
*/
lastToken: Token,
lastToken: Token;

/**
* The currently focused non-ignored token.
*/
token: Token,
token: Token;

/**
* The (1-indexed) line containing the current token.
*/
line: number,
line: number;

/**
* The character offset at which the current line begins.
*/
lineStart: number,
lineStart: number;

constructor(source: Source) {
const startOfFileToken = new Tok(TokenKind.SOF, 0, 0, 0, 0, null);

this.source = source;
this.lastToken = startOfFileToken;
this.token = startOfFileToken;
this.line = 1;
this.lineStart = 0;
}

/**
* Advances the token stream to the next non-ignored token.
*/
advance(): Token,
advance(): Token {
this.lastToken = this.token;
const token = (this.token = this.lookahead());
return token;
}

/**
* Looks ahead and returns the next non-ignored token, but does not change
* the Lexer's state.
*/
lookahead(): Token,

...
};
lookahead(): Token {
let token = this.token;
if (token.kind !== TokenKind.EOF) {
do {
// Note: next is only mutable during parsing, so we cast to allow this.
token = token.next || ((token: any).next = readToken(this, token));
} while (token.kind === TokenKind.COMMENT);
}
return token;
}
}

/**
* @internal
Expand Down
4 changes: 2 additions & 2 deletions src/language/parser.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { Kind } from './kinds';
import { Source } from './source';
import { DirectiveLocation } from './directiveLocation';
import { type TokenKindEnum, TokenKind } from './tokenKind';
import { type Lexer, createLexer, isPunctuatorTokenKind } from './lexer';
import { Lexer, isPunctuatorTokenKind } from './lexer';
import {
type Location,
type Token,
Expand Down Expand Up @@ -177,7 +177,7 @@ class Parser {
`Must provide Source. Received: ${inspect(sourceObj)}`,
);

this._lexer = createLexer(sourceObj);
this._lexer = new Lexer(sourceObj);
this._options = options || {};
}

Expand Down
4 changes: 2 additions & 2 deletions src/utilities/__tests__/stripIgnoredCharacters-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import invariant from '../../jsutils/invariant';

import { parse } from '../../language/parser';
import { Source } from '../../language/source';
import { createLexer } from '../../language/lexer';
import { Lexer } from '../../language/lexer';

import { stripIgnoredCharacters } from '../stripIgnoredCharacters';

Expand Down Expand Up @@ -59,7 +59,7 @@ const nonPunctuatorTokens = [
];

function lexValue(str) {
const lexer = createLexer(new Source(str));
const lexer = new Lexer(new Source(str));
const value = lexer.advance().value;

invariant(lexer.advance().kind === '<EOF>', 'Expected EOF');
Expand Down
4 changes: 2 additions & 2 deletions src/utilities/stripIgnoredCharacters.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import inspect from '../jsutils/inspect';

import { Source } from '../language/source';
import { TokenKind } from '../language/tokenKind';
import { createLexer, isPunctuatorTokenKind } from '../language/lexer';
import { Lexer, isPunctuatorTokenKind } from '../language/lexer';
import {
dedentBlockStringValue,
getBlockStringIndentation,
Expand Down Expand Up @@ -71,7 +71,7 @@ export function stripIgnoredCharacters(source: string | Source): string {
}

const body = sourceObj.body;
const lexer = createLexer(sourceObj);
const lexer = new Lexer(sourceObj);
let strippedBody = '';

let wasLastAddedTokenNonPunctuator = false;
Expand Down
3 changes: 1 addition & 2 deletions tstypes/index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ export {
printLocation,
printSourceLocation,
// Lex
createLexer,
Lexer,
TokenKind,
// Parse
parse,
Expand Down Expand Up @@ -206,7 +206,6 @@ export {
} from './language';

export {
Lexer,
ParseOptions,
SourceLocation,
Location,
Expand Down
2 changes: 1 addition & 1 deletion tstypes/language/index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ export { printLocation, printSourceLocation } from './printLocation';

export { Kind, KindEnum } from './kinds';
export { TokenKind, TokenKindEnum } from './tokenKind';
export { createLexer, Lexer } from './lexer';
export { Lexer } from './lexer';
export { parse, parseValue, parseType, ParseOptions } from './parser';
export { print } from './printer';
export {
Expand Down
9 changes: 3 additions & 6 deletions tstypes/language/lexer.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,7 @@ import { Source } from './source';
* EOF, after which the lexer will repeatedly return the same EOF token
* whenever called.
*/
export function createLexer(source: Source): Lexer;

/**
* The return type of createLexer.
*/
export interface Lexer {
export class Lexer {
source: Source;

/**
Expand All @@ -38,6 +33,8 @@ export interface Lexer {
*/
lineStart: number;

constructor(source: Source);

/**
* Advances the token stream to the next non-ignored token.
*/
Expand Down