Skip to content

Commit

Permalink
[#96] handle lexer errors
Browse files Browse the repository at this point in the history
  • Loading branch information
z7zmey committed Aug 20, 2019
1 parent 5110c36 commit c97ca17
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 11 deletions.
31 changes: 20 additions & 11 deletions scanner/lexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ package scanner
import (
"bufio"
"bytes"
"go/token"
t "go/token"
"io"
"unicode"
Expand Down Expand Up @@ -62,23 +63,31 @@ func Rune2Class(r rune) int {
return classOther
}

func (l *Lexer) lexErrorFunc(p token.Pos, msg string) {
pos := position.NewPosition(
l.File.Line(p),
l.File.Line(p),
int(p),
int(p),
)
l.Errors = append(l.Errors, errors.NewError(msg, pos))
}

// NewLexer the Lexer constructor
func NewLexer(src io.Reader, fName string) *Lexer {
file := t.NewFileSet().AddFile(fName, -1, 1<<31-3)
lx, err := lex.New(file, bufio.NewReader(src), lex.RuneClass(Rune2Class))
if err != nil {
panic(err)
}

return &Lexer{
Lexer: lx,
lexer := &Lexer{
StateStack: []int{0},
PhpDocComment: "",
FreeFloating: nil,
heredocLabel: "",
tokenBytesBuf: &bytes.Buffer{},
TokenPool: &TokenPool{},
}

file := t.NewFileSet().AddFile(fName, -1, 1<<31-3)
lx, err := lex.New(file, bufio.NewReader(src), lex.RuneClass(Rune2Class), lex.ErrorFunc(lexer.lexErrorFunc))
if err != nil {
panic(err)
}
lexer.Lexer = lx
return lexer
}

func (l *Lexer) Error(msg string) {
Expand Down
12 changes: 12 additions & 0 deletions scanner/scanner_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1420,3 +1420,15 @@ func TestIgnoreControllCharactersAtStringVarOffset(t *testing.T) {
actual = lv.Tkn.Value
assert.DeepEqual(t, expected, actual)
}

func TestBomInMiddleOfFile(t *testing.T) {
src := "<?php \xEF\xBB\xBF $a;"

lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lv := &lval{}

lexer.Lex(lv)

assert.Assert(t, len(lexer.Errors) > 0)
assert.Assert(t, lexer.Errors[0].String() == "unicode (UTF-8) BOM in middle of file at line 1")
}

0 comments on commit c97ca17

Please sign in to comment.