Skip to content

Commit

Permalink
handle with simple mathematical operation in lexical analysis
Browse files Browse the repository at this point in the history
  • Loading branch information
@JunNishimura
JunNishimura committed Sep 20, 2024
1 parent 423aab6 commit ba87afe
Show file tree
Hide file tree
Showing 3 changed files with 176 additions and 5 deletions.
10 changes: 10 additions & 0 deletions lexer/lexer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,16 @@ func (l *Lexer) NextToken() token.Token {
tok = newToken(token.LBRACE, l.curChar)
case '}':
tok = newToken(token.RBRACE, l.curChar)
case '[':
tok = newToken(token.LBRACKET, l.curChar)
case ']':
tok = newToken(token.RBRACKET, l.curChar)
case '"':
tok = newToken(token.DOUBLE_QUOTE, l.curChar)
case ':':
tok = newToken(token.COLON, l.curChar)
case ',':
tok = newToken(token.COMMA, l.curChar)
case '+':
if isDigit(l.peekChar()) {
l.readChar()
Expand All @@ -56,6 +62,10 @@ func (l *Lexer) NextToken() token.Token {
} else {
tok = newToken(token.MINUS, l.curChar)
}
case '*':
tok = newToken(token.ASTERISK, l.curChar)
case '/':
tok = newToken(token.SLASH, l.curChar)
case 0:
tok.Literal = ""
tok.Type = token.EOF
Expand Down
152 changes: 152 additions & 0 deletions lexer/lexer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,158 @@ func TestSingleProgram(t *testing.T) {
{Type: token.EOF, Literal: ""},
},
},
{
name: "mathematical operation: addition",
input: `
{
"command": {
"symbol": "+",
"args": [1, 2]
}
}`,
expected: []token.Token{
{Type: token.LBRACE, Literal: "{"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COMMAND, Literal: "command"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COLON, Literal: ":"},
{Type: token.LBRACE, Literal: "{"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.SYMBOL, Literal: "symbol"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COLON, Literal: ":"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.PLUS, Literal: "+"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COMMA, Literal: ","},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.ARGS, Literal: "args"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COLON, Literal: ":"},
{Type: token.LBRACKET, Literal: "["},
{Type: token.INT, Literal: "1"},
{Type: token.COMMA, Literal: ","},
{Type: token.INT, Literal: "2"},
{Type: token.RBRACKET, Literal: "]"},
{Type: token.RBRACE, Literal: "}"},
{Type: token.RBRACE, Literal: "}"},
{Type: token.EOF, Literal: ""},
},
},
{
name: "mathematical operation: subtraction",
input: `
{
"command": {
"symbol": "-",
"args": [1, 2]
}
}`,
expected: []token.Token{
{Type: token.LBRACE, Literal: "{"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COMMAND, Literal: "command"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COLON, Literal: ":"},
{Type: token.LBRACE, Literal: "{"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.SYMBOL, Literal: "symbol"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COLON, Literal: ":"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.MINUS, Literal: "-"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COMMA, Literal: ","},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.ARGS, Literal: "args"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COLON, Literal: ":"},
{Type: token.LBRACKET, Literal: "["},
{Type: token.INT, Literal: "1"},
{Type: token.COMMA, Literal: ","},
{Type: token.INT, Literal: "2"},
{Type: token.RBRACKET, Literal: "]"},
{Type: token.RBRACE, Literal: "}"},
{Type: token.RBRACE, Literal: "}"},
{Type: token.EOF, Literal: ""},
},
},
{
name: "mathematical operation: multiplication",
input: `
{
"command": {
"symbol": "*",
"args": [1, 2]
}
}`,
expected: []token.Token{
{Type: token.LBRACE, Literal: "{"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COMMAND, Literal: "command"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COLON, Literal: ":"},
{Type: token.LBRACE, Literal: "{"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.SYMBOL, Literal: "symbol"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COLON, Literal: ":"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.ASTERISK, Literal: "*"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COMMA, Literal: ","},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.ARGS, Literal: "args"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COLON, Literal: ":"},
{Type: token.LBRACKET, Literal: "["},
{Type: token.INT, Literal: "1"},
{Type: token.COMMA, Literal: ","},
{Type: token.INT, Literal: "2"},
{Type: token.RBRACKET, Literal: "]"},
{Type: token.RBRACE, Literal: "}"},
{Type: token.RBRACE, Literal: "}"},
{Type: token.EOF, Literal: ""},
},
},
{
name: "mathematical operation: division",
input: `
{
"command": {
"symbol": "/",
"args": [1, 2]
}
}`,
expected: []token.Token{
{Type: token.LBRACE, Literal: "{"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COMMAND, Literal: "command"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COLON, Literal: ":"},
{Type: token.LBRACE, Literal: "{"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.SYMBOL, Literal: "symbol"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COLON, Literal: ":"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.SLASH, Literal: "/"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COMMA, Literal: ","},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.ARGS, Literal: "args"},
{Type: token.DOUBLE_QUOTE, Literal: "\""},
{Type: token.COLON, Literal: ":"},
{Type: token.LBRACKET, Literal: "["},
{Type: token.INT, Literal: "1"},
{Type: token.COMMA, Literal: ","},
{Type: token.INT, Literal: "2"},
{Type: token.RBRACKET, Literal: "]"},
{Type: token.RBRACE, Literal: "}"},
{Type: token.RBRACE, Literal: "}"},
{Type: token.EOF, Literal: ""},
},
},
}

for _, tt := range tests {
Expand Down
19 changes: 14 additions & 5 deletions token/token.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,23 +11,32 @@ const (
ILLEGAL = "ILLEGAL"
EOF = "EOF"

INT = "INT"
SYMBOL = "SYMBOL"
INT = "INT"
SYMBOL = "SYMBOL"
COMMAND = "COMMAND"
ARGS = "ARGS"

PLUS = "+"
MINUS = "-"
PLUS = "+"
MINUS = "-"
ASTERISK = "*"
SLASH = "/"

LBRACE = "{"
RBRACE = "}"
LBRACKET = "["
RBRACKET = "]"
DOUBLE_QUOTE = "\""
COLON = ":"
COMMA = ","

// reserved tokens
ATOM = "ATOM"
)

var reservedWords = map[string]TokenType{
"atom": ATOM,
"atom": ATOM,
"command": COMMAND,
"args": ARGS,
}

func LookupStringTokenType(word string) TokenType {
Expand Down

0 comments on commit ba87afe

Please sign in to comment.