From 40737671324c2c14ab4c6835bb9501e026817691 Mon Sep 17 00:00:00 2001 From: Udo Klimaschewski Date: Sun, 12 Feb 2023 10:29:16 +0100 Subject: [PATCH] adds a token validation for infix operators to tokenizer (#348) --- .../com/ezylang/evalex/parser/Tokenizer.java | 22 +++++ .../parser/ShuntingYardExceptionsTest.java | 94 ++++++------------- 2 files changed, 49 insertions(+), 67 deletions(-) diff --git a/src/main/java/com/ezylang/evalex/parser/Tokenizer.java b/src/main/java/com/ezylang/evalex/parser/Tokenizer.java index 7de16989..45164893 100644 --- a/src/main/java/com/ezylang/evalex/parser/Tokenizer.java +++ b/src/main/java/com/ezylang/evalex/parser/Tokenizer.java @@ -16,6 +16,7 @@ package com.ezylang.evalex.parser; import static com.ezylang.evalex.parser.Token.TokenType.BRACE_OPEN; +import static com.ezylang.evalex.parser.Token.TokenType.INFIX_OPERATOR; import com.ezylang.evalex.config.ExpressionConfiguration; import com.ezylang.evalex.config.FunctionDictionaryIfc; @@ -75,6 +76,7 @@ public List parse() throws ParseException { throw new ParseException(currentToken, "Missing operator"); } } + validateToken(currentToken); tokens.add(currentToken); currentToken = getNextToken(); } @@ -90,6 +92,26 @@ public List parse() throws ParseException { return tokens; } + private void validateToken(Token currentToken) throws ParseException { + Token previousToken = getPreviousToken(); + if (previousToken != null + && previousToken.getType() == INFIX_OPERATOR + && invalidTokenAfterInfixOperator(currentToken)) { + throw new ParseException(currentToken, "Unexpected token after infix operator"); + } + } + + private boolean invalidTokenAfterInfixOperator(Token token) { + switch (token.getType()) { + case INFIX_OPERATOR: + case BRACE_CLOSE: + case COMMA: + return true; + default: + return false; + } + } + private Token getNextToken() throws ParseException { // blanks are always skipped. diff --git a/src/test/java/com/ezylang/evalex/parser/ShuntingYardExceptionsTest.java b/src/test/java/com/ezylang/evalex/parser/ShuntingYardExceptionsTest.java index fe413ee2..8f237231 100644 --- a/src/test/java/com/ezylang/evalex/parser/ShuntingYardExceptionsTest.java +++ b/src/test/java/com/ezylang/evalex/parser/ShuntingYardExceptionsTest.java @@ -15,12 +15,7 @@ */ package com.ezylang.evalex.parser; -import static com.ezylang.evalex.parser.Token.TokenType.FUNCTION_PARAM_START; -import static com.ezylang.evalex.parser.Token.TokenType.INFIX_OPERATOR; -import static com.ezylang.evalex.parser.Token.TokenType.POSTFIX_OPERATOR; -import static com.ezylang.evalex.parser.Token.TokenType.PREFIX_OPERATOR; -import static com.ezylang.evalex.parser.Token.TokenType.STRUCTURE_SEPARATOR; -import static com.ezylang.evalex.parser.Token.TokenType.VARIABLE_OR_CONSTANT; +import static com.ezylang.evalex.parser.Token.TokenType.*; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.ezylang.evalex.Expression; @@ -29,6 +24,9 @@ import java.util.Arrays; import java.util.List; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; class ShuntingYardExceptionsTest extends BaseParserTest { @@ -143,75 +141,37 @@ void testFunctionTooManyParameters() { .hasMessage("Too many parameters for function"); } - @Test - void testTooManyOperands() { - Expression expression = new Expression("1 2"); - - assertThatThrownBy(expression::evaluate) - .isInstanceOf(ParseException.class) - .hasMessage("Too many operands"); - } - - @Test - void testTooManyOperandsString() { - Expression expression = new Expression("Hello World"); + @ParameterizedTest + @ValueSource( + strings = { + "Hello, World", + "Hello ROUND(1,2) + (1 + 1)", + "Hello ROUND(1,2)", + "Hello 1 + (1 + 1)", + "Hello 1 + 1", + "Hello World", + "Hello 1", + "1 2" + }) + void testTooManyOperands(String expressionString) { + Expression expression = new Expression(expressionString); assertThatThrownBy(expression::evaluate) .isInstanceOf(ParseException.class) .hasMessage("Too many operands"); } - @Test - void testTooManyOperandsStringWithNumbers() { - Expression expression = new Expression("Hello 1"); - - assertThatThrownBy(expression::evaluate) - .isInstanceOf(ParseException.class) - .hasMessage("Too many operands"); - } - - @Test - void testTooManyOperandsStringWithNumbersAndOperators() { - Expression expression = new Expression("Hello 1 + 1"); - - assertThatThrownBy(expression::evaluate) - .isInstanceOf(ParseException.class) - .hasMessage("Too many operands"); - } - - @Test - void testTooManyOperandsStringWithNumbersAndOperatorsAndBraces() { - Expression expression = new Expression("Hello 1 + (1 + 1)"); + @ParameterizedTest + @CsvSource( + delimiter = ':', + value = {"(x+y)*(a-) : 10", "a** : 3", "5+, : 3"}) + void testInvalidTokenAfterInfixOperator(String expressionString, int position) { + Expression expression = new Expression(expressionString); assertThatThrownBy(expression::evaluate) .isInstanceOf(ParseException.class) - .hasMessage("Too many operands"); - } - - @Test - void testTooManyOperandsStringWithFunctions() { - Expression expression = new Expression("Hello ROUND(1,2)"); - - assertThatThrownBy(expression::evaluate) - .isInstanceOf(ParseException.class) - .hasMessage("Too many operands"); - } - - @Test - void testTooManyOperandsStringWithFunctionsAndBraces() { - Expression expression = new Expression("Hello ROUND(1,2) + (1 + 1)"); - - assertThatThrownBy(expression::evaluate) - .isInstanceOf(ParseException.class) - .hasMessage("Too many operands"); - } - - @Test - void testTooManyOperandsStringWithSpecialCharacters() { - Expression expression = new Expression("Hello, World"); - - assertThatThrownBy(expression::evaluate) - .isInstanceOf(ParseException.class) - .hasMessage("Too many operands"); + .hasMessage("Unexpected token after infix operator") + .extracting("startPosition") + .isEqualTo(position); } }