From bdae896ea6f3b0ce1beeef5fc5dac149be0e9e22 Mon Sep 17 00:00:00 2001 From: Snownee Date: Wed, 20 Dec 2023 02:22:55 +0800 Subject: [PATCH 1/4] adds single quote string literal --- .../evalex/config/ExpressionConfiguration.java | 3 +++ .../com/ezylang/evalex/parser/Tokenizer.java | 10 ++++++++-- .../config/ExpressionConfigurationTest.java | 9 +++++++++ .../ezylang/evalex/parser/BaseParserTest.java | 6 ++++++ .../parser/TokenizerStringLiteralTest.java | 17 +++++++++++++++++ 5 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/ezylang/evalex/config/ExpressionConfiguration.java b/src/main/java/com/ezylang/evalex/config/ExpressionConfiguration.java index 9f901f8e..c0b7f517 100644 --- a/src/main/java/com/ezylang/evalex/config/ExpressionConfiguration.java +++ b/src/main/java/com/ezylang/evalex/config/ExpressionConfiguration.java @@ -222,6 +222,9 @@ public class ExpressionConfiguration { /** Support for implicit multiplication, like in (a+b)(b+c) are allowed or not. */ @Builder.Default @Getter private final boolean implicitMultiplicationAllowed = true; + /** Support for single quote string literals, like in 'Hello World' are allowed or not. */ + @Builder.Default @Getter private final boolean singleQuoteStringLiteralsAllowed = false; + /** * The power of operator precedence, can be set higher {@link * OperatorIfc#OPERATOR_PRECEDENCE_POWER_HIGHER} or to a custom value. diff --git a/src/main/java/com/ezylang/evalex/parser/Tokenizer.java b/src/main/java/com/ezylang/evalex/parser/Tokenizer.java index de7eba33..65e279a6 100644 --- a/src/main/java/com/ezylang/evalex/parser/Tokenizer.java +++ b/src/main/java/com/ezylang/evalex/parser/Tokenizer.java @@ -144,7 +144,7 @@ private Token getNextToken() throws ParseException { } // we have a token start, identify and parse it - if (currentChar == '"') { + if (isAtStringLiteralStart()) { return parseStringLiteral(); } else if (currentChar == '(') { return parseBraceOpen(); @@ -464,6 +464,7 @@ private Token parseIdentifier() throws ParseException { } Token parseStringLiteral() throws ParseException { + int startChar = currentChar; int tokenStartIndex = currentColumnIndex; StringBuilder tokenValue = new StringBuilder(); // skip starting quote @@ -473,7 +474,7 @@ Token parseStringLiteral() throws ParseException { if (currentChar == '\\') { consumeChar(); tokenValue.append(escapeCharacter(currentChar)); - } else if (currentChar == '"') { + } else if (currentChar == startChar) { inQuote = false; } else { tokenValue.append((char) currentChar); @@ -584,6 +585,11 @@ private boolean isAtIdentifierChar() { return Character.isLetter(currentChar) || Character.isDigit(currentChar) || currentChar == '_'; } + private boolean isAtStringLiteralStart() { + return currentChar == '"' + || currentChar == '\'' && configuration.isSingleQuoteStringLiteralsAllowed(); + } + private void skipBlanks() { if (currentChar == -2) { // consume first character of expression diff --git a/src/test/java/com/ezylang/evalex/config/ExpressionConfigurationTest.java b/src/test/java/com/ezylang/evalex/config/ExpressionConfigurationTest.java index 0f28526a..72f1dc24 100644 --- a/src/test/java/com/ezylang/evalex/config/ExpressionConfigurationTest.java +++ b/src/test/java/com/ezylang/evalex/config/ExpressionConfigurationTest.java @@ -54,6 +54,7 @@ void testDefaultSetup() { .isEqualTo(ExpressionConfiguration.DECIMAL_PLACES_ROUNDING_UNLIMITED); assertThat(configuration.isStripTrailingZeros()).isTrue(); assertThat(configuration.isAllowOverwriteConstants()).isTrue(); + assertThat(configuration.isSingleQuoteStringLiteralsAllowed()).isFalse(); } @Test @@ -158,6 +159,14 @@ void testStructuresAllowed() { assertThat(configuration.isStructuresAllowed()).isFalse(); } + @Test + void testSingleQuoteStringLiteralsAllowed() { + ExpressionConfiguration configuration = + ExpressionConfiguration.builder().singleQuoteStringLiteralsAllowed(true).build(); + + assertThat(configuration.isSingleQuoteStringLiteralsAllowed()).isTrue(); + } + @Test void testImplicitMultiplicationAllowed() { ExpressionConfiguration configuration = diff --git a/src/test/java/com/ezylang/evalex/parser/BaseParserTest.java b/src/test/java/com/ezylang/evalex/parser/BaseParserTest.java index 5eb87844..1596f1da 100644 --- a/src/test/java/com/ezylang/evalex/parser/BaseParserTest.java +++ b/src/test/java/com/ezylang/evalex/parser/BaseParserTest.java @@ -28,6 +28,12 @@ public abstract class BaseParserTest { TestConfigurationProvider.StandardConfigurationWithAdditionalTestOperators; void assertAllTokensParsedCorrectly(String input, Token... expectedTokens) throws ParseException { + assertAllTokensParsedCorrectly(input, configuration, expectedTokens); + } + + void assertAllTokensParsedCorrectly( + String input, ExpressionConfiguration configuration, Token... expectedTokens) + throws ParseException { List tokensParsed = new Tokenizer(input, configuration).parse(); assertThat(tokensParsed).containsExactly(expectedTokens); diff --git a/src/test/java/com/ezylang/evalex/parser/TokenizerStringLiteralTest.java b/src/test/java/com/ezylang/evalex/parser/TokenizerStringLiteralTest.java index 4026b127..78f3c874 100644 --- a/src/test/java/com/ezylang/evalex/parser/TokenizerStringLiteralTest.java +++ b/src/test/java/com/ezylang/evalex/parser/TokenizerStringLiteralTest.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.ezylang.evalex.Expression; +import com.ezylang.evalex.config.ExpressionConfiguration; import com.ezylang.evalex.parser.Token.TokenType; import org.junit.jupiter.api.Test; @@ -96,4 +97,20 @@ void testErrorUnmatchedQuoteOffset() { .isInstanceOf(ParseException.class) .hasMessage("Closing quote not found"); } + + @Test + void testSingleQuoteAllowed() throws ParseException { + assertThatThrownBy(() -> new Tokenizer("'hello'", configuration).parse()) + .isInstanceOf(ParseException.class); + + ExpressionConfiguration config = + ExpressionConfiguration.builder().singleQuoteStringLiteralsAllowed(true).build(); + + assertAllTokensParsedCorrectly( + "'\"Hello\", ' + \"'World'\"", + config, + new Token(1, "\"Hello\", ", TokenType.STRING_LITERAL), + new Token(13, "+", TokenType.INFIX_OPERATOR), + new Token(15, "'World'", TokenType.STRING_LITERAL)); + } } From bda247ab7450b24cf1cbcb7c2ea8d97dc25aeb04 Mon Sep 17 00:00:00 2001 From: Snownee Date: Fri, 22 Dec 2023 16:32:53 +0800 Subject: [PATCH 2/4] changes based on the reviews --- docs/concepts/datatypes.md | 4 ++ docs/configuration/configuration.md | 8 +++- .../parser/TokenizerStringLiteralTest.java | 47 ++++++++++++++++++- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/docs/concepts/datatypes.md b/docs/concepts/datatypes.md index 63b433e1..894d91c6 100644 --- a/docs/concepts/datatypes.md +++ b/docs/concepts/datatypes.md @@ -64,6 +64,10 @@ Any instance of _java.lang.CharSequence_ or _java.lang.Character_ will automatic a _STRING_ datatype. Conversion will be done by invoking the _toString()_ method on the input object. +By default, the string literal delimiter is the double quote character ("). You can also use both +`"` and `'` as string literal delimiters by changing the configuration. See +chapter [Configuration](../configuration/configuration.html) for details. + ### DATE_TIME Any instance of _java.time.Instant_, _java.time.LocalDate_, _java.time.LocalDateTime_, _java.time.ZoneDateTime_, diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 9bb9484c..5dc56a98 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -25,6 +25,7 @@ ExpressionConfiguration configuration=ExpressionConfiguration.builder() .powerOfPrecedence(OperatorIfc.OPERATOR_PRECEDENCE_POWER) .stripTrailingZeros(true) .structuresAllowed(true) + .singleQuoteStringLiteralsAllowed(false) .build(); Expression expression=new Expression("2.128 + a",configuration); @@ -88,7 +89,7 @@ See the reference chapter for a list: [Default Constants](../references/constant ### Evaluation Value Converter The converter to use when converting different data types to an _EvaluationValue_. -The _DefaultEvaluationValueConverter_ is used by default. +The _DefaultEvaluationValueConverter_ is used by default. ### Function Dictionary @@ -163,9 +164,14 @@ no operator or function is defined for this character. ### Zone Id The time zone id. By default, the system default zone ID is used. + ```java ExpressionConfiguration configuration=ExpressionConfiguration.builder() .zoneId(ZoneId.of("Europe/Berlin")) .build(); ``` +### Single Quote String Literals + +Specifies if the single quote character (') can be used as a string literal delimiter, not only the +double quote character (") (default is false). If set to true, the expression will throw a _ParseException_. diff --git a/src/test/java/com/ezylang/evalex/parser/TokenizerStringLiteralTest.java b/src/test/java/com/ezylang/evalex/parser/TokenizerStringLiteralTest.java index 78f3c874..8c222d36 100644 --- a/src/test/java/com/ezylang/evalex/parser/TokenizerStringLiteralTest.java +++ b/src/test/java/com/ezylang/evalex/parser/TokenizerStringLiteralTest.java @@ -99,10 +99,14 @@ void testErrorUnmatchedQuoteOffset() { } @Test - void testSingleQuoteAllowed() throws ParseException { + void testSingleQuoteAllowed() { assertThatThrownBy(() -> new Tokenizer("'hello'", configuration).parse()) - .isInstanceOf(ParseException.class); + .isInstanceOf(ParseException.class) + .hasMessage("Undefined operator '''"); + } + @Test + void testSingleQuoteOperation() throws ParseException { ExpressionConfiguration config = ExpressionConfiguration.builder().singleQuoteStringLiteralsAllowed(true).build(); @@ -113,4 +117,43 @@ void testSingleQuoteAllowed() throws ParseException { new Token(13, "+", TokenType.INFIX_OPERATOR), new Token(15, "'World'", TokenType.STRING_LITERAL)); } + + @Test + void testErrorUnmatchedSingleQuoteStart() { + ExpressionConfiguration config = + ExpressionConfiguration.builder().singleQuoteStringLiteralsAllowed(true).build(); + + assertThatThrownBy(() -> new Tokenizer("'hello", config).parse()) + .isInstanceOf(ParseException.class) + .hasMessage("Closing quote not found"); + } + + @Test + void testErrorUnmatchedSingleQuoteOffset() { + ExpressionConfiguration config = + ExpressionConfiguration.builder().singleQuoteStringLiteralsAllowed(true).build(); + + assertThatThrownBy(() -> new Tokenizer("test 'hello", config).parse()) + .isInstanceOf(ParseException.class) + .hasMessage("Closing quote not found"); + } + + @Test + void testErrorUnmatchedDelimiters() { + ExpressionConfiguration config = + ExpressionConfiguration.builder().singleQuoteStringLiteralsAllowed(true).build(); + + assertThatThrownBy(() -> new Tokenizer("'test\"", config).parse()) + .isInstanceOf(ParseException.class) + .hasMessage("Closing quote not found"); + } + + @Test + void testEscapeSingleQuoteCharacter() throws ParseException { + ExpressionConfiguration config = + ExpressionConfiguration.builder().singleQuoteStringLiteralsAllowed(true).build(); + + assertAllTokensParsedCorrectly( + "' \\' \\' \\' '", config, new Token(1, " ' ' ' ", TokenType.STRING_LITERAL)); + } } From 2ee4baae453518168a07ed2fb2a2d6b1b09e0e48 Mon Sep 17 00:00:00 2001 From: Snownee Date: Sat, 23 Dec 2023 18:19:40 +0800 Subject: [PATCH 3/4] move chapter and fix a typo --- docs/configuration/configuration.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 5dc56a98..c8ef0318 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -150,6 +150,11 @@ ExpressionConfiguration configuration=ExpressionConfiguration.builder() Expression expression=new Expression("-2^2",configuration); ``` +### Single Quote String Literals + +Specifies if the single quote character (') can be used as a string literal delimiter, not only the +double quote character (") (default is false). If set to false, the expression will throw a _ParseException_. + ### Strip Trailing Zeros If set to true (default), then the trailing decimal zeros in a number result will be stripped. @@ -170,8 +175,3 @@ ExpressionConfiguration configuration=ExpressionConfiguration.builder() .zoneId(ZoneId.of("Europe/Berlin")) .build(); ``` - -### Single Quote String Literals - -Specifies if the single quote character (') can be used as a string literal delimiter, not only the -double quote character (") (default is false). If set to true, the expression will throw a _ParseException_. From 98517eeedcbc8e6ca7a01e98cc4028d0875e189a Mon Sep 17 00:00:00 2001 From: uklimaschewski Date: Sat, 23 Dec 2023 11:30:13 +0100 Subject: [PATCH 4/4] added clarification --- docs/configuration/configuration.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index c8ef0318..445faa26 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -152,8 +152,9 @@ ExpressionConfiguration configuration=ExpressionConfiguration.builder() ### Single Quote String Literals -Specifies if the single quote character (') can be used as a string literal delimiter, not only the -double quote character (") (default is false). If set to false, the expression will throw a _ParseException_. +Specifies if the single quote character (') also can be used as a string literal delimiter, not only the +double quote character (") (default is false). +If set to false, the parser will throw a _ParseException_, if a single quote is used. ### Strip Trailing Zeros