Merge pull request python#51 from isidentical/more-simplifications

isidentical · Mar 24, 2023 · b41a3b8 · b41a3b8
2 parents f1246db + bd7483e
commit b41a3b8
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 6 deletions.
diff --git a/Include/internal/pycore_token.h b/Include/internal/pycore_token.h
@@ -90,6 +90,9 @@ extern "C" {
                                  (x) == NEWLINE   || \
                                  (x) == INDENT    || \
                                  (x) == DEDENT)
+#define ISSTRINGLIT(x)          ((x) == STRING           || \
+                                 (x) == FSTRING_MIDDLE   || \
+                                 (x) == FSTRING_END)
 
 
 // Symbols exported for test_peg_generator

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
@@ -1563,7 +1563,12 @@ token_setup(struct tok_state *tok, struct token *token, int type, const char *st
 {
     assert((start == NULL && end == NULL) || (start != NULL && end != NULL));
     token->level = tok->level;
-    token->lineno = type == STRING ? tok->first_lineno : (type == FSTRING_MIDDLE || type == FSTRING_END ? tok->fstring_first_constant_lineno : tok->lineno);
+    if (ISSTRINGLIT(type)) {
+        token->lineno = tok->first_lineno;
+    }
+    else {
+        token->lineno = tok->lineno;
+    }
     token->end_lineno = tok->lineno;
     token->col_offset = token->end_col_offset = -1;
     token->start = start;
@@ -2423,7 +2428,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
     const char *p_start = NULL;
     const char *p_end = NULL;
     tok->start = tok->cur;
-    tok->fstring_first_constant_lineno = tok->lineno;
+    tok->first_lineno = tok->lineno;
     tok->starting_col_offset = tok->col_offset;
 
     // If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize

diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
@@ -82,8 +82,6 @@ struct tok_state {
     int lineno;         /* Current line number */
     int first_lineno;   /* First line of a single line or multi line string
                            expression (cf. issue 16806) */
-    int fstring_first_constant_lineno; /* First line number of a single line or multiline
-                                            constant part of an f-string*/
     int starting_col_offset; /* The column offset at the beginning of a token */
     int col_offset;     /* Current col offset */
     int level;          /* () [] {} Parentheses nesting level */

diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
@@ -86,8 +86,8 @@ tokenizeriter_next(tokenizeriterobject *it)
         Py_DECREF(str);
         return NULL;
     }
-    const char *line_start = type == STRING ? it->tok->multi_line_start : it->tok->line_start;
-    int lineno = type == STRING ? it->tok->first_lineno : it->tok->lineno;
+    const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start;
+    int lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;
     int end_lineno = it->tok->lineno;
     int col_offset = -1;
     int end_col_offset = -1;

diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py
@@ -80,6 +80,9 @@ def update_file(file, content):
                                  (x) == NEWLINE   || \\
                                  (x) == INDENT    || \\
                                  (x) == DEDENT)
+#define ISSTRINGLIT(x)          ((x) == STRING           || \\
+                                 (x) == FSTRING_MIDDLE   || \\
+                                 (x) == FSTRING_END)
 
 
 // Symbols exported for test_peg_generator