diff --git a/Include/internal/pycore_token.h b/Include/internal/pycore_token.h index 3d635d0e5fd0e8..2f6af74cb9c01b 100644 --- a/Include/internal/pycore_token.h +++ b/Include/internal/pycore_token.h @@ -90,6 +90,9 @@ extern "C" { (x) == NEWLINE || \ (x) == INDENT || \ (x) == DEDENT) +#define ISSTRINGLIT(x) ((x) == STRING || \ + (x) == FSTRING_MIDDLE || \ + (x) == FSTRING_END) // Symbols exported for test_peg_generator diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index f40d78ea1ba298..f41cbb6f5a5942 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1563,7 +1563,12 @@ token_setup(struct tok_state *tok, struct token *token, int type, const char *st { assert((start == NULL && end == NULL) || (start != NULL && end != NULL)); token->level = tok->level; - token->lineno = type == STRING ? tok->first_lineno : (type == FSTRING_MIDDLE || type == FSTRING_END ? tok->fstring_first_constant_lineno : tok->lineno); + if (ISSTRINGLIT(type)) { + token->lineno = tok->first_lineno; + } + else { + token->lineno = tok->lineno; + } token->end_lineno = tok->lineno; token->col_offset = token->end_col_offset = -1; token->start = start; @@ -2423,7 +2428,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct const char *p_start = NULL; const char *p_end = NULL; tok->start = tok->cur; - tok->fstring_first_constant_lineno = tok->lineno; + tok->first_lineno = tok->lineno; tok->starting_col_offset = tok->col_offset; // If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 0c542775572bd6..010ec4fbd8b191 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -82,8 +82,6 @@ struct tok_state { int lineno; /* Current line number */ int first_lineno; /* First line of a single line or multi line string expression (cf. issue 16806) */ - int fstring_first_constant_lineno; /* First line number of a single line or multiline - constant part of an f-string*/ int starting_col_offset; /* The column offset at the beginning of a token */ int col_offset; /* Current col offset */ int level; /* () [] {} Parentheses nesting level */ diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 8daa9877254e2e..416dc5971bca3d 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -86,8 +86,8 @@ tokenizeriter_next(tokenizeriterobject *it) Py_DECREF(str); return NULL; } - const char *line_start = type == STRING ? it->tok->multi_line_start : it->tok->line_start; - int lineno = type == STRING ? it->tok->first_lineno : it->tok->lineno; + const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start; + int lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno; int end_lineno = it->tok->lineno; int col_offset = -1; int end_col_offset = -1; diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py index fc12835b7762ad..4a06c8398bd346 100755 --- a/Tools/build/generate_token.py +++ b/Tools/build/generate_token.py @@ -80,6 +80,9 @@ def update_file(file, content): (x) == NEWLINE || \\ (x) == INDENT || \\ (x) == DEDENT) +#define ISSTRINGLIT(x) ((x) == STRING || \\ + (x) == FSTRING_MIDDLE || \\ + (x) == FSTRING_END) // Symbols exported for test_peg_generator