Merge remote-tracking branch 'origin/master' into cpython-int-tests

brython-dev · Mar 5, 2024 · 2bf8608 · 2bf8608
2 parents c900314 + 30bc073
commit 2bf8608
Show file tree

Hide file tree

Showing 91 changed files with 42,999 additions and 50,033 deletions.
diff --git a/README.md b/README.md
@@ -62,7 +62,7 @@ You can choose the latest stable release :
 
 ```html
 <script type="text/javascript"
-    src="https://cdn.jsdelivr.net/npm/[email protected].1/brython.min.js">
+    src="https://cdn.jsdelivr.net/npm/[email protected].3/brython.min.js">
 </script>
 ```
 
@@ -72,7 +72,7 @@ with the [available stdlib](https://github.com/brython-dev/brython/tree/master/w
 
 ```html
 <script type="text/javascript"
-    src="https://cdn.jsdelivr.net/npm/[email protected].1/brython_stdlib.js">
+    src="https://cdn.jsdelivr.net/npm/[email protected].3/brython_stdlib.js">
 </script>
 ```
 

diff --git a/npm/package.json b/npm/package.json
@@ -1,6 +1,6 @@
 {
   "name": "brython",
-  "version": "3.12.1",
+  "version": "3.12.3",
   "description": "Python 3 in the browser",
   "main": "brython.js",
   "scripts": {

diff --git a/releases/index.html b/releases/index.html
@@ -7,7 +7,7 @@
 <script type="text/javascript" src="brython_stdlib.js"></script>
 </head>
 
-<body onload="brython(1)">
+<body>
 <script type="text/python">
 from browser import document
 

diff --git a/scripts/adapt_grammar_actions.py b/scripts/adapt_grammar_actions.py
@@ -8,11 +8,12 @@ def transform_action(action):
     action2 = re.sub(r'\(\(.*_ty\) (.*?)\)', r'\1', action1)
     action3 = re.sub(r'\([^(]+ \*\)', '', action2)
     action4 = re.sub(r'\([a-z_]*\*?\)_Py', '_Py', action3)
-    action5 = re.sub(r'([a-z_]+)\*', r'\1', action4)
+    # the negative lookahead below is for case 'except*' in invalid_try_stmt
+    action5 = re.sub(r"([a-z_]+)\*(?!')", r'\1', action4)
     action6 = re.sub(r'\s*->\s*', '.', action5)
     action7 = re.sub('_PyPegen_', '$B._PyPegen.', action6)
+    action7 = re.sub('PyPegen_', '$B.PyPegen.', action7)
     action8 = re.sub('_PyAST_', 'new $B._PyAST.', action7)
-    #action9 = re.sub(operators_re, r'$B.ast.\1', action8)
     action9 = re.sub(r'([a-z]+)_ty\b', r'$B.ast.\1', action8)
 
     for name in operators + ['Module']:
@@ -23,7 +24,7 @@ def transform_action(action):
 
     for name in parser_constants:
         action9 = re.sub(rf'\b{name}\b', '$B.parser_constants.' + name, action9)
-        
+
     # remove parameter types, eg
     # "$B._PyPegen.joined_str(p, a, (asdl_expr_seq)b, c)"
     # replaced by
@@ -60,7 +61,9 @@ def transform_action(action):
     if type_decl:
         action9 = type_decl.groups()[0]
 
-    action9 = action9.replace('void', '_void')
+    action9 = action9.replace('void', 'NULL')
+
+    action9 = re.sub(r'RAISE_(.*?)\s*\(([^p])', r'RAISE_\1(p, \2', action9)
     action10 = action9.strip('{}')
 
     return action10
@@ -88,7 +91,8 @@ def transform_action(action):
     'asdl_type_param_seq',
     'AugOperator', 'Py_Ellipsis', 'Py_False', 'Py_True', 'Py_None',
     'PyExc_SyntaxError',
-    'STAR_TARGETS', 'DEL_TARGETS', 'FOR_TARGETS'
+    'STAR_TARGETS', 'DEL_TARGETS', 'FOR_TARGETS',
+    'PyBytes_AS_STRING'
     ]
 
 helper_functions = [
@@ -105,6 +109,7 @@ def transform_action(action):
     "RAISE_SYNTAX_ERROR_INVALID_TARGET",
     "_RAISE_SYNTAX_ERROR_INVALID_TARGET",
     "RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN",
+    "RAISE_SYNTAX_ERROR_STARTING_FROM",
     "asdl_seq_LEN",
     "asdl_seq_GET"]
 

diff --git a/scripts/experiments/parser.py b/scripts/experiments/parser.py
@@ -5,6 +5,12 @@
 import shlex
 import string
 
+grammar_file = 'python.mini.gram2'
+
+
+src = "x + (1 + (y * (7 - 6)))"
+src = 'global x, y'
+
 primitives = {}
 
 operators = set()
@@ -19,6 +25,10 @@ class Keyword(str):
     def __str__(self):
         return f"'{str.__str__(self)}'"
 
+class Frozen:
+
+    def __init__(self, ge):
+        self.ge = ge
 
 class Punctuation(str):
     pass
@@ -84,7 +94,9 @@ def parse_grammar_options(line):
     return option
 
 grammar = {}
-with open('python.mini.gram2', encoding='utf-8') as f:
+
+options = []
+with open(grammar_file, encoding='utf-8') as f:
     rule = None
     for line in f:
         if not line.strip():
@@ -103,11 +115,13 @@ def parse_grammar_options(line):
 
     grammar[rule] = options
 
+def show_option(option):
+    return '  | ' + ' '.join(str(x) for x in option)
+
 for rule, options in grammar.items():
     print(rule)
     for option in options:
-        print(option)
-        print('  |', ' '.join(str(x) for x in option))
+        print(show_option(option))
 
 print('operators', operators)
 print('keywords', keywords)
@@ -159,17 +173,22 @@ def add_to_list(t1, t2):
         if item not in t1:
             t1.append(item)
 
+class Candidate(list):
+
+    def append(self, item):
+        list.append(self, Candidate(item))
+
 def can_follow(e):
     # list of elements that can follow grammar element "e"
-    result = []
+    result = Candidate()
     test = False # isinstance(e, Separated)
     if test:
         print('can follow', e)
     for rule, options in grammar.items():
         for onum, option in enumerate(options):
             if e is None:
                 if option[0] == option[0].upper() or \
-                        isinstance(option[0], (Operator, Keyword)):
+                        isinstance(option[0], (Operator, Keyword, Frozen)):
                     result.append([rule, onum, 0])
             else:
                 for inum, ge in enumerate(option[:-1]):
@@ -224,11 +243,6 @@ def can_follow(e):
 
 candidates = []
 
-
-src = "x + (1 + (y * (7 - 6)))"
-src = "global x"
-
-
 last_item = None
 
 def starters(r):
@@ -243,11 +257,19 @@ def starters(r):
             add_to_list(result, starters(equiv))
     return result
 
+def follower(candidate):
+    rule, onum, inum = candidate
+    result = []
+    option = grammar[rule][onum][:]
+    ge = option[inum]
+    if option[-1].startswith('{'):
+        option.pop()
 
-
-def match(token, ge):
+def match(token, candidate):
     # check if token matches the grammar expression referenced by candidate
     # returns a list of candidates for the next token
+    rule, onum, inum = candidate
+    ge = grammar[rule][onum][inum]
     tok_type = _token.tok_name[token.type]
     test = tok_type == 'OP' and token.string == '='
     if test:
@@ -259,17 +281,22 @@ def match(token, ge):
     elif ge in keywords:
         if tok_type == 'NAME' and token.string == ge:
             return ge
-
     elif isinstance(ge, Separated):
-        if not hasattr(ge, 'expect'):
-            ge.expect = 'repeated'
-        if ge.expect == 'repeated' and tok_type == ge:
-            ge.expect = 'separator'
+        if not hasattr(candidate, 'expect'):
+            candidate.expect = 'repeated'
+        if candidate.expect == 'repeated' and tok_type == ge:
+            candidate.expect = 'separator'
             return ge
-        elif ge.expect == 'separator' and tok_type == 'OP' and \
+        elif candidate.expect == 'separator' and tok_type == 'OP' and \
                 token.string == ge.separator:
-            ge.expect = 'repeated'
+            candidate.expect = 'repeated'
             return ge
+    elif ge == '&':
+        print('match with ge', ge)
+        next_candidate = candidate[:]
+        next_candidate[-1] += 1
+        if next_ge := match(token, next_candidate):
+            return Frozen(next_ge)
     elif ge == ge.upper():
         if tok_type == ge or (ge.endswith('?') and tok_type == ge[:-1]):
             return ge
@@ -298,7 +325,9 @@ def match(token, ge):
 def show_candidates(candidates):
     for rule, onum, inum in candidates:
         option = grammar[rule][onum]
-        print('  ', rule, option, 'expect', option[inum])
+        print('  ', rule, show_option(option), 'expect', option[inum])
+
+
 
 for token in tokenize.tokenize(io.BytesIO(src.encode('utf-8')).readline):
     print()
@@ -310,11 +339,15 @@ def show_candidates(candidates):
     show_candidates(candidates)
     matches = []
     new_candidates = []
-    for rule, onum, inum in candidates:
+    for candidate in candidates:
+        rule, onum, inum = candidate
         ge = grammar[rule][onum][inum]
-        if new_ge := match(token, ge):
+        if new_ge := match(token, candidate):
             print('token matches with grammar expression', ge, 'new', new_ge)
             matches.append(new_ge)
+            if isinstance(new_ge, Frozen):
+                add_to_list(new_candidates, can_follow(new_ge.ge))
+                break
             add_to_list(new_candidates, can_follow(new_ge))
 
     print('new candidates')