Parseable sequences with in-place reader extend (#78)

* Op::Extend and Op::Capture

This makes all inline-sequences consuming input to areas with an in-place reader extend, which avoid the outer frame to break because input is not indicated to be parsed.

Nevertheless, programs like

```tokay
(Int ',' Int) | Ident   # will accept: 1,2a
```
will accept inputs like `1,2a`, which is obviously not a wanted behavior.

* Adding working test-case

* Starting over with a `@(...)` parseable area

This reverts the first attempt, but uses a new syntactic element to define a sequence / block as an area which extends the reader.

examples/tokay.tok

@@ -243,10 +243,11 @@ TokenLiteral : @{
 
 TokenAtom : @{
     TokenLiteral
-    ParseletInstance '(' _ ___ CallArguments? ___ expect ')'  ast("call")
-    ParseletInstance
     InlineBlock
+    '@' _ InlineBlock  ast("area")
     Block
+    ParseletInstance '(' _ ___ CallArguments? ___ expect ')'  ast("call")
+    ParseletInstance
 }
 
 Token1 : @{  # todo: Token1 can be renamed back to Token again when #31 is fixed and Self is used.

src/compiler/ast.rs

@@ -644,6 +644,12 @@ fn traverse_node(compiler: &mut Compiler, node: &Dict) -> ImlOp {
             ImlOp::from(vec![expr, alias, ImlOp::from(Op::MakeAlias)])
         }
 
+        // area -----------------------------------------------------------
+        "area" => {
+            let body = traverse(compiler, &node["children"]);
+            ImlOp::seq(vec![body, ImlOp::from(Op::Extend)], false)
+        }
+
         // assign ---------------------------------------------------------
         assign if assign.starts_with("assign") => {
             let children = node["children"].borrow();
@@ -1416,13 +1422,12 @@ fn traverse_node(compiler: &mut Compiler, node: &Dict) -> ImlOp {
                 ));
             }
 
-            // Lists are definitive lists with a given length
+            // Lists are definitive lists with a given length and only non-aliased values
             if emit == "list" {
                 ops.push(Op::MakeList(children.len()).into());
                 ImlOp::from(ops)
             }
             // In most cases, lists are parsed as sequences;
-            // inline-sequence always have a higher return severity.
             else {
                 ImlOp::seq(ops, true)
             }

src/compiler/iml/op.rs

@@ -442,8 +442,7 @@ impl ImlOp {
                     item.compile(ops, linker);
                 }
 
-                // Create a frame and collect in collection-mode,
-                // when there's more than one operation inside ret.
+                // Check if the sequence exists of more than one operational instruction
                 if *collection
                     && ops[start..]
                         .iter()
@@ -572,14 +571,15 @@ impl ImlOp {
                         // Kleene
                         ops.extend(vec![
                             Op::Frame(0),            // The overall capture
-                            Op::Frame(body_len + 5), // The fused capture for repetition
+                            Op::Frame(body_len + 6), // The fused capture for repetition
                         ]);
                         ops.extend(body_ops); // here comes the body
                         ops.extend(vec![
                             Op::ForwardIfConsumed(2), // When consumed we can commit and jump backward
-                            Op::Forward(3),           // otherwise leave the loop
-                            Op::Commit,
-                            Op::Backward(body_len + 3), // repeat the body
+                            Op::Forward(4),           // otherwise leave the loop
+                            Op::Capture,
+                            Op::Extend,
+                            Op::Backward(body_len + 4), // repeat the body
                             Op::Close,
                             Op::Collect,
                             Op::Close,
@@ -592,14 +592,15 @@ impl ImlOp {
                         ops.extend(vec![
                             Op::ForwardIfConsumed(2), // If nothing was consumed, then...
                             Op::Next,                 //...reject
-                            Op::Frame(body_len + 5),  // The fused capture for repetition
+                            Op::Frame(body_len + 6),  // The fused capture for repetition
                         ]);
                         ops.extend(body_ops); // here comes the body again inside the repetition
                         ops.extend(vec![
                             Op::ForwardIfConsumed(2), // When consumed we can commit and jump backward
-                            Op::Forward(3),           // otherwise leave the loop
-                            Op::Commit,
-                            Op::Backward(body_len + 3), // repeat the body
+                            Op::Forward(4),           // otherwise leave the loop
+                            Op::Capture,
+                            Op::Extend,
+                            Op::Backward(body_len + 4), // repeat the body
                             Op::Close,
                             Op::Collect,
                             Op::Close,

src/compiler/parser.rs

@@ -290,11 +290,12 @@ impl Parser {
 
         (TokenAtom = {
             TokenLiteral,
+            InlineBlock,
+            ["@", _, InlineBlock, (call ast[(value "area")])],
+            Block,
             [ParseletInstance, "(", _, ___, (opt CallArguments), ___, (expect ")"),
                 (call ast[(value "call")])],
-            ParseletInstance,
-            InlineBlock,
-            Block
+            ParseletInstance
         }),
 
         (Token = {

src/test.rs

@@ -640,8 +640,9 @@ fn examples() {
         Ok(Some(value!(24)))
     );
 
-    // todo: Move this to a separate function
+    // todo: Move stuff below to a separate function
     testcase("tests/test_piped_grammar.tok");
+    testcase("tests/test_inline_parseable_sequence.tok");
 }
 
 // Tests for control flow -------------------------------------------------------------------------

src/vm/context.rs

@@ -128,12 +128,7 @@ impl<'runtime, 'program, 'reader, 'parselet> Context<'runtime, 'program, 'reader
         self.push(value)
     }
 
-    // Return current frame
-    pub fn frame(&self) -> &Frame {
-        &self.frame
-    }
-
-    // Return top-level frame
+    /// Return top-level frame
     pub fn frame0(&self) -> &Frame {
         if self.frames.is_empty() {
             &self.frame
@@ -142,6 +137,15 @@ impl<'runtime, 'program, 'reader, 'parselet> Context<'runtime, 'program, 'reader
         }
     }
 
+    /// Return mutable top-level frame
+    pub fn frame0_mut(&mut self) -> &mut Frame {
+        if self.frames.is_empty() {
+            &mut self.frame
+        } else {
+            &mut self.frames[0]
+        }
+    }
+
     /** Return a capture by index as RefValue. */
     pub fn get_capture(&mut self, pos: usize) -> Option<RefValue> {
         let frame0 = self.frame0();

src/vm/op.rs

@@ -22,7 +22,8 @@ pub(crate) enum Op {
 
     // Capture frames
     Frame(usize), // Start new frame with optional relative forward address fuse
-    Commit,       // Commit frame
+    Capture,      // Reset frame capture to current stack size, saving captures
+    Extend,       // Extend frame's reader to current position
     Reset,        // Reset frame
     Close,        // Close frame
     Collect,      // Collect stack values from current frame
@@ -194,8 +195,12 @@ impl Op {
                     Ok(Accept::Next)
                 }
 
-                Op::Commit => {
+                Op::Capture => {
                     context.frame.capture_start = context.runtime.stack.len();
+                    Ok(Accept::Next)
+                }
+
+                Op::Extend => {
                     context.frame.reader_start = context.runtime.reader.tell();
                     Ok(Accept::Next)
                 }

tests/test_inline_parseable_sequence.tok

@@ -0,0 +1,10 @@
+print("stage 1 " + repr(x))
+#Int _ x = (Int expect EOF  ast("test")) ast("yo")
+x = @(Int _ Int ast("test") | Void)
+print("stage 2 " + repr(x))
+#end x
+#---
+#1 2
+#---
+#stage 1 void
+#stage 2 (emit => "test", children => (1, 2), offset => 0, row => 1, col => 1, stop_offset => 3, stop_row => 1, stop_col => 4)