[ysh] Enable closures in a loop

i.e. an enclosing frame for each iteration.

This is shopt -s for_loop_frames, and it's on in ysh:upgrade.

Document it.

core/state.py

@@ -898,11 +898,10 @@ def _MakeArgvCell(argv):
 
 class ctx_LoopFrame(object):
 
-    def __init__(self, mem, name1):
-        # type: (Mem, str) -> None
+    def __init__(self, mem, do_new_frame):
+        # type: (Mem, bool) -> None
         self.mem = mem
-        self.name1 = name1
-        self.do_new_frame = name1 == '__hack__'
+        self.do_new_frame = do_new_frame
 
         if self.do_new_frame:
             to_enclose = self.mem.var_stack[-1]

demo/survey-closure.sh

@@ -108,7 +108,7 @@ loops() {
   console.assert(functions[1]() === 1, "Test 4.2 failed");
   console.assert(functions[2]() === 2, "Test 4.3 failed");
 
-  console.log(functions[2]())
+  console.log(functions[1]())
   '
 
   echo 'LOOPS PYTHON'
@@ -137,7 +137,7 @@ for i in range(3):
   actual = functions[i]()
   assert i == actual, "%d != %d" % (i, actual)
 
-print(functions[2]())
+print(functions[1]())
     '
 }
 

demo/survey-loop.sh

@@ -1,18 +1,20 @@
 #!/usr/bin/env bash
 #
-# Survey loop behavior
+# Survey loop behavior - mutating the container while iterating over it
+#
+# List - Python and JS are consistent
+# Dict - Python and JS are different - Python 3 introduced version number check
+#   JS and Python 2 may have "unknown" results
 #
 # Usage:
-#   demo/survey-str-api.sh <function name>
+#   demo/survey-loop.sh <function name>
 
 set -o nounset
 set -o pipefail
 set -o errexit
 
 source build/dev-shell.sh  # python3 in $PATH
 
-# Python and JS string and regex replacement APIs
-
 mutate-py() {
   echo ---
   echo PY

doc/ref/chap-cmd-lang.md

@@ -669,17 +669,31 @@ The `io.stdin` object iterates over lines:
     }
     # lines are buffered, so it's much faster than `while read --raw-line`
 
+---
+
+(This section is based on [A Tour of YSH](../ysh-tour.html).)
+
+#### Closing Over the Loop Variable
+
+Each iteration of a `for` loop creates a new frame, which may be captured.
+
+    var x = 42  # outside the loop
+    for i in (0 ..< 3) {
+      var j = i + 2
+
+      var expr = ^"$x: i = $i, j = $j"  # captures x, i, and j
+
+      my-task {
+        echo "$x: i = $i, j = $j"       # also captures x, i, and j
+      }
+    }
 
 #### Mutating Containers in a `for` Loop
 
 - If you append or remove from a `List` while iterating over it, the loop **will** be affected.
 - If you mutate a `Dict` while iterating over it, the loop will **not** be
   affected.
 
----
-
-(This section is based on [A Tour of YSH](../ysh-tour.html).)
-
 ### ysh-while
 
 You can use an expression as the condition:

doc/ref/chap-option.md

@@ -205,8 +205,9 @@ Details on each option:
       xtrace_rich             Hierarchical and process tracing
       xtrace_details (-u)     Disable most tracing with +
       dashglob (-u)           Disabled to avoid files like -rf
-      no_exported             Environ doesn't correspond to exported (-x) vars
-
+      env_obj                 Init ENV Obj at startup; use it when starting
+                              child processes
+      for_loop_frames         YSH can create closures from loop vars
 
 <h3 id="ysh:all">ysh:all</h3>
 
@@ -232,7 +233,7 @@ Details on options that are not in `ysh:upgrade` and `strict:all`:
                               ... source  unset  printf  [un]alias
                               ... getopts
     X old_syntax (-u)         ( )   ${x%prefix}  ${a[@]}   $$
-      env_obj                 Populate the ENV object
+      no_exported             Environ doesn't correspond to exported (-x) vars
       no_init_globals         At startup, don't set vars like PWD, SHELLOPTS
       simple_echo             echo doesn't accept flags -e -n
       simple_eval_builtin     eval takes exactly 1 argument

frontend/option_def.py

@@ -130,6 +130,9 @@ def DoneWithImplementedOptions(self):
 
     # create ENV at startup; read from it when starting processes
     ('env_obj', False),
+
+    # Can create closures from loop variables, like JS / C# / Go
+    ('for_loop_frames', False),
 ]
 
 # TODO: Add strict_arg_parse?  For example, 'trap 1 2 3' shouldn't be

osh/cmd_eval.py

@@ -1304,7 +1304,8 @@ def _DoForEach(self, node):
         status = 0  # in case we loop zero times
         with ctx_LoopLevel(self):
             while True:
-                with state.ctx_LoopFrame(self.mem, name1.name):
+                with state.ctx_LoopFrame(self.mem,
+                                         self.exec_opts.for_loop_frames()):
                     first = it2.FirstValue()
                     #log('first %s', first)
                     if first is None:  # for StdinIterator

spec/loop.test.sh

@@ -6,20 +6,23 @@ fun() {
   for i; do
     echo $i
   done
+  echo "finished=$i"
 }
 fun 1 2 3
 ## STDOUT:
 1
 2
 3
+finished=3
 ## END
 
 #### empty for loop (has "in")
 set -- 1 2 3
 for i in ; do
   echo $i
 done
-## stdout-json: ""
+## STDOUT:
+## END
 
 #### for loop with invalid identifier
 # should be compile time error, but runtime error is OK too
@@ -37,10 +40,12 @@ done
 for in in a b c; do
   echo $in
 done
+echo finished=$in
 ## STDOUT:
 a
 b
 c
+finished=c
 ## END
 
 #### Tilde expansion within for loop

spec/ysh-closures.test.sh

@@ -41,6 +41,9 @@ p
 ## END
 
 #### Expr Closures in a Loop !
+
+# see demo/survey-closure.sh
+
 shopt --set ysh:upgrade
 
 proc task (; tasks, expr) {
@@ -50,8 +53,7 @@ proc task (; tasks, expr) {
 func makeTasks() {
   var tasks = []
   var x = 'x'
-  for __hack__ in (0 ..< 3) {
-    var i = __hack__
+  for i in (0 ..< 3) {
     var j = i + 2
     task (tasks, ^"$x: i = $i, j = $j")
   }
@@ -82,8 +84,7 @@ proc task (; tasks; ; b) {
 func makeTasks() {
   var tasks = []
   var x = 'x'
-  for __hack__ in (0 ..< 3) {
-    var i = __hack__
+  for i in (0 ..< 3) {
     var j = i + 2
     task (tasks) { echo "$x: i = $i, j = $j" }
   }