Merge Preparation: Try-Finally to Integrated EOP

Changelog.md

@@ -1,5 +1,27 @@
 # Motoko compiler changelog
 
+## 0.12.0 (to come)
+
+* motoko (`moc`)
+
+  * feat: `finally` clauses for `try` expressions (#4507).
+
+    A trailing `finally` clause to `try`/`catch` expressions facilitates structured
+    resource deallocation (e.g. acquired locks, etc.) and similar cleanups in the
+    presence of control-flow expressions (`return`, `break`, `continue`, `throw`).
+
+    _Note_: `finally`-expressions that are in scope will be executed even if an execution
+    path _following_ an `await`-expression traps. This feature, not available before in Motoko,
+    allows programmers to implement cleanups even in the presence of traps. For trapping 
+    execution paths prior to any `await`, the replica-provided state roll back mechanism must 
+    ensure no cleanup is required.
+
+    The relevant security best practices are accessible at
+    https://internetcomputer.org/docs/current/developer-docs/security/security-best-practices/inter-canister-calls#recommendation
+
+    BREAKING CHANGE (Minor): `finally` is now a reserved keyword,
+    programs using this identifier will break.
+
 ## 0.11.3 (2024-07-16)
 
 * motoko (`moc`)

doc/md/examples/grammar.txt

@@ -217,6 +217,8 @@
     'if' <exp_nullary> <exp_nest>
     'if' <exp_nullary> <exp_nest> 'else' <exp_nest>
     'try' <exp_nest> <catch>
+    'try' <exp_nest> <catch> 'finally' <exp_nest>
+    'try' <exp_nest> 'finally' <exp_nest>
     'throw' <exp_nest>
     'switch' <exp_nullary> '{' <list(<case>, ';')> '}'
     'while' <exp_nullary> <exp_nest>

doc/md/reference/language-manual.md

@@ -91,7 +91,7 @@ The following keywords are reserved and may not be used as identifiers:
 ``` bnf
 
 actor and assert async async* await await* break case catch class
-composite continue debug debug_show do else flexible false for
+composite continue debug debug_show do else false flexible finally for
 from_candid func if ignore import in module not null object or label
 let loop private public query return shared stable switch system throw
 to_candid true try type var while with
@@ -514,6 +514,9 @@ The syntax of an expression is as follows:
   await* <block-or-exp>                          Await a delayed computation (only in async)
   throw <exp>                                    Raise an error (only in async)
   try <block-or-exp> catch <pat> <block-or-exp>  Catch an error (only in async)
+  try <block-or-exp> finally <block-or-exp>      Guard with cleanup
+  try <block-or-exp> catch <pat> <block-or-exp> finally <block-or-exp>
+                                                 Catch an error (only in async) and cleanup
   assert <block-or-exp>                          Assertion
   <exp> : <typ>                                  Type annotation
   <dec>                                          Declaration
@@ -2547,6 +2550,11 @@ Because the [`Error`](../base/Error.md) type is opaque, the pattern match cannot
 
 :::
 
+The `try` expression can be provided with a `finally` cleanup clause to facilitate structured rollback of temporary state changes (e.g. to release a lock). 
+The preceding `catch` clause may be omitted in the presence of a `finally` clause.
+
+This form is `try <block-or-exp1> (catch <pat> <block-or-exp2>)? finally <block-or-exp3>`, and evaluation proceeds as above with the crucial addition that every control-flow path leaving `<block-or-exp1>` or `<block-or-exp2>` will execute the unit-valued `<block-or-exp3>` before the entire `try` expression obtains its value. The cleanup expression will additionaly also be executed when the processing after an intervening `await` (directly, or indirectly as `await*`) traps.
+
 See [Error type](#error-type).
 
 ### Assert

emacs/motoko-mode.el

@@ -47,6 +47,7 @@
                  "debug"
                  "debug_show"
                  "else"
+                 "finally"
                  "flexible"
                  "for"
                  "from_candid"

rts/motoko-rts/src/continuation_table.rs

@@ -120,7 +120,7 @@ pub unsafe fn remember_continuation<M: Memory>(mem: &mut M, ptr: Value) -> usize
 
 // Position of the future in explicit self-send ContinuationTable entries
 // Invariant: keep this synchronised with compiler.ml (see future_array_index)
-const FUTURE_ARRAY_INDEX: usize = 2;
+const FUTURE_ARRAY_INDEX: usize = 3;
 
 #[no_mangle]
 pub unsafe extern "C" fn peek_future_continuation(idx: usize) -> Value {

src/codegen/compile_classical.ml

@@ -9220,39 +9220,39 @@ module FuncDec = struct
     ))
 
   let message_start env sort = match sort with
-      | Type.Shared Type.Write ->
-        Lifecycle.trans env Lifecycle.InUpdate
-      | Type.Shared Type.Query ->
-        Lifecycle.trans env Lifecycle.InQuery
-      | Type.Shared Type.Composite ->
-        Lifecycle.trans env Lifecycle.InComposite
+      | Type.(Shared Write) ->
+        Lifecycle.(trans env InUpdate)
+      | Type.(Shared Query) ->
+        Lifecycle.(trans env InQuery)
+      | Type.(Shared Composite) ->
+        Lifecycle.(trans env InComposite)
       | _ -> assert false
 
   let message_cleanup env sort = match sort with
-      | Type.Shared Type.Write ->
+      | Type.(Shared Write) ->
         GC.collect_garbage env ^^
-        Lifecycle.trans env Lifecycle.Idle
-      | Type.Shared Type.Query ->
-        Lifecycle.trans env Lifecycle.PostQuery
-      | Type.Shared Type.Composite ->
+        Lifecycle.(trans env Idle)
+      | Type.(Shared Query) ->
+        Lifecycle.(trans env PostQuery)
+      | Type.(Shared Composite) ->
         (* Stay in composite query state such that callbacks of 
         composite queries can also use the memory reserve. 
         The state is isolated since memory changes of queries 
         are rolled back by the IC runtime system. *)
-        Lifecycle.trans env Lifecycle.InComposite
+        Lifecycle.(trans env InComposite)
       | _ -> assert false
 
   let callback_start env =
-    Lifecycle.is_in env Lifecycle.InComposite ^^
+    Lifecycle.(is_in env InComposite) ^^
     G.if0
       (G.nop)
-      (message_start env (Type.Shared Type.Write))
+      (message_start env Type.(Shared Write))
 
   let callback_cleanup env =
-    Lifecycle.is_in env Lifecycle.InComposite ^^
+    Lifecycle.(is_in env InComposite) ^^
     G.if0
       (G.nop)
-      (message_cleanup env (Type.Shared Type.Write))
+      (message_cleanup env Type.(Shared Write))
 
   let compile_const_message outer_env outer_ae sort control args mk_body ret_tys at : E.func_with_names =
     let ae0 = VarEnv.mk_fun_ae outer_ae in
@@ -9454,7 +9454,7 @@ module FuncDec = struct
     (* result is a function that accepts a list of closure getters, from which
        the first and second must be the reply and reject continuations. *)
     fun closure_getters ->
-      let (set_cb_index, get_cb_index) = new_local env "cb_index" in
+      let set_cb_index, get_cb_index = new_local env "cb_index" in
       Arr.lit env Tagged.T closure_getters ^^
       ContinuationTable.remember env ^^
       set_cb_index ^^
@@ -9482,7 +9482,12 @@ module FuncDec = struct
     Func.define_built_in env name ["env", I32Type] [] (fun env ->
         G.i (LocalGet (nr 0l)) ^^
         ContinuationTable.recall env ^^
-        G.i Drop);
+        Arr.load_field env 2l ^^ (* get the cleanup closure *)
+        let set_closure, get_closure = new_local env "closure" in
+        set_closure ^^ get_closure ^^
+        Closure.prepare_closure_call env ^^
+        get_closure ^^
+        Closure.call_closure env 0 0);
     compile_unboxed_const (E.add_fun_ptr env (E.built_in env name))
 
   let ic_call_threaded env purpose get_meth_pair push_continuations
@@ -9531,29 +9536,29 @@ module FuncDec = struct
     | _ ->
       E.trap_with env (Printf.sprintf "cannot perform %s when running locally" purpose)
 
-  let ic_call env ts1 ts2 get_meth_pair get_arg get_k get_r =
+  let ic_call env ts1 ts2 get_meth_pair get_arg get_k get_r get_c =
     ic_call_threaded
       env
       "remote call"
       get_meth_pair
-      (closures_to_reply_reject_callbacks env ts2 [get_k; get_r])
+      (closures_to_reply_reject_callbacks env ts2 [get_k; get_r; get_c])
       (fun _ -> get_arg ^^ Serialization.serialize env ts1)
 
-  let ic_call_raw env get_meth_pair get_arg get_k get_r =
+  let ic_call_raw env get_meth_pair get_arg get_k get_r get_c =
     ic_call_threaded
       env
       "raw call"
       get_meth_pair
-      (closures_to_raw_reply_reject_callbacks env [get_k; get_r])
+      (closures_to_raw_reply_reject_callbacks env [get_k; get_r; get_c])
       (fun _ -> get_arg ^^ Blob.as_ptr_len env)
 
-  let ic_self_call env ts get_meth_pair get_future get_k get_r =
+  let ic_self_call env ts get_meth_pair get_future get_k get_r get_c =
     ic_call_threaded
       env
       "self call"
       get_meth_pair
-      (* Storing the tuple away, future_array_index = 2, keep in sync with rts/continuation_table.rs *)
-      (closures_to_reply_reject_callbacks env ts [get_k; get_r; get_future])
+      (* Storing the tuple away, future_array_index = 3, keep in sync with rts/continuation_table.rs *)
+      (closures_to_reply_reject_callbacks env ts [get_k; get_r; get_c; get_future])
       (fun get_cb_index ->
         get_cb_index ^^
         BoxedSmallWord.box env Type.Nat32 ^^
@@ -11865,7 +11870,7 @@ and compile_prim_invocation (env : E.t) ae p es at =
   | ICCallerPrim, [] ->
     SR.Vanilla, IC.caller env
 
-  | ICCallPrim, [f;e;k;r] ->
+  | ICCallPrim, [f;e;k;r;c] ->
     SR.unit, begin
     (* TBR: Can we do better than using the notes? *)
     let _, _, _, ts1, _ = Type.as_func f.note.Note.typ in
@@ -11874,19 +11879,22 @@ and compile_prim_invocation (env : E.t) ae p es at =
     let (set_arg, get_arg) = new_local env "arg" in
     let (set_k, get_k) = new_local env "k" in
     let (set_r, get_r) = new_local env "r" in
+    let (set_c, get_c) = new_local env "c" in
     let add_cycles = Internals.add_cycles env ae in
     compile_exp_vanilla env ae f ^^ set_meth_pair ^^
     compile_exp_vanilla env ae e ^^ set_arg ^^
     compile_exp_vanilla env ae k ^^ set_k ^^
     compile_exp_vanilla env ae r ^^ set_r ^^
-    FuncDec.ic_call env ts1 ts2 get_meth_pair get_arg get_k get_r add_cycles
+    compile_exp_vanilla env ae c ^^ set_c ^^
+    FuncDec.ic_call env ts1 ts2 get_meth_pair get_arg get_k get_r get_c add_cycles
     end
-  | ICCallRawPrim, [p;m;a;k;r] ->
+  | ICCallRawPrim, [p;m;a;k;r;c] ->
     SR.unit, begin
-    let (set_meth_pair, get_meth_pair) = new_local env "meth_pair" in
-    let (set_arg, get_arg) = new_local env "arg" in
-    let (set_k, get_k) = new_local env "k" in
-    let (set_r, get_r) = new_local env "r" in
+    let set_meth_pair, get_meth_pair = new_local env "meth_pair" in
+    let set_arg, get_arg = new_local env "arg" in
+    let set_k, get_k = new_local env "k" in
+    let set_r, get_r = new_local env "r" in
+    let set_c, get_c = new_local env "c" in
     let add_cycles = Internals.add_cycles env ae in
     compile_exp_vanilla env ae p ^^
     compile_exp_vanilla env ae m ^^ Text.to_blob env ^^
@@ -11895,7 +11903,8 @@ and compile_prim_invocation (env : E.t) ae p es at =
     compile_exp_vanilla env ae a ^^ set_arg ^^
     compile_exp_vanilla env ae k ^^ set_k ^^
     compile_exp_vanilla env ae r ^^ set_r ^^
-    FuncDec.ic_call_raw env get_meth_pair get_arg get_k get_r add_cycles
+    compile_exp_vanilla env ae c ^^ set_c ^^
+    FuncDec.ic_call_raw env get_meth_pair get_arg get_k get_r get_c add_cycles
     end
 
   | ICMethodNamePrim, [] ->
@@ -12104,11 +12113,12 @@ and compile_exp_with_hint (env : E.t) ae sr_hint exp =
     let return_arity = List.length return_tys in
     let mk_body env1 ae1 = compile_exp_as env1 ae1 (StackRep.of_arity return_arity) e in
     FuncDec.lit env ae x sort control captured args mk_body return_tys exp.at
-  | SelfCallE (ts, exp_f, exp_k, exp_r) ->
+  | SelfCallE (ts, exp_f, exp_k, exp_r, exp_c) ->
     SR.unit,
     let (set_future, get_future) = new_local env "future" in
     let (set_k, get_k) = new_local env "k" in
     let (set_r, get_r) = new_local env "r" in
+    let (set_c, get_c) = new_local env "c" in
     let mk_body env1 ae1 = compile_exp_as env1 ae1 SR.unit exp_f in
     let captured = Freevars.captured exp_f in
     let add_cycles = Internals.add_cycles env ae in
@@ -12118,13 +12128,15 @@ and compile_exp_with_hint (env : E.t) ae sr_hint exp =
 
     compile_exp_vanilla env ae exp_k ^^ set_k ^^
     compile_exp_vanilla env ae exp_r ^^ set_r ^^
+    compile_exp_vanilla env ae exp_c ^^ set_c ^^
 
     FuncDec.ic_self_call env ts
       IC.(get_self_reference env ^^
           actor_public_field env async_method_name)
       get_future
       get_k
       get_r
+      get_c
       add_cycles
   | ActorE (ds, fs, _, _) ->
     fatal "Local actors not supported by backend"