Support toplevel evaluation

This supports defining new types, new methods, assigning
variables in modules, and creating new modules.

src/ASTInterpreter2.jl

@@ -7,7 +7,7 @@ using REPL.LineEdit
 using REPL
 import Base: +, convert, isless
 using Core: CodeInfo, SSAValue, SlotNumber, TypeMapEntry, SimpleVector, LineInfoNode, GotoNode, Slot,
-            GeneratedFunctionStub, MethodInstance
+            GeneratedFunctionStub, MethodInstance, NewvarNode, TypeName
 using Markdown
 
 export @enter, @make_stack, @interpret, Compiled, JuliaStackFrame
@@ -135,6 +135,8 @@ const genframedict = Dict{Tuple{Method,Type},JuliaFrameCode}() # the same for @g
 
 const junk = JuliaStackFrame[]      # to allow re-use of allocated memory (this is otherwise a bottleneck)
 
+const empty_svec = Core.svec()
+
 include("localmethtable.jl")
 include("interpret.jl")
 include("builtins.jl")
@@ -280,7 +282,7 @@ function DebuggerFramework.print_next_state(io::IO, state, frame::JuliaStackFram
         expr = expr.args[2]
     end
     if isexpr(expr, :call) || isexpr(expr, :return)
-        expr.args = map(var->maybe_quote(@eval_rhs(true, frame, var, frame.pc[])), expr.args)
+        expr.args = map(var->maybe_quote(@lookup(frame, var)), expr.args)
     end
     if isa(expr, Expr)
         for (i, arg) in enumerate(expr.args)
@@ -518,6 +520,27 @@ function prepare_call(f, allargs; enter_generated = false)
     return framecode, args, lenv, argtypes
 end
 
+function prepare_thunk(mod::Module, thunk::Expr)
+    if isexpr(thunk, :thunk)
+        framecode = JuliaFrameCode(mod, thunk.args[1])
+    elseif isa(thunk, Expr)
+        error("expected thunk or module, got ", thunk.head)
+    else
+        error("expected expression, got ", typeof(thunk))
+    end
+    return prepare_locals(framecode, [])
+end
+
+function prepare_toplevel(mod::Module, expr::Expr)
+    if expr.head == :using || expr.head == :import || expr.head == :export
+        Core.eval(mod, expr)
+        return nothing
+    else
+        thunk = Meta.lower(mod, expr)
+    end
+    return prepare_thunk(mod, thunk)
+end
+
 """
     framecode, frameargs, lenv, argtypes = determine_method_for_expr(expr; enter_generated = false)
 
@@ -598,6 +621,24 @@ function replace_ssa!(stmt, ssalookup)
     return nothing
 end
 
+function renumber_ssa!(stmts::Vector{Any}, ssalookup)
+    for (i, stmt) in enumerate(stmts)
+        if isa(stmt, GotoNode)
+            new_code[i] = GotoNode(ssalookup[stmt.label])
+        elseif isa(stmt, SSAValue)
+            new_code[i] = SSAValue(ssalookup[stmt.id])
+        elseif isa(stmt, NewSSAValue)
+            new_code[i] = SSAValue(stmt.id)
+        elseif isa(stmt, Expr)
+            replace_ssa!(stmt, ssalookup)
+            if stmt.head == :gotoifnot && isa(stmt.args[2], Int)
+                stmt.args[2] = ssalookup[stmt.args[2]]
+            end
+        end
+    end
+    return stmts
+end
+
 function lookup_global_refs!(ex::Expr)
     for (i, a) in enumerate(ex.args)
         if isa(a, GlobalRef)
@@ -645,41 +686,28 @@ function optimize!(code::CodeInfo, mod::Module)
         end
     end
 
-    ## Un-nest :call expressions (so that there will be only one :call per line)
-    # This will allow us to re-use args-buffers rather than having to allocate new ones each time.
-    old_code, old_codelocs = code.code, code.codelocs
-    code.code = new_code = eltype(old_code)[]
-    code.codelocs = new_codelocs = Int32[]
-    ssainc = fill(1, length(old_code))
-    for (i, stmt) in enumerate(old_code)
-        loc = old_codelocs[i]
-        inner = extract_inner_call!(stmt, length(new_code)+1)
-        while inner !== nothing
-            push!(new_code, inner)
-            push!(new_codelocs, loc)
-            ssainc[i] += 1
-            inner = extract_inner_call!(stmt, length(new_code)+1)
-        end
-        push!(new_code, stmt)
-        push!(new_codelocs, loc)
-    end
-    # Fix all the SSAValues and GotoNodes
-    ssalookup = cumsum(ssainc)
-    for (i, stmt) in enumerate(new_code)
-        if isa(stmt, GotoNode)
-            new_code[i] = GotoNode(ssalookup[stmt.label])
-        elseif isa(stmt, SSAValue)
-            new_code[i] = SSAValue(ssalookup[stmt.id])
-        elseif isa(stmt, NewSSAValue)
-            new_code[i] = SSAValue(stmt.id)
-        elseif isa(stmt, Expr)
-            replace_ssa!(stmt, ssalookup)
-            if stmt.head == :gotoifnot && isa(stmt.args[2], Int)
-                stmt.args[2] = ssalookup[stmt.args[2]]
-            end
-        end
-    end
-    code.ssavaluetypes = length(new_code)
+    # ## Un-nest :call expressions (so that there will be only one :call per line)
+    # # This will allow us to re-use args-buffers rather than having to allocate new ones each time.
+    # old_code, old_codelocs = code.code, code.codelocs
+    # code.code = new_code = eltype(old_code)[]
+    # code.codelocs = new_codelocs = Int32[]
+    # ssainc = fill(1, length(old_code))
+    # for (i, stmt) in enumerate(old_code)
+    #     loc = old_codelocs[i]
+    #     inner = extract_inner_call!(stmt, length(new_code)+1)
+    #     while inner !== nothing
+    #         push!(new_code, inner)
+    #         push!(new_codelocs, loc)
+    #         ssainc[i] += 1
+    #         inner = extract_inner_call!(stmt, length(new_code)+1)
+    #     end
+    #     push!(new_code, stmt)
+    #     push!(new_codelocs, loc)
+    # end
+    # # Fix all the SSAValues and GotoNodes
+    # ssalookup = cumsum(ssainc)
+    # renumber_ssa!(new_code, ssalookup)
+    # code.ssavaluetypes = length(new_code)
     return code
 end
 
@@ -868,7 +896,7 @@ function maybe_step_through_wrapper!(stack)
             pc = next_call!(Compiled(), frame, pc)
         end
         stack[1] = JuliaStackFrame(JuliaFrameCode(frame.code; wrapper=true), frame, pc)
-        newcall = Expr(:call, map(x->@eval_rhs(true, frame, x, pc), last.args)...)
+        newcall = Expr(:call, map(x->@lookup(frame, x), last.args)...)
         pushfirst!(stack, enter_call_expr(newcall))
         return maybe_step_through_wrapper!(stack)
     end
@@ -877,6 +905,29 @@ end
 
 lower(mod, arg) = false ? expand(arg) : Meta.lower(mod, arg)
 
+function interpret!(stack, mod::Module, expr::Expr)
+    if expr.head != :toplevel
+        expr = Expr(:toplevel, expr)
+    end
+    local ret
+    for arg in expr.args
+        if isexpr(arg, :module)
+            # Create the module
+            innermod = Core.eval(mod, Expr(:module, arg.args[1], arg.args[2], quote end))
+            # Interpret the module body at toplevel inside the new module
+            modbody = Expr(:toplevel)
+            modbody.args = arg.args[3].args
+            ret = interpret!(stack, innermod, modbody)
+        elseif isa(arg, Expr)
+            frame = ASTInterpreter2.prepare_toplevel(mod, arg)
+            frame === nothing && continue
+            ret = ASTInterpreter2.finish_and_return!(stack, frame, true)
+        end
+    end
+
+    return ret
+end
+
 # This is a version of gen_call_with_extracted_types, except that is passes back the call expression
 # for further processing.
 function extract_args(__module__, ex0)

src/commands.jl

@@ -2,7 +2,7 @@ function perform_return!(state)
     returning_frame = state.stack[1]
     returning_expr = pc_expr(returning_frame)
     @assert isexpr(returning_expr, :return)
-    val = @eval_rhs(true, returning_frame, returning_expr.args[1], returning_frame.pc[])
+    val = @lookup(returning_frame, returning_expr.args[1])
     if length(state.stack) != 1
         calling_frame = state.stack[2]
         if returning_frame.code.generator
@@ -71,8 +71,7 @@ function DebuggerFramework.execute_command(state, frame::JuliaStackFrame, cmd::U
         if isa(expr, Expr)
             if is_call(expr)
                 isexpr(expr, :(=)) && (expr = expr.args[2])
-                args = map(x->isa(x, QuoteNode) ? x.value :
-                @eval_rhs(true, frame, x, pc), expr.args)
+                args = map(x->isa(x, QuoteNode) ? x.value : @lookup(frame, x), expr.args)
                 expr = Expr(:call, args...)
                 f = (expr.args[1] == Core._apply) ? expr.args[2] : expr.args[1]
                 ok = true
@@ -139,8 +138,7 @@ function DebuggerFramework.execute_command(state, frame::JuliaStackFrame, ::Val{
     if isa(expr, Expr)
         if is_call(expr)
             isexpr(expr, :(=)) && (expr = expr.args[2])
-            args = map(x->isa(x, QuoteNode) ? x.value :
-                @eval_rhs(true, frame, x, frame.pc[]), expr.args)
+            args = map(x->isa(x, QuoteNode) ? x.value : @lookup(frame, x), expr.args)
             f = args[1]
             if f == Core._apply
                 f = to_function(args[2])

src/generate_builtins.jl

@@ -21,7 +21,7 @@ function generate_fcall(f, table, id)
             end
             argcall = ""
             for i = 1:nargs
-                argcall *= "@eval_rhs(true, frame, args[$(i+1)])"
+                argcall *= "@lookup(frame, args[$(i+1)])"
                 if i < nargs
                     argcall *= ", "
                 end
@@ -53,7 +53,7 @@ function getargs(args, frame)
     nargs = length(args)-1  # skip f
     callargs = resize!(frame.callargs, nargs)
     for i = 1:nargs
-        callargs[i] = @eval_rhs(true, frame, args[i+1])
+        callargs[i] = @lookup(frame, args[i+1])
     end
     return callargs
 end
@@ -74,7 +74,7 @@ function maybe_evaluate_builtin(frame, call_expr)
     if isa(fex, QuoteNode)
         f = fex.value
     else
-        f = @eval_rhs(true, frame, fex)
+        f = @lookup(frame, fex)
     end
     # Builtins and intrinsics have empty method tables. We can circumvent
     # a long "switch" check by looking for this.
@@ -97,7 +97,7 @@ function maybe_evaluate_builtin(frame, call_expr)
             print(io,
 """
     $head f === $f
-        return Some{Any}(ntuple(i->@eval_rhs(true, frame, args[i+1]), length(args)-1))
+        return Some{Any}(ntuple(i->@lookup(frame, args[i+1]), length(args)-1))
 """)
             continue
         end