From 6cae15e352311b1b657319e760f798368b12801b Mon Sep 17 00:00:00 2001 From: Thibaut Lienart Date: Tue, 11 Jul 2023 14:15:35 +0200 Subject: [PATCH] REPL mode for code blocks (#1035) * repl mode #818 --- docs/code/index.md | 109 ++++++++++++++++++++++++ src/converter/markdown/blocks.jl | 6 +- src/eval/codeblock.jl | 127 ++++++++++++++++++++++++---- src/manager/franklin.jl | 5 ++ src/manager/write_page.jl | 13 +++ src/parser/markdown/tokens.jl | 35 ++++++-- src/regexes.jl | 2 +- src/utils/html.jl | 30 +++++++ src/utils/vars.jl | 2 + test/eval/repl.jl | 139 +++++++++++++++++++++++++++++++ test/runtests.jl | 1 + 11 files changed, 442 insertions(+), 27 deletions(-) create mode 100644 test/eval/repl.jl diff --git a/docs/code/index.md b/docs/code/index.md index 87bbdbd36..438c4d185 100644 --- a/docs/code/index.md +++ b/docs/code/index.md @@ -277,6 +277,115 @@ savefig(joinpath(@OUTPUT, "sinc.svg")) # hide **Note**: If you wish to use `Plots.jl` and deploy to GitHub pages, you will need to modify the `.github/workflows/Deploy.yml` by adding `env: GKSwstype: "100"` before the ` - name: Build and Deploy` line. [Here](https://github.com/storopoli/Bayesian-Julia/blob/master/.github/workflows/Deploy.yml) is an example. +### Auto and REPL mode + +You can use `!` and `>` to indicate respectively a code that should be run automatically +and the output appended immediately after, or the same but with a REPL-style display: + +````plaintext +```! +x = 5 +y = x^2 +``` +```` + +for instance gives: + +```! +x = 5 +y = x^2 +``` + +In a similar way: + +````plaintext +```> +x = 5 +y = x^2 +``` +```` + +gives + +```> +x = 5 +y = x^2 +``` + +**Shell, Pkg, Help**, these modes are also experimentally supported: + +_Pkg mode_ : + +```` +```] +st +``` +```` + +gives + +```] +st +``` + +_Shell mode_ : (**note**: in a multi-line setting, each line is assumed to be a separate command) + +```` +```; +echo "hello!" +date +``` +```` + +gives + +```; +echo "hello!" +date +``` + +~~~ + +~~~ + +_Help mode_ : (**note**: only single line cell blocks will work properly) + +```` +```? +im +``` +```` + +```? +im +``` + +**Note**: for the `help` mode above, the output is HTML corresponding to the julia +docs, it's wrapped in a `julia-help` div which you should style, the above style +for instance corresponds to the following CSS: + +```css +.julia-help { + background-color: #fffee0; + padding: 10px; + font-style: italic; +} +.julia-help h1,h2,h3 { + font-size: 1em; + font-weight: 500; +} +``` + ### Troubleshooting A few things can go wrong when attempting to use and evaluate code blocks. diff --git a/src/converter/markdown/blocks.jl b/src/converter/markdown/blocks.jl index 21cb0eb31..34482843a 100644 --- a/src/converter/markdown/blocks.jl +++ b/src/converter/markdown/blocks.jl @@ -14,7 +14,11 @@ function convert_block(β::AbstractBlock, lxdefs::Vector{LxDef})::AS βn == :CODE_INLINE && return html_code_inline(stent(β) |> htmlesc) βn == :CODE_BLOCK_LANG && return resolve_code_block(β.ss) - βn == :CODE_BLOCK! && return resolve_code_block(β.ss, shortcut=true) + βn == :CODE_BLOCK! && return resolve_code_block(β.ss; shortcut=true) + βn == :CODE_REPL && return resolve_code_block(β.ss; repl=true) + βn == :CODE_PKG && return resolve_code_block(β.ss; pkg=true) + βn == :CODE_HELP && return resolve_code_block(β.ss; help=true) + βn == :CODE_SHELL && return resolve_code_block(β.ss; shell=true) βn == :CODE_BLOCK && return html_code(stent(β), "{{fill lang}}") βn == :CODE_BLOCK_IND && return convert_indented_code_block(β.ss) diff --git a/src/eval/codeblock.jl b/src/eval/codeblock.jl index 93a6f5915..cca39eb08 100644 --- a/src/eval/codeblock.jl +++ b/src/eval/codeblock.jl @@ -9,8 +9,10 @@ $SIGNATURES Take a fenced code block and return a tuple with the language, the relative path (if any) and the code. """ -function parse_fenced_block(ss::SubString, shortcut=false)::Tuple - if shortcut +function parse_fenced_block(ss::SubString; shortcut=false, + repl=false, shell=false, pkg=false, help=false)::Tuple + + if any((shortcut, repl, shell, pkg, help)) lang = locvar(:lang)::String cntr = locvar(:fd_evalc)::Int rpath = "_ceval_$cntr" @@ -86,9 +88,16 @@ Helper function to process the content of a code block. Return the html corresponding to the code block, possibly after having evaluated the code. """ -function resolve_code_block(ss::SubString; shortcut=false)::String +function resolve_code_block( + ss::SubString; + shortcut=false, + repl=false, + pkg=false, + shell=false, + help=false + )::String # 1. what kind of code is it - lang, rpath, code = parse_fenced_block(ss, shortcut) + lang, rpath, code = parse_fenced_block(ss; shortcut, repl, pkg, shell, help) # 1.a if no rpath is given, code should not be evaluated isnothing(rpath) && return html_code(code, lang) # 1.b if not julia code, eval is not supported @@ -105,10 +114,11 @@ function resolve_code_block(ss::SubString; shortcut=false)::String # languages, can't do the module trick so will need to keep track # of that virtually. There will need to be a branching over lang=="julia" # vs rest here. + repl_code_chunks = Pair{String,String}[] # 2. here we have Julia code, assess whether to run it or not # if not, just return the code as a html block - if should_eval(code, rpath) + if any((shortcut, repl, shell, help, pkg)) || should_eval(code, rpath) # 3. here we have code that should be (re)evaluated # >> retrieve the modulename, the module may not exist # (& may not need to) @@ -117,6 +127,7 @@ function resolve_code_block(ss::SubString; shortcut=false)::String mod = ismodule(modname) ? getfield(Main, Symbol(modname)) : newmodule(modname) + # >> retrieve the code paths cp = form_codepaths(rpath) # >> write the code to file @@ -129,22 +140,106 @@ function resolve_code_block(ss::SubString; shortcut=false)::String out = ifelse(locvar(:auto_code_path)::Bool, cp.out_dir, bk) isdir(out) || mkpath(out) cd(out) - # >> eval the code in the relevant module (this creates output/) - res = run_code(mod, code, cp.out_path; strip_code=false) - cd(bk) - # >> write res to file - # >> this weird thing with QuoteNode is to make sure that the proper - # "show" method is called... - io = IOBuffer() - Core.eval(mod, quote show($(io), "text/plain", $(QuoteNode(res))) end) - write(cp.res_path, take!(io)) + + if repl + # imitating https://github.com/JuliaLang/julia/blob/fe2eeadc0b382508bef7e77ab517789ea844e708/stdlib/REPL/src/REPL.jl#L429-L430 + chunk_code = "" + chunk_ast = nothing + for line in split(code, r"\r?\n", keepempty=false) + chunk_code *= line * "\n" + chunk_ast = Base.parse_input_line(chunk_code) + if (isa(chunk_ast, Expr) && chunk_ast.head === :incomplete) + continue + else + # we have a complete chunk of code + # >> eval the code in the relevant module (this creates output/) + res = run_code(mod, chunk_code, cp.out_path; strip_code=false) + cd(bk) + # >> write res to string (see further down) + io = IOBuffer() + Core.eval(mod, quote show($(io), "text/plain", $(QuoteNode(res))) end) + stdout_str = read(cp.out_path, String) + res_str = String(take!(io)) + res_str = ifelse(res_str == "nothing", "", res_str * "\n") + push!(repl_code_chunks, + chunk_code => stdout_str * res_str + ) + # reset for the next chunk + chunk_code = "" + chunk_ast = nothing + end + end + + # NOTE: shell, pkg, and help mode are currently fairly rudimentary + # and should be considered experimental + + elseif shell + for line in split(code, '\n', keepempty=false) + a = tempname() + open(a, "w") do outf + redirect_stdout(outf) do + redirect_stderr(outf) do + Base.repl_cmd(Cmd(string.(split(line))), nothing) + end + end + end + push!(repl_code_chunks, + line => String(strip(read(a, String))) * "\n" + ) + end + + elseif pkg + for line in split(code, '\n', keepempty=false) + a = tempname() + pname = splitpath(Pkg.project().path)[end-1] + open(a, "w") do outf + redirect_stdout(outf) do + redirect_stderr(outf) do + Pkg.REPLMode.pkgstr(string(line)) + end + end + end + push!(repl_code_chunks, + "($(pname)) pkg> " * line => String(strip(read(a, String))) * "\n" + ) + end + + elseif help + # NOTE: this is pretty crap there should be a better way to just + # reproduce what `?` but the code for the Docs module is opaque to me. + r = eval(Meta.parse("@doc $code")) + push!(repl_code_chunks, + code => replace(Markdown.html(r), + "" => "", + "" => "", + "language-jldoctest" => "language-julia-repl" + ) + ) + + else + # >> eval the code in the relevant module (this creates output/) + res = run_code(mod, code, cp.out_path; strip_code=false) + cd(bk) + # >> write res to file + # >> this weird thing with QuoteNode is to make sure that the proper + # "show" method is called... + io = IOBuffer() + Core.eval(mod, quote show($(io), "text/plain", $(QuoteNode(res))) end) + write(cp.res_path, take!(io)) + end # >> since we've evaluated a code block, toggle scope as stale set_var!(LOCAL_VARS, "fd_eval", true) end - # >> finally return as html - if locvar(:showall)::Bool || shortcut + # >> finally return as html either with or without output + # --- with + if any((repl, shell, help, pkg)) + s = repl ? :repl : shell ? :shell : help ? :help : :pkg + return html_repl_code(repl_code_chunks, s) + + elseif shortcut || locvar(:showall)::Bool return html_code(code, lang) * reprocess("\\show{$rpath}", [GLOBAL_LXDEFS["\\show"]]) end + # --- without return html_code(code, lang) end diff --git a/src/manager/franklin.jl b/src/manager/franklin.jl index a1edef450..9bbf8a6f1 100644 --- a/src/manager/franklin.jl +++ b/src/manager/franklin.jl @@ -106,9 +106,11 @@ function serve(; clear::Bool = false, # check if a Project.toml file is available, if so activate the folder flag_env = false + if isfile(joinpath(FOLDER_PATH[], "Project.toml")) Pkg.activate(".") flag_env = true + set_var!(GLOBAL_VARS, "_has_base_project", true) end # construct the set of files to watch @@ -210,11 +212,14 @@ function fd_fullpass(watched_files::NamedTuple, join_to_prepath::String="")::Int # reset global page variables and latex definitions # NOTE: need to keep track of pre-path if specified, see optimize prepath = get(GLOBAL_VARS, "prepath", "") + has_base_project = globvar(:_has_base_project) + def_GLOBAL_VARS!() def_GLOBAL_LXDEFS!() empty!.((RSS_ITEMS, SITEMAP_DICT)) # reinsert prepath if specified isempty(prepath) || (GLOBAL_VARS["prepath"] = prepath) + set_var!(GLOBAL_VARS, "_has_base_project", has_base_project) # process configuration file (see also `process_mddefs!`) # note the order (utils the config) is important, see also #774 diff --git a/src/manager/write_page.jl b/src/manager/write_page.jl index d21d57c3c..fed5e2eba 100644 --- a/src/manager/write_page.jl +++ b/src/manager/write_page.jl @@ -174,6 +174,19 @@ function convert_and_write(root::String, file::String, head::String, # conversion content = convert_md(read(fpath, String)) + # check if the active project has changed, if so change it back + blackhole = IOBuffer() + if globvar(:_has_base_project)::Bool + if Pkg.project().path != joinpath(FOLDER_PATH[], "Project.toml") + Pkg.activate(".", io=blackhole) + end + else + if isnothing(match(r"v\d\.\d+", splitpath(Pkg.project().path)[end-1])) + Pkg.activate(io=blackhole) + end + end + + # adding document variables to the dictionary # note that some won't change and so it's not necessary to do this every # time but it takes negligible time to do this so ¯\_(ツ)_/¯ diff --git a/src/parser/markdown/tokens.jl b/src/parser/markdown/tokens.jl index 00b9bf5bb..91eb3dbdf 100644 --- a/src/parser/markdown/tokens.jl +++ b/src/parser/markdown/tokens.jl @@ -87,16 +87,20 @@ const MD_TOKENS = LittleDict{Char, Vector{TokenFinder}}( ], '`' => [ isexactly("`", ('`',), false) => :CODE_SINGLE, # `⎵ isexactly("``", ('`',), false) => :CODE_DOUBLE, # ``⎵* + isexactly("```", SPACE_CHAR) => :CODE_TRIPLE, # ```⎵* + isexactly("```!", SPACE_CHAR) => :CODE_TRIPLE!, # ```!⎵* + isexactly("```>", SPACE_CHAR) => :CODE_REPL, # ```>⎵* + isexactly("```?", SPACE_CHAR) => :CODE_HELP, # ```?⎵* + isexactly("```;", SPACE_CHAR) => :CODE_SHELL, # ```;⎵* + isexactly("```]", SPACE_CHAR) => :CODE_PKG, # ```]⎵* # 3+ can be named - isexactly("```", SPACE_CHAR) => :CODE_TRIPLE, # ```⎵* - isexactly("```!", SPACE_CHAR) => :CODE_TRIPLE!,# ```!⎵* - is_language(3) => :CODE_LANG3, # ```lang* - isexactly("````", SPACE_CHAR) => :CODE_QUAD, # ````⎵* - is_language(4) => :CODE_LANG4, # ````lang* - isexactly("`````", SPACE_CHAR) => :CODE_PENTA, # `````⎵* - is_language(5) => :CODE_LANG5, # `````lang* + is_language(3) => :CODE_LANG3, # ```lang* + isexactly("````", SPACE_CHAR) => :CODE_QUAD, # ````⎵* + is_language(4) => :CODE_LANG4, # ````lang* + isexactly("`````", SPACE_CHAR) => :CODE_PENTA, # `````⎵* + is_language(5) => :CODE_LANG5, # `````lang* ], - '*' => [ incrlook(is_hr3) => :HORIZONTAL_RULE, + '*' => [ incrlook(is_hr3) => :HORIZONTAL_RULE, ] ) # end dict #= NOTE @@ -148,6 +152,10 @@ const MD_OCB = [ OCProto(:CODE_BLOCK_LANG, :CODE_LANG4, (:CODE_QUAD,) ), OCProto(:CODE_BLOCK_LANG, :CODE_LANG5, (:CODE_PENTA,) ), OCProto(:CODE_BLOCK!, :CODE_TRIPLE!, (:CODE_TRIPLE,) ), + OCProto(:CODE_REPL, :CODE_REPL, (:CODE_TRIPLE,) ), + OCProto(:CODE_HELP, :CODE_HELP, (:CODE_TRIPLE,) ), + OCProto(:CODE_SHELL, :CODE_SHELL, (:CODE_TRIPLE,) ), + OCProto(:CODE_PKG, :CODE_PKG, (:CODE_TRIPLE,) ), OCProto(:CODE_BLOCK, :CODE_TRIPLE, (:CODE_TRIPLE,) ), OCProto(:CODE_BLOCK, :CODE_QUAD, (:CODE_QUAD,) ), OCProto(:CODE_BLOCK, :CODE_PENTA, (:CODE_PENTA,) ), @@ -256,7 +264,16 @@ CODE_BLOCKS_NAMES List of names of code blocks environments. """ -const CODE_BLOCKS_NAMES = (:CODE_BLOCK_LANG, :CODE_BLOCK, :CODE_BLOCK!, :CODE_BLOCK_IND) +const CODE_BLOCKS_NAMES = ( + :CODE_BLOCK_LANG, + :CODE_BLOCK, + :CODE_BLOCK!, + :CODE_REPL, + :CODE_HELP, + :CODE_PKG, + :CODE_SHELL, + :CODE_BLOCK_IND +) """ MD_CLOSEP diff --git a/src/regexes.jl b/src/regexes.jl index 804c1934e..9879da645 100644 --- a/src/regexes.jl +++ b/src/regexes.jl @@ -69,7 +69,7 @@ const FN_DEF_PAT = r"^\[\^[\p{L}0-9_]+\](:)?$" CODE blocks ===================================================== =# -const CODE_3!_PAT = r"```\!\s*\n?((?:.|\n)*)```" +const CODE_3!_PAT = r"```(?:[\!\>\?\;\]])\s*\n?((?:.|\n)*)```" const CODE_3_PAT = Regex( "```([a-zA-Z][a-zA-Z-]*)" * # language diff --git a/src/utils/html.jl b/src/utils/html.jl index 609691ae3..e9502d594 100644 --- a/src/utils/html.jl +++ b/src/utils/html.jl @@ -109,6 +109,36 @@ Convenience function to introduce inline code. """ html_code_inline(c::AS) = "$c" + +""" + html_repl_code +""" +function html_repl_code(chunks::Vector{Pair{String,String}}, s::Symbol)::String + isempty(chunks) && return "" + io = IOBuffer() + print(io, "
")
+    prefix = s == :repl ? "julia> " :
+             s == :help ? "help?> " :
+             s == :pkg ? "" : # the project name is recuperated in resolve_block
+             "shell> "
+    if s == :help
+        code, result = chunks[1]
+        println(io, prefix * htmlesc(strip(code)))
+        println(io, "
") + println(io, "
") + println(io, result) + println(io, "
") + else + for (code, result) in chunks + println(io, prefix * htmlesc(strip(code))) + println(io, result) + end + println(io, "") + end + return String(take!(io)) +end + + """ html_err diff --git a/src/utils/vars.jl b/src/utils/vars.jl index a96fa73cb..b31775f57 100644 --- a/src/utils/vars.jl +++ b/src/utils/vars.jl @@ -70,6 +70,8 @@ const GLOBAL_VARS_DEFAULT = [ # ----------------------------------------------------- # LEGACY "div_content" => dpair(""), # see build_page + # + "_has_base_project" => dpair(false), ] const GLOBAL_VARS_ALIASES = LittleDict( diff --git a/test/eval/repl.jl b/test/eval/repl.jl new file mode 100644 index 000000000..dea8d2bbc --- /dev/null +++ b/test/eval/repl.jl @@ -0,0 +1,139 @@ +@testset "repl" begin + s = """ + ```> + x = 5 + y = 7 + x + ``` + some explanation + ```> + z = y * 2 + ``` + """ |> fd2html + + @test isapproxstr(s, """ + 
julia> x = 5
+        5
+
+        julia> y = 7 + x
+        12
+
+
+ +

some explanation

+ 

+        julia> z = y * 2
+        24
+
+
+ """) + + s = """ + ```> + println("hello") + x = 5 + ``` + """ |> fd2html + @test isapproxstr(s, """ + 
julia> println("hello")
+        hello
+        
+        julia> x = 5
+        5
+        
+
+ """) + + s = """ + ```> + x = 5; + ``` + """ |> fd2html + @test isapproxstr(s, """ + 
julia> x = 5;
+
+
+ """ + ) +end + +@testset "help" begin + s = """ + ```? + im + ``` + """ |> fd2html + + @test occursin(""" + 
help?> im
+
+
+ 
im
+

The imaginary unit.

+ """, s) +end + +@testset "shell" begin + s = """ + ```; + echo "foo" + ``` + """ |> fd2html + @test isapproxstr(s, """ + 
shell> echo "foo"
+        "foo"
+
+ """) + # multiline + s = """ + ```; + echo abc + echo "abc" + ``` + """ |> fd2html + @test isapproxstr(s, """ + 
shell> echo abc
+        abc
+        
+        shell> echo "abc"
+        "abc"
+        
+
+ """) +end + +@testset "pkg" begin + s = """ + ```] + activate --temp + add StableRNGs + st + ``` + + this is now in the activated stuff + + ```! + using StableRNGs + round(rand(StableRNG(1)), sigdigits=3) + ``` + + """ |> fd2html + + # first block + @test occursin( + """pkg> activate --temp""", s + ) + @test occursin( + """pkg> add StableRNGs\nResolving package versions...""", s + ) + @test occursin( + """pkg> st\nStatus""", s + ) + + # second block uses StableRNG + @test occursin( + """
using StableRNGs
+        round(rand(StableRNG(1)), sigdigits=3)
0.585
""", + s + ) + Pkg.activate() +end diff --git a/test/runtests.jl b/test/runtests.jl index fc9b26206..7acb336cf 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -56,6 +56,7 @@ include("eval/codeblock.jl") include("eval/eval.jl") include("eval/integration.jl") include("eval/extras.jl") +include("eval/repl.jl") # LATEX println("LATEX")