Skip to content

Commit

Permalink
adding experimental tight integration for literate, closes #256 (#257)
Browse files Browse the repository at this point in the history
* adding experimental integration for literate, closes #256
  • Loading branch information
tlienart authored Oct 9, 2019
1 parent 5245abb commit a261aeb
Show file tree
Hide file tree
Showing 9 changed files with 211 additions and 20 deletions.
1 change: 1 addition & 0 deletions Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
JuDocTemplates = "6793090a-55ae-11e9-0511-73b91164f4ea"
Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
LiveServer = "16fef848-5104-11e9-1b77-fb7a48bbb589"
Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
Expand Down
4 changes: 4 additions & 0 deletions src/JuDoc.jl
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ using DelimitedFiles: readdlm
using OrderedCollections
using Pkg
using DocStringExtensions: SIGNATURES, TYPEDEF
using Literate: markdown

import Logging
import LiveServer
Expand Down Expand Up @@ -126,4 +127,7 @@ include("misc_html.jl")
# ERROR TYPES
include("error_types.jl")

# INTEGRATION
include("integration/literate.jl")

end # module
25 changes: 19 additions & 6 deletions src/converter/lx.jl
Original file line number Diff line number Diff line change
Expand Up @@ -228,10 +228,19 @@ function resolve_lx_input_hlcode(rpath::AS, lang::AS)::String

end
end
hideall && take!(io_in) # discard the content
hideall && take!(io_in) # discard the content
code = String(take!(io_in))
isempty(code) && return ""
endswith(code, "\n") && (code = chop(code, tail=1))
return html_code(code, lang)
html = html_code(code, lang)
if LOCAL_PAGE_VARS["showall"].first
fd, fn = splitdir(fpath)
res = read(joinpath(fd, "output", splitext(fn)[1] * ".res"), String)
if !isempty(res)
html *= html_div("code_output", html_code(res))
end
end
return html
end


Expand Down Expand Up @@ -259,9 +268,11 @@ function resolve_lx_input_plainoutput(rpath::AS, reproc::Bool=false; code::Bool=
out_file = joinpath(dir, "output", fname * ".out")
# check if the output file exists
isfile(out_file) || throw(ErrorException("I found an \\input but no relevant output file."))
# return the content in a pre block
reproc || return html_code(read(out_file, String))
return read(out_file, String) * EOS
# return the content in a pre block (if non empty)
content = read(out_file, String)
isempty(content) && return ""
reproc || return html_code(content)
return content * EOS
end


Expand All @@ -277,7 +288,9 @@ function resolve_lx_input_textfile(rpath::AS)::String
fpath = resolve_assets_rpath(rpath; canonical=true)
isempty(splitext(fpath)[2]) && (fpath *= ".md")
isfile(fpath) || throw(ErrorException("I found a \\textinput but no relevant file."))
return read(fpath, String) * EOS
content = read(fpath, String)
isempty(content) && return ""
return content * EOS
end


Expand Down
21 changes: 20 additions & 1 deletion src/converter/lx_simple.jl
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,24 @@ function resolve_lx_textinput(lxc::LxCom)::String
end


"""
$SIGNATURES
Internal function to resolve a `\\literate{rpath}` see [`literate_to_judoc`](@ref).
"""
function resolve_lx_literate(lxc::LxCom)::String
rpath = strip(content(lxc.braces[1]))
path = resolve_assets_rpath(rpath; canonical=true)
endswith(path, ".jl") || (path *= ".jl")
if !isfile(path)
@warn "File not found when trying to resolve a \\literate command ($path)."
return ""
end
opath = literate_to_judoc(path)
return read(opath, String) * EOS
end


"""
$SIGNATURES
Expand Down Expand Up @@ -172,4 +190,5 @@ Same as [`LXCOM_SIMPLE`](@ref) except the output is re-processed before being in
const LXCOM_SIMPLE_REPROCESS = LittleDict{String, Function}(
"\\textoutput" => resolve_lx_textoutput, # include output generated by code and reproc
"\\textinput" => resolve_lx_textinput,
)
"\\literate" => resolve_lx_literate,
)
12 changes: 10 additions & 2 deletions src/converter/md_blocks.jl
Original file line number Diff line number Diff line change
Expand Up @@ -161,11 +161,13 @@ function eval_and_resolve_code(code::AS, rpath::AS;
out_path, fname = splitdir(path)
out_path = mkpath(joinpath(out_path, "output"))
out_name = splitext(fname)[1] * ".out"
res_name = splitext(fname)[1] * ".res"
res_path = joinpath(out_path, res_name)
out_path = joinpath(out_path, out_name)

# if we're in the no-eval case, check that the relevant files are
# there otherwise do re-eval with re-write
if !eval && isfile(path) && isfile(out_path)
if !eval && isfile(path) && isfile(out_path) && isfile(res_path)
# just return the resolved code block
return resolve_lx_input_hlcode(rpath, "julia")
end
Expand All @@ -174,15 +176,21 @@ function eval_and_resolve_code(code::AS, rpath::AS;
print(rpad("\r→ evaluating code [...] ($(CUR_PATH[]), $rpath)", 79))
# - execute the code while redirecting stdout to file
Logging.disable_logging(Logging.LogLevel(3_000))
res = nothing
open(out_path, "w") do outf # for stdout
redirect_stdout(outf) do
try
res = try
Main.include(path)
catch e
print("There was an error running the code:\n$(e.error)")
end
end
end
open(res_path, "w") do outf
redirect_stdout(outf) do
show(res)
end
end
Logging.disable_logging(Logging.Debug)
print(rpad("\r→ evaluating code [✓]", 79)*"\r")

Expand Down
45 changes: 45 additions & 0 deletions src/integration/literate.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
const LITERATE_JULIA_FENCE = "```julia"
const LITERATE_JULIA_FENCE_L = length(LITERATE_JULIA_FENCE)
const LITERATE_JULIA_FENCE_R = Regex(LITERATE_JULIA_FENCE)

"""
$SIGNATURES
Take a markdown string generated by literate and post-process it to number each code block
and mark them as eval-ed ones.
"""
function literate_post_process(s::String)::String
isempty(s) && return s
em = eachmatch(LITERATE_JULIA_FENCE_R, s)
buf = IOBuffer()
head = 1
c = 1
for m in em
write(buf, SubString(s, head, prevind(s, m.offset)))
write(buf, "```julia:ex$c\n")
head = nextind(s, m.offset + LITERATE_JULIA_FENCE_L)
c += 1
end
lis = lastindex(s)
head < lis && write(buf, SubString(s, head, lis))
return String(take!(buf))
end


"""
$SIGNATURES
Take a Literate.jl script and transform it into a JuDoc-markdown file.
"""
function literate_to_judoc(fpath::String)::String
outpath = joinpath(PATHS[:assets], "literate")
isdir(outpath) || mkdir(outpath)
# don't show infos
Logging.disable_logging(Logging.LogLevel(Logging.Info))
markdown(fpath, outpath, documenter=false,
postprocess=literate_post_process, credit=false)
# bring back logging
Logging.disable_logging(Logging.LogLevel(Logging.Debug))
fname = splitdir(fpath)[2]
return joinpath(outpath, splitext(fname)[1] * ".md")
end
23 changes: 14 additions & 9 deletions src/jd_vars.jl
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,17 @@ is processed.
LOCAL_PAGE_VARS["date"] = Pair(Date(1), (String, Date, Nothing))
LOCAL_PAGE_VARS["lang"] = Pair("julia", (String,)) # default lang for indented code
LOCAL_PAGE_VARS["reflinks"] = Pair(true, (Bool,)) # whether there are reflinks or not
LOCAL_PAGE_VARS["freezecode"] = Pair(false, (Bool,)) # no-reevaluation of the code

# CODE EVALUATION
#
LOCAL_PAGE_VARS["reeval"] = Pair(false, (Bool,)) # whether to always re-evals all on pg
LOCAL_PAGE_VARS["freezecode"] = Pair(false, (Bool,)) # no-reevaluation of the code
LOCAL_PAGE_VARS["showall"] = Pair(false, (Bool,)) # like a notebook on each cell
# the jd_* should not be assigned externally
LOCAL_PAGE_VARS["jd_code_scope"] = code_scope
LOCAL_PAGE_VARS["jd_code_head"] = Pair(Ref(0), (Ref{Int},))
LOCAL_PAGE_VARS["jd_code_eval"] = Pair(Ref(false), (Ref{Bool},)) # toggle reeval
LOCAL_PAGE_VARS["jd_code"] = Pair("", (String,)) # just the script

# RSS 2.0 item specs:
# only title, link and description must be defined
Expand Down Expand Up @@ -123,13 +133,6 @@ is processed.
LOCAL_PAGE_VARS["jd_mtime"] = Pair(Date(1), (Date,)) # time of last modification
LOCAL_PAGE_VARS["jd_rpath"] = Pair("", (String,)) # local path to file src/[...]/blah.md

# Internal vars for code blocks
LOCAL_PAGE_VARS["jd_code_scope"] = code_scope
LOCAL_PAGE_VARS["jd_code_head"] = Pair(Ref(0), (Ref{Int},))
LOCAL_PAGE_VARS["jd_code_eval"] = Pair(Ref(false), (Ref{Bool},)) # toggle reeval
LOCAL_PAGE_VARS["reeval"] = Pair(false, (Bool,)) # always reeval on pg
LOCAL_PAGE_VARS["jd_code"] = Pair("", (String,)) # just the script

# If there are GLOBAL vars that are defined, they take precedence
local_keys = keys(LOCAL_PAGE_VARS)
for k in keys(GLOBAL_PAGE_VARS)
Expand Down Expand Up @@ -230,12 +233,14 @@ the site. See [`resolve_lxcom`](@ref).
# inclusion
GLOBAL_LXDEFS["\\input"] = LxDef("\\input", 2, EMPTY_SS)
GLOBAL_LXDEFS["\\output"] = LxDef("\\output", 1, EMPTY_SS)
GLOBAL_LXDEFS["\\codeoutput"] = LxDef("\\codeoutput", 1, subs("@@code_output \\output{#1}@@"))
GLOBAL_LXDEFS["\\textoutput"] = LxDef("\\textoutput", 1, EMPTY_SS)
GLOBAL_LXDEFS["\\textinput"] = LxDef("\\textinput", 1, EMPTY_SS)
GLOBAL_LXDEFS["\\textinput"] = LxDef("\\textinput", 1, EMPTY_SS)
GLOBAL_LXDEFS["\\figalt"] = LxDef("\\figalt", 2, EMPTY_SS)
GLOBAL_LXDEFS["\\fig"] = LxDef("\\fig", 1, subs("\\figalt{}{#1}"))
GLOBAL_LXDEFS["\\file"] = LxDef("\\file", 2, subs("[#1]()"))
GLOBAL_LXDEFS["\\tableinput"] = LxDef("\\tableinput", 2, EMPTY_SS)
GLOBAL_LXDEFS["\\literate"] = LxDef("\\literate", 1, EMPTY_SS)
# text formatting
GLOBAL_LXDEFS["\\underline"] = LxDef("\\underline", 1,
subs("~~~<span style=\"text-decoration:underline;\">!#1</span>~~~"))
Expand Down
93 changes: 93 additions & 0 deletions test/integration/literate.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
scripts = joinpath(J.PATHS[:folder], "scripts")
cd(td); J.set_paths!(); mkpath(scripts)

@testset "Literate-a" begin
# Post processing: numbering of julia blocks
s = raw"""
A
```julia
B
```
C
```julia
D
```
"""
@test J.literate_post_process(s) == """
A
```julia:ex1
B
```
C
```julia:ex2
D
```
"""
end

@testset "Literate-b" begin
# Literate to JuDoc
s = raw"""
# # Rational numbers
#
# In julia rational numbers can be constructed with the `//` operator.
# Lets define two rational numbers, `x` and `y`:
## Define variable x and y
x = 1//3
y = 2//5
# When adding `x` and `y` together we obtain a new rational number:
z = x + y
"""
path = joinpath(scripts, "tutorial.jl")
write(path, s)
opath = J.literate_to_judoc(path)
@test endswith(opath, joinpath(J.PATHS[:assets], "literate", "tutorial.md"))
out = read(opath, String)
@test out == """
# Rational numbers
In julia rational numbers can be constructed with the `//` operator.
Lets define two rational numbers, `x` and `y`:
```julia:ex1
# Define variable x and y
x = 1//3
y = 2//5
```
When adding `x` and `y` together we obtain a new rational number:
```julia:ex2
z = x + y
```
"""

# Use of `\literate` command
h = raw"""
@def hascode = true
@def showall = true
\literate{/scripts/tutorial.jl}
""" |> jd2html_td
@test isapproxstr(h, """
<h1 id="rational_numbers"><a href="/index.html#rational_numbers">Rational numbers</a></h1>
<p>In julia rational numbers can be constructed with the <code>//</code> operator. Lets define two rational numbers, <code>x</code> and <code>y</code>:</p>
<pre><code class="language-julia"># Define variable x and y
x = 1//3
y = 2//5</code></pre>
<div class="code_output"><pre><code>2//5</code></pre></div>
<p>When adding <code>x</code> and <code>y</code> together we obtain a new rational number:</p>
<pre><code class="language-julia">z = x + y</code></pre>
<div class="code_output"><pre><code>11//15</code></pre></div>
""")
end
7 changes: 5 additions & 2 deletions test/runtests.jl
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using JuDoc, Test, Markdown, Dates, Random
using JuDoc, Test, Markdown, Dates, Random, Literate
const J = JuDoc
const D = joinpath(dirname(dirname(pathof(JuDoc))), "test", "_dummies")

Expand Down Expand Up @@ -41,7 +41,7 @@ include("converter/lx_input.jl")
include("converter/lx_simple.jl")
println("🍺")

println("INTEGRATION")
println("GLOBAL")
include("global/cases1.jl")
include("global/cases2.jl")
include("global/ordering.jl")
Expand All @@ -65,5 +65,8 @@ begin
end
cd(dirname(dirname(pathof(JuDoc))))

println("INTEGRATION")
include("integration/literate.jl")

flush_td()
println("😅 😅 😅 😅")

0 comments on commit a261aeb

Please sign in to comment.