From 3768334ef72183c425c99143b27f1b8e7612e3a6 Mon Sep 17 00:00:00 2001 From: Thibaut Lienart Date: Wed, 9 Oct 2019 13:45:09 +0200 Subject: [PATCH 1/7] explicit use of plaintext to avoid automatic tagging with hilight --- src/misc_html.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/misc_html.jl b/src/misc_html.jl index 05e0635e9..03182176b 100644 --- a/src/misc_html.jl +++ b/src/misc_html.jl @@ -64,7 +64,7 @@ Convenience function to introduce a code block. """ function html_code(c::AS, lang::AS="") isempty(c) && return "" - isempty(lang) && return "
$c
" + isempty(lang) && return "
$c
" return "
$c
" end From 5264c2e90ec906e53026a886baab8727b1b23199 Mon Sep 17 00:00:00 2001 From: Thibaut Lienart Date: Wed, 9 Oct 2019 14:03:15 +0200 Subject: [PATCH 2/7] explicit use of plaintext and relevant test fixes --- src/converter/lx.jl | 1 + src/converter/md_blocks.jl | 4 ++-- test/converter/eval.jl | 16 ++++++++-------- test/converter/lx_input.jl | 4 ++-- test/global/eval.jl | 32 ++++++++++++++++---------------- test/integration/literate.jl | 4 ++-- test/misc.jl | 2 +- 7 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/converter/lx.jl b/src/converter/lx.jl index 1b3e49851..ddda33d03 100644 --- a/src/converter/lx.jl +++ b/src/converter/lx.jl @@ -266,6 +266,7 @@ function show_res(rpath::AS)::String if !isempty(stdo) endswith(stdo, "\n") || (stdo *= "\n") end + res == "nothing" && (res = "") return html_div("code_output", html_code(stdo * res)) end diff --git a/src/converter/md_blocks.jl b/src/converter/md_blocks.jl index c4f8f8fa3..3755a1881 100644 --- a/src/converter/md_blocks.jl +++ b/src/converter/md_blocks.jl @@ -173,7 +173,7 @@ function eval_and_resolve_code(code::AS, rpath::AS; end write(path, MESSAGE_FILE_GEN * code) - print(rpad("\r→ evaluating code [...] ($(CUR_PATH[]), $rpath)", 79)) + print(rpad("\r→ evaluating code [...] ($(CUR_PATH[]), $rpath)", 79) * "\r") # - execute the code while redirecting stdout to file Logging.disable_logging(Logging.LogLevel(3_000)) res = nothing @@ -192,7 +192,7 @@ function eval_and_resolve_code(code::AS, rpath::AS; end end Logging.disable_logging(Logging.Debug) - print(rpad("\r→ evaluating code [✓]", 79)*"\r") + print(rpad("\r→ evaluating code [✓]", 79) * "\r") # resolve the code block (highlighting) and return it return resolve_lx_input_hlcode(rpath, "julia") diff --git a/test/converter/eval.jl b/test/converter/eval.jl index eea399263..275184e0f 100644 --- a/test/converter/eval.jl +++ b/test/converter/eval.jl @@ -43,7 +43,7 @@ 
a = 5
             print(a^2)
then: - 
25
+ 
25
done.

""") end @@ -91,7 +91,7 @@ end 
a = 5
             print(a^2)
then: - 
25
+ 
25
done.

""") # ------------ @@ -122,7 +122,7 @@ end 
a = 5
             print(a^2)
then: - 
25
done.

""") + 
25
done.

""") end @testset "Eval (module)" begin @@ -138,7 +138,7 @@ end done. """ * J.EOS |> seval # dot(a, a) == 54 - @test occursin("then: 
54
done.", h) + @test occursin("""then: 
54
done.""", h) end @testset "Eval (img)" begin @@ -166,7 +166,7 @@ end done. """ * J.EOS |> seval # errors silently - @test occursin("then: 
There was an error running the code:\nDomainError", h)
+    @test occursin("then: 
There was an error running the code:\nDomainError", h)
 end
 
 @testset "Eval (nojl)" begin
@@ -202,7 +202,7 @@ end
             println("Is this a file? $(isfile(fn))")
             include(abspath(fn))
             println("Now: $a")
-            rm(fn)
 done. 
Is this a file? true
+            rm(fn)
 done. 
Is this a file? true
             Now: 2
             

             """)
@@ -222,7 +222,7 @@ end
     @test isapproxstr(h, """
         
a = 5
         a *= 2

-        
10

+        
10

         """)
 
     # Show with stdout
@@ -240,7 +240,7 @@ end
         
a = 5
         println("hello")
         a *= 2

-        
hello
+        
hello
         10

         """)
 end
diff --git a/test/converter/lx_input.jl b/test/converter/lx_input.jl
index ff9367fc1..d722bf969 100644
--- a/test/converter/lx_input.jl
+++ b/test/converter/lx_input.jl
@@ -27,7 +27,7 @@
     write(plain1, "2")
 
     r = J.resolve_lx_input_plainoutput("script1.jl", code=true)
-    @test r == "
2
"
+    @test r == "
2
"
 end
 
 
@@ -52,7 +52,7 @@ end
     h = J.convert_html(m, J.PageVars())
 
     @test occursin("
Some string 
$(read(joinpath(J.PATHS[:assets], "index", "code", "s1.jl"), String))
", h)
-    @test occursin("Then maybe 
$(read(joinpath(J.PATHS[:assets], "index", "code",  "output", "s1.out"), String))
", h)
+    @test occursin("Then maybe 
$(read(joinpath(J.PATHS[:assets], "index", "code",  "output", "s1.out"), String))
", h)
     @test occursin("Finally img: \"\" done.", h)
 end
 
diff --git a/test/global/eval.jl b/test/global/eval.jl
index 792197b39..dff3e2ce1 100644
--- a/test/global/eval.jl
+++ b/test/global/eval.jl
@@ -24,7 +24,7 @@
     h = foo |> jd2html_td
 
     @test isapproxstr(h, """
-                
println(randn())
 
$a

+                
println(randn())
 
$a

                 """)
 
     # TEXT MODIFICATION + IN SCOPE --> NO REEVAL
@@ -34,7 +34,7 @@
     h = foo |> jd2html_td
 
     @test isapproxstr(h, """
-                
println(randn())
 
$a
 etc
+                
println(randn())
 
$a
 etc
                 """)
 
     # CODE ADDITION + IN SCOPE --> NO REEVAL OF FIRST BLOCK
@@ -53,9 +53,9 @@
     h = foo |> jd2html_td
 
     @test isapproxstr(h, """
-                
println(randn())
 
$a
 etc
-                
println(randn())
 
$b

-                
println(randn())
 
$c

+                
println(randn())
 
$a
 etc
+                
println(randn())
 
$b

+                
println(randn())
 
$c

                 """)
 
     # CODE MODIFICATION + IN SCOPE --> REEVAL OF BLOCK AND AFTER
@@ -80,10 +80,10 @@
     h = foo |> jd2html_td
 
     @test isapproxstr(h, """
-                
println(randn())
 
$a

+                
println(randn())
 
$a

                 
# modif
-                println(randn())
 
$d

-                
println(randn())
 
$e

+                println(randn())
 
$d

+                
println(randn())
 
$e

                 """)
 
     # FROZEN CODE --> WILL USE CODE FROM BEFORE even though we changed it (no re-eval)
@@ -109,10 +109,10 @@
     h = foo |> jd2html_td
 
     @test isapproxstr(h, """
-                
println(randn())
 
$a

+                
println(randn())
 
$a

                 
# modif
-                println(randn())
 
$d

-                
println(randn())
 
$e

+                println(randn())
$d
+ 
println(randn())
$e
""") end @@ -137,10 +137,10 @@ end @test isapproxstr(h, """ 
a = 5
             println(a)
- 
5
+ 
5
a += 3
             println(a)
- 
8
+ 
8
""") @test J.LOCAL_PAGE_VARS["jd_code"].first == """ @@ -173,13 +173,13 @@ end @test isapproxstr(h, """ 
a = 5
             println(a)
- 
5
+ 
5
a += 1
             println(a)
- 
6
+ 
6
a += 3
             println(a)
- 
9
+ 
9
""") @test J.LOCAL_PAGE_VARS["jd_code"].first == """ a = 5 diff --git a/test/integration/literate.jl b/test/integration/literate.jl index 4e7a34717..5c92ec67a 100644 --- a/test/integration/literate.jl +++ b/test/integration/literate.jl @@ -85,9 +85,9 @@ end 
# Define variable x and y
         x = 1//3
         y = 2//5
- 
2//5
+ 
2//5

When adding x and y together we obtain a new rational number:

z = x + y
- 
11//15
+ 
11//15
""") end diff --git a/test/misc.jl b/test/misc.jl index 22335f7a0..008585de2 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -112,7 +112,7 @@ end @test J.html_ahref_key("cc", "dd") == "dd" @test J.html_div("dn","ct") == "
ct
" @test J.html_img("src", "alt") == "\"alt\"" - @test J.html_code("code") == "
code
" + @test J.html_code("code") == "
code
" @test J.html_code("code", "lang") == "
code
" @test J.html_err("blah") == "

// blah //

" end From cbcdbc1e0d195d32dade6542fd110bbe4d1baff7 Mon Sep 17 00:00:00 2001 From: Thibaut Lienart Date: Wed, 9 Oct 2019 16:23:36 +0200 Subject: [PATCH 3/7] better path handling with literate, check if file has changed and if it has mark for reeval --- src/JuDoc.jl | 5 ++++- src/converter/lx_simple.jl | 10 +++------- src/converter/md_blocks.jl | 3 --- src/integration/literate.jl | 36 +++++++++++++++++++++++++++++------- src/manager/dir_utils.jl | 12 +++++++++++- src/misc_utils.jl | 3 ++- test/integration/literate.jl | 4 +++- 7 files changed, 52 insertions(+), 21 deletions(-) diff --git a/src/JuDoc.jl b/src/JuDoc.jl index c3fb70349..908f3d91a 100644 --- a/src/JuDoc.jl +++ b/src/JuDoc.jl @@ -9,11 +9,11 @@ using DelimitedFiles: readdlm using OrderedCollections using Pkg using DocStringExtensions: SIGNATURES, TYPEDEF -using Literate: markdown import Logging import LiveServer import Base.push! +import Literate export serve, publish, cleanpull, newsite, optimize, jd2html @@ -75,6 +75,9 @@ const CUR_PATH_WITH_EVAL = Ref("") """Shorter name for a type that we use everywhere""" const AS = AbstractString +"""Convenience constant for an automatic message to add to code files.""" +const MESSAGE_FILE_GEN = "# This file was generated by JuDoc, do not modify it. # hide\n" + # ----------------------------------------------------------------------------- include("build.jl") # check if user has Node/minify diff --git a/src/converter/lx_simple.jl b/src/converter/lx_simple.jl index 2977fcada..04a98370b 100644 --- a/src/converter/lx_simple.jl +++ b/src/converter/lx_simple.jl @@ -53,13 +53,9 @@ Internal function to resolve a `\\literate{rpath}` see [`literate_to_judoc`](@re """ function resolve_lx_literate(lxc::LxCom)::String rpath = strip(content(lxc.braces[1])) - path = resolve_assets_rpath(rpath; canonical=true) - endswith(path, ".jl") || (path *= ".jl") - if !isfile(path) - @warn "File not found when trying to resolve a \\literate command ($path)." - return "" - end - opath = literate_to_judoc(path) + opath, haschanged = literate_to_judoc(rpath) + # if has changed, mark the page for re-evaluation + set_var!(LOCAL_PAGE_VARS, "reeval", true) return read(opath, String) * EOS end diff --git a/src/converter/md_blocks.jl b/src/converter/md_blocks.jl index 3755a1881..5cc7449b3 100644 --- a/src/converter/md_blocks.jl +++ b/src/converter/md_blocks.jl @@ -125,9 +125,6 @@ function convert_header(β::OCBlock)::String return html_hk(hk, html_ahref_key(rstitle, title); id=rstitle) end -"""Convenience constant for an automatic message to add to code files.""" -const MESSAGE_FILE_GEN = "# This file was generated by JuDoc, do not modify it. # hide\n" - """Convenience function to increment the eval' code block counter""" increment_code_head() = (LOCAL_PAGE_VARS["jd_code_head"].first[] += 1) diff --git a/src/integration/literate.jl b/src/integration/literate.jl index a660d9b93..0af87a2d2 100644 --- a/src/integration/literate.jl +++ b/src/integration/literate.jl @@ -12,6 +12,7 @@ function literate_post_process(s::String)::String isempty(s) && return s em = eachmatch(LITERATE_JULIA_FENCE_R, s) buf = IOBuffer() + write(buf, "\n") head = 1 c = 1 for m in em @@ -31,15 +32,36 @@ $SIGNATURES Take a Literate.jl script and transform it into a JuDoc-markdown file. """ -function literate_to_judoc(fpath::String)::String - outpath = joinpath(PATHS[:assets], "literate") +function literate_to_judoc(rpath::AS)::Tuple{String,Bool} + startswith(rpath, "/") || error("Literate expects a paths starting with '/'") + # rpath is of the form "/scripts/[path/]tutorial[.jl]" + # split it so that when recombining it will lead to valid path inc windows + srpath = split(rpath, '/')[2:end] # discard empty first since starts with "/" + fpath = joinpath(PATHS[:folder], srpath...) + endswith(fpath, ".jl") || (fpath *= ".jl") + if !isfile(fpath) + @warn "File not found when trying to convert a literate file ($fpath)." + return "" + end + outpath = joinpath(PATHS[:assets], "literate", srpath[2:end-1]...) isdir(outpath) || mkdir(outpath) - # don't show infos + # retrieve the file name + fname = splitext(splitdir(fpath)[2])[1] + spath = joinpath(outpath, fname * "_script.jl") + prev = "" + if isfile(spath) + prev = read(spath, String) + end + # don't show Literate's infos Logging.disable_logging(Logging.LogLevel(Logging.Info)) - markdown(fpath, outpath, documenter=false, - postprocess=literate_post_process, credit=false) + Literate.markdown(fpath, outpath; documenter=false, + postprocess=literate_post_process, credit=false) + Literate.script(fpath, outpath; documenter=false, + name=fname * "_script", credit=false) # bring back logging Logging.disable_logging(Logging.LogLevel(Logging.Debug)) - fname = splitdir(fpath)[2] - return joinpath(outpath, splitext(fname)[1] * ".md") + # see if things have changed + haschanged = (read(spath, String) == prev) + # return path to md file + return joinpath(outpath, fname * ".md"), haschanged end diff --git a/src/manager/dir_utils.jl b/src/manager/dir_utils.jl index abb45e5a4..3df437cc2 100644 --- a/src/manager/dir_utils.jl +++ b/src/manager/dir_utils.jl @@ -85,7 +85,17 @@ function scan_input_dir!(md_files::TrackedFiles, html_files::TrackedFiles, end end # infastructure files (src/_css/* and src/_html_parts/*) - for d ∈ [:src_css, :src_html], (root, _, files) ∈ walkdir(PATHS[d]) + for d ∈ (:src_css, :src_html), (root, _, files) ∈ walkdir(PATHS[d]) + for file ∈ files + isfile(joinpath(root, file)) || continue + fname, fext = splitext(file) + # skipping files that are not of the type INFRA_FILES + fext ∉ INFRA_FILES && continue + add_if_new_file!(infra_files, root=>file, verb) + end + end + # literate script files if any + for d ∈ (:src_css, :src_html), (root, _, files) ∈ walkdir(PATHS[d]) for file ∈ files isfile(joinpath(root, file)) || continue fname, fext = splitext(file) diff --git a/src/misc_utils.jl b/src/misc_utils.jl index c949c8e5b..0001f93d7 100644 --- a/src/misc_utils.jl +++ b/src/misc_utils.jl @@ -1,5 +1,6 @@ - +# # Convenience functions to work with strings and substrings +# """ $(SIGNATURES) diff --git a/test/integration/literate.jl b/test/integration/literate.jl index 5c92ec67a..7af43615c 100644 --- a/test/integration/literate.jl +++ b/test/integration/literate.jl @@ -17,6 +17,7 @@ cd(td); J.set_paths!(); mkpath(scripts) ``` """ @test J.literate_post_process(s) == """ + A ```julia:ex1 @@ -49,10 +50,11 @@ end """ path = joinpath(scripts, "tutorial.jl") write(path, s) - opath = J.literate_to_judoc(path) + opath, = J.literate_to_judoc("/scripts/tutorial") @test endswith(opath, joinpath(J.PATHS[:assets], "literate", "tutorial.md")) out = read(opath, String) @test out == """ + # Rational numbers In julia rational numbers can be constructed with the `//` operator. From e4245f8857f95e11ac9bc5858e4c3724a05e4426 Mon Sep 17 00:00:00 2001 From: Thibaut Lienart Date: Wed, 9 Oct 2019 23:19:12 +0200 Subject: [PATCH 4/7] more work on literate integration and suppression of output with show --- src/converter/lx.jl | 25 +++++++++++++++++++++++++ src/converter/lx_simple.jl | 10 +++++++++- src/converter/md.jl | 5 +++++ src/integration/literate.jl | 2 +- src/jd_vars.jl | 5 +++++ test/integration/literate.jl | 1 + 6 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/converter/lx.jl b/src/converter/lx.jl index ddda33d03..7ef5d92c4 100644 --- a/src/converter/lx.jl +++ b/src/converter/lx.jl @@ -252,6 +252,27 @@ function resolve_lx_input_othercode(rpath::AS, lang::AS)::String end +""" +$SIGNATURES + +Internal function to check if a code should suppress the final show. +""" +function check_suppress_show(code::AS) + scode = strip(code) + scode[end] == ';' && return true + # last line ? + lastline = scode + i = findlast(e -> e in (';','\n'), scode) + if !isnothing(i) + lastline = strip(scode[nextind(scode, i):end]) + end + startswith(lastline, "@show ") && return true + startswith(lastline, "println(") && return true + startswith(lastline, "print(") && return true + return false +end + + """ $SIGNATURES @@ -262,6 +283,10 @@ function show_res(rpath::AS)::String fd, fn = splitdir(fpath) stdo = read(joinpath(fd, "output", splitext(fn)[1] * ".out"), String) res = read(joinpath(fd, "output", splitext(fn)[1] * ".res"), String) + # check if there's a final `;` or if the last line is a print, println or show + # in those cases, ignore the result file + code = strip(read(splitext(fpath)[1] * ".jl", String)) + check_suppress_show(code) && (res = "") isempty(stdo) && isempty(res) && return "" if !isempty(stdo) endswith(stdo, "\n") || (stdo *= "\n") diff --git a/src/converter/lx_simple.jl b/src/converter/lx_simple.jl index 04a98370b..54a528e50 100644 --- a/src/converter/lx_simple.jl +++ b/src/converter/lx_simple.jl @@ -55,7 +55,15 @@ function resolve_lx_literate(lxc::LxCom)::String rpath = strip(content(lxc.braces[1])) opath, haschanged = literate_to_judoc(rpath) # if has changed, mark the page for re-evaluation - set_var!(LOCAL_PAGE_VARS, "reeval", true) + if haschanged + set_var!(LOCAL_PAGE_VARS, "reeval", true) + else + # page has not changed, check if literate is the only source of code + # and in that case skip eval of all code blocks via freezecode + if LOCAL_PAGE_VARS["literate_only"].first + set_var!(LOCAL_PAGE_VARS, "freezecode", true) + end + end return read(opath, String) * EOS end diff --git a/src/converter/md.jl b/src/converter/md.jl index 137eb69c8..deb15fab6 100644 --- a/src/converter/md.jl +++ b/src/converter/md.jl @@ -93,6 +93,11 @@ function convert_md(mds::String, pre_lxdefs::Vector{LxDef}=Vector{LxDef}(); strip(prod(c*"\n" for c in codes))) end + # if no title is specified, grab the first header if there is one + if isnothing(LOCAL_PAGE_VARS["title"]) && !isempty(PAGE_HEADERS) + set_var!(LOCAL_PAGE_VARS, "title", first(values(PAGE_HEADERS))[1]) + end + # Return the string + judoc variables return hstring, jd_vars end diff --git a/src/integration/literate.jl b/src/integration/literate.jl index 0af87a2d2..fa1b8742d 100644 --- a/src/integration/literate.jl +++ b/src/integration/literate.jl @@ -61,7 +61,7 @@ function literate_to_judoc(rpath::AS)::Tuple{String,Bool} # bring back logging Logging.disable_logging(Logging.LogLevel(Logging.Debug)) # see if things have changed - haschanged = (read(spath, String) == prev) + haschanged = (read(spath, String) != prev) # return path to md file return joinpath(outpath, fname * ".md"), haschanged end diff --git a/src/jd_vars.jl b/src/jd_vars.jl index b6d1f27f0..1bd9501b7 100644 --- a/src/jd_vars.jl +++ b/src/jd_vars.jl @@ -99,6 +99,11 @@ is processed. LOCAL_PAGE_VARS["reeval"] = Pair(false, (Bool,)) # whether to always re-evals all on pg LOCAL_PAGE_VARS["freezecode"] = Pair(false, (Bool,)) # no-reevaluation of the code LOCAL_PAGE_VARS["showall"] = Pair(false, (Bool,)) # like a notebook on each cell + # NOTE: when using literate, `literate_only` will assume that it's the only source of + # code, so if it doesn't see change there, it will freeze the code to avoid an eval, this will + # cause problems if there's more code on the page than from just the call to \literate + # in such cases set literate_only to false. + LOCAL_PAGE_VARS["literate_only"] = Pair(true, (Bool,)) # the jd_* should not be assigned externally LOCAL_PAGE_VARS["jd_code_scope"] = code_scope LOCAL_PAGE_VARS["jd_code_head"] = Pair(Ref(0), (Ref{Int},)) diff --git a/test/integration/literate.jl b/test/integration/literate.jl index 7af43615c..27a051df0 100644 --- a/test/integration/literate.jl +++ b/test/integration/literate.jl @@ -78,6 +78,7 @@ end h = raw""" @def hascode = true @def showall = true + @def reeval = true \literate{/scripts/tutorial.jl} """ |> jd2html_td From 89425e2b2ddb30e857e2c516406073d52eb234e2 Mon Sep 17 00:00:00 2001 From: Thibaut Lienart Date: Thu, 10 Oct 2019 00:27:18 +0200 Subject: [PATCH 5/7] literate files are now tracked --- src/converter/lx_simple.jl | 7 +++---- src/jd_paths.jl | 1 + src/manager/dir_utils.jl | 20 ++++++++++--------- src/manager/judoc.jl | 41 +++++++++++++++++++++++++++++++------- src/misc_utils.jl | 13 ++++++++++++ test/manager/utils.jl | 5 +++-- test/runtests.jl | 1 + 7 files changed, 66 insertions(+), 22 deletions(-) diff --git a/src/converter/lx_simple.jl b/src/converter/lx_simple.jl index 54a528e50..2571ae6b2 100644 --- a/src/converter/lx_simple.jl +++ b/src/converter/lx_simple.jl @@ -54,16 +54,15 @@ Internal function to resolve a `\\literate{rpath}` see [`literate_to_judoc`](@re function resolve_lx_literate(lxc::LxCom)::String rpath = strip(content(lxc.braces[1])) opath, haschanged = literate_to_judoc(rpath) - # if has changed, mark the page for re-evaluation - if haschanged - set_var!(LOCAL_PAGE_VARS, "reeval", true) - else + if !haschanged # page has not changed, check if literate is the only source of code # and in that case skip eval of all code blocks via freezecode if LOCAL_PAGE_VARS["literate_only"].first set_var!(LOCAL_PAGE_VARS, "freezecode", true) end end + # if haschanged=true then this will be handled cell by cell + # comparing with cell files following `eval_and_resolve_code` return read(opath, String) * EOS end diff --git a/src/jd_paths.jl b/src/jd_paths.jl index 006772af6..8afce1113 100644 --- a/src/jd_paths.jl +++ b/src/jd_paths.jl @@ -55,6 +55,7 @@ function set_paths!()::LittleDict{Symbol,String} PATHS[:css] = joinpath(PATHS[:folder], "css") PATHS[:libs] = joinpath(PATHS[:folder], "libs") PATHS[:assets] = joinpath(PATHS[:folder], "assets") + PATHS[:literate] = joinpath(PATHS[:folder], "scripts") return PATHS end diff --git a/src/manager/dir_utils.jl b/src/manager/dir_utils.jl index 3df437cc2..6bf1fd58a 100644 --- a/src/manager/dir_utils.jl +++ b/src/manager/dir_utils.jl @@ -51,7 +51,7 @@ propagates verbosity. """ function scan_input_dir!(md_files::TrackedFiles, html_files::TrackedFiles, other_files::TrackedFiles, infra_files::TrackedFiles, - verb::Bool=false)::Nothing + literate_files::TrackedFiles, verb::Bool=false)::Nothing # top level files (src/*) for file ∈ readdir(PATHS[:src]) isfile(joinpath(PATHS[:src], file)) || continue @@ -94,14 +94,16 @@ function scan_input_dir!(md_files::TrackedFiles, html_files::TrackedFiles, add_if_new_file!(infra_files, root=>file, verb) end end - # literate script files if any - for d ∈ (:src_css, :src_html), (root, _, files) ∈ walkdir(PATHS[d]) - for file ∈ files - isfile(joinpath(root, file)) || continue - fname, fext = splitext(file) - # skipping files that are not of the type INFRA_FILES - fext ∉ INFRA_FILES && continue - add_if_new_file!(infra_files, root=>file, verb) + # literate script files if any, note that the folder may not exist + if isdir(PATHS[:literate]) + for (root, _, files) ∈ walkdir(PATHS[:literate]) + for file ∈ files + isfile(joinpath(root, file)) || continue + fname, fext = splitext(file) + # skipping files that are not script file + fext != ".jl" && continue + add_if_new_file!(literate_files, root=>file, verb) + end end end return nothing diff --git a/src/manager/judoc.jl b/src/manager/judoc.jl index c9951757c..20eaba20d 100644 --- a/src/manager/judoc.jl +++ b/src/manager/judoc.jl @@ -93,12 +93,14 @@ function jd_setup(; clear::Bool=true)::NamedTuple # . recovering the list of files in the input dir we care about # -- these are stored in dictionaries, the key is the full path and the value is the time of # last change (useful for continuous monitoring) - md_files = TrackedFiles() - html_files = TrackedFiles() - other_files = TrackedFiles() - infra_files = TrackedFiles() + md_files = TrackedFiles() + html_files = TrackedFiles() + other_files = TrackedFiles() + infra_files = TrackedFiles() + literate_files = TrackedFiles() # named tuples of all the watched files - watched_files = (md=md_files, html=html_files, other=other_files, infra=infra_files) + watched_files = (md=md_files, html=html_files, other=other_files, + infra=infra_files, literate=literate_files) # fill the dictionaries scan_input_dir!(watched_files...) return watched_files @@ -228,12 +230,37 @@ function jd_loop(cycle_counter::Int, ::LiveServer.FileWatcher, watched_files::Na start = time() jd_fullpass(watched_files; clear=false, verb=false, prerender=false) verb && (print_final(rpad("✔ full pass...", 15), start); println("")) + # if it's a literate file + elseif haskey(watched_files[:literate], fpair) + fmsg = fmsg * rpad("→ updating... ", 15) + verb && print("\r" * fmsg) + start = time() + # + literate_path = joinpath(fpair...) + # + head = read(joinpath(PATHS[:src_html], "head.html"), String) + pg_foot = read(joinpath(PATHS[:src_html], "page_foot.html"), String) + foot = read(joinpath(PATHS[:src_html], "foot.html"), String) + # process all md files that have `\literate` with something that matches + + for mdfpair in keys(watched_files.md) + mdfpath = joinpath(mdfpair...) + # read the content and look for `\\literate{...}` + content = read(mdfpath, String) + for m in eachmatch(r"\\literate\{(.*?)\}", content) + if endswith(literate_path, m.captures[1]) + process_file(:md, mdfpair, head, pg_foot, foot, cur_t; + clear=false, prerender=false) + break + end + end + end + verb && print_final(fmsg, start) else fmsg = fmsg * rpad("→ updating... ", 15) verb && print("\r" * fmsg) start = time() - # TODO, ideally these would only be read if they've changed. Not super important - # but just not necessary. (Fixing may be a bit of a pain though) + # head = read(joinpath(PATHS[:src_html], "head.html"), String) pg_foot = read(joinpath(PATHS[:src_html], "page_foot.html"), String) foot = read(joinpath(PATHS[:src_html], "foot.html"), String) diff --git a/src/misc_utils.jl b/src/misc_utils.jl index 0001f93d7..5a08a3750 100644 --- a/src/misc_utils.jl +++ b/src/misc_utils.jl @@ -1,3 +1,16 @@ +""" +$SIGNATURES + +Specify the folder for the Literate scripts, by default this is `scripts/`. +""" +function literate_folder(rp::String="") + isempty(rp) && return PATHS[:literate] + path = joinpath(PATHS[:assets], rp) + !isdir(path) && error("Specified Literate path not found ($rp -- $path)") + PATHS[:assets] = path + return path +end + # # Convenience functions to work with strings and substrings # diff --git a/test/manager/utils.jl b/test/manager/utils.jl index 747f2851d..e85490581 100644 --- a/test/manager/utils.jl +++ b/test/manager/utils.jl @@ -36,8 +36,9 @@ end html_files = empty(md_files) other_files = empty(md_files) infra_files = empty(md_files) - watched_files = [md_files, html_files, other_files, infra_files] - JuDoc.scan_input_dir!(md_files, html_files, other_files, infra_files, true) + literate_files = empty(md_files) + watched_files = [md_files, html_files, other_files, infra_files, literate_files] + JuDoc.scan_input_dir!(md_files, html_files, other_files, infra_files, literate_files, true) @test haskey(md_files, JuDoc.PATHS[:src_pages]=>"blah.md") @test md_files[JuDoc.PATHS[:src_pages]=>"blah.md"] == mtime(temp_blah) == stat(temp_blah).mtime @test html_files[JuDoc.PATHS[:src_pages]=>"temp.html"] == mtime(temp_html) diff --git a/test/runtests.jl b/test/runtests.jl index acd288aa3..4046e6fa2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -70,4 +70,5 @@ println("INTEGRATION") include("integration/literate.jl") flush_td() +cd(joinpath(dirname(dirname(pathof(JuDoc))))) println("😅 😅 😅 😅") From b80566986c725e6cf51e9f429a921e07d5d53f32 Mon Sep 17 00:00:00 2001 From: Thibaut Lienart Date: Thu, 10 Oct 2019 14:48:10 +0200 Subject: [PATCH 6/7] more tests --- src/JuDoc.jl | 2 +- src/converter/lx_simple.jl | 1 + src/converter/md.jl | 4 +-- src/integration/literate.jl | 2 +- src/misc_utils.jl | 4 +-- src/parser/html_blocks.jl | 2 +- test/coverage/extras1.jl | 68 ++++++++++++++++++++++++++++++++++++ test/integration/literate.jl | 23 ++++++++++-- test/runtests.jl | 4 +++ 9 files changed, 100 insertions(+), 10 deletions(-) create mode 100644 test/coverage/extras1.jl diff --git a/src/JuDoc.jl b/src/JuDoc.jl index 94b2a1bd6..33066e65c 100644 --- a/src/JuDoc.jl +++ b/src/JuDoc.jl @@ -16,7 +16,7 @@ import Base.push! import NodeJS import Literate -export serve, publish, cleanpull, newsite, optimize, jd2html +export serve, publish, cleanpull, newsite, optimize, jd2html, literate_folder # ----------------------------------------------------------------------------- # diff --git a/src/converter/lx_simple.jl b/src/converter/lx_simple.jl index 2571ae6b2..234c8d53c 100644 --- a/src/converter/lx_simple.jl +++ b/src/converter/lx_simple.jl @@ -54,6 +54,7 @@ Internal function to resolve a `\\literate{rpath}` see [`literate_to_judoc`](@re function resolve_lx_literate(lxc::LxCom)::String rpath = strip(content(lxc.braces[1])) opath, haschanged = literate_to_judoc(rpath) + isempty(opath) && return "~~~"*html_err("Literate file matching '$rpath' not found.")*"~~~" if !haschanged # page has not changed, check if literate is the only source of code # and in that case skip eval of all code blocks via freezecode diff --git a/src/converter/md.jl b/src/converter/md.jl index deb15fab6..e9ff8f7be 100644 --- a/src/converter/md.jl +++ b/src/converter/md.jl @@ -341,7 +341,7 @@ function process_md_defs(blocks::Vector{OCBlock}, isconfig::Bool, # Find all markdown definitions (MD_DEF) blocks mddefs = filter(β -> (β.name == :MD_DEF), blocks) # empty container for the assignments - assignments = Vector{Pair{String, String}}(undef, length(mddefs)) + assignments = Vector{Pair{String, String}}() # go over the blocks, and extract the assignment for (i, mdd) ∈ enumerate(mddefs) matched = match(MD_DEF_PAT, mdd.ss) @@ -351,7 +351,7 @@ function process_md_defs(blocks::Vector{OCBlock}, isconfig::Bool, continue end vname, vdef = matched.captures[1:2] - assignments[i] = (String(vname) => String(vdef)) + push!(assignments, (String(vname) => String(vdef))) end # if we're currently looking at the config file, update the global page var dictionary # GLOBAL_PAGE_VARS and store the latex definition globally as well in GLOBAL_LXDEFS diff --git a/src/integration/literate.jl b/src/integration/literate.jl index fa1b8742d..ca2fb01a0 100644 --- a/src/integration/literate.jl +++ b/src/integration/literate.jl @@ -41,7 +41,7 @@ function literate_to_judoc(rpath::AS)::Tuple{String,Bool} endswith(fpath, ".jl") || (fpath *= ".jl") if !isfile(fpath) @warn "File not found when trying to convert a literate file ($fpath)." - return "" + return "", true end outpath = joinpath(PATHS[:assets], "literate", srpath[2:end-1]...) isdir(outpath) || mkdir(outpath) diff --git a/src/misc_utils.jl b/src/misc_utils.jl index 5a08a3750..cf953f0c7 100644 --- a/src/misc_utils.jl +++ b/src/misc_utils.jl @@ -5,9 +5,9 @@ Specify the folder for the Literate scripts, by default this is `scripts/`. """ function literate_folder(rp::String="") isempty(rp) && return PATHS[:literate] - path = joinpath(PATHS[:assets], rp) + path = joinpath(PATHS[:folder], rp) !isdir(path) && error("Specified Literate path not found ($rp -- $path)") - PATHS[:assets] = path + PATHS[:literate] = path return path end diff --git a/src/parser/html_blocks.jl b/src/parser/html_blocks.jl index 400632b06..d6737dc70 100644 --- a/src/parser/html_blocks.jl +++ b/src/parser/html_blocks.jl @@ -130,7 +130,7 @@ function find_html_cdblocks(qblocks::Vector{AbstractBlock} # look forward until next `{{end}} block k = findfirst(cβ -> (typeof(cβ) == HEnd), qblocks[i+1:end]) if isnothing(k) - throw(HTMLBlockError("Found an {{if(n)def ...}} block but no matching {{end}} block.")) + throw(HTMLBlockError("Found an {{is(not)def ...}} block but no matching {{end}} block.")) end k += i endβ = qblocks[k] diff --git a/test/coverage/extras1.jl b/test/coverage/extras1.jl new file mode 100644 index 000000000..3c5e58115 --- /dev/null +++ b/test/coverage/extras1.jl @@ -0,0 +1,68 @@ +@testset "Converter-lx" begin + cd(td) + @test_throws ArgumentError J.check_input_rpath("aldjfk") +end + +@testset "Converter-html" begin + @test_throws J.HTMLFunctionError J.convert_html("{{fill bb cc}}", J.PageVars()) + @test_throws J.HTMLFunctionError J.convert_html("{{insert bb cc}}", J.PageVars()) + @test_throws J.HTMLFunctionError J.convert_html("{{href aa}}", J.PageVars()) + @test (@test_logs (:warn, "Unknown dictionary name aa in {{href ...}}. Ignoring") J.convert_html("{{href aa bb}}", J.PageVars())) == "??" + @test_throws J.HTMLBlockError J.convert_html("{{if asdf}}{{end}}", J.PageVars()) + @test_throws J.HTMLBlockError J.convert_html("{{if asdf}}", J.PageVars()) + @test_throws J.HTMLBlockError J.convert_html("{{isdef asdf}}", J.PageVars()) + @test_throws J.HTMLBlockError J.convert_html("{{ispage asdf}}", J.PageVars()) +end + +@testset "Converter-md" begin + s = """ + @def blah + """ + @test (@test_logs (:warn, "Found delimiters for an @def environment but it didn't have the right @def var = ... format. Verify (ignoring for now).") (s |> jd2html_td)) == "" + + s = """ + Blah + [^1]: hello + """ |> jd2html_td + @test isapproxstr(s, "

Blah

") +end + +@testset "Judoc" begin + cd(td); mkpath("foo"); cd("foo"); + @test_throws ArgumentError serve(single=true) + cd(td) +end + +@testset "RSS" begin + J.set_var!(J.GLOBAL_PAGE_VARS, "website_descr", "") + J.RSS_DICT["hello"] = J.RSSItem("","","","","","","",Date(1)) + @test (@test_logs (:warn, """ + I found RSS items but the RSS feed is not properly described: + at least one of the following variables has not been defined in + your config.md: `website_title`, `website_descr`, `website_url`. + The feed will not be (re)generated.""") J.rss_generator()) === nothing +end + + +@testset "parser-lx" begin + s = raw""" + \newcommand{hello}{hello} + """ + @test_throws J.LxDefError (s |> jd2html) + s = raw""" + \foo + """ + @test_throws J.LxComError (s |> jd2html) + s = raw""" + \newcommand{\foo}[2]{hello #1 #2} + \foo{a} {} + """ + @test_throws J.LxComError (s |> jd2html) +end + +@testset "ocblocks" begin + s = raw""" + @@foo + """ + @test_throws J.OCBlockError (s |> jd2html) +end diff --git a/test/integration/literate.jl b/test/integration/literate.jl index 27a051df0..9dcef4bf7 100644 --- a/test/integration/literate.jl +++ b/test/integration/literate.jl @@ -1,6 +1,12 @@ -scripts = joinpath(J.PATHS[:folder], "scripts") +scripts = joinpath(J.PATHS[:folder], "literate-scripts") cd(td); J.set_paths!(); mkpath(scripts) +@testset "Literate-0" begin + @test_throws ErrorException literate_folder("foo/") + litpath = literate_folder("literate-scripts/") + @test litpath == joinpath(J.PATHS[:folder], "literate-scripts/") +end + @testset "Literate-a" begin # Post processing: numbering of julia blocks s = raw""" @@ -50,7 +56,7 @@ end """ path = joinpath(scripts, "tutorial.jl") write(path, s) - opath, = J.literate_to_judoc("/scripts/tutorial") + opath, = J.literate_to_judoc("/literate-scripts/tutorial") @test endswith(opath, joinpath(J.PATHS[:assets], "literate", "tutorial.md")) out = read(opath, String) @test out == """ @@ -80,7 +86,7 @@ end @def showall = true @def reeval = true - \literate{/scripts/tutorial.jl} + \literate{/literate-scripts/tutorial.jl} """ |> jd2html_td @test isapproxstr(h, """

Rational numbers

@@ -94,3 +100,14 @@ end 
11//15
""") end + +@testset "Literate-c" begin + s = raw""" + \literate{foo} + """ + @test_throws ErrorException (s |> jd2html) + # s = raw""" + # \literate{/foo} + # """ + # a = @test_logs (:warn, "File not found when trying to convert a literate file ($(joinpath(J.PATHS[:folder], "foo.jl"))).") (s |> jd2html_td) +end diff --git a/test/runtests.jl b/test/runtests.jl index 4046e6fa2..940545403 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -71,4 +71,8 @@ include("integration/literate.jl") flush_td() cd(joinpath(dirname(dirname(pathof(JuDoc))))) + +println("COVERAGE") +include("coverage/extras1.jl") + println("😅 😅 😅 😅") From 51e603435e75b69f351ced0b10c3f8dc035ffa15 Mon Sep 17 00:00:00 2001 From: Thibaut Lienart Date: Thu, 10 Oct 2019 14:53:37 +0200 Subject: [PATCH 7/7] system error.. --- test/coverage/extras1.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/coverage/extras1.jl b/test/coverage/extras1.jl index 3c5e58115..84532050b 100644 --- a/test/coverage/extras1.jl +++ b/test/coverage/extras1.jl @@ -1,6 +1,7 @@ @testset "Converter-lx" begin cd(td) - @test_throws ArgumentError J.check_input_rpath("aldjfk") + # Exception instead of ArgumentError as may fail with system error + @test_throws Exception J.check_input_rpath("aldjfk") end @testset "Converter-html" begin