Ongoing fixes while battle testing Literate integration

src/JuDoc.jl

@@ -9,14 +9,14 @@ using DelimitedFiles: readdlm
 using OrderedCollections
 using Pkg
 using DocStringExtensions: SIGNATURES, TYPEDEF
-using Literate: markdown
 
 import Logging
 import LiveServer
 import Base.push!
 import NodeJS
+import Literate
 
-export serve, publish, cleanpull, newsite, optimize, jd2html
+export serve, publish, cleanpull, newsite, optimize, jd2html, literate_folder
 
 # -----------------------------------------------------------------------------
 #
@@ -76,6 +76,9 @@ const CUR_PATH_WITH_EVAL = Ref("")
 """Shorter name for a type that we use everywhere"""
 const AS = AbstractString
 
+"""Convenience constant for an automatic message to add to code files."""
+const MESSAGE_FILE_GEN = "# This file was generated by JuDoc, do not modify it. # hide\n"
+
 # -----------------------------------------------------------------------------
 
 include("build.jl") # check if user has Node/minify

src/converter/lx.jl

@@ -252,6 +252,27 @@ function resolve_lx_input_othercode(rpath::AS, lang::AS)::String
 end
 
 
+"""
+$SIGNATURES
+
+Internal function to check if a code should suppress the final show.
+"""
+function check_suppress_show(code::AS)
+    scode = strip(code)
+    scode[end] == ';' && return true
+    # last line ?
+    lastline = scode
+    i = findlast(e -> e in (';','\n'), scode)
+    if !isnothing(i)
+        lastline = strip(scode[nextind(scode, i):end])
+    end
+    startswith(lastline, "@show ")   && return true
+    startswith(lastline, "println(") && return true
+    startswith(lastline, "print(")   && return true
+    return false
+end
+
+
 """
 $SIGNATURES
 
@@ -262,10 +283,15 @@ function show_res(rpath::AS)::String
     fd, fn = splitdir(fpath)
     stdo   = read(joinpath(fd, "output", splitext(fn)[1] * ".out"), String)
     res    = read(joinpath(fd, "output", splitext(fn)[1] * ".res"), String)
+    # check if there's a final `;` or if the last line is a print, println or show
+    # in those cases, ignore the result file
+    code = strip(read(splitext(fpath)[1] * ".jl", String))
+    check_suppress_show(code) && (res = "")
     isempty(stdo) && isempty(res) && return ""
     if !isempty(stdo)
         endswith(stdo, "\n") || (stdo *= "\n")
     end
+    res == "nothing" && (res = "")
     return html_div("code_output", html_code(stdo * res))
 end
 

src/converter/lx_simple.jl

@@ -53,13 +53,17 @@ Internal function to resolve a `\\literate{rpath}` see [`literate_to_judoc`](@re
 """
 function resolve_lx_literate(lxc::LxCom)::String
     rpath = strip(content(lxc.braces[1]))
-    path  = resolve_assets_rpath(rpath; canonical=true)
-    endswith(path, ".jl") || (path *= ".jl")
-    if !isfile(path)
-        @warn "File not found when trying to resolve a \\literate command ($path)."
-        return ""
+    opath, haschanged = literate_to_judoc(rpath)
+    isempty(opath) && return "~~~"*html_err("Literate file matching '$rpath' not found.")*"~~~"
+    if !haschanged
+        # page has not changed, check if literate is the only source of code
+        # and in that case skip eval of all code blocks via freezecode
+        if LOCAL_PAGE_VARS["literate_only"].first
+            set_var!(LOCAL_PAGE_VARS, "freezecode", true)
+        end
     end
-    opath = literate_to_judoc(path)
+    # if haschanged=true then this will be handled cell by cell
+    # comparing with cell files following `eval_and_resolve_code`
     return read(opath, String) * EOS
 end
 

src/converter/md.jl

@@ -93,6 +93,11 @@ function convert_md(mds::String, pre_lxdefs::Vector{LxDef}=Vector{LxDef}();
             strip(prod(c*"\n" for c in codes)))
     end
 
+    # if no title is specified, grab the first header if there is one
+    if isnothing(LOCAL_PAGE_VARS["title"]) && !isempty(PAGE_HEADERS)
+        set_var!(LOCAL_PAGE_VARS, "title", first(values(PAGE_HEADERS))[1])
+    end
+
     # Return the string + judoc variables
     return hstring, jd_vars
 end
@@ -336,7 +341,7 @@ function process_md_defs(blocks::Vector{OCBlock}, isconfig::Bool,
     # Find all markdown definitions (MD_DEF) blocks
     mddefs = filter(β -> (β.name == :MD_DEF), blocks)
     # empty container for the assignments
-    assignments = Vector{Pair{String, String}}(undef, length(mddefs))
+    assignments = Vector{Pair{String, String}}()
     # go over the blocks, and extract the assignment
     for (i, mdd) ∈ enumerate(mddefs)
         matched = match(MD_DEF_PAT, mdd.ss)
@@ -346,7 +351,7 @@ function process_md_defs(blocks::Vector{OCBlock}, isconfig::Bool,
             continue
         end
         vname, vdef = matched.captures[1:2]
-        assignments[i] = (String(vname) => String(vdef))
+        push!(assignments, (String(vname) => String(vdef)))
     end
     # if we're currently looking at the config file, update the global page var dictionary
     # GLOBAL_PAGE_VARS and store the latex definition globally as well in GLOBAL_LXDEFS

src/converter/md_blocks.jl

@@ -125,9 +125,6 @@ function convert_header(β::OCBlock)::String
     return html_hk(hk, html_ahref_key(rstitle, title); id=rstitle)
 end
 
-"""Convenience constant for an automatic message to add to code files."""
-const MESSAGE_FILE_GEN = "# This file was generated by JuDoc, do not modify it. # hide\n"
-
 """Convenience function to increment the eval' code block counter"""
 increment_code_head() = (LOCAL_PAGE_VARS["jd_code_head"].first[] += 1)
 
@@ -173,7 +170,7 @@ function eval_and_resolve_code(code::AS, rpath::AS;
     end
 
     write(path, MESSAGE_FILE_GEN * code)
-    print(rpad("\r→ evaluating code [...] ($(CUR_PATH[]), $rpath)", 79))
+    print(rpad("\r→ evaluating code [...] ($(CUR_PATH[]), $rpath)", 79) * "\r")
     # - execute the code while redirecting stdout to file
     Logging.disable_logging(Logging.LogLevel(3_000))
     res = nothing
@@ -192,7 +189,7 @@ function eval_and_resolve_code(code::AS, rpath::AS;
         end
     end
     Logging.disable_logging(Logging.Debug)
-    print(rpad("\r→ evaluating code [✓]", 79)*"\r")
+    print(rpad("\r→ evaluating code [✓]", 79) * "\r")
 
     # resolve the code block (highlighting) and return it
     return resolve_lx_input_hlcode(rpath, "julia")

src/integration/literate.jl

@@ -12,6 +12,7 @@ function literate_post_process(s::String)::String
     isempty(s) && return s
     em   = eachmatch(LITERATE_JULIA_FENCE_R, s)
     buf  = IOBuffer()
+    write(buf, "<!--This file was generated, do not modify it.-->\n")
     head = 1
     c    = 1
     for m in em
@@ -31,15 +32,36 @@ $SIGNATURES
 
 Take a Literate.jl script and transform it into a JuDoc-markdown file.
 """
-function literate_to_judoc(fpath::String)::String
-    outpath = joinpath(PATHS[:assets], "literate")
+function literate_to_judoc(rpath::AS)::Tuple{String,Bool}
+    startswith(rpath, "/") || error("Literate expects a paths starting with '/'")
+    # rpath is of the form "/scripts/[path/]tutorial[.jl]"
+    # split it so that when recombining it will lead to valid path inc windows
+    srpath = split(rpath, '/')[2:end] # discard empty first since starts with "/"
+    fpath  = joinpath(PATHS[:folder], srpath...)
+    endswith(fpath, ".jl") || (fpath *= ".jl")
+    if !isfile(fpath)
+        @warn "File not found when trying to convert a literate file ($fpath)."
+        return "", true
+    end
+    outpath = joinpath(PATHS[:assets], "literate", srpath[2:end-1]...)
     isdir(outpath) || mkdir(outpath)
-    # don't show infos
+    # retrieve the file name
+    fname = splitext(splitdir(fpath)[2])[1]
+    spath = joinpath(outpath, fname * "_script.jl")
+    prev  = ""
+    if isfile(spath)
+        prev = read(spath, String)
+    end
+    # don't show Literate's infos
     Logging.disable_logging(Logging.LogLevel(Logging.Info))
-    markdown(fpath, outpath, documenter=false,
-             postprocess=literate_post_process, credit=false)
+    Literate.markdown(fpath, outpath; documenter=false,
+                      postprocess=literate_post_process, credit=false)
+    Literate.script(fpath, outpath; documenter=false,
+                      name=fname * "_script", credit=false)
     # bring back logging
     Logging.disable_logging(Logging.LogLevel(Logging.Debug))
-    fname = splitdir(fpath)[2]
-    return joinpath(outpath, splitext(fname)[1] * ".md")
+    # see if things have changed
+    haschanged = (read(spath, String) != prev)
+    # return path to md file
+    return joinpath(outpath, fname * ".md"), haschanged
 end

src/jd_paths.jl

@@ -55,6 +55,7 @@ function set_paths!()::LittleDict{Symbol,String}
     PATHS[:css]       = joinpath(PATHS[:folder],  "css")
     PATHS[:libs]      = joinpath(PATHS[:folder],  "libs")
     PATHS[:assets]    = joinpath(PATHS[:folder],  "assets")
+    PATHS[:literate]  = joinpath(PATHS[:folder],  "scripts")
 
     return PATHS
 end

src/jd_vars.jl

@@ -99,6 +99,11 @@ is processed.
     LOCAL_PAGE_VARS["reeval"]        = Pair(false,  (Bool,)) # whether to always re-evals all on pg
     LOCAL_PAGE_VARS["freezecode"]    = Pair(false,  (Bool,)) # no-reevaluation of the code
     LOCAL_PAGE_VARS["showall"]       = Pair(false,  (Bool,)) # like a notebook on each cell
+    # NOTE: when using literate, `literate_only` will assume that it's the only source of
+    # code, so if it doesn't see change there, it will freeze the code to avoid an eval, this will
+    # cause problems if there's more code on the page than from just the call to \literate
+    # in such cases set literate_only to false.
+    LOCAL_PAGE_VARS["literate_only"] = Pair(true,       (Bool,))
     # the jd_* should not be assigned externally
     LOCAL_PAGE_VARS["jd_code_scope"] = code_scope
     LOCAL_PAGE_VARS["jd_code_head"]  = Pair(Ref(0),     (Ref{Int},))

src/manager/dir_utils.jl

@@ -51,7 +51,7 @@ propagates verbosity.
 """
 function scan_input_dir!(md_files::TrackedFiles, html_files::TrackedFiles,
                          other_files::TrackedFiles, infra_files::TrackedFiles,
-                         verb::Bool=false)::Nothing
+                         literate_files::TrackedFiles, verb::Bool=false)::Nothing
     # top level files (src/*)
     for file ∈ readdir(PATHS[:src])
         isfile(joinpath(PATHS[:src], file)) || continue
@@ -85,7 +85,7 @@ function scan_input_dir!(md_files::TrackedFiles, html_files::TrackedFiles,
         end
     end
     # infastructure files (src/_css/* and src/_html_parts/*)
-    for d ∈ [:src_css, :src_html], (root, _, files) ∈ walkdir(PATHS[d])
+    for d ∈ (:src_css, :src_html), (root, _, files) ∈ walkdir(PATHS[d])
         for file ∈ files
             isfile(joinpath(root, file)) || continue
             fname, fext = splitext(file)
@@ -94,6 +94,18 @@ function scan_input_dir!(md_files::TrackedFiles, html_files::TrackedFiles,
             add_if_new_file!(infra_files, root=>file, verb)
         end
     end
+    # literate script files if any, note that the folder may not exist
+    if isdir(PATHS[:literate])
+        for (root, _, files) ∈ walkdir(PATHS[:literate])
+            for file ∈ files
+                isfile(joinpath(root, file)) || continue
+                fname, fext = splitext(file)
+                # skipping files that are not script file
+                fext != ".jl" && continue
+                add_if_new_file!(literate_files, root=>file, verb)
+            end
+        end
+    end
     return nothing
 end
 

src/manager/judoc.jl

@@ -93,12 +93,14 @@ function jd_setup(; clear::Bool=true)::NamedTuple
     # . recovering the list of files in the input dir we care about
     # -- these are stored in dictionaries, the key is the full path and the value is the time of
     # last change (useful for continuous monitoring)
-    md_files    = TrackedFiles()
-    html_files  = TrackedFiles()
-    other_files = TrackedFiles()
-    infra_files = TrackedFiles()
+    md_files       = TrackedFiles()
+    html_files     = TrackedFiles()
+    other_files    = TrackedFiles()
+    infra_files    = TrackedFiles()
+    literate_files = TrackedFiles()
     # named tuples of all the watched files
-    watched_files = (md=md_files, html=html_files, other=other_files, infra=infra_files)
+    watched_files = (md=md_files, html=html_files, other=other_files,
+                     infra=infra_files, literate=literate_files)
     # fill the dictionaries
     scan_input_dir!(watched_files...)
     return watched_files
@@ -228,12 +230,37 @@ function jd_loop(cycle_counter::Int, ::LiveServer.FileWatcher, watched_files::Na
                 start = time()
                 jd_fullpass(watched_files; clear=false, verb=false, prerender=false)
                 verb && (print_final(rpad("✔ full pass...", 15), start); println(""))
+            # if it's a literate file
+            elseif haskey(watched_files[:literate], fpair)
+                fmsg = fmsg * rpad("→ updating... ", 15)
+                verb && print("\r" * fmsg)
+                start = time()
+                #
+                literate_path = joinpath(fpair...)
+                #
+                head    = read(joinpath(PATHS[:src_html], "head.html"), String)
+                pg_foot = read(joinpath(PATHS[:src_html], "page_foot.html"), String)
+                foot    = read(joinpath(PATHS[:src_html], "foot.html"), String)
+                # process all md files that have `\literate` with something that matches
+
+                for mdfpair in keys(watched_files.md)
+                    mdfpath = joinpath(mdfpair...)
+                    # read the content and look for `\\literate{...}`
+                    content = read(mdfpath, String)
+                    for m in eachmatch(r"\\literate\{(.*?)\}", content)
+                        if endswith(literate_path, m.captures[1])
+                            process_file(:md, mdfpair, head, pg_foot, foot, cur_t;
+                                         clear=false, prerender=false)
+                            break
+                        end
+                    end
+                end
+                verb && print_final(fmsg, start)
             else
                 fmsg = fmsg * rpad("→ updating... ", 15)
                 verb && print("\r" * fmsg)
                 start = time()
-                # TODO, ideally these would only be read if they've changed. Not super important
-                # but just not necessary. (Fixing may be a bit of a pain though)
+                #
                 head    = read(joinpath(PATHS[:src_html], "head.html"), String)
                 pg_foot = read(joinpath(PATHS[:src_html], "page_foot.html"), String)
                 foot    = read(joinpath(PATHS[:src_html], "foot.html"), String)

src/misc_html.jl

@@ -64,7 +64,7 @@ Convenience function to introduce a code block.
 """
 function html_code(c::AS, lang::AS="")
     isempty(c) && return ""
-    isempty(lang) && return "<pre><code>$c</code></pre>"
+    isempty(lang) && return "<pre><code class=\"plaintext\">$c</code></pre>"
     return "<pre><code class=\"language-$lang\">$c</code></pre>"
 end
 

src/misc_utils.jl

@@ -1,5 +1,19 @@
+"""
+$SIGNATURES
+
+Specify the folder for the Literate scripts, by default this is `scripts/`.
+"""
+function literate_folder(rp::String="")
+    isempty(rp) && return PATHS[:literate]
+    path = joinpath(PATHS[:folder], rp)
+    !isdir(path) && error("Specified Literate path not found ($rp -- $path)")
+    PATHS[:literate] = path
+    return path
+end
 
+#
 # Convenience functions to work with strings and substrings
+#
 
 """
 $(SIGNATURES)

src/parser/html_blocks.jl

@@ -130,7 +130,7 @@ function find_html_cdblocks(qblocks::Vector{AbstractBlock}
         # look forward until next `{{end}} block
         k = findfirst(cβ -> (typeof(cβ) == HEnd), qblocks[i+1:end])
         if isnothing(k)
-            throw(HTMLBlockError("Found an {{if(n)def ...}} block but no matching {{end}} block."))
+            throw(HTMLBlockError("Found an {{is(not)def ...}} block but no matching {{end}} block."))
         end
         k += i
         endβ = qblocks[k]