From 5eaac6ecf99535f8423147c6b6c336605afec40d Mon Sep 17 00:00:00 2001 From: Thibaut Lienart Date: Mon, 13 Apr 2020 10:16:52 +0200 Subject: [PATCH] v07 init (#443) --- .travis.yml | 2 - NEWS.md | 63 ++++++++++++++++++-- Project.toml | 18 +++--- README.md | 2 +- src/Franklin.jl | 9 ++- src/converter/html/blocks.jl | 98 +++++++++++++++++++------------- src/converter/html/functions.jl | 59 ++++++++++++------- src/converter/html/html.jl | 1 + src/converter/latex/hyperrefs.jl | 2 +- src/converter/latex/io.jl | 20 +++---- src/converter/latex/latex.jl | 13 ++++- src/converter/markdown/blocks.jl | 4 +- src/converter/markdown/md.jl | 51 ++++------------- src/converter/markdown/mddefs.jl | 39 +++++++++++++ src/converter/markdown/utils.jl | 9 ++- src/eval/module.jl | 6 +- src/manager/file_utils.jl | 67 +++++++++++++++++++--- src/manager/franklin.jl | 42 +++++++++----- src/parser/latex/blocks.jl | 31 +++++++--- src/parser/latex/tokens.jl | 23 ++++++-- src/parser/markdown/indent.jl | 87 ++++++++++++++-------------- src/parser/markdown/tokens.jl | 1 - src/parser/ocblocks.jl | 4 -- src/parser/tokens.jl | 6 ++ src/utils/misc.jl | 28 --------- src/utils/vars.jl | 68 ++++++++++++++++++++-- test/converter/markdown3.jl | 7 ++- test/coverage/extras1.jl | 1 - test/parser/2-blocks.jl | 2 +- test/parser/indentation++.jl | 63 ++++++++++++++++++++ test/runtests.jl | 13 ++++- test/templating/fill.jl | 34 +++++++++++ test/templating/for.jl | 42 ++++++++++++++ test/test_utils.jl | 9 ++- test/utils_file/basic.jl | 72 +++++++++++++++++++++++ 35 files changed, 735 insertions(+), 261 deletions(-) create mode 100644 src/converter/markdown/mddefs.jl create mode 100644 test/parser/indentation++.jl create mode 100644 test/templating/fill.jl create mode 100644 test/templating/for.jl create mode 100644 test/utils_file/basic.jl diff --git a/.travis.yml b/.travis.yml index 6f11424f6..98eacd811 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,6 @@ language: julia os: - linux julia: - - 1.1 - - 1.2 - 1.3 - 1.4 - nightly diff --git a/NEWS.md b/NEWS.md index 375bcc3f1..c9ca20d2d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,12 +1,67 @@ # NEWS -This document keeps track of breaking changes and what you can do if you update and things don't work anymore. - -Notes are in reverse chronological order. +This document keeps track of **key new features**, **breaking changes** and what you can do if you update and things don't work anymore. You can also check out [this issue](https://github.com/tlienart/Franklin.jl/issues/323) with a more granular list of things that are being worked on / have been fixed / added. -### v0.6+ (new functionalities) +## v0.7 + +### Breaking changes + +* indented code blocks are now opt-in, prefer fenced code blocks with backticks `` ` ``. This helps avoid ambiguities in parsing. If you do want indented code blocks on a page use `@def indented_code = true`, if you want them everywhere, put that definition in your `config.md`. +* markers for comments now **must** be separated with a character distinct from `-` so `` is ok, `--->` is not. Rule of thumb: use a whitespace. + +**Note**: supported Julia versions: 1.3, **1.4**, 1.5. + +### New stuff + +The main new stuff are: + +* you can unpack in template for loops `{{for (x, y, z) in iterator}}`, +* you can use `{{var}}` as a short for `{{fill var}}`, you can use `{{...}}` blocks directly in markdown, +* variable definitions (`@def`) can now be multi-line, the secondary lines **must** be indented, +* you can access page variables from other pages using `{{fill var path}}` where `path` is the relative path to the other file so for instance `{{fill var index}}` or `{{fill var blog/page1}}`, this can be combined with a for loop for instance: + +``` +@def paths = ["index", "blog/page1"] +{{for p in paths}} + {{fill var p}} +{{end}} +``` + +And maybe most importantly, the addition of a `utils.jl` file to complement the `config.md`. In that file you can define variables and functions that can be used elsewhere on your site; they are evaluated in a `Utils` module which is imported in all evaluated code blocks. + +* if you define a variable `var = 5`, it will be accessible everywhere, taking priority over any other page variable (including global), you can call `{{fill var}}` or use it in an evaluated code block with `Utils.var`. +* if you define a function `foo() = print("hello")`, you can use it in an evaluated code block with `Utils.foo()` +* if you define a function `hfun_foo() = return "blah"`, you can use it as a template function `{{foo}}`, they have access to page variable names so this would also be valid: + +```julia +function hfun_bar(vname) # vname is a vector of string here + val = locvar(vname[1]) + return round(sqrt(val), digits=2) +end +``` + +which you can call with `{{bar var}}`. + +* if you define a function `lx_baz` you can use `\baz{...}` and have the function directly act on the input string, the syntax to use must conform to the following example: + +```julia +function lx_baz(com, _) + # keep this first line + brace_content = Franklin.content(com.braces[1]) # input string + # Now do whatever you want with the content: + return uppercase(brace_content) +end +``` + +which you can call with `\baz{some string}`. + + + +--- + +## v0.6+ * addition of an `ignore` global page variable to ignore files and directories, addition of a `div_content` page variable to change the name of the main container div. * Multiline markdown definitions are now allowed, the lines **must** be indented with four spaces e.g. this is ok: diff --git a/Project.toml b/Project.toml index a698183d5..2ecd1e4ca 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Franklin" uuid = "713c75ef-9fc9-4b05-94a9-213340da978e" authors = ["Thibaut Lienart "] -version = "0.6.17" +version = "0.7.0" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" @@ -19,14 +19,14 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [compat] -DocStringExtensions = "^0.8" -FranklinTemplates = "^0.5, 0.6" -HTTP = "^0.8" -Literate = "^2.2" -LiveServer = "^0.3" -NodeJS = "^0.6, ^1" -OrderedCollections = "^1.1" -julia = "^1.1" +DocStringExtensions = "0.8" +FranklinTemplates = "0.6" +HTTP = "0.8" +Literate = "2.2" +LiveServer = "0.3" +NodeJS = "0.6,1" +OrderedCollections = "1.1" +julia = "1.1" [extras] DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" diff --git a/README.md b/README.md index 3dc4a6ce0..6a4c0ca0f 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ See [the docs](https://franklinjl.org) for more information and examples. ## Getting started -With Julia ≥ 1.1: +With Julia ≥ 1.3: ```julia pkg> add Franklin diff --git a/src/Franklin.jl b/src/Franklin.jl index 81690db02..80935f693 100644 --- a/src/Franklin.jl +++ b/src/Franklin.jl @@ -1,6 +1,7 @@ module Franklin using FranklinTemplates +using FranklinTemplates: filecmp using Markdown using Markdown: htmlesc @@ -36,9 +37,10 @@ export jd2html # = fd2html # CONSTANTS # -# Obtained via `dig www...`; may change over time; see check_ping -# we check in sequence, one should work... this may need to be updated -# over time. +# These are common IP addresses that we can quickly ping to see if the +# user seems online. This is used in `verify_links`. The IPs were +# obtained via `dig www...`; they may change over time; see `check_ping` +# we check in sequence, one should work if the user is online... const IP_CHECK = ( "172.217.21.132" => "Google", # google "140.82.118.4" => "GitHub", # github @@ -139,6 +141,7 @@ include("eval/literate.jl") # > markdown include("converter/markdown/blocks.jl") include("converter/markdown/utils.jl") +include("converter/markdown/mddefs.jl") include("converter/markdown/md.jl") # > latex include("converter/latex/latex.jl") diff --git a/src/converter/html/blocks.jl b/src/converter/html/blocks.jl index a7937e8dd..80a78a7d5 100644 --- a/src/converter/html/blocks.jl +++ b/src/converter/html/blocks.jl @@ -157,56 +157,37 @@ end """ $SIGNATURES -Process a for block. +Process a for block (for a variable iterate). """ function process_html_for(hs::AS, qblocks::Vector{AbstractBlock}, i::Int)::Tuple{String,Int,Int} # check that the iterable is known β_open = qblocks[i] - vname = β_open.vname - iname = β_open.iname - if !haskey(LOCAL_VARS, iname) + vname = β_open.vname # x or (x, v) + iname = β_open.iname # var + + if iname ∉ UTILS_NAMES && !haskey(LOCAL_VARS, iname) throw(HTMLBlockError("The iterable '$iname' is not recognised. " * "Please make sure it's defined.")) end - - # try to close the for loop - if i == length(qblocks) - throw(HTMLBlockError("Could not close the conditional block " * - "starting with '$(qblocks[i].ss)'.")) - end - - init_idx = i - content = "" - - # inbalance keeps track of whether we've managed to find a - # matching {{end}}. It increases if it sees other opening {{if..}} - # and decreases if it sees a {{end}} - inbalance = 1 - while i < length(qblocks) && inbalance > 0 - i += 1 - inbalance += hbalance(qblocks[i]) - end - # we've exhausted the candidate qblocks and not found an appropriate {{end}} - if inbalance > 0 - throw(HTMLBlockError("Could not close the conditional block " * - "starting with '$(qblocks[init_idx].ss)'.")) + if iname ∈ UTILS_NAMES # can only happen if Utils is defined. + iter = getfield(Main.Utils, Symbol(iname)) + else + iter = locvar(iname) end - # we've found the closing {{end}} and index `i` - β_close = qblocks[i] - i_close = i - isempty(locvar(iname)) && @goto final_step + i_close, β_close = get_for_body(i, qblocks) + isempty(iter) && @goto final_step # is vname a single variable or multiple variables? # --> {{for v in iterate}} - # --> {{for (v1, v2) in iterate }} + # --> {{for (v1, v2) in iterate }} (unpacking) vnames = [vname] if startswith(vname, "(") vnames = strip.(split(vname[2:end-1], ",")) end # check that the first element of the iterate has the same length - el1 = first(locvar(iname)) + el1 = first(iter) length(vnames) in (1, length(el1)) || throw(HTMLBlockError("In a {{for ...}}, the first element of" * "the iterate has length $(length(el1)) but tried to unpack" * @@ -222,22 +203,57 @@ function process_html_for(hs::AS, qblocks::Vector{AbstractBlock}, isempty(strip(inner)) && @goto final_step content = "" if length(vnames) == 1 - for value in locvar(iname) + rx1 = Regex("{{\\s*fill\\s+$vname\\s*}}") # {{ fill v}} + rx2 = Regex("{{\\s*fill\\s+(\\S+)\\s+$vname\\s*}}") # {{ fill x v}} + for v in iter # at the moment we only consider {{fill ...}} - content *= replace(inner, - Regex("({{\\s*fill\\s+$vname\\s*}})") => "$value") + tmp = replace(inner, rx1 => "$v") + tmp = replace(tmp, rx2 => SubstitutionString("{{fill \\1 $v}}")) + content *= tmp end else - for value in locvar(iname) - temp = inner - for (vname, value) in zip(vnames, value) - temp = replace(temp, - Regex("({{\\s*fill\\s+$vname\\s*}})") => "$value") + for values in iter # each element of the iter can be unpacked + tmp = inner + for (vname, v) in zip(vnames, values) + rx1 = Regex("{{\\s*fill\\s+$vname\\s*}}") + rx2 = Regex("{{\\s*fill\\s+(\\S+)\\s+$vname\\s*}}") + tmp = replace(tmp, rx1 => "$v") + tmp = replace(tmp, rx2 => SubstitutionString("{{fill \\1 $v}}")) end - content *= temp + content *= tmp end end @label final_step head = nextind(hs, to(β_close)) return convert_html(content), head, i_close end + +""" +$SIGNATURES + +Extract the body of a for loop, keeping track of balancing. +""" +function get_for_body(i::Int, qblocks::Vector{AbstractBlock}) + # try to close the for loop + if i == length(qblocks) + throw(HTMLBlockError("Could not close the block starting with" * + "'$(qblocks[i].ss)'.")) + end + init_idx = i + content = "" + # inbalance keeps track of whether we've managed to find a + # matching {{end}}. It increases if it sees other opening {{if..}} + # and decreases if it sees a {{end}} + inb = 1 + while i < length(qblocks) && inb > 0 + i += 1 + inb += hbalance(qblocks[i]) + end + # we've exhausted the candidate qblocks and not found a matching {{end}} + if inb > 0 + throw(HTMLBlockError("Could not close the block starting with" * + "'$(qblocks[init_idx].ss)'.")) + end + # we've found the closing {{end}} and index `i` + return i, qblocks[i] +end diff --git a/src/converter/html/functions.jl b/src/converter/html/functions.jl index ea1c1743a..7feb8d37d 100644 --- a/src/converter/html/functions.jl +++ b/src/converter/html/functions.jl @@ -5,19 +5,20 @@ Helper function to process an individual block when it's a `HFun` such as `{{ fill author }}`. See also [`convert_html`](@ref). """ function convert_html_fblock(β::HFun)::String - # try to find a function `hfun_...` fun = Symbol("hfun_" * lowercase(β.fname)) - isdefined(Franklin, fun) && return eval(:($fun($β.params))) - - # if zero parameters, see if can fill - if isempty(β.params) && !isnothing(locvar(β.fname)) + ex = isempty(β.params) ? :($fun()) : :($fun($β.params)) + # see if a hfun was defined in utils + if isdefined(Main, :Utils) && isdefined(Main.Utils, fun) + res = Core.eval(Main.Utils, ex) + return string(res) + end + # see if a hfun was defined internally + isdefined(Franklin, fun) && return eval(ex) + # if zero parameters, see if can fill (case: {{vname}}) + if isempty(β.params) && + (!isnothing(locvar(β.fname)) || β.fname in UTILS_NAMES) return hfun_fill([β.fname]) end - - # XXX Future - # XXX see if there's an externally defined hfun - # XXX isdefined(Utils, fun) && Utils.eval(:($fun($β.params))) - # if we get here, then the function name is unknown, warn and ignore @warn "I found a function block '{{$(β.fname) ...}}' but I don't " * "recognise the function name. Ignoring." @@ -29,24 +30,40 @@ end """ $(SIGNATURES) -H-Function of the form `{{ fill vname }}` to plug in the content of a -franklin-var `vname` (assuming it can be represented as a string). +H-Function of the form `{{ fill vname }}` or `{{ fill vname rpath}}` to plug in +the content of a franklin-var `vname` (assuming it can be represented as a +string). """ function hfun_fill(params::Vector{String})::String # check params - if length(params) != 1 - throw(HTMLFunctionError("I found a {{fill ...}} with more than one parameter. Verify.")) + if length(params) > 2 || isempty(params) + throw(HTMLFunctionError("{{fill ...}} should have one or two " * + "($(length(params)) given). Verify.")) end # form the fill repl = "" vname = params[1] - if haskey(LOCAL_VARS, vname) - # retrieve the value stored - tmp_repl = locvar(vname) - isnothing(tmp_repl) || (repl = string(tmp_repl)) - else - @warn "I found a '{{fill $vname}}' but I do not know the " * - "variable '$vname'. Ignoring." + if length(params) == 1 + if vname in UTILS_NAMES + repl = string(getfield(Main.Utils, Symbol(vname))) + else + tmp_repl = locvar(vname) + if isnothing(tmp_repl) + @warn "I found a '{{fill $vname}}' but I do not know the " * + "variable '$vname'. Ignoring." + else + repl = string(tmp_repl) + end + end + else # two parameters, look in a path + rpath = params[2] + tmp_repl = pagevar(rpath, vname) + if isnothing(tmp_repl) + @warn "I found a '{{fill $vname $rpath}}' but I do not know the " * + "variable '$vname' or the path '$rpath'. Ignoring." + else + repl = string(tmp_repl) + end end return repl end diff --git a/src/converter/html/html.jl b/src/converter/html/html.jl index 6b31eeefc..bf9ded222 100644 --- a/src/converter/html/html.jl +++ b/src/converter/html/html.jl @@ -44,6 +44,7 @@ function fd2html_v(st::AS; internal::Bool=false, dir::String="")::Tuple{String,Dict} isempty(st) && return st if !internal + empty!(ALL_PAGE_VARS) FOLDER_PATH[] = isempty(dir) ? mktempdir() : dir set_paths!() def_GLOBAL_LXDEFS!() diff --git a/src/converter/latex/hyperrefs.jl b/src/converter/latex/hyperrefs.jl index f9a1bb815..88c226d4c 100644 --- a/src/converter/latex/hyperrefs.jl +++ b/src/converter/latex/hyperrefs.jl @@ -76,7 +76,7 @@ function lx_label(lxc::LxCom, _) end function lx_biblabel(lxc::LxCom, _)::String - name = refstring(strip(content(lxc.braces[1]))) + name = refstring(stent(lxc.braces[1])) PAGE_BIBREFS[name] = content(lxc.braces[2]) return "" end diff --git a/src/converter/latex/io.jl b/src/converter/latex/io.jl index 9b793a5b2..5634f0bb2 100644 --- a/src/converter/latex/io.jl +++ b/src/converter/latex/io.jl @@ -20,8 +20,8 @@ Different actions can be taken based on the qualifier: e.g.: `\\input{plot:4}{ex2}`. """ function lx_input(lxc::LxCom, _) - qualifier = lowercase(strip(content(lxc.braces[1]))) - rpath = strip(content(lxc.braces[2])) + qualifier = lowercase(stent(lxc.braces[1])) + rpath = stent(lxc.braces[2]) # check the qualifier if startswith(qualifier, "plot") # check if id is given e.g. \input{plot:5}{ex2} @@ -82,7 +82,7 @@ At most one will be true. See [`lx_show`](@ref). """ function lx_output(lxc::LxCom, lxd::Vector{LxDef}; reproc::Bool=false, res::Bool=false) - rpath = strip(content(lxc.braces[1])) + rpath = stent(lxc.braces[1]) cpaths = form_codepaths(rpath) outpath = cpaths.out_path respath = cpaths.res_path @@ -123,7 +123,7 @@ If you just want to include text as a plaintext block use `\\input{plaintext}{rpath}` instead. """ function lx_textinput(lxc::LxCom, lxd::Vector{LxDef}) - rpath = strip(content(lxc.braces[1])) + rpath = stent(lxc.braces[1]) input = "" try fp, = resolve_rpath(rpath) @@ -138,11 +138,11 @@ end Resolve a `\\figalt{alt}{rpath}` (find a fig and include it with alt). """ function lx_figalt(lxc::LxCom, _) - rpath = strip(content(lxc.braces[2])) - alt = strip(content(lxc.braces[1])) + rpath = stent(lxc.braces[2]) + alt = stent(lxc.braces[1]) path = parse_rpath(rpath; canonical=false, code=true) fdir, fext = splitext(path) - + # there are several cases # A. a path with no extension --> guess extension # B. a path with extension --> use that @@ -181,8 +181,8 @@ end Resolve a `\\tableinput{header}{rpath}` (find a table+header and include it). """ function lx_tableinput(lxc::LxCom, _) - rpath = strip(content(lxc.braces[2])) - header = strip(content(lxc.braces[1])) + rpath = stent(lxc.braces[2]) + header = stent(lxc.braces[1]) path = parse_rpath(rpath; canonical=false) fdir, fext = splitext(path) # copy-paste from resolve_lx_figalt() @@ -215,7 +215,7 @@ end Resolve a `\\literate{rpath}` (find a literate script and insert it). """ function lx_literate(lxc::LxCom, lxd::Vector{LxDef}) - rpath = strip(content(lxc.braces[1])) + rpath = stent(lxc.braces[1]) opath, haschanged = literate_to_franklin(rpath) # check file is there if isempty(opath) diff --git a/src/converter/latex/latex.jl b/src/converter/latex/latex.jl index c7a35c794..60b764908 100644 --- a/src/converter/latex/latex.jl +++ b/src/converter/latex/latex.jl @@ -10,7 +10,16 @@ function resolve_lxcom(lxc::LxCom, lxdefs::Vector{LxDef}; # retrieve the definition the command points to lxd = getdef(lxc) # it will be `nothing` in math mode, let KaTeX have it - lxd === nothing && return lxc.ss + if lxd === nothing + name = getname(lxc) # `\\cite` -> `cite` + fun = Symbol("lx_" * name) + @show fun + if isdefined(Main, :Utils) && isdefined(Main.Utils, fun) + return Core.eval(Main.Utils, :($fun($lxc, $lxdefs))) + else + return lxc.ss + end + end # otherwise it may be # -> empty, in which case try to find a specific internal definition or # return an empty string (see `commands.jl`) @@ -29,7 +38,7 @@ function resolve_lxcom(lxc::LxCom, lxdefs::Vector{LxDef}; # non-empty case, take the definition and iteratively replace any `#...` partial = lxd for (i, brace) in enumerate(lxc.braces) - cont = strip(content(brace)) + cont = stent(brace) # space-sensitive 'unsafe' one partial = replace(partial, "!#$i" => cont) # space-insensitive 'safe' one (e.g. `\mathbb#1`) diff --git a/src/converter/markdown/blocks.jl b/src/converter/markdown/blocks.jl index 3442c3ac6..74d9934a1 100644 --- a/src/converter/markdown/blocks.jl +++ b/src/converter/markdown/blocks.jl @@ -15,7 +15,7 @@ function convert_block(β::AbstractBlock, lxdefs::Vector{LxDef})::AS βn == :CODE_INLINE && return html_code_inline(content(β) |> htmlesc) βn == :CODE_BLOCK_LANG && return resolve_code_block(β.ss) βn == :CODE_BLOCK_IND && return convert_indented_code_block(β.ss) - βn == :CODE_BLOCK && return html_code(strip(content(β)), "{{fill lang}}") + βn == :CODE_BLOCK && return html_code(stent(β), "{{fill lang}}") βn == :ESCAPE && return chop(β.ss, head=3, tail=3) βn == :FOOTNOTE_REF && return convert_footnote_ref(β) @@ -30,7 +30,7 @@ function convert_block(β::AbstractBlock, lxdefs::Vector{LxDef})::AS # Div block --> need to process the block as a sub-element if βn == :DIV - raw_cont = strip(content(β)) + raw_cont = stent(β) cont = convert_md(raw_cont, lxdefs; isrecursive=true, has_mddefs=false) divname = chop(otok(β).ss, head=2, tail=0) diff --git a/src/converter/markdown/md.jl b/src/converter/markdown/md.jl index 41fe7915e..23022ae60 100644 --- a/src/converter/markdown/md.jl +++ b/src/converter/markdown/md.jl @@ -25,7 +25,9 @@ function convert_md(mds::AS, isrecursive::Bool=false, isinternal::Bool=false, isconfig::Bool=false, - has_mddefs::Bool=true )::String + has_mddefs::Bool=true, + only_mddefs::Bool=false, # see pagevar + )::String # instantiate page dictionaries isrecursive || isinternal || set_page_env() # if we're given a substring, force it to a string @@ -99,6 +101,8 @@ function convert_md(mds::AS, if has_mddefs process_mddefs(blocks, isconfig) end + #> 4.0 if only_mddefs, terminate early, we don't care about the rest + only_mddefs && return "" #> 4.b if config, update global lxdefs as well if isconfig for lxd ∈ lxdefs @@ -121,6 +125,11 @@ function convert_md(mds::AS, # and add them to the blocks to insert fnrefs = filter(τ -> τ.name == :FOOTNOTE_REF, tokens) + # Discard indented blocks unless locvar("indented_code") + if !locvar("indented_code") + filter!(b -> b.name != :CODE_BLOCK_IND, blocks) + end + #> 1. Merge all the blocks that will need further processing before # insertion b2insert = merge_blocks(lxcoms, deactivate_divs(blocks), @@ -366,43 +375,3 @@ function convert_inter_html(ihtml::AS, # return the full string return String(take!(htmls)) end - - -""" -$(SIGNATURES) - -Convenience function to process markdown definitions `@def ...` as appropriate. -Depending on `isconfig`, will update `GLOBAL_VARS` or `LOCAL_VARS`. - -**Arguments** - -* `blocks`: vector of active docs -* `isconfig`: whether the file being processed is the config file - (--> global page variables) -""" -function process_mddefs(blocks::Vector{OCBlock}, isconfig::Bool)::Nothing - # Find all markdown definitions (MD_DEF) blocks - mddefs = filter(β -> (β.name == :MD_DEF), blocks) - # empty container for the assignments - assignments = Vector{Pair{String, String}}() - # go over the blocks, and extract the assignment - for (i, mdd) ∈ enumerate(mddefs) - inner = strip(content(mdd)) - m = match(ASSIGN_PAT, inner) - if isnothing(m) - @warn "Found delimiters for an @def environment but it didn't " * - "have the right @def var = ... format. Verify (ignoring " * - "for now)." - continue - end - vname, vdef = m.captures[1:2] - push!(assignments, (String(vname) => String(vdef))) - end - # if in config file, update `GLOBAL_VARS` and `GLOBAL_LXDEFS` - if isconfig - set_vars!(GLOBAL_VARS, assignments) - else - set_vars!(LOCAL_VARS, assignments) - end - return nothing -end diff --git a/src/converter/markdown/mddefs.jl b/src/converter/markdown/mddefs.jl new file mode 100644 index 000000000..b14ad53a1 --- /dev/null +++ b/src/converter/markdown/mddefs.jl @@ -0,0 +1,39 @@ +""" +$(SIGNATURES) + +Convenience function to process markdown definitions `@def ...` as appropriate. +Depending on `isconfig`, will update `GLOBAL_VARS` or `LOCAL_VARS`. + +**Arguments** + +* `blocks`: vector of active docs +* `isconfig`: whether the file being processed is the config file + (--> global page variables) +""" +function process_mddefs(blocks::Vector{OCBlock}, isconfig::Bool)::Nothing + # Find all markdown definitions (MD_DEF) blocks + mddefs = filter(β -> (β.name == :MD_DEF), blocks) + # empty container for the assignments + assignments = Vector{Pair{String, String}}() + # go over the blocks, and extract the assignment + for (i, mdd) ∈ enumerate(mddefs) + inner = stent(mdd) + m = match(ASSIGN_PAT, inner) + if isnothing(m) + @warn "Found delimiters for an @def environment but it didn't " * + "have the right @def var = ... format. Verify (ignoring " * + "for now)." + continue + end + vname, vdef = m.captures[1:2] + push!(assignments, (String(vname) => String(vdef))) + end + # if in config file, update `GLOBAL_VARS` and `GLOBAL_LXDEFS` + if isconfig + set_vars!(GLOBAL_VARS, assignments) + else + set_vars!(LOCAL_VARS, assignments) + ALL_PAGE_VARS[splitext(locvar("fd_rpath"))[1]] = deepcopy(LOCAL_VARS) + end + return nothing +end diff --git a/src/converter/markdown/utils.jl b/src/converter/markdown/utils.jl index 62a6f5657..b495a9a43 100644 --- a/src/converter/markdown/utils.jl +++ b/src/converter/markdown/utils.jl @@ -10,7 +10,14 @@ function md2html(ss::AS; stripp::Bool=false)::AS # if there's nothing, return that... isempty(ss) && return ss # Use Julia's Markdown parser followed by Julia's MD->HTML conversion - partial = ss |> fix_inserts |> Markdown.parse |> Markdown.html + partial = ss |> fix_inserts |> Markdown.parse + # take over from the parsing of indented blocks + for (i, c) in enumerate(partial.content) + c isa Markdown.Code || continue + partial.content[i] = Markdown.Paragraph(Any[c.code]) + end + # Use Julia's MD->HTML conversion + partial = partial |> Markdown.html # Markdown.html transforms {{ with HTML entities but we don't want that partial = replace(partial, r"{{" => "{{") partial = replace(partial, r"}}" => "}}") diff --git a/src/eval/module.jl b/src/eval/module.jl index 459f7d68b..c4944f68d 100644 --- a/src/eval/module.jl +++ b/src/eval/module.jl @@ -36,7 +36,11 @@ function newmodule(name::String)::Module redirect_stderr(outf) do mod = Core.eval(Main, Meta.parse(""" module $name - import Franklin: @OUTPUT, fdplotly, locvar + import Franklin + import Franklin: @OUTPUT, fdplotly, locvar, pagevar + if isdefined(Main, :Utils) && typeof(Main.Utils) == Module + import ..Utils + end end """)) end diff --git a/src/manager/file_utils.jl b/src/manager/file_utils.jl index 50cfdc369..2c2c8d8f6 100644 --- a/src/manager/file_utils.jl +++ b/src/manager/file_utils.jl @@ -1,9 +1,9 @@ """ $(SIGNATURES) -Checks for a `config.md` file in `PATHS[:src]` and uses it to set the global -variables referenced in `GLOBAL_VARS` it also sets the global latex commands -via `GLOBAL_LXDEFS`. If the configuration file is not found a warning is shown. +Checks for a `config.md` file and uses it to set the global variables +referenced in `GLOBAL_VARS` it also sets the global latex commands via +`GLOBAL_LXDEFS`. If the configuration file is not found a warning is shown. The keyword `init` is used internally to distinguish between the first call where only structural variables are considered (e.g. controlling folder structure). @@ -34,6 +34,36 @@ function process_config(; init::Bool=false)::Nothing return nothing end +""" +$(SIGNATURES) + +Checks for a `utils.jl` file and uses it to set global computed variables, +functions and html functions. Whatever is defined in `utils.jl` takes +precedence over things defined internally in Franklin or in the global vars; +in particular users can redefine the behaviour of `hfuns` though that's not +recommended. +""" +function process_utils() + utils_path_v1 = joinpath(FOLDER_PATH[], "src", "utils.jl") + utils_path_v2 = joinpath(FOLDER_PATH[], "utils.jl") + if isfile(utils_path_v2) + utils = utils_path_v2 + elseif isfile(utils_path_v1) + utils = utils_path_v1 + else + return nothing + end + # wipe / create module Utils + newmodule("Utils") + Core.eval(Main.Utils, :(include($utils))) + # # keep track of utils names + ns = String.(names(Main.Utils, all=true)) + filter!(n -> n[1] != '#' && n ∉ ("eval", "include", "Utils"), ns) + empty!(UTILS_NAMES) + append!(UTILS_NAMES, ns) + return nothing +end + """ $(SIGNATURES) @@ -199,10 +229,33 @@ Convenience function to assemble the html out of its parts. build_page(head::String, content::String, pgfoot::String, foot::String) = "$head\n
\n$content\n$pgfoot\n
\n$foot" -function set_cur_rpath(fpath::String) + +""" + get_rpath(fpath) + +Extracts the relative file system path out of the full system path to a file +currently being processed. Does not start with a path separator. +So `[some_fs_path]/blog/page.md` --> `blog/page.md`. +""" +function get_rpath(fpath::String) root = path(ifelse(FD_ENV[:STRUCTURE] < v"0.2", :src, :folder)) - cur_rpath = fpath[lastindex(root)+length(PATH_SEP)+1:end] - FD_ENV[:CUR_PATH] = cur_rpath - set_var!(LOCAL_VARS, "fd_rpath", FD_ENV[:CUR_PATH]) + return fpath[lastindex(root)+length(PATH_SEP)+1:end] +end + +""" + set_cur_rpath(fpath) + +Takes the path to the current file and sets the `fd_rpath` local page variable +as well as the `FD_ENV[:CUR_PATH]` variable (used for conditional blocks +depending on URL for instance). +""" +function set_cur_rpath(fpath::String; isrelative::Bool=false) + if isrelative + rpath = fpath + else + rpath = get_rpath(fpath) + end + FD_ENV[:CUR_PATH] = rpath + set_var!(LOCAL_VARS, "fd_rpath", rpath) return nothing end diff --git a/src/manager/franklin.jl b/src/manager/franklin.jl index 390d75217..9d746e846 100644 --- a/src/manager/franklin.jl +++ b/src/manager/franklin.jl @@ -3,7 +3,7 @@ $SIGNATURES Clear the environment dictionaries. """ -clear_dicts() = empty!.((GLOBAL_LXDEFS, GLOBAL_VARS, LOCAL_VARS)) +clear_dicts() = empty!.((GLOBAL_LXDEFS, GLOBAL_VARS, LOCAL_VARS, ALL_PAGE_VARS)) """ $(SIGNATURES) @@ -14,18 +14,24 @@ Keyword arguments: * `clear=false`: whether to remove any existing output directory * `verb=false`: whether to display messages -* `port=8000`: the port to use for the local server (should pick a number between 8000 and 9000) +* `port=8000`: the port to use for the local server (should pick a number + between 8000 and 9000) * `single=false`: whether to run a single pass or run continuously * `prerender=false`: whether to pre-render javascript (KaTeX and highlight.js) * `nomess=false`: suppresses all messages (internal use). -* `isoptim=false`: whether we're in an optimisation phase or not (if so, links are fixed in case - of a project website, see [`write_page`](@ref). -* `no_fail_prerender=true`: whether, in a prerendering phase, ignore errors and try to produce an output +* `isoptim=false`: whether we're in an optimisation phase or not (if so, + links are fixed in case of a project website, see + [`write_page`](@ref). +* `no_fail_prerender=true`: whether, in a prerendering phase, ignore errors and + try to produce an output * `eval_all=false`: whether to force re-evaluation of all code blocks -* `silent=false`: switch this on to suppress all output (including eval statements). -* `cleanup=true`: whether to clear environment dictionaries, see [`cleanup`](@ref). -* `on_write(pg, fd_vars)`: callback function after the page is rendered, passing as arguments - the rendered page and the page variables +* `silent=false`: switch this on to suppress all output (including eval + statements). +* `cleanup=true`: whether to clear environment dictionaries, see + [`cleanup`](@ref). +* `on_write(pg, fd_vars)`: callback function after the page is rendered, + passing as arguments the rendered page and the page + variables """ function serve(; clear::Bool=false, verb::Bool=false, @@ -143,8 +149,8 @@ function fd_setup(; clear::Bool=true)::NamedTuple prepare_output_dir(clear) # . recovering the list of files in the input dir we care about - # -- these are stored in dictionaries, the key is the full path and the value is the time of - # last change (useful for continuous monitoring) + # -- these are stored in dictionaries, the key is the full path and the + # value is the time of last change (useful for continuous monitoring) md_pages = TrackedFiles() html_pages = TrackedFiles() other_files = TrackedFiles() @@ -195,6 +201,7 @@ function fd_fullpass(watched_files::NamedTuple; clear::Bool=false, # process configuration file (see also `process_mddefs!`) process_config() + process_utils() # form page segments root_key = ifelse(FD_ENV[:STRUCTURE] < v"0.2", :src, :folder) @@ -263,7 +270,12 @@ function fd_loop(cycle_counter::Int, ::LiveServer.FileWatcher, # anything, we just remove the file reference from the corresponding # dictionary. for d ∈ watched_files, (fpair, _) ∈ d - isfile(joinpath(fpair...)) || delete!(d, fpair) + fpath = joinpath(fpair...) + if !isfile(fpath) + delete!(d, fpair) + rp = splitext(get_rpath(fpath))[1] + haskey(ALL_PAGE_VARS, rp) && delete!(ALL_PAGE_VARS, rp) + end end # 2) scan the input folder, if new files have been added then this will # update the dictionaries @@ -280,10 +292,14 @@ function fd_loop(cycle_counter::Int, ::LiveServer.FileWatcher, cur_t <= t && continue # if there was then the file has been modified and should be # re-processed + copied - fmsg = rpad("→ file $(fpath[length(FOLDER_PATH[])+1:end]) was modified ", 30) + fmsg = rpad("→ file $(fpath[length(FOLDER_PATH[])+1:end]) was " * + "modified ", 30) verb && print(fmsg) dict[fpair] = cur_t + # Reprocess utils.jl in Utils module + process_utils() + # if it's an infra_file trigger a fullpass as potentially # the whole website depends upon it (e.g. CSS) if haskey(watched_files[:infra], fpair) diff --git a/src/parser/latex/blocks.jl b/src/parser/latex/blocks.jl index 22b416ed1..b65cdfd9a 100644 --- a/src/parser/latex/blocks.jl +++ b/src/parser/latex/blocks.jl @@ -74,7 +74,7 @@ function find_lxdefs(tokens::Vector{Token}, blocks::Vector{OCBlock}) # keep track of the command name, definition and where it stops lxname = matched.captures[1] - lxdef = strip(content(defining_braces)) + lxdef = stent(defining_braces) todef = to(defining_braces) # store the new latex command push!(lxdefs, LxDef(lxname, lxnarg, lxdef, fromτ, todef)) @@ -119,7 +119,9 @@ function get_lxdef_ref(lxname::SubString, lxdefs::Vector{LxDef}, fromlx = from(lxname) + offset filter!(k -> (fromlx > from(lxdefs[k])), ks) if isempty(ks) - if !inmath + if isdefined(Main, :Utils) && isdefined(Main.Utils, Symbol("lx_$(lxname[2:end])")) + return Ref(nothing) + elseif !inmath throw(LxComError("Command '$lxname' was used before it was defined.")) end # not found but inmath --> let KaTex deal with it @@ -149,11 +151,22 @@ function find_lxcoms(tokens::Vector{Token}, lxdefs::Vector{LxDef}, # 1. look for the definition given the command name lxname = τ.ss lxdefref = get_lxdef_ref(lxname, lxdefs, inmath, offset) + derived = false # will only be nothing in a 'inmath' --> no failure, just ignore token - isnothing(lxdefref[]) && continue - - # 2. retrieve number of arguments - lxnarg = getindex(lxdefref).narg + if isnothing(lxdefref[]) + lxn = lxname[2:end] + fun = Symbol("lx_" * lxn) + if isdefined(Main, :Utils) && isdefined(Main.Utils, fun) + # take a single bracket + lxnarg = 1 + derived = true + else + continue + end + else + lxnarg = getindex(lxdefref).narg + end + # 2. explore with # arguments # 2.a there are none if lxnarg == 0 push!(lxcoms, LxCom(lxname, lxdefref)) @@ -188,7 +201,11 @@ function find_lxcoms(tokens::Vector{Token}, lxdefs::Vector{LxDef}, from_c = from(τ) to_c = to(cand_braces[end]) str_c = subs(str(τ), from_c, to_c) - push!(lxcoms, LxCom(str_c, lxdefref, cand_braces)) + if derived + push!(lxcoms, LxCom(str_c, nothing, cand_braces)) + else + push!(lxcoms, LxCom(str_c, lxdefref, cand_braces)) + end # deactivate tokens in the span of the command (will be # reparsed later) diff --git a/src/parser/latex/tokens.jl b/src/parser/latex/tokens.jl index 6fe2e5d94..635c2b541 100644 --- a/src/parser/latex/tokens.jl +++ b/src/parser/latex/tokens.jl @@ -45,9 +45,9 @@ $(TYPEDEF) A `LxCom` has a similar content as a `Block`, with the addition of the definition and a vector of brace blocks. """ struct LxCom <: AbstractBlock - ss ::SubString # \\command - lxdef ::Ref{LxDef} # definition of the command - braces::Vector{OCBlock} # relevant {...} associated with the command + ss ::SubString # \\command + lxdef ::Union{Nothing,Ref{LxDef}} # definition of the command + braces::Vector{OCBlock} # relevant {...} with the command end LxCom(ss, def) = LxCom(ss, def, Vector{OCBlock}()) from(lxc::LxCom) = from(lxc.ss) @@ -58,10 +58,23 @@ to(lxc::LxCom) = to(lxc.ss) For a given `LxCom`, retrieve the definition attached to the corresponding `LxDef` via the reference. """ -getdef(lxc::LxCom)::AS = getindex(lxc.lxdef).def +function getdef(lxc::LxCom)::Union{Nothing,AS} + if isnothing(lxc.lxdef) + return nothing + end + return getindex(lxc.lxdef).def +end """ For a given `LxCom`, retrieve the name of the command via the reference. Example: `\\cite` --> `cite`. """ -getname(lxc::LxCom)::String = String(getindex(lxc.lxdef).name)[2:end] +function getname(lxc::LxCom)::String + if isnothing(lxc.lxdef) + s = String(lxc.ss) + j = findfirst('{', s) + return lxc.ss[2:prevind(s, j)] + end + return String(getindex(lxc.lxdef).name)[2:end] +end +# XXX if nothing lxc.lxdef then extract from lxc.ss but extract before {} diff --git a/src/parser/markdown/indent.jl b/src/parser/markdown/indent.jl index ba70063b8..bc5ec724e 100644 --- a/src/parser/markdown/indent.jl +++ b/src/parser/markdown/indent.jl @@ -1,49 +1,6 @@ """ $(SIGNATURES) -In initial phase, discard all `:LR_INDENT` that don't meet the requirements for -an indented code block. I.e.: where the first line is not preceded by a blank -line and then subsequently followed by indented lines. -""" -function filter_lr_indent!(vt::Vector{Token}, s::String)::Nothing - tind_idx = [i for i in eachindex(vt) if vt[i].name == :LR_INDENT] - isempty(tind_idx) && return nothing - disable = zeros(Bool, length(tind_idx)) - open = false - i = 1 - while i <= length(tind_idx) - t = vt[tind_idx[i]] - if !open - # check if previous line is blank - prev_idx = prevind(s, from(t)) - if prev_idx < 1 || s[prev_idx] != '\n' - disable[i] = true - else - open = true - end - i += 1 - else - # if touching previous line, keep open and go to next - prev = vt[tind_idx[i-1]] - if prev.lno + 1 == t.lno - i += 1 - else - # otherwise close, and continue (will go to !open) - open = false - continue - end - end - end - for i in eachindex(disable) - disable[i] || continue - vt[tind_idx[i]] = Token(:LINE_RETURN, subs(vt[tind_idx[i]].ss, 1)) - end - return nothing -end - -""" -$(SIGNATURES) - Find markers for indented lines (i.e. a line return followed by a tab or 4 spaces). """ @@ -86,6 +43,48 @@ function find_indented_blocks!(tokens::Vector{Token}, st::String)::Nothing return nothing end +""" +$(SIGNATURES) + +In initial phase, discard all `:LR_INDENT` that don't meet the requirements for +an indented code block. I.e.: where the first line is not preceded by a blank +line and then subsequently followed by indented lines. +""" +function filter_lr_indent!(vt::Vector{Token}, s::String)::Nothing + tind_idx = [i for i in eachindex(vt) if vt[i].name == :LR_INDENT] + isempty(tind_idx) && return nothing + disable = zeros(Bool, length(tind_idx)) + open = false + i = 1 + while i <= length(tind_idx) + t = vt[tind_idx[i]] + if !open + # check if previous line is blank + prev_idx = prevind(s, from(t)) + if prev_idx < 1 || s[prev_idx] != '\n' + disable[i] = true + else + open = true + end + i += 1 + else + # if touching previous line, keep open and go to next + prev = vt[tind_idx[i-1]] + if prev.lno + 1 == t.lno + i += 1 + else + # otherwise close, and continue (will go to !open) + open = false + continue + end + end + end + for i in eachindex(disable) + disable[i] || continue + vt[tind_idx[i]] = Token(:LINE_RETURN, subs(vt[tind_idx[i]].ss, 1)) + end + return nothing +end """ $SIGNATURES @@ -145,7 +144,7 @@ $SIGNATURES Discard any indented block that is within a larger block to avoid ambiguities (see #285). """ -function filter_indented_blocks!(blocks::Vector{OCBlock})::Nothing\ +function filter_indented_blocks!(blocks::Vector{OCBlock})::Nothing # retrieve the indices of the indented blocks idx = [i for i in eachindex(blocks) if blocks[i].name == :CODE_BLOCK_IND] isempty(idx) && return nothing diff --git a/src/parser/markdown/tokens.jl b/src/parser/markdown/tokens.jl index bb4d34f3c..12a9b63ee 100644 --- a/src/parser/markdown/tokens.jl +++ b/src/parser/markdown/tokens.jl @@ -21,7 +21,6 @@ const MD_1C_TOKENS_LX = LittleDict{Char, Symbol}( '}' => :LXB_CLOSE ) - """ MD_TOKENS diff --git a/src/parser/ocblocks.jl b/src/parser/ocblocks.jl index 21fcee046..bc4ab4d3c 100644 --- a/src/parser/ocblocks.jl +++ b/src/parser/ocblocks.jl @@ -77,12 +77,8 @@ vector of vectors of ocblocks. """ function find_all_ocblocks(tokens::Vector{Token}, ocplist::Vector{OCProto}; inmath=false) - ocbs_all = Vector{OCBlock}() for ocp ∈ ocplist - if ocp.name == :CODE_BLOCK_IND && !locvar("indented_code") - continue - end ocbs, tokens = find_ocblocks(tokens, ocp; inmath=inmath) append!(ocbs_all, ocbs) end diff --git a/src/parser/tokens.jl b/src/parser/tokens.jl index ae0d75abc..0eafec0cb 100644 --- a/src/parser/tokens.jl +++ b/src/parser/tokens.jl @@ -124,6 +124,12 @@ function content(ocb::OCBlock)::SubString return subs(s, cfrom, cto) end +""" +$(SIGNATURES) + +Convenience function to strip the content of a block. +""" +stent = strip ∘ content #= FUNCTIONS / CONSTANTS THAT HELP DEFINE TOKENS diff --git a/src/utils/misc.jl b/src/utils/misc.jl index a66cf5b75..d28fe0a25 100644 --- a/src/utils/misc.jl +++ b/src/utils/misc.jl @@ -222,34 +222,6 @@ function rprint(s::AS)::Nothing return nothing end -""" - filecmp(path1, path2) - -Takes 2 absolute paths and check if the files are different. -This code was suggested by Steven J. Johnson on discourse: -https://discourse.julialang.org/t/how-to-obtain-the-result-of-a-diff-between-2-files-in-a-loop/23784/4 -""" -function filecmp(path1::AbstractString, path2::AbstractString) - stat1, stat2 = stat(path1), stat(path2) - if !(isfile(stat1) && isfile(stat2)) || filesize(stat1) != filesize(stat2) - return false - end - stat1 == stat2 && return true # same file - open(path1, "r") do file1 - open(path2, "r") do file2 - buf1 = Vector{UInt8}(undef, 32768) - buf2 = similar(buf1) - while !eof(file1) && !eof(file2) - n1 = readbytes!(file1, buf1) - n2 = readbytes!(file2, buf2) - n1 != n2 && return false - 0 != Base._memcmp(buf1, buf2, n1) && return false - end - return eof(file1) == eof(file2) - end - end -end - """ check_ping(ipaddr) diff --git a/src/utils/vars.jl b/src/utils/vars.jl index d452b2306..301c855dc 100644 --- a/src/utils/vars.jl +++ b/src/utils/vars.jl @@ -55,7 +55,7 @@ const LOCAL_VARS_DEFAULT = [ "date" => Pair(Date(1), (String, Date, Nothing)), "lang" => Pair("julia", (String,)), # default lang indented code "reflinks" => Pair(true, (Bool,)), # are there reflinks? - "indented_code" => Pair(true, (Bool,)), # support indented code? + "indented_code" => Pair(false, (Bool,)), # support indented code? # ----------------- # TABLE OF CONTENTS "mintoclevel" => Pair(1, (Int,)), # set to 2 to ignore h1 @@ -116,16 +116,72 @@ function def_LOCAL_VARS!()::Nothing end """ + locvar(name) + Convenience function to get the value associated with a local var. Return `nothing` if the variable is not found. """ -locvar(name::String) = haskey(LOCAL_VARS, name) ? LOCAL_VARS[name].first : nothing +function locvar(name::Union{Symbol,String}) + name = String(name) + return haskey(LOCAL_VARS, name) ? LOCAL_VARS[name].first : nothing +end """ + globvar(name) + Convenience function to get the value associated with a global var. Return `nothing` if the variable is not found. """ -globvar(name::String) = haskey(GLOBAL_VARS, name) ? GLOBAL_VARS[name].first : nothing +function globvar(name::Union{Symbol,String}) + name = String(name) + return haskey(GLOBAL_VARS, name) ? GLOBAL_VARS[name].first : nothing +end + + +""" +Dict to keep track of all pages and their vars. Each key is a relative path +to a page, values are PageVars. +""" +const ALL_PAGE_VARS = Dict{String,PageVars}() + +""" + pagevar(rpath, name) + +Convenience function to get the value associated with a var available to a page +corresponding to `rpath`. So for instance if `blog/index.md` has `@def var = 0` +then this can be accessed with `pagevar("var", "blog/index")`. +If `rpath` is not yet a key of `ALL_PAGE_VARS` then maybe the page hasn't been +processed yet so force a pass over that page. +""" +function pagevar(rpath::AS, name::Union{Symbol,String}) + rpath = splitext(rpath)[1] + if !haskey(ALL_PAGE_VARS, rpath) + # does there exist a file with a `.md` ? if so go over it + # otherwise return nothing + fpath = rpath * ".md" + candpath = FD_ENV[:STRUCTURE] < v"0.2" ? + joinpath(path(:src), fpath) : + joinpath(path(:folder), fpath) + isfile(candpath) || return nothing + # store curpath + bk_path = locvar("fd_rpath") + # set temporary cur path (so that defs go to the right place) + set_cur_rpath(rpath, isrelative=true) + # effectively we only care about the mddefs + convert_md(read(fpath, String), only_mddefs=true, isinternal=true) + # re-set the cur path to what it was before + set_cur_rpath(bk_path) + end + name = String(name) + haskey(ALL_PAGE_VARS[rpath], name) || return nothing + return ALL_PAGE_VARS[rpath][name].first +end + +""" +Keep track of the names declared in the Utils module. +""" +const UTILS_NAMES = Vector{String}() + """ Keep track of seen headers. The key is the refstring, the value contains the @@ -187,8 +243,8 @@ Convenience functions related to the page vars $(SIGNATURES) Convenience function taking a `DateTime` object and returning the corresponding -formatted string with the format contained in `LOCAL_VARS["date_format"]` and -with the locale data provided in `date_months`, `date_shortmonths`, `date_days`, +formatted string with the format contained in `LOCAL_VARS["date_format"]` and +with the locale data provided in `date_months`, `date_shortmonths`, `date_days`, and `date_shortdays` local variables. If `short` variations are not provided, automatically construct them using the first three letters of the names in `date_months` and `date_days`. @@ -214,7 +270,7 @@ function fd_date(d::DateTime) shortmonths = first.(months, 3) end # set locale for this page - Dates.LOCALES["date_locale"] = Dates.DateLocale(months, shortmonths, + Dates.LOCALES["date_locale"] = Dates.DateLocale(months, shortmonths, days, shortdays) return Dates.format(d, format, locale="date_locale") end diff --git a/test/converter/markdown3.jl b/test/converter/markdown3.jl index 128f85df7..0651b61c8 100644 --- a/test/converter/markdown3.jl +++ b/test/converter/markdown3.jl @@ -100,10 +100,10 @@ end st = raw"""A `single` and ``double ` double`` B""" steps = explore_md_steps(st) blocks, tokens = steps[:ocblocks] + @test blocks[2].name == :CODE_INLINE + @test F.content(blocks[2]) == "double ` double" @test blocks[1].name == :CODE_INLINE @test F.content(blocks[1]) == "single" - @test F.content(blocks[2]) == "double ` double" - @test blocks[2].name == :CODE_INLINE st = raw"""A `single` and ``double ` double`` and ``` triple ``` B""" steps = explore_md_steps(st) @@ -225,6 +225,7 @@ end @testset "IndCode" begin # issue 207 st = raw""" + @def indented_code = true A a = 1+1 @@ -248,6 +249,7 @@ end

""") st = raw""" + @def indented_code = true A `single` and ```python blah``` and a = 1+1 @@ -276,6 +278,7 @@ end """) st = raw""" + @def indented_code = true A function foo() diff --git a/test/coverage/extras1.jl b/test/coverage/extras1.jl index 68ee05185..9cde34d4e 100644 --- a/test/coverage/extras1.jl +++ b/test/coverage/extras1.jl @@ -5,7 +5,6 @@ end @testset "Conv-html" begin - @test_throws F.HTMLFunctionError F.convert_html("{{fill bb cc}}") @test_throws F.HTMLFunctionError F.convert_html("{{insert bb cc}}") @test_throws F.HTMLFunctionError F.convert_html("{{href aa}}") @test (@test_logs (:warn, "Unknown dictionary name aa in {{href ...}}. Ignoring") F.convert_html("{{href aa bb}}")) == "??" diff --git a/test/parser/2-blocks.jl b/test/parser/2-blocks.jl index 714907ae7..573dc05ba 100644 --- a/test/parser/2-blocks.jl +++ b/test/parser/2-blocks.jl @@ -4,7 +4,7 @@ vh = s -> (t = vfn(s); F.validate_headers!(t); t) fib = s -> (t = vh(s); F.find_indented_blocks!(t, s); t) fib2 = s -> (t = fib(s); F.filter_lr_indent!(t, s); t) -blk = s -> (F.def_LOCAL_VARS!(); F.find_all_ocblocks(fib2(s), F.MD_OCB_ALL)) +blk = s -> (F.def_LOCAL_VARS!(); F.set_var!(F.LOCAL_VARS, "indented_code", true); F.find_all_ocblocks(fib2(s), F.MD_OCB_ALL)) blk2 = s -> ((b, t) = blk(s); F.merge_indented_blocks!(b, s); b) blk3 = s -> (b = blk2(s); F.filter_indented_blocks!(b); b) diff --git a/test/parser/indentation++.jl b/test/parser/indentation++.jl new file mode 100644 index 000000000..0a303d117 --- /dev/null +++ b/test/parser/indentation++.jl @@ -0,0 +1,63 @@ +@testset "indentation" begin + mds = """ + A + B + C + D""" + + tokens = F.find_tokens(mds, F.MD_TOKENS, F.MD_1C_TOKENS) + F.find_indented_blocks!(tokens, mds) + toks = deepcopy(tokens) + + @test tokens[1].name == :LR_INDENT + @test tokens[2].name == :LR_INDENT + @test tokens[3].name == :LINE_RETURN + + ocp = F.OCProto(:CODE_BLOCK_IND, :LR_INDENT, (:LINE_RETURN,), false) + + blocks, tokens = F.find_ocblocks(tokens, ocp) + + @test blocks[1].name == :CODE_BLOCK_IND + @test F.content(blocks[1]) == "B\n C" + + blocks, tokens = F.find_all_ocblocks(toks, [ocp]) + + @test blocks[1].name == :CODE_BLOCK_IND + @test F.content(blocks[1]) == "B\n C" + + mds = """ + @def indented_code = true + A + + B + C + D""" + steps = explore_md_steps(mds) + toks = steps[:tokenization].tokens + @test toks[4].name == toks[5].name == :LR_INDENT + blk = steps[:ocblocks].blocks + @test blk[2].name == :CODE_BLOCK_IND + b2i = steps[:b2insert].b2insert + @test b2i[2].name == :CODE_BLOCK_IND + @test isapproxstr(mds |> fd2html_td, """ +

A 

B
+        C
D

+ """) +end + +@testset "ind+lx" begin + s = raw""" + \newcommand{\julia}[1]{ + ```julia + #1 + ``` + } + Hello + \julia{a=5 + x=3} + """ |> fd2html_td + @test isapproxstr(s, """ +

Hello 

a=5
+        x=3

+ """) +end diff --git a/test/runtests.jl b/test/runtests.jl index 5d26305d9..6848f0cf8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,9 +7,11 @@ F.FD_ENV[:SILENT_MODE] = true F.FD_ENV[:STRUCTURE] = v"0.1" # legacy, it's switched up in the tests. # UTILS -println("UTILS") +println("UTILS-1") include("utils/folder_structure.jl") include("utils/paths_vars.jl"); include("test_utils.jl") +# --- +println("UTILS-2") include("utils/misc.jl") include("utils/errors.jl") include("utils/html.jl") @@ -35,6 +37,7 @@ include("parser/markdown+latex.jl") include("parser/markdown-extra.jl") include("parser/footnotes+links.jl") include("parser/latex++.jl") +include("parser/indentation++.jl") println("🍺") # EVAL @@ -97,6 +100,14 @@ begin end cd(dirname(dirname(pathof(Franklin)))) +println("TEMPLATING") +include("templating/for.jl") +# XXX XXX +include("templating/fill.jl") + +println("UTILS FILE") +include("utils_file/basic.jl") + println("INTEGRATION") include("integration/literate.jl") include("integration/literate_fs2.jl") diff --git a/test/templating/fill.jl b/test/templating/fill.jl new file mode 100644 index 000000000..39289e68d --- /dev/null +++ b/test/templating/fill.jl @@ -0,0 +1,34 @@ +fs2() +write(joinpath("_layout", "head.html"), "") +write(joinpath("_layout", "foot.html"), "") +write(joinpath("_layout", "page_foot.html"), "") +write("config.md", "") +write("index.md", """ + @def var = 5 + """) +write("page2.md", """ + @def var = 7 + """) + +@testset "fill2" begin + empty!(F.ALL_PAGE_VARS) + @test F.pagevar("page2", :var) == 7 + @test F.pagevar("index", :var) == 5 + @test F.pagevar("page2.md", :var) == 7 + @test F.pagevar("index.md", :var) == 5 + @test F.pagevar("page2.html", :var) == 7 + @test F.pagevar("index.html", :var) == 5 + s = """ + {{fill var page2}} + """ + r = fd2html(s; internal=true) + @test isapproxstr(r, "7") + s = """ + @def paths = ["page2", "index"] + {{for p in paths}} + {{fill var p}} + {{end}} + """ + r = fd2html(s; internal=true) + @test isapproxstr(r, "7 5") +end diff --git a/test/templating/for.jl b/test/templating/for.jl new file mode 100644 index 000000000..e5ac27d66 --- /dev/null +++ b/test/templating/for.jl @@ -0,0 +1,42 @@ +@testset "for-basic" begin + s = """ + @def v1 = [1, 2, 3] + ~~~ + {{for v in v1}} + v: {{fill v}} + {{end}} + ~~~ + """ |> fd2html_td + @test isapproxstr(s, """ + v: 1 + v: 2 + v: 3 + """) + + s = """ + @def v1 = [[1,2], [3,4]] + ~~~ + {{for (a,b) in v1}} + a: {{fill a}} b: {{fill b}} + {{end}} + ~~~ + """ |> fd2html_td + @test isapproxstr(s, """ + a: 1 b: 2 + a: 3 b: 4 + """) + + s = """ + @def v_1 = ("a"=>1, "b"=>2, "c"=>3) + ~~~ + {{for (a,b) in v_1}} + a: {{fill a}} b: {{fill b}} + {{end}} + ~~~ + """ |> fd2html_td + @test isapproxstr(s, """ + a: a b: 1 + a: b b: 2 + a: c b: 3 + """) +end diff --git a/test/test_utils.jl b/test/test_utils.jl index d59d8f07d..e7f33cc5c 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -64,7 +64,7 @@ function explore_md_steps(mds) F.validate_headers!(tokens) hrules = F.find_hrules!(tokens) F.find_indented_blocks!(tokens, mds) - steps[:tokenization] = (tokens=tokens,) + steps[:tokenization] = (tokens=tokens,) # ocblocks blocks, tokens = F.find_all_ocblocks(tokens, F.MD_OCB) @@ -73,7 +73,6 @@ function explore_md_steps(mds) blocks2, tokens = F.find_all_ocblocks(tokens, vcat(F.MD_OCB2, F.MD_OCB_MATH)) append!(blocks, blocks2) F.deactivate_inner_blocks!(blocks) - F.merge_indented_blocks!(blocks, mds) F.filter_indented_blocks!(blocks) steps[:ocblocks] = (blocks=blocks, tokens=tokens) @@ -95,12 +94,18 @@ function explore_md_steps(mds) dbb = F.find_double_brace_blocks(toks_pre_ocb) + F.process_mddefs(blocks, false) + sp_chars = F.find_special_chars(tokens) steps[:spchars] = (spchars=sp_chars,) fnrefs = filter(τ -> τ.name == :FOOTNOTE_REF, tokens) steps[:fnrefs] = (fnrefs=fnrefs,) + if !F.locvar("indented_code") + filter!(b -> b.name != :CODE_BLOCK_IND, blocks) + end + b2insert = F.merge_blocks(lxcoms, F.deactivate_divs(blocks), sp_chars, fnrefs, dbb, hrules) steps[:b2insert] = (b2insert=b2insert,) diff --git a/test/utils_file/basic.jl b/test/utils_file/basic.jl new file mode 100644 index 000000000..c301b6a5f --- /dev/null +++ b/test/utils_file/basic.jl @@ -0,0 +1,72 @@ +# Tests for what happens in the `utils.jl` + +fs2() +write(joinpath("_layout", "head.html"), "") +write(joinpath("_layout", "foot.html"), "") +write(joinpath("_layout", "page_foot.html"), "") +write("config.md", "") +fdi(s) = fd2html(s; internal=true) + +@testset "utils1" begin + write("utils.jl", """ + x = 5 + """) + F.process_utils() + s = """ + {{fill x}} + """ |> fdi + @test isapproxstr(s, "5") + s = """ + {{x}} + """ |> fdi + @test isapproxstr(s, "5") +end + +@testset "utils:hfun" begin + write("utils.jl", """ + hfun_foo() = return "blah" + """) + F.process_utils() + s = """ + {{foo}} + """ |> fdi + @test isapproxstr(s, "blah") + + write("utils.jl", """ + function hfun_bar(vname) + val = locvar(vname[1]) + return round(sqrt(val), digits=2) + end + """) + F.process_utils() + s = """ + @def xx = 25 + {{bar xx}} + """ |> fdi + @test isapproxstr(s, "5.0") +end + +@testset "utils:lxfun" begin + write("utils.jl", """ + function lx_foo(lxc::Franklin.LxCom, _) + return uppercase(Franklin.content(lxc.braces[1])) + end + """) + F.process_utils() + s = raw""" + \foo{bar} + """ |> fdi + @test isapproxstr(s, "BAR") + + write("utils.jl", """ + function lx_baz(com, _) + brace_content = Franklin.content(com.braces[1]) + return uppercase(brace_content) + end + """) + F.process_utils() + s = raw""" + \baz{bar} + """ |> fdi + @test isapproxstr(s, "BAR") +end