Taxonomy (second attempt)

Project.toml

@@ -1,7 +1,7 @@
 name = "Franklin"
 uuid = "713c75ef-9fc9-4b05-94a9-213340da978e"
 authors = ["Thibaut Lienart <[email protected]>"]
-version = "0.7.10"
+version = "0.8.0"
 
 [deps]
 Crayons = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
@@ -22,7 +22,7 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 [compat]
 Crayons = "4"
 DocStringExtensions = "0.8"
-FranklinTemplates = "0.6"
+FranklinTemplates = "0.6,0.7"
 HTTP = "0.8"
 Literate = "2.2"
 LiveServer = "0.3"

src/Franklin.jl

@@ -21,7 +21,7 @@ import HTTP
 import Random
 
 export serve, publish, cleanpull, newsite, optimize, fd2html,
-       literate_folder, verify_links, @OUTPUT
+       literate_folder, verify_links, @OUTPUT, get_url
 
 # Extra functions
 export lunr

src/converter/html/blocks.jl

@@ -76,6 +76,16 @@ function process_html_cond(hs::AS, qblocks::Vector{AbstractBlock},
                 # pages/blah => pub/blah
                 rpath = replace(rpath, Regex("^pages") => "pub")
             end
+            # are we currently on a tag page?
+            if !isempty(locvar("fd_tag"))
+                tag = locvar("fd_tag")
+                if FD_ENV[:STRUCTURE] < v"0.2"
+                    rpath = "/pub/tag/$tag/"
+                else
+                    rpath = "/tag/$tag/"
+                end
+            end
+
             # compare with β.pages
             inpage = any(p -> match_url(rpath, p), βi.pages)
 

src/converter/html/functions.jl

@@ -188,21 +188,15 @@ end
 """
 $(SIGNATURES)
 
-H-Function of the form `{{list tag}}`.
+H-Function of the form `{{taglist}}`.
 """
-function hfun_list(params::Vector{String})::String
-    if length(params) != 1
-        throw(HTMLFunctionError("I found a {{list ...}} block and expected 1 " *
-                                "parameter but got $(length(params)). Verify."))
-    end
-    tag = params[1]
-    TAG_PAGES = globvar("fd_tag_pages")
+function hfun_taglist()::String
+    tag = locvar(:fd_tag)
 
     c = IOBuffer()
-    write(c, "<h1>Tag: $tag</h1>")
     write(c, "<ul>")
 
-    rpaths = TAG_PAGES[tag]
+    rpaths = globvar("fd_tag_pages")[tag]
     sorter(p) = begin
         pvd = pagevar(p, "date")
         if isnothing(pvd)
@@ -219,11 +213,10 @@ function hfun_list(params::Vector{String})::String
         if isnothing(title)
             title = "/$rpath/"
         end
-
-        (:hfun_list, "(in loop) $rpath - $title") |> logger
-
-        write(c, "<li><a href=\"/$rpath/\">$title</a></li>")
+        url = get_url(rpath)
+        write(c, "<li><a href=\"$url\">$title</a></li>")
     end
     write(c, "</ul>")
+
     return String(take!(c))
 end

src/converter/html/html.jl

@@ -12,7 +12,8 @@ function convert_html(hs::AS; isoptim::Bool=false)::String
 
     # Find hblocks ({{ ... }})
     hblocks, tokens = find_all_ocblocks(tokens, HTML_OCB)
-    filter!(hb -> hb.name != :COMMENT, hblocks)
+    deactivate_inner_blocks!(hblocks, (:COMMENT, :SCRIPT))
+    filter!(hb -> hb.name ∉ (:COMMENT, :SCRIPT), hblocks)
 
     # Find qblocks (qualify the hblocks)
     qblocks = qualify_html_hblocks(hblocks)

src/converter/markdown/blocks.jl

@@ -34,7 +34,7 @@ function convert_block(β::AbstractBlock, lxdefs::Vector{LxDef})::AS
         cont     = convert_md(raw_cont, lxdefs;
                               isrecursive=true, has_mddefs=false)
         divname  = chop(otok(β).ss, head=2, tail=0)
-        return html_div(divname, cont)
+        return html_div(replace(divname, ","=>" "), cont)
     end
 
     # default case, ignore block (should not happen)

src/converter/markdown/md.jl

@@ -70,7 +70,7 @@ function convert_md(mds::AS,
     #>> a'. find the rest
     blocks2, tokens = find_all_ocblocks(tokens, vcat(MD_OCB2, MD_OCB_MATH))
     append!(blocks, blocks2)
-    deactivate_inner_blocks!(blocks)
+    ranges = deactivate_inner_blocks!(blocks)
     #>> b. merge CODE_BLOCK_IND which are separated by emptyness
     merge_indented_blocks!(blocks, mds)
     #>> b'. only keep indented code blocks which are not contained in larger
@@ -102,7 +102,7 @@ function convert_md(mds::AS,
     #> 3[ex]. find double brace blocks, note we do it on pre_ocb tokens
     # as the step `find_all_ocblocks` possibly found and deactivated {...}.
     dbb = find_double_brace_blocks(toks_pre_ocb)
-
+    deactivate_inner_dbb!(dbb, ranges)
     # ------------------------------------------------------------------------
     #> 4. Page variable definition (mddefs), also if in config, update lxdefs
     if has_mddefs
@@ -238,6 +238,8 @@ const INSERT_LEN = length(INSERT_)
 
 
 """
+$SIGNATURES
+
 Form an intermediate MD file where special blocks are replaced by a marker
 (`INSERT`) indicating that a piece will need to be plugged in there later.
 
@@ -377,7 +379,8 @@ function convert_inter_html(ihtml::AS,
             head = ifelse(ihtml[head] in (' ', '>'), nextind(ihtml, head), head)
         end
         # store the resolved block
-        write(htmls, convert_block(blocks[i], lxdefs))
+        resolved = convert_block(blocks[i], lxdefs)
+        write(htmls, resolved)
     end
     # store whatever is after the last INSERT if anything
     (head ≤ strlen) && write(htmls, subs(ihtml, head:strlen))

src/converter/markdown/mddefs.jl

@@ -63,39 +63,42 @@ function process_mddefs(blocks::Vector{OCBlock}, isconfig::Bool,
 
     (:process_mddefs, "assignments done & copy to ALL_PAGE_VARS") |> logger
 
-    # # TAGS
-    # tags = Set(unique(locvar("tags")))
-    # # Cases:
-    # # 1. that page did not have tags
-    # #   a. tags is empty --> do nothing
-    # #   b. tags is not empty register them and update all
-    # # 2. that page did have tags
-    # #   a. tags are unchanged --> do nothing
-    # #   b. check which ones change and update those
-    # PAGE_TAGS = globvar("fd_page_tags")
-    # if isnothing(PAGE_TAGS)
-    #     isempty(tags) && return nothing
-    #     set_var!(GLOBAL_VARS, "fd_page_tags", DTAG((rpath => tags,)))
-    # elseif !haskey(PAGE_TAGS, rpath)
-    #     isempty(tags) && return nothing
-    #     PAGE_TAGS[rpath] = tags
-    # else
-    #     old_tags = PAGE_TAGS[rpath]
-    #     if isempty(tags)
-    #         delete!(PAGE_TAGS, rpath)
-    #     else
-    #         PAGE_TAGS[rpath] = tags
-    #     end
-    #     # we will need to update all tag pages that were or are related to
-    #     # the current rpath, indeed properties of the page may have changed
-    #     # and affect these derived pages (that's why it's a union here
-    #     # and not a setdiff).
-    #     tags = union(old_tags, tags)
-    # end
-    # # In the full pass each page is processed first (without generating tag
-    # # pages) and then, when all tags have been gathered, generate_tag_pages
-    # # is called (see `fd_fullpass`).
-    # # During the serve loop, we want to trigger on page change.
-    # pagevar || FD_ENV[:FULL_PASS] || generate_tag_pages(tags)
+    # TAGS
+    tags = Set(unique(locvar("tags")))
+    # Cases:
+    # 0. there was no page tags before
+    #   a. tags is empty --> do nothing
+    #   b. tags is not empty --> initialise global page tags + add rpath=>tags
+    # 1. that page did not have tags
+    #   a. tags is empty --> do nothing
+    #   b. tags is not empty register them and update all
+    # 2. that page did have tags
+    #   a. tags are unchanged --> do nothing
+    #   b. check which ones change and update those
+    PAGE_TAGS = globvar("fd_page_tags")
+    if isnothing(PAGE_TAGS)
+        isempty(tags) && return nothing
+        set_var!(GLOBAL_VARS, "fd_page_tags", DTAG((rpath => tags,)))
+    elseif !haskey(PAGE_TAGS, rpath)
+        isempty(tags) && return nothing
+        PAGE_TAGS[rpath] = tags
+    else
+        old_tags = PAGE_TAGS[rpath]
+        if isempty(tags)
+            delete!(PAGE_TAGS, rpath)
+        else
+            PAGE_TAGS[rpath] = tags
+        end
+        # we will need to update all tag pages that were or are related to
+        # the current rpath, indeed properties of the page may have changed
+        # and affect these derived pages (that's why it's a union here
+        # and not a setdiff).
+        tags = union(old_tags, tags)
+    end
+    # In the full pass each page is processed first (without generating tag
+    # pages) and then, when all tags have been gathered, generate_tag_pages
+    # is called (see `fd_fullpass`).
+    # During the serve loop, we want to trigger on page change.
+    pagevar || FD_ENV[:FULL_PASS] || generate_tag_pages(tags)
     return nothing
 end

src/converter/markdown/tags.jl

@@ -6,42 +6,74 @@ refresh (in which case only the pages associated to those tags will be
 refreshed) or an empty set in which case all tags will be (re)-generated.
 Note that if a tag is removed from a page, it may be that `refresh_tags`
 contains a tag which should effectively be removed because no page have it.
-(This only matters at the end in the call to `write_tag_pages`).
+(This only matters at the end in the call to `write_tag_page`).
+Note: the `clean_tags` cleans up orphan tags (tags that wouldn't be pointing
+to any page anymore).
 """
 function generate_tag_pages(refresh_tags=Set{String}())::Nothing
-    # filter out pages that may not exist anymore
+    # if there are no page tags, cleanup and finish
     PAGE_TAGS = globvar("fd_page_tags")
-    isnothing(PAGE_TAGS) && return nothing
+    isnothing(PAGE_TAGS) && return clean_tags()
+    # if there are page tags, eliminiate the pages that don't exist anymore
     for rpath in keys(PAGE_TAGS)
         isfile(rpath * ".md") || delete!(PAGE_TAGS, rpath)
     end
-    isempty(PAGE_TAGS) && return nothing
+    # if there's nothing left, clean up and finish
+    isempty(PAGE_TAGS) && return clean_tags()
 
-    # Get the dictionary tag -> [rp1, rp2...]
+    # Here we have a non-empty PAGE_TAGS dictionary where the keys are
+    # paths of existing files, and values are the tags associated with
+    # these files.
+    # Get the inverse dictionary {tag -> [rp1, rp2...]}
     TAG_PAGES = invert_dict(PAGE_TAGS)
     # store it in globvar
     set_var!(GLOBAL_VARS, "fd_tag_pages", TAG_PAGES)
     all_tags = collect(keys(TAG_PAGES))
 
-    if !isdir(path(:tag))
-        mkpath(path(:tag))
-    else
-        # there may be tags in `refresh_tags` that are not in all_tags
-        # -> in that case delete those pages
-        rm_tags = filter(t -> t ∉ all_tags, refresh_tags)
-        # there may also be existing dirs which shouldn't be there anymore
-        # because the tags have been deleted, these two things combine but
-        # can independently happen (e.g. on the main generate_tag_pages)
-        for dirname in union(setdiff(readdir(path(:tag)), all_tags), rm_tags)
-            rm(joinpath(path(:tag), dirname), recursive=true)
-        end
-    end
+    # some tags may have been given to refresh which don't have any
+    # pages linking to them anymore, these tags will have to be cleaned up
+    rm_tags = filter(t -> t ∉ all_tags, refresh_tags)
 
-    # check which tags should be refreshed, note that
-    update_tags = isempty(refresh_tags) ?
-                    all_tags :
+    # check which tags should be refreshed
+    update_tags = isempty(refresh_tags) ? all_tags :
                     filter(t -> t ∈ all_tags, refresh_tags)
-    write_tag_pages(update_tags)
+
+    # Generate the tag folder if it doesn't exist already
+    isdir(path(:tag)) || mkpath(path(:tag))
+    # Generate the tag layout page if it doesn't exist (it should...)
+    isfile(joinpath(path(:layout), "tag.html")) || write_default_tag_layout()
+    # write each tag, note that they are necessarily in TAG_PAGES
+    for tag in update_tags
+        write_tag_page(tag)
+    end
+    return clean_tags(rm_tags)
+end
+
+"""
+    clean_tags(rm_tags=nothing)
+
+Check the content of the tag folder and remove orphan pages.
+"""
+function clean_tags(rm_tags=nothing)::Nothing
+    isdir(path(:tag)) || return nothing
+    PAGE_TAGS = globvar("fd_page_tags")
+    if isnothing(PAGE_TAGS) || isempty(PAGE_TAGS)
+        all_tags = []
+    else
+        all_tags = union(values(PAGE_TAGS)...)
+    end
+    # check what folders are present
+    all_tag_folders = readdir(path(:tag))
+    # check what folders shouldn't be present
+    orphans = setdiff(all_tags, all_tag_folders)
+    if !isnothing(rm_tags)
+        orphans = union(orphans, rm_tags)
+    end
+    # clean up
+    for dirname in orphans
+        dirpath = joinpath(path(:tag), dirname)
+        isdir(dirpath) && rm(dirpath, recursive=true)
+    end
     return nothing
 end
 
@@ -50,27 +82,49 @@ $SIGNATURES
 
 Internal function to (re)write the tag pages corresponding to `update_tags`.
 """
-function write_tag_pages(update_tags)::Nothing
+function write_tag_page(tag)::Nothing
+    # make `fd_tag` available to that page generation
+    # also allows {{istag tagname}} ... {{end}}
+    set_var!(LOCAL_VARS, "fd_tag", tag)
+
     layout_key  = ifelse(FD_ENV[:STRUCTURE] < v"0.2", :src_html, :layout)
     layout      = path(layout_key)
-    head        = read(joinpath(layout, "head.html"),      String)
-    pg_foot     = read(joinpath(layout, "page_foot.html"), String)
-    foot        = read(joinpath(layout, "foot.html"),      String)
+    content     = read(joinpath(layout, "tag.html"), String)
 
-    # XXX this needs to be fixed, basically tag pages should have some
-    # form of page variable scope as they use the `head` and `foot` which
-    # may use some `{{ ...  }}`. At the moment the tag page uses the scope
-    # of the last-processed page (ambient LOCAL_VARS) which is not good.
+    dir = joinpath(path(:tag), tag)
+    isdir(dir) || mkdir(dir)
 
-    for tag in update_tags
-        # check if `tag/$tag` exists otherwise create it
-        dir = joinpath(path(:tag), tag)
-        isdir(dir) || mkdir(dir)
-        # assemble the page using the hfun list (which can be overwritten
-        # by the user)
-        page = build_page(head, "{{list $tag}}", pg_foot, foot)
-        # write the processed page.
-        write(joinpath(dir, "index.html"), convert_html(page))
-    end
+    write(joinpath(dir, "index.html"), convert_html(content))
+
+    # reset `fd_tag`
+    set_var!(LOCAL_VARS, "fd_tag", "")
+    return nothing
+end
+
+
+"""
+    write_default_tag_layout()
+
+If `_layout/tag.html` is not defined, input a basic default one indicating
+that the user has to modify it. This will help users transition when they may
+have used a template that did not define `_layout/tag.html`.
+"""
+function write_default_tag_layout()::Nothing
+    html = """
+        <!doctype html>
+        <html lang="en">
+        <head>
+          <meta charset="UTF-8">
+          <meta name="viewport" content="width=device-width, initial-scale=1">
+          <title>Tag: {{fill fd_tag}}</title>
+        </head>
+        <body>
+          <div class="{{div_content}} tagpage">
+            {{taglist}}
+          </div>
+        </body>
+        </html>
+        """
+    write(joinpath(path(:layout), "tag.html"), html)
     return nothing
 end

src/eval/module.jl

@@ -37,7 +37,8 @@ function newmodule(name::String)::Module
             mod = Core.eval(Main, Meta.parse("""
                 module $name
                     import Franklin
-                    import Franklin: @OUTPUT, fdplotly, locvar, pagevar
+                    import Franklin: @OUTPUT, fdplotly, locvar,
+                                     pagevar, globvar, fd2html, get_url
                     if isdefined(Main, :Utils) && typeof(Main.Utils) == Module
                         import ..Utils
                     end