refactor HTMLWriter search index construction

.gitignore

@@ -1,7 +1,7 @@
 *.jl.cov
 *.jl.*.cov
 *.jl.mem
-/Manifest.toml
+Manifest.toml
 
 test/examples/builds/
 test/formats/builds/
@@ -11,4 +11,3 @@ test/errors/build/
 docs/build/
 docs/pdf/build/
 docs/site/
-docs/Manifest.toml

.travis.yml

@@ -27,7 +27,7 @@ jobs:
       name: "HTML"
       after_success: skip
     - script:
-        - julia --project=docs/pdf/ -e 'using Pkg; Pkg.instantiate()'
+        - julia --project=docs/pdf/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
         - julia --project=docs/pdf/ docs/pdf/make.jl
       name: "PDF"
       after_success: skip

CHANGELOG.md

@@ -42,6 +42,8 @@
 * ![Enhancement][badge-enhancement] Reporting of failed doctests are now using the logging
   system to be consistent with the rest of Documenter's output. ([#958][github-958])
 
+* ![Enhancement][badge-enhancement] The construction of the search index in the HTML output has been refactored to make it easier to use with other search backends in the future. The structure of the generated search index has also been modified, which can yield slightly different search results. Documenter now depends on the lightweight [JSON.jl][json-jl] package. ([#966][github-966])
+
 * ![Bugfix][badge-bugfix] Paths in `include` calls in `@eval`, `@example`, `@repl` and `jldoctest`
   blocks are now interpreted to be relative `pwd`, which is set to the output directory of the
   resulting file. ([#941][github-941])
@@ -265,11 +267,13 @@
 [github-958]: https://github.com/JuliaDocs/Documenter.jl/pull/958
 [github-959]: https://github.com/JuliaDocs/Documenter.jl/pull/959
 [github-960]: https://github.com/JuliaDocs/Documenter.jl/pull/960
+[github-966]: https://github.com/JuliaDocs/Documenter.jl/pull/966
 [github-967]: https://github.com/JuliaDocs/Documenter.jl/pull/967
 [github-971]: https://github.com/JuliaDocs/Documenter.jl/pull/971
 
 [documenterlatex]: https://github.com/JuliaDocs/DocumenterLaTeX.jl
 [documentermarkdown]: https://github.com/JuliaDocs/DocumenterMarkdown.jl
+[json-jl]: https://github.com/JuliaIO/JSON.jl
 
 
 [badge-breaking]: https://img.shields.io/badge/BREAKING-red.svg

Project.toml

@@ -6,6 +6,7 @@ version = "0.21.2"
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"

REQUIRE

@@ -1,2 +1,3 @@
 julia 0.7
 DocStringExtensions 0.2
+JSON 0.19

docs/Project.toml

@@ -1,2 +1,3 @@
 [deps]
+Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 DocumenterTools = "35a29f4d-8980-5a13-9543-d66fff28ecb8"

src/Writers/HTMLWriter.jl

@@ -41,6 +41,7 @@ then it is intended as the page title. This has two consequences:
 module HTMLWriter
 
 import Markdown
+import JSON
 
 import ...Documenter:
     Anchors,
@@ -149,6 +150,16 @@ const google_fonts = "https://fonts.googleapis.com/css?family=Lato|Roboto+Mono"
 const fontawesome_css = "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.6.3/css/font-awesome.min.css"
 const highlightjs_css = "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"
 
+struct SearchRecord
+    src :: String
+    page :: Documents.Page
+    loc :: String
+    category :: String
+    title :: String
+    page_title :: String
+    text :: String
+end
+
 """
 [`HTMLWriter`](@ref)-specific globals that are passed to [`domify`](@ref) and
 other recursive functions.
@@ -160,12 +171,54 @@ mutable struct HTMLContext
     scripts :: Vector{String}
     documenter_js :: String
     search_js :: String
-    search_index :: IOBuffer
+    search_index :: Vector{SearchRecord}
     search_index_js :: String
     search_navnode :: Documents.NavNode
     local_assets :: Vector{String}
 end
-HTMLContext(doc, settings=HTML()) = HTMLContext(doc, settings, "", [], "", "", IOBuffer(), "", Documents.NavNode("search", "Search", nothing), [])
+
+HTMLContext(doc, settings=HTML()) = HTMLContext(doc, settings, "", [], "", "", [], "", Documents.NavNode("search", "Search", nothing), [])
+
+function SearchRecord(ctx::HTMLContext, navnode; loc="", title=nothing, category="page", text="")
+    page_title = mdflatten(pagetitle(ctx, navnode))
+    if title === nothing
+        title = page_title
+    end
+    SearchRecord(
+        pretty_url(ctx, get_url(ctx, navnode.page)),
+        getpage(ctx, navnode),
+        loc,
+        lowercase(category),
+        title,
+        page_title,
+        text
+    )
+end
+
+function SearchRecord(ctx::HTMLContext, navnode, node::Markdown.Header)
+    a = getpage(ctx, navnode).mapping[node]
+    SearchRecord(ctx, navnode;
+        loc="$(a.id)-$(a.nth)",
+        title=mdflatten(node),
+        category="section")
+end
+
+function SearchRecord(ctx, navnode, node)
+    SearchRecord(ctx, navnode; text=mdflatten(node))
+end
+
+function JSON.lower(rec::SearchRecord)
+    # Replace any backslashes in links, if building the docs on Windows
+    src = replace(rec.src, '\\' => '/')
+    ref = string(src, '#', rec.loc)
+    Dict{String, String}(
+        "location" => ref,
+        "page" => rec.page_title,
+        "title" => rec.title,
+        "category" => rec.category,
+        "text" => rec.text
+    )
+end
 
 """
 Returns a page (as a [`Documents.Page`](@ref) object) using the [`HTMLContext`](@ref).
@@ -203,9 +256,13 @@ function render(doc::Documents.Document, settings::HTML=HTML())
     render_search(ctx)
 
     open(joinpath(doc.user.build, ctx.search_index_js), "w") do io
-        println(io, "var documenterSearchIndex = {\"docs\": [\n")
-        write(io, String(take!(ctx.search_index)))
-        println(io, "]}")
+        println(io, "var documenterSearchIndex = {\"docs\":")
+        # convert Vector{SearchRecord} to a JSON string, and escape two Unicode
+        # characters since JSON is not a JS subset, and we want JS here
+        # ref http://timelessrepo.com/json-isnt-a-javascript-subset
+        escapes = ('\u2028' => "\\u2028", '\u2029' => "\\u2029")
+        js = reduce(replace, escapes, init=JSON.json(ctx.search_index))
+        println(io, js, "\n}")
     end
 end
 
@@ -658,101 +715,11 @@ Converts recursively a [`Documents.Page`](@ref), `Markdown` or Documenter
 """
 function domify(ctx, navnode)
     page = getpage(ctx, navnode)
-    sib = SearchIndexBuffer(ctx, navnode)
-    ret = map(page.elements) do elem
-        search_append(sib, elem)
+    map(page.elements) do elem
+        rec = SearchRecord(ctx, navnode, elem)
+        push!(ctx.search_index, rec)
         domify(ctx, navnode, page.mapping[elem])
     end
-    search_flush(sib)
-    ret
-end
-
-mutable struct SearchIndexBuffer
-    ctx :: HTMLContext
-    src :: String
-    page :: Documents.Page
-    loc :: String
-    category :: Symbol
-    title :: String
-    page_title :: String
-    buffer :: IOBuffer
-    function SearchIndexBuffer(ctx, navnode)
-        page_title = mdflatten(pagetitle(ctx, navnode))
-        new(
-            ctx,
-            pretty_url(ctx, get_url(ctx, navnode.page)),
-            getpage(ctx, navnode),
-            "",
-            :page,
-            page_title,
-            page_title,
-            IOBuffer()
-        )
-    end
-end
-
-function search_append(sib, node::Markdown.Header)
-    search_flush(sib)
-    sib.category = :section
-    sib.title = mdflatten(node)
-    a = sib.page.mapping[node]
-    sib.loc = "$(a.id)-$(a.nth)"
-end
-
-search_append(sib, node) = mdflatten(sib.buffer, node)
-
-function search_flush(sib)
-    # Replace any backslashes in links, if building the docs on Windows
-    src = replace(sib.src, '\\' => '/')
-    ref = "$(src)#$(sib.loc)"
-    text = String(take!(sib.buffer))
-    println(sib.ctx.search_index, """
-    {
-        "location": "$(jsescape(ref))",
-        "page": "$(jsescape(sib.page_title))",
-        "title": "$(jsescape(sib.title))",
-        "category": "$(jsescape(lowercase(string(sib.category))))",
-        "text": "$(jsescape(text))"
-    },
-    """)
-end
-
-"""
-Replaces some of the characters in the string with escape sequences so that the strings
-would be valid JS string literals, as per the
-[ECMAScript® 2017 standard](https://www.ecma-international.org/ecma-262/8.0/index.html#sec-literals-string-literals).
-
-Note that it always escapes both potential `"` and `'` closing quotes.
-"""
-function jsescape(s)
-    b = IOBuffer()
-    # From the ECMAScript® 2017 standard:
-    #
-    # > All code points may appear literally in a string literal except for the closing
-    # > quote code points, U+005C (REVERSE SOLIDUS), U+000D (CARRIAGE RETURN), U+2028 (LINE
-    # > SEPARATOR), U+2029 (PARAGRAPH SEPARATOR), and U+000A (LINE FEED).
-    #
-    # https://www.ecma-international.org/ecma-262/8.0/index.html#sec-literals-string-literals
-    for c in s
-        if c === '\u000a'     # LINE FEED,       i.e. \n
-            write(b, "\\n")
-        elseif c === '\u000d' # CARRIAGE RETURN, i.e. \r
-            write(b, "\\r")
-        elseif c === '\u005c' # REVERSE SOLIDUS, i.e. \
-            write(b, "\\\\")
-        elseif c === '\u0022' # QUOTATION MARK,  i.e. "
-            write(b, "\\\"")
-        elseif c === '\u0027' # APOSTROPHE,      i.e. '
-            write(b, "\\'")
-        elseif c === '\u2028' # LINE SEPARATOR
-            write(b, "\\u2028")
-        elseif c === '\u2029' # PARAGRAPH SEPARATOR
-            write(b, "\\u2029")
-        else
-            write(b, c)
-        end
-    end
-    String(take!(b))
 end
 
 function domify(ctx, navnode, node)
@@ -837,12 +804,13 @@ function domify(ctx, navnode, node::Documents.DocsNode)
     @tags a code div section span
 
     # push to search index
-    sib = SearchIndexBuffer(ctx, navnode)
-    sib.loc = node.anchor.id
-    sib.title = string(node.object.binding)
-    sib.category = Symbol(Utilities.doccat(node.object))
-    mdflatten(sib.buffer, node.docstr)
-    search_flush(sib)
+    rec = SearchRecord(ctx, navnode;
+        loc=node.anchor.id,
+        title=string(node.object.binding),
+        category=Utilities.doccat(node.object),
+        text = mdflatten(node.docstr))
+
+    push!(ctx.search_index, rec)
 
     section[".docstring"](
         div[".docstring-header"](