From d59687ac44026f4ec9d7ca8cc6d017bdbc91819e Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Sun, 17 Feb 2019 16:09:12 +0100 Subject: [PATCH 1/5] refactor HTMLWriter search index constructionThe mutable `SearchIndexBuffer` that held an `IOBuffer` that was filledwith JSON text is replaced by the immutable `SearchRecord`.The `HTMLContext` now instead holds a `Vector{SearchRecord}`, which isserialized to JSON at the end, done so with the help of`JSON.lower(::SearchRecord)`.This should make it easier to possibly upload the search index to asearch server in the future. --- Project.toml | 1 + REQUIRE | 1 + src/Writers/HTMLWriter.jl | 172 +++++++++++++++----------------------- test/htmlwriter.jl | 18 +--- 4 files changed, 72 insertions(+), 120 deletions(-) diff --git a/Project.toml b/Project.toml index 11f43c5102..e44c9bcc24 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "0.21.2" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" diff --git a/REQUIRE b/REQUIRE index 4e2a4a0ceb..86371d876b 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,2 +1,3 @@ julia 0.7 DocStringExtensions 0.2 +JSON 0.19 diff --git a/src/Writers/HTMLWriter.jl b/src/Writers/HTMLWriter.jl index 036e7e790d..3040170367 100644 --- a/src/Writers/HTMLWriter.jl +++ b/src/Writers/HTMLWriter.jl @@ -41,6 +41,7 @@ then it is intended as the page title. This has two consequences: module HTMLWriter import Markdown +import JSON import ...Documenter: Anchors, @@ -149,6 +150,16 @@ const google_fonts = "https://fonts.googleapis.com/css?family=Lato|Roboto+Mono" const fontawesome_css = "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.6.3/css/font-awesome.min.css" const highlightjs_css = "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css" +struct SearchRecord + src :: String + page :: Documents.Page + loc :: String + category :: String + title :: String + page_title :: String + text :: String +end + """ [`HTMLWriter`](@ref)-specific globals that are passed to [`domify`](@ref) and other recursive functions. @@ -160,12 +171,54 @@ mutable struct HTMLContext scripts :: Vector{String} documenter_js :: String search_js :: String - search_index :: IOBuffer + search_index :: Vector{SearchRecord} search_index_js :: String search_navnode :: Documents.NavNode local_assets :: Vector{String} end -HTMLContext(doc, settings=HTML()) = HTMLContext(doc, settings, "", [], "", "", IOBuffer(), "", Documents.NavNode("search", "Search", nothing), []) + +HTMLContext(doc, settings=HTML()) = HTMLContext(doc, settings, "", [], "", "", [], "", Documents.NavNode("search", "Search", nothing), []) + +function SearchRecord(ctx::HTMLContext, navnode; loc="", title=nothing, category="page", text="") + page_title = mdflatten(pagetitle(ctx, navnode)) + if title === nothing + title = page_title + end + SearchRecord( + pretty_url(ctx, get_url(ctx, navnode.page)), + getpage(ctx, navnode), + loc, + lowercase(category), + title, + page_title, + text + ) +end + +function SearchRecord(ctx::HTMLContext, navnode, node::Markdown.Header) + a = getpage(ctx, navnode).mapping[node] + SearchRecord(ctx, navnode; + loc="$(a.id)-$(a.nth)", + title=mdflatten(node), + category="section") +end + +function SearchRecord(ctx, navnode, node) + SearchRecord(ctx, navnode; text=mdflatten(node)) +end + +function JSON.lower(rec::SearchRecord) + # Replace any backslashes in links, if building the docs on Windows + src = replace(rec.src, '\\' => '/') + ref = string(src, '#', rec.loc) + Dict{String, String}( + "location" => ref, + "page" => rec.page_title, + "title" => rec.title, + "category" => rec.category, + "text" => rec.text + ) +end """ Returns a page (as a [`Documents.Page`](@ref) object) using the [`HTMLContext`](@ref). @@ -203,9 +256,9 @@ function render(doc::Documents.Document, settings::HTML=HTML()) render_search(ctx) open(joinpath(doc.user.build, ctx.search_index_js), "w") do io - println(io, "var documenterSearchIndex = {\"docs\": [\n") - write(io, String(take!(ctx.search_index))) - println(io, "]}") + println(io, "var documenterSearchIndex = {\"docs\":") + JSON.print(io, ctx.search_index) + println(io, "\n}") end end @@ -658,101 +711,11 @@ Converts recursively a [`Documents.Page`](@ref), `Markdown` or Documenter """ function domify(ctx, navnode) page = getpage(ctx, navnode) - sib = SearchIndexBuffer(ctx, navnode) - ret = map(page.elements) do elem - search_append(sib, elem) + map(page.elements) do elem + rec = SearchRecord(ctx, navnode, elem) + push!(ctx.search_index, rec) domify(ctx, navnode, page.mapping[elem]) end - search_flush(sib) - ret -end - -mutable struct SearchIndexBuffer - ctx :: HTMLContext - src :: String - page :: Documents.Page - loc :: String - category :: Symbol - title :: String - page_title :: String - buffer :: IOBuffer - function SearchIndexBuffer(ctx, navnode) - page_title = mdflatten(pagetitle(ctx, navnode)) - new( - ctx, - pretty_url(ctx, get_url(ctx, navnode.page)), - getpage(ctx, navnode), - "", - :page, - page_title, - page_title, - IOBuffer() - ) - end -end - -function search_append(sib, node::Markdown.Header) - search_flush(sib) - sib.category = :section - sib.title = mdflatten(node) - a = sib.page.mapping[node] - sib.loc = "$(a.id)-$(a.nth)" -end - -search_append(sib, node) = mdflatten(sib.buffer, node) - -function search_flush(sib) - # Replace any backslashes in links, if building the docs on Windows - src = replace(sib.src, '\\' => '/') - ref = "$(src)#$(sib.loc)" - text = String(take!(sib.buffer)) - println(sib.ctx.search_index, """ - { - "location": "$(jsescape(ref))", - "page": "$(jsescape(sib.page_title))", - "title": "$(jsescape(sib.title))", - "category": "$(jsescape(lowercase(string(sib.category))))", - "text": "$(jsescape(text))" - }, - """) -end - -""" -Replaces some of the characters in the string with escape sequences so that the strings -would be valid JS string literals, as per the -[ECMAScript® 2017 standard](https://www.ecma-international.org/ecma-262/8.0/index.html#sec-literals-string-literals). - -Note that it always escapes both potential `"` and `'` closing quotes. -""" -function jsescape(s) - b = IOBuffer() - # From the ECMAScript® 2017 standard: - # - # > All code points may appear literally in a string literal except for the closing - # > quote code points, U+005C (REVERSE SOLIDUS), U+000D (CARRIAGE RETURN), U+2028 (LINE - # > SEPARATOR), U+2029 (PARAGRAPH SEPARATOR), and U+000A (LINE FEED). - # - # https://www.ecma-international.org/ecma-262/8.0/index.html#sec-literals-string-literals - for c in s - if c === '\u000a' # LINE FEED, i.e. \n - write(b, "\\n") - elseif c === '\u000d' # CARRIAGE RETURN, i.e. \r - write(b, "\\r") - elseif c === '\u005c' # REVERSE SOLIDUS, i.e. \ - write(b, "\\\\") - elseif c === '\u0022' # QUOTATION MARK, i.e. " - write(b, "\\\"") - elseif c === '\u0027' # APOSTROPHE, i.e. ' - write(b, "\\'") - elseif c === '\u2028' # LINE SEPARATOR - write(b, "\\u2028") - elseif c === '\u2029' # PARAGRAPH SEPARATOR - write(b, "\\u2029") - else - write(b, c) - end - end - String(take!(b)) end function domify(ctx, navnode, node) @@ -837,12 +800,13 @@ function domify(ctx, navnode, node::Documents.DocsNode) @tags a code div section span # push to search index - sib = SearchIndexBuffer(ctx, navnode) - sib.loc = node.anchor.id - sib.title = string(node.object.binding) - sib.category = Symbol(Utilities.doccat(node.object)) - mdflatten(sib.buffer, node.docstr) - search_flush(sib) + rec = SearchRecord(ctx, navnode; + loc=node.anchor.id, + title=string(node.object.binding), + category=Utilities.doccat(node.object), + text = mdflatten(node.docstr)) + + push!(ctx.search_index, rec) section[".docstring"]( div[".docstring-header"]( diff --git a/test/htmlwriter.jl b/test/htmlwriter.jl index abc58aa706..82c6ab7dd8 100644 --- a/test/htmlwriter.jl +++ b/test/htmlwriter.jl @@ -2,7 +2,8 @@ module HTMLWriterTests using Test -import Documenter.Writers.HTMLWriter: jsescape, generate_version_file, expand_versions +import JSON +import Documenter.Writers.HTMLWriter: generate_version_file, expand_versions function verify_version_file(versionfile, entries) @test isfile(versionfile) @@ -16,21 +17,6 @@ function verify_version_file(versionfile, entries) end @testset "HTMLWriter" begin - @test jsescape("abc123") == "abc123" - @test jsescape("▶αβγ") == "▶αβγ" - @test jsescape("") == "" - - @test jsescape("a\nb") == "a\\nb" - @test jsescape("\r\n") == "\\r\\n" - @test jsescape("\\") == "\\\\" - - @test jsescape("\"'") == "\\\"\\'" - - # Ref: #639 - @test jsescape("\u2028") == "\\u2028" - @test jsescape("\u2029") == "\\u2029" - @test jsescape("policy to
 delete.") == "policy to\\u2028 delete." - mktempdir() do tmpdir versionfile = joinpath(tmpdir, "versions.js") versions = ["stable", "dev", From 2dcb10281535c3642e964a235faea448b13d3e01 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Sun, 24 Feb 2019 14:56:30 +0100 Subject: [PATCH 2/5] Travis dev Documenter for docs/pdf as well To try to fix error about missing JSON --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a72b55a111..cd62c046c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ jobs: name: "HTML" after_success: skip - script: - - julia --project=docs/pdf/ -e 'using Pkg; Pkg.instantiate()' + - julia --project=docs/pdf/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()' - julia --project=docs/pdf/ docs/pdf/make.jl name: "PDF" after_success: skip From 301edc38362f49e11b5b698b5a259a2ac119d0b6 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Sun, 24 Feb 2019 19:44:42 +0100 Subject: [PATCH 3/5] remove checked in Manifest And git ignore them all. Also adds Documenter to the Project.toml in docs. --- .gitignore | 3 +- docs/Project.toml | 1 + docs/pdf/Manifest.toml | 93 ------------------------------------------ 3 files changed, 2 insertions(+), 95 deletions(-) delete mode 100644 docs/pdf/Manifest.toml diff --git a/.gitignore b/.gitignore index 69ef5417e2..0b84a18baf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ *.jl.cov *.jl.*.cov *.jl.mem -/Manifest.toml +Manifest.toml test/examples/builds/ test/formats/builds/ @@ -11,4 +11,3 @@ test/errors/build/ docs/build/ docs/pdf/build/ docs/site/ -docs/Manifest.toml diff --git a/docs/Project.toml b/docs/Project.toml index c12fbd9d91..2742e84b27 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,2 +1,3 @@ [deps] +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" DocumenterTools = "35a29f4d-8980-5a13-9543-d66fff28ecb8" diff --git a/docs/pdf/Manifest.toml b/docs/pdf/Manifest.toml deleted file mode 100644 index bd3b22b388..0000000000 --- a/docs/pdf/Manifest.toml +++ /dev/null @@ -1,93 +0,0 @@ -[[Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" - -[[Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" - -[[Distributed]] -deps = ["LinearAlgebra", "Random", "Serialization", "Sockets"] -uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" - -[[DocStringExtensions]] -deps = ["LibGit2", "Markdown", "Pkg", "Test"] -git-tree-sha1 = "1df01539a1c952cef21f2d2d1c092c2bcf0177d7" -uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.6.0" - -[[Documenter]] -deps = ["Base64", "DocStringExtensions", "InteractiveUtils", "LibGit2", "Logging", "Markdown", "Pkg", "REPL", "Random", "Test", "Unicode"] -path = "../.." -uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "0.20.0+" - -[[DocumenterLaTeX]] -deps = ["Documenter"] -git-tree-sha1 = "653299370be20ff580bccd707dc9f360c0852d7f" -repo-rev = "fe/ltx" -repo-url = "https://github.com/JuliaDocs/DocumenterLaTeX.jl.git" -uuid = "cd674d7a-5f81-5cf3-af33-235ef1834b99" -version = "0.1.0" - -[[DocumenterTools]] -deps = ["Base64", "DocStringExtensions", "LibGit2", "Pkg", "Test"] -git-tree-sha1 = "f5803a9c2c23ff226e8eab2df7ac4c75e77a0d53" -uuid = "35a29f4d-8980-5a13-9543-d66fff28ecb8" -version = "0.1.0" - -[[InteractiveUtils]] -deps = ["LinearAlgebra", "Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" - -[[LibGit2]] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" - -[[Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[LinearAlgebra]] -deps = ["Libdl"] -uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - -[[Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[Markdown]] -deps = ["Base64"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" - -[[Pkg]] -deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" - -[[Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" - -[[REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets"] -uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[Random]] -deps = ["Serialization"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" - -[[Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - -[[Test]] -deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] -uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[UUIDs]] -deps = ["Random"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" - -[[Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" From 73281c287c3ecfc13af99601e1d19358026f9be8 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Tue, 26 Feb 2019 09:54:47 +0100 Subject: [PATCH 4/5] convert JSON into valid JS By escaping two Unicode characters --- src/Writers/HTMLWriter.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Writers/HTMLWriter.jl b/src/Writers/HTMLWriter.jl index 3040170367..444b640b97 100644 --- a/src/Writers/HTMLWriter.jl +++ b/src/Writers/HTMLWriter.jl @@ -257,8 +257,12 @@ function render(doc::Documents.Document, settings::HTML=HTML()) open(joinpath(doc.user.build, ctx.search_index_js), "w") do io println(io, "var documenterSearchIndex = {\"docs\":") - JSON.print(io, ctx.search_index) - println(io, "\n}") + # convert Vector{SearchRecord} to a JSON string, and escape two Unicode + # characters since JSON is not a JS subset, and we want JS here + # ref http://timelessrepo.com/json-isnt-a-javascript-subset + escapes = ('\u2028' => "\\u2028", '\u2029' => "\\u2029") + js = reduce(replace, escapes, init=JSON.json(ctx.search_index)) + println(io, js, "\n}") end end From 3688874893380fbb3e9edbd66e979fc1036b5d88 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Sun, 3 Mar 2019 17:03:41 +1300 Subject: [PATCH 5/5] Add CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a645f49f4b..54f642c238 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ * ![Enhancement][badge-enhancement] Reporting of failed doctests are now using the logging system to be consistent with the rest of Documenter's output. ([#958][github-958]) +* ![Enhancement][badge-enhancement] The construction of the search index in the HTML output has been refactored to make it easier to use with other search backends in the future. The structure of the generated search index has also been modified, which can yield slightly different search results. Documenter now depends on the lightweight [JSON.jl][json-jl] package. ([#966][github-966]) + * ![Bugfix][badge-bugfix] Paths in `include` calls in `@eval`, `@example`, `@repl` and `jldoctest` blocks are now interpreted to be relative `pwd`, which is set to the output directory of the resulting file. ([#941][github-941]) @@ -265,11 +267,13 @@ [github-958]: https://github.com/JuliaDocs/Documenter.jl/pull/958 [github-959]: https://github.com/JuliaDocs/Documenter.jl/pull/959 [github-960]: https://github.com/JuliaDocs/Documenter.jl/pull/960 +[github-966]: https://github.com/JuliaDocs/Documenter.jl/pull/966 [github-967]: https://github.com/JuliaDocs/Documenter.jl/pull/967 [github-971]: https://github.com/JuliaDocs/Documenter.jl/pull/971 [documenterlatex]: https://github.com/JuliaDocs/DocumenterLaTeX.jl [documentermarkdown]: https://github.com/JuliaDocs/DocumenterMarkdown.jl +[json-jl]: https://github.com/JuliaIO/JSON.jl [badge-breaking]: https://img.shields.io/badge/BREAKING-red.svg