Adjust generated code to work better with cross-references and be more complete

.github/workflows/generate_bindings.yml

@@ -29,11 +29,23 @@ jobs:
       - name: Run the generator
         run: ./gen/generate.sh
 
-      - name: Create Pull Request
+      - name: Create Pull Request (on push)
+        if: ${{ github.event_name == 'push' }}
         uses: peter-evans/create-pull-request@v6
         with:
           commit-message: "Regenerate bindings"
           title: "Regenerate bindings"
           reviewers: |
             quinnj
             Octogonapus
+
+      - name: Create Pull Request (on PR)
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: peter-evans/create-pull-request@v6
+        with:
+          base: ${{ github.head_ref }}
+          commit-message: "Regenerate bindings"
+          title: "Regenerate bindings"
+          reviewers: |
+            quinnj
+            Octogonapus

docs/make.jl

@@ -11,6 +11,8 @@ makedocs(;
         prettyurls=get(ENV, "CI", "false") == "true",
         canonical="https://github.com/JuliaServices/LibAwsCommon.jl",
         assets=String[],
+        size_threshold=2_000_000, # 2 MB, we generate about 1 MB page
+        size_threshold_warn=2_000_000,
     ),
     pages=["Home" => "index.md"],
 )

gen/Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.10.2"
 manifest_format = "2.0"
-project_hash = "17529965341e648e57dcbc863dd11933859fd679"
+project_hash = "981bed63f0072e15b9d8cfedf2a309eaddd66050"
 
 [[deps.ArgTools]]
 uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
@@ -21,9 +21,9 @@ version = "0.5.0"
 
 [[deps.Clang]]
 deps = ["CEnum", "Clang_jll", "Downloads", "Pkg", "TOML"]
-git-tree-sha1 = "846054622cb22aa63b5d51b5d84ec04b42d4d587"
+git-tree-sha1 = "2397d5da17ba4970f772a9888b208a0a1d77eb5d"
 uuid = "40e3b903-d033-50b4-a0cc-940c62c95e31"
-version = "0.17.8"
+version = "0.18.3"
 
 [[deps.Clang_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "TOML", "Zlib_jll", "libLLVM_jll"]
@@ -57,9 +57,9 @@ version = "1.3.1"
 
 [[deps.Git_jll]]
 deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"]
-git-tree-sha1 = "12945451c5d0e2d0dca0724c3a8d6448b46bbdf9"
+git-tree-sha1 = "d18fb8a1f3609361ebda9bf029b60fd0f120c809"
 uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb"
-version = "2.44.0+1"
+version = "2.44.0+2"
 
 [[deps.HistoricalStdlibVersions]]
 git-tree-sha1 = "c8b04a26eaa706b4da6968dfc27ae2d030547cba"

gen/Project.toml

@@ -4,6 +4,6 @@ JLLPrefixes = "afc68a34-7891-4c5a-9da1-1c62935e7b0d"
 aws_c_common_jll = "73048d1d-b8c4-5092-a58d-866c5e8d1e50"
 
 [compat]
-Clang = "0.17"
+Clang = "0.18.3"
 JLLPrefixes = "0.3"
 aws_c_common_jll = "=0.9.14"

gen/generate.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 dir=$(dirname "$0")
 julia --project="$dir" -e 'using Pkg; Pkg.instantiate()'
-julia --project="$dir" "$dir/generator.jl"
+julia --project="$dir" -t auto "$dir/generator.jl"

gen/generator.jl

@@ -5,44 +5,95 @@ import aws_c_common_jll
 
 cd(@__DIR__)
 
-function remove_itt_symbols!(dag::ExprDAG)
-    for i in eachindex(dag.nodes)
-        node = dag.nodes[i]
-        for expr in get_exprs(node)
-            node_name = if expr.head == :function
-                if expr.args[1].args[1] isa Expr # function is Module.name instead of just name
-                    expr.args[1].args[1].args[2]
-                else
-                    expr.args[1].args[1]
-                end
-            elseif expr.head == :struct
-                if expr.args[2] isa Expr # struct has type parameter
-                    expr.args[2].args[1]
-                else
-                    expr.args[2]
-                end
-            elseif expr.head == :const
+function node_is_ignored(node)
+    for expr in get_exprs(node)
+        node_name = if expr.head == :function
+            if expr.args[1].args[1] isa Expr # function is Module.name instead of just name
+                expr.args[1].args[1].args[2]
+            else
                 expr.args[1].args[1]
             end
-            # remove the node by renaming it to IGNORED, which we include in the generator's ignorelist
-            if contains(lowercase(string(node_name)), "itt")
-                dag.nodes[i] = ExprNode(:IGNORED, node.type, node.cursor, node.exprs, node.premature_exprs, node.adj)
+        elseif expr.head == :struct
+            if expr.args[2] isa Expr # struct has type parameter
+                expr.args[2].args[1]
+            else
+                expr.args[2]
             end
+        elseif expr.head == :const
+            expr.args[1].args[1]
+        end
+        return contains(lowercase(string(node_name)), "itt")
+    end
+    return false
+end
+
+function remove_itt_symbols!(dag::ExprDAG)
+    for i in eachindex(dag.nodes)
+        # remove the node by renaming it to IGNORED, which we include in the generator's ignorelist
+        node = dag.nodes[i]
+        if node_is_ignored(node)
+            dag.nodes[i] = ExprNode(:IGNORED, node.type, node.cursor, node.exprs, node.premature_exprs, node.adj)
         end
     end
     return nothing
 end
 
+const refs_to_remove = ("AWS_CONTAINER_OF", "AWS_STATIC_STRING_FROM_LITERAL",)
+
+# This is called if the docs generated from the extract_c_comment_style method did not generate any lines.
+# We need to generate at least some docs so that cross-references work with Documenter.jl.
+function get_docs(node, docs)
+    # The macro node types (except for MacroDefault) seem to not generate code, but they will still emit docs and then
+    # you end up with docs stacked on top of each other, which is a Julia LoadError.
+    if node.type isa Generators.AbstractMacroNodeType && !(node.type isa Generators.MacroDefault)
+        return String[]
+    end
+
+    # don't generate empty docs because it makes Documenter.jl mad
+    if isempty(docs)
+        return ["Documentation not found."]
+    end
+
+    # remove references to things which don't exist because it causes Documenter.jl's cross_references check to fail
+    for ref in refs_to_remove
+        for doci in eachindex(docs)
+            docs[doci] = replace(docs[doci], "[`$ref`](@ref)" => "`$ref`")
+        end
+    end
+
+    # fix other random stuff
+    for doci in eachindex(docs)
+        # fix some code that gets bogus references inserted
+        docs[doci] = replace(docs[doci], "for (struct [`aws_hash_iter`](@ref) iter = [`aws_hash_iter_begin`](@ref)(&map); ![`aws_hash_iter_done`](@ref)(&iter); [`aws_hash_iter_next`](@ref)(&iter)) { const key\\_type key = *(const key\\_type *)iter.element.key; value\\_type value = *(value\\_type *)iter.element.value; // etc. }" => "`for (struct aws_hash_iter iter = aws_hash_iter_begin(&map); !aws_hash_iter_done(&iter); aws_hash_iter_next(&iter)) { const key\\_type key = *(const key\\_type *)iter.element.key; value\\_type value = *(value\\_type *)iter.element.value; // etc. }`")
+    end
+
+    return docs
+end
+
+function should_skip_target(target)
+    # aws_c_common_jll does not support i686 windows https://github.com/JuliaPackaging/Yggdrasil/blob/bbab3a916ae5543902b025a4a873cf9ee4a7de68/A/aws_c_common/build_tarballs.jl#L48-L49
+    return target == "i686-w64-mingw32"
+end
+
+# download toolchains in parallel
+Threads.@threads for target in JLLEnvs.JLL_ENV_TRIPLES
+    if should_skip_target(target)
+        continue
+    end
+    get_default_args(target) # downloads the toolchain
+end
+
 for target in JLLEnvs.JLL_ENV_TRIPLES
-    if target == "i686-w64-mingw32"
-        # aws_c_common_jll does not support i686 windows https://github.com/JuliaPackaging/Yggdrasil/blob/bbab3a916ae5543902b025a4a873cf9ee4a7de68/A/aws_c_common/build_tarballs.jl#L48-L49
+    if should_skip_target(target)
         continue
     end
     options = load_options(joinpath(@__DIR__, "generator.toml"))
     options["general"]["output_file_path"] = joinpath(@__DIR__, "..", "lib", "$target.jl")
+    options["general"]["callback_documentation"] = get_docs
 
     header_dirs = []
     args = get_default_args(target)
+    push!(args, "-fparse-all-comments")
     inc = JLLEnvs.get_pkg_include_dir(aws_c_common_jll, target)
     push!(args, "-I$inc")
     push!(header_dirs, inc)