Merge branch 'variadic' into bitfield

docs/make.jl

@@ -12,7 +12,8 @@ makedocs(;
     ),
     pages=[
         "Introduction" => "index.md",
-        "Tutorial" => "tutorial.md",
+        "Generator Tutorial" => "generator.md",
+        "LibClang Tutorial" => "tutorial.md",
         "API Reference" => "api.md",
     ],
 )

docs/src/generator.md

@@ -0,0 +1,32 @@
+# Generator Tutorial
+
+## Variadic Function
+With the help of `@ccall` macro, variadic C functions can be from Julia, for example, `@ccall printf("%d\n"::Cstring; 123::Cint)::Cint` can be used to call the C function `printf`. Note that those arguments after the semicolon `;` are variadic arguments.
+
+If `wrap_variadic_function` in `codegen` section of options is set to `true`, `Clang.jl` will generate wrappers for variadic C functions, for example, `printf` will be wrapped as:
+```julia
+@generated function printf(fmt, va_list...)
+        :(@ccall(libexample.printf(fmt::Ptr{Cchar}; $(to_c_type_pairs(va_list)...))::Cint))
+    end
+```
+It can be called just like normal Julia functions without specifying types: `LibExample.printf("%d\n", 123)`.
+
+!!! note
+    Although variadic functions are supported, the C type `va_list` can not be used from Julia.
+
+### Type Correspondence
+
+However, variadic C functions must be called with the correct argument types. The most useful ones are listed below:
+
+| C type                              | ccall signature                                  | Julia type                             |
+|-------------------------------------|--------------------------------------------------|----------------------------------------|
+| Integers and floating point numbers | the same type                                    | the same type                          |
+| Struct `T`                          | a concrete Julia struct `T` with the same layout | `T`                                    |
+| Pointer (`T*`)                      | `Ref{T}` or `Ptr{T}`                             | `Ref{T}` or `Ptr{T}` or any array type |
+| String (`char*`)                    | `Cstring` or `Ptr{Cchar}`                        | `String`                               |
+
+As observed from the table, if you want to pass strings or arrays to C, you need to annotate the type as `Ptr{T}` or `Ref{T}` (or `Cstring`), otherwise the struct that represents the `String` or `Array`type instead of the buffer itself will be passed. There are two methods to pass arguments of these types:
+* directly use @ccall macro: `@ccall printf("%s\n"; "hello"::Cstring)::Cint`. You can also create wrappers for common use cases of this.
+* overload `to_c_type` to map Julia type to correct ccall signature type: add `to_c_type(::Type{String}) = Cstring` to prologue (prologue can be added by setting `prologue_file_path` in options). Then all arguments of String will be annotated as `Cstring`.
+
+For a complete tutorial on calling C functions, refer to [Calling C and Fortran Code](https://docs.julialang.org/en/v1/manual/calling-c-and-fortran-code/#Calling-C-and-Fortran-Code) in the Julia manual.

docs/src/index.md

@@ -53,23 +53,57 @@ for header in find_std_headers()
 end
 ```
 
-#### Backward compatibility
-If you miss those old behaviors before v0.8, please `Pkg.pin` the package to v0.8 and
-make the following change in your old generator script:
-```julia
-using Clang: CLANG_INCLUDE
-using Clang.Deprecated.wrap_c
-using Clang.Deprecated.cindex
-```
 
 ## Build a custom C-bindings generator
 A custom C-bindings generator tends to be used on large codebases, often with multiple API versions to support. Building a generator requires some customization effort, so for small libraries the initial
 investment may not pay off.
 
 The above-mentioned C-bindings generator only exposes several entry points for customization.
 In fact, it's actually not that hard to directly build your own C-bindings generator,
-for example, the following script is used for generating `LibClang`, you could refer to [Tutorial](@ref) for
+for example, the following script is used for generating `LibClang`, you could refer to [LibClang Tutorial](@ref) for
 further details.
+
+Write a config file `generator.toml`:
+```
+[general]
+library_name = "libclang"
+output_file_path = "./LibClang.jl"
+module_name = "LibClang"
+jll_pkg_name = "Clang_jll"
+export_symbol_prefixes = ["CX", "clang_"]
+```
+
+and a Julia script `generator.jl`:
+```julia
+using Clang.Generators
+using Clang.LibClang.Clang_jll
+
+cd(@__DIR__)
+
+include_dir = joinpath(Clang_jll.artifact_dir, "include") |> normpath
+clang_dir = joinpath(include_dir, "clang-c")
+
+options = load_options(joinpath(@__DIR__, "generator.toml"))
+
+# add compiler flags, e.g. "-DXXXXXXXXX"
+args = get_default_args()
+push!(args, "-I$include_dir")
+
+headers = [joinpath(clang_dir, header) for header in readdir(clang_dir) if endswith(header, ".h")]
+# there is also an experimental `detect_headers` function for auto-detecting top-level headers in the directory
+# headers = detect_headers(clang_dir, args)
+
+# create context
+ctx = create_context(headers, args, options)
+
+# run generator
+build!(ctx)
+```
+
+Please refer to [this toml file](https://github.com/JuliaInterop/Clang.jl/blob/master/gen/generator.toml) for a full list of configuration options.
+
+### v0.12 example
+The generator above is the recommended generator for v0.14 and above. There was another generator wrappper before, here is an example.
 ```julia
 using Clang
 using Clang_jll # `pkg> activate Clang`

docs/src/tutorial.md

@@ -1,4 +1,4 @@
-# Tutorial
+# LibClang Tutorial
 Clang is an open-source compiler built on the LLVM framework and targeting C, C++, and Objective-C (LLVM is also the JIT backend for Julia). Due to a highly modular design, Clang has in recent years become the core of a growing number of projects utilizing pieces of the compiler, such as tools for source-to-source translation, static analysis and security evaluation, and editor tools for code completion, formatting, etc.
 
 While LLVM and Clang are written in C++, the Clang project maintains a C-exported interface called "libclang" which provides access to the abstract syntax tree and type representations. Thanks to the ubiquity of support for C calling conventions, a number of languages have utilized libclang as a basis for tooling related to C and C++.
@@ -35,17 +35,17 @@ Parsing and querying the fields of this struct requires just a few lines of code
 ```julia
 julia> using Clang
 
-julia> trans_unit = parse_header("example.h")
+julia> trans_unit = Clang.parse_header(Index(), "example.h")
 TranslationUnit(Ptr{Nothing} @0x00007fe13cdc8a00, Index(Ptr{Nothing} @0x00007fe13cc8dde0, 0, 1))
 
-julia> root_cursor = getcursor(trans_unit)
+julia> root_cursor = Clang.getTranslationUnitCursor(trans_unit)
 CLCursor (CLTranslationUnit) example.h
 
-julia> struct_cursor = search(root_cursor, "ExStruct")[1]
+julia> struct_cursor = search(root_cursor, "ExStruct") |> only
 CLCursor (CLStructDecl) ExStruct
 
 julia> for c in children(struct_cursor)  # print children
-           println("Cursor: ", c, "\n  Kind: ", kind(c), "\n  Name: ", name(c), "\n  Type: ", getCursorType(c))
+           println("Cursor: ", c, "\n  Kind: ", kind(c), "\n  Name: ", name(c), "\n  Type: ", Clang.getCursorType(c))
        end
 Cursor: CLCursor (CLFieldDecl) kind
   Kind: CXCursor_FieldDecl(6)
@@ -154,7 +154,7 @@ To find the cursor for this function declaration, we use function `search` to re
 ```julia
 julia> using Clang.LibClang  # CXCursor_FunctionDecl is exposed from LibClang
 
-julia> fdecl = search(root_cursor, CXCursor_FunctionDecl)[end]
+julia> fdecl = search(root_cursor, CXCursor_FunctionDecl) |> only
 CLCursor (CLFunctionDecl) ExFunction(int, char *, float *)
 
 julia> fdecl_children = [c for c in children(fdecl)]
@@ -166,15 +166,15 @@ julia> fdecl_children = [c for c in children(fdecl)]
 ```
 The first three children are `CLParmDecl` cursors with the same name as the arguments in the function signature. Checking the types of the `CLParmDecl` cursors indicates a similarity to the function signature:
 ```julia
-julia> [getCursorType(t) for t in fdecl_children[1:3]]
+julia> [Clang.getCursorType(t) for t in fdecl_children[1:3]]
 3-element Array{CLType,1}:
  CLType (CLInt)     
  CLType (CLPointer)
  CLType (CLPointer)
 ```
 And, finally, retrieving the target type of each `CLPointer` argument confirms that these cursors represent the function argument type declaration:
 ```julia
-julia> [getPointeeType(getCursorType(t)) for t in fdecl_children[2:3]]
+julia> [Clang.getPointeeType(Clang.getCursorType(t)) for t in fdecl_children[2:3]]
 2-element Array{CLType,1}:
  CLType (CLChar_S)
  CLType (CLFloat)  
@@ -188,11 +188,11 @@ printind(ind::Int, st...) = println(join([repeat(" ", 2*ind), st...]))
 printobj(cursor::CLCursor) = printobj(0, cursor)
 printobj(t::CLType) = join(typeof(t), " ", spelling(t))
 printobj(t::CLInt) = t
-printobj(t::CLPointer) = getPointeeType(t)
+printobj(t::CLPointer) = Clang.getPointeeType(t)
 printobj(ind::Int, t::CLType) = printind(ind, printobj(t))
 
 function printobj(ind::Int, cursor::Union{CLFieldDecl, CLParmDecl})
-    printind(ind+1, typeof(cursor), " ", printobj(getCursorType(cursor)), " ", name(cursor))
+    printind(ind+1, typeof(cursor), " ", printobj(Clang.getCursorType(cursor)), " ", name(cursor))
 end
 
 function printobj(ind::Int, node::Union{CLCursor, CLStructDecl, CLCompoundStmt,

gen/generator.toml

@@ -132,6 +132,9 @@ opaque_as_mutable_struct = true
 # if true, use Julia 1.5's new `@ccall` macro 
 use_ccall_macro = true
 
+# if true, variadic functions are wrapped with `@ccall` macro. Otherwise variadic functions are ignored.
+wrap_variadic_function = false
+
 # generate getproperty/setproperty! methods for the types in the following list
 field_access_method_list = []
 

src/generator/codegen.jl

@@ -19,19 +19,28 @@ function _get_func_name(cursor, options)
     return library_name
 end
 
-function emit!(dag::ExprDAG, node::ExprNode{FunctionProto}, options::Dict; args...)
+"Get argument names and types. Return (names, types)."
+function _get_func_arg(cursor, options, dag)
+    # argument names
     conflict_syms = get(options, "function_argument_conflict_symbols", [])
-
-    cursor = node.cursor
-
-    library_name = _get_func_name(cursor, options)
-    library_expr = Meta.parse(library_name)
-
-    func_name = Symbol(spelling(cursor))
     func_args = get_function_args(cursor)
-    arg_types = [getArgType(getCursorType(cursor), i - 1) for i in 1:length(func_args)]
-    ret_type = translate(tojulia(getCursorResultType(cursor)), options)
+    # handle unnamed args
+    arg_names = Vector{Symbol}(undef, length(func_args))
+    for (i, arg) in enumerate(func_args)
+        ns = Symbol(name(arg))
+        safe_name = make_name_safe(ns)
+        safe_name = isempty(safe_name) ? "arg$i" : safe_name
+        # handle name collisions
+        if haskey(dag.tags, ns) || haskey(dag.ids, ns) || haskey(dag.ids_extra, ns)
+            safe_name *= "_"
+        elseif safe_name ∈ conflict_syms
+            safe_name = "_" * safe_name
+        end
+        arg_names[i] = Symbol(safe_name)
+    end
 
+    # argument types
+    arg_types = [getArgType(getCursorType(cursor), i - 1) for i in 1:length(func_args)]
     args = Union{Expr,Symbol}[translate(tojulia(arg), options) for arg in arg_types]
     for (i, arg) in enumerate(args)
         # array function arguments should decay to pointers
@@ -50,21 +59,23 @@ function emit!(dag::ExprDAG, node::ExprNode{FunctionProto}, options::Dict; args.
             end
         end
     end
+    return arg_names, args
+end
 
-    # handle unnamed args
-    arg_names = Vector{Symbol}(undef, length(func_args))
-    for (i, arg) in enumerate(func_args)
-        ns = Symbol(name(arg))
-        safe_name = make_name_safe(ns)
-        safe_name = isempty(safe_name) ? "arg$i" : safe_name
-        # handle name collisions
-        if haskey(dag.tags, ns) || haskey(dag.ids, ns) || haskey(dag.ids_extra, ns)
-            safe_name *= "_"
-        elseif safe_name ∈ conflict_syms
-            safe_name = "_" * safe_name
-        end
-        arg_names[i] = Symbol(safe_name)
-    end
+function _get_func_return_type(cursor, options)
+    translate(tojulia(getCursorResultType(cursor)), options)
+end
+
+function emit!(dag::ExprDAG, node::ExprNode{FunctionProto}, options::Dict; args...)
+    cursor = node.cursor
+
+    library_name = _get_func_name(cursor, options)
+    library_expr = Meta.parse(library_name)
+
+    func_name = Symbol(spelling(cursor))
+
+    arg_names, args = _get_func_arg(cursor, options, dag)
+    ret_type = _get_func_return_type(cursor, options)
 
     is_strict_typed = get(options, "is_function_strictly_typed", false)
     signature = if is_strict_typed
@@ -131,7 +142,35 @@ function emit!(dag::ExprDAG, node::ExprNode{FunctionNoProto}, options::Dict; arg
 end
 
 function emit!(dag::ExprDAG, node::ExprNode{FunctionVariadic}, options::Dict; args...)
-    # TODO: add impl
+    # @ccall is needed to support variadic argument
+    use_ccall_macro = get(options, "use_ccall_macro", true)
+    if use_ccall_macro
+        cursor = node.cursor
+        is_strict_typed = get(options, "is_function_strictly_typed", false)
+        arg_names, args = _get_func_arg(cursor, options, dag)
+        ret_type = _get_func_return_type(cursor, options)
+        library_name = _get_func_name(cursor, options)
+        library_expr = Meta.parse(library_name)
+        func_name = Symbol(spelling(cursor))
+
+        signature = is_strict_typed ? efunsig(func_name, arg_names, args) : Expr(:call, func_name, arg_names...)
+        push!(signature.args, :(va_list...))
+
+        fixed_args = map(arg_names, args) do name, type
+            :($name::$type)
+        end
+        va_list_expr = Expr(:$, :(to_c_type_pairs(va_list)...))
+        ccall_body = :($library_expr.$func_name($(fixed_args...); $va_list_expr)::$ret_type)
+        ccall_expr = Expr(:macrocall, Symbol("@ccall"), nothing, ccall_body) 
+        body = quote
+            $(Meta.quot(ccall_expr))
+        end
+
+        generated = Expr(:function, signature, body)
+        ex = Expr(:macrocall, Symbol("@generated"), nothing, generated)
+        rm_line_num_node!(ex)
+        push!(node.exprs, ex)
+    end
     return dag
 end
 

src/generator/passes.jl

@@ -959,6 +959,7 @@ ProloguePrinter(file::AbstractString; info=true) = ProloguePrinter(file, info)
 
 function (x::ProloguePrinter)(dag::ExprDAG, options::Dict)
     general_options = get(options, "general", Dict())
+    codegen_options = get(options, "codegen", Dict())
     log_options = get(general_options, "log", Dict())
     show_info = get(log_options, "ProloguePrinter_log", x.show_info)
     module_name = get(general_options, "module_name", "")
@@ -967,6 +968,7 @@ function (x::ProloguePrinter)(dag::ExprDAG, options::Dict)
     prologue_file_path = get(general_options, "prologue_file_path", "")
     use_native_enum = get(general_options, "use_julia_native_enum_type", false)
     print_CEnum = get(general_options, "print_using_CEnum", true)
+    wrap_variadic_function = get(codegen_options, "wrap_variadic_function", false)
 
     show_info && @info "[ProloguePrinter]: print to $(x.file)"
     open(x.file, "w") do io
@@ -995,6 +997,16 @@ function (x::ProloguePrinter)(dag::ExprDAG, options::Dict)
             println(io, "using CEnum")
             println(io)
         end
+
+        if wrap_variadic_function
+            println(io, """
+            to_c_type(t::Type) = t
+            to_c_type_pairs(va_list) = map(enumerate(to_c_type.(va_list))) do (ind, type)
+                :(va_list[\$ind]::\$type)
+            end
+            """)
+        end
+
         # print prelogue patches
         if !isempty(prologue_file_path)
             println(io, read(prologue_file_path, String))

src/generator/print.jl

@@ -23,6 +23,16 @@ function pretty_print(io, node::ExprNode{FunctionProto}, options::Dict)
     return nothing
 end
 
+function pretty_print(io, node::ExprNode{FunctionVariadic}, options::Dict)
+    isempty(node.exprs) && return
+    println(io, "# automatic type deduction for variadic arguments may not be what you want, please use with caution")
+    for expr in node.exprs
+        println(io, expr)
+        println(io)
+    end
+    return nothing
+end
+
 function pretty_print(io, node::ExprNode{FunctionNoProto}, options::Dict)
     @assert !isempty(node.exprs)
     file, line, col = get_file_line_column(node.cursor)

src/trans_unit.jl

@@ -111,7 +111,7 @@ See also [`parse_headers`](@ref).
 function parse_header(
     index::Index,
     header::AbstractString,
-    args::Vector{String}=[],
+    args::Vector{String}=String[],
     flags=CXTranslationUnit_None,
 )
     # TODO: support parsing in-memory with CXUnsavedFile arg