Add support for multipart/form-data POST data parsing (#41)

* Add support for multipart/form-data POST data parsing

* client and server handling of multipart/form-data

- client and server handling of multipart/form-data
- update test to add petstore upload_file
- add tests for forms

* also test required form param condition

Updated test code to test required form params. Test code re-generated with fixed codegen.
Bumped version in prep for tagging

---------

Co-authored-by: tan <[email protected]>

Project.toml

@@ -4,7 +4,7 @@ keywords = ["Swagger", "OpenAPI", "REST"]
 license = "MIT"
 desc = "OpenAPI server and client helper for Julia"
 authors = ["JuliaHub Inc."]
-version = "0.1.12"
+version = "0.1.13"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

src/client.jl

@@ -7,6 +7,7 @@ using MbedTLS
 using Dates
 using TimeZones
 using LibCURL
+using HTTP
 
 import Base: convert, show, summary, getproperty, setproperty!, iterate
 import ..OpenAPI: APIModel, UnionAPIModel, OneOfAPIModel, AnyOfAPIModel, APIClientImpl, OpenAPIException, InvocationException, to_json, from_json, validate_property, property_type
@@ -227,12 +228,40 @@ function prep_args(ctx::Ctx)
     isempty(ctx.file) && (ctx.body === nothing) && isempty(ctx.form) && !("Content-Length" in keys(ctx.header)) && (ctx.header["Content-Length"] = "0")
     headers = ctx.header
     body = nothing
+
+    header_pairs = [convert(HTTP.Header, p) for p in headers]
+    content_type_set = HTTP.header(header_pairs, "Content-Type", nothing)
+    if !isnothing(content_type_set)
+        content_type_set = lowercase(content_type_set)
+    end
+
     if !isempty(ctx.form)
-        headers["Content-Type"] = "application/x-www-form-urlencoded"
-        body = URIs.escapeuri(ctx.form)
+        if !isnothing(content_type_set) && content_type_set !== "multipart/form-data" && content_type_set !== "application/x-www-form-urlencoded"
+            throw(OpenAPIException("Content type already set to $content_type_set. To send form data, it must be multipart/form-data or application/x-www-form-urlencoded."))
+        end
+        if isnothing(content_type_set)
+            if !isempty(ctx.file)
+                headers["Content-Type"] = content_type_set = "multipart/form-data"
+            else
+                headers["Content-Type"] = content_type_set = "application/x-www-form-urlencoded"
+            end
+        end
+        if content_type_set == "application/x-www-form-urlencoded"
+            body = URIs.escapeuri(ctx.form)
+        else
+            # we shall process it along with file uploads where we send multipart/form-data
+        end
     end
 
-    if !isempty(ctx.file)
+    if !isempty(ctx.file) || (content_type_set == "multipart/form-data")
+        if !isnothing(content_type_set) && content_type_set !== "multipart/form-data"
+            throw(OpenAPIException("Content type already set to $content_type_set. To send file, it must be multipart/form-data."))
+        end
+
+        if isnothing(content_type_set)
+            headers["Content-Type"] = content_type_set = "multipart/form-data"
+        end
+
         # use a separate downloader for file uploads
         # until we have something like https://github.com/JuliaLang/Downloads.jl/pull/148
         downloader = Downloads.Downloader()
@@ -249,19 +278,25 @@ function prep_args(ctx::Ctx)
                 LibCURL.curl_mime_filedata(part, _v)
                 # TODO: make provision to call curl_mime_type in future?
             end
+            for (_k,_v) in ctx.form
+                # add multipart sections for form data as well
+                part = LibCURL.curl_mime_addpart(mime)
+                LibCURL.curl_mime_name(part, _k)
+                LibCURL.curl_mime_data(part, _v, length(_v))
+            end
             Downloads.Curl.setopt(easy, LibCURL.CURLOPT_MIMEPOST, mime)
         end
         kwargs[:downloader] = downloader
     end
 
     if ctx.body !== nothing
         (isempty(ctx.form) && isempty(ctx.file)) || throw(OpenAPIException("Can not send both form-encoded data and a request body"))
-        if is_json_mime(get(ctx.header, "Content-Type", "application/json"))
+        if is_json_mime(something(content_type_set, "application/json"))
             body = to_json(ctx.body)
-        elseif ("application/x-www-form-urlencoded" == ctx.header["Content-Type"]) && isa(ctx.body, Dict)
+        elseif ("application/x-www-form-urlencoded" == content_type_set) && isa(ctx.body, Dict)
             body = URIs.escapeuri(ctx.body)
-        elseif isa(ctx.body, APIModel) && isempty(get(ctx.header, "Content-Type", ""))
-            headers["Content-Type"] = "application/json"
+        elseif isa(ctx.body, APIModel) && isnothing(content_type_set)
+            headers["Content-Type"] = content_type_set = "application/json"
             body = to_json(ctx.body)
         else
             body = ctx.body

src/server.jl

@@ -37,6 +37,18 @@ function get_param(source::Dict, name::String, required::Bool)
     return val
 end
 
+function get_param(source::Vector{HTTP.Forms.Multipart}, name::String, required::Bool)
+    ind = findfirst(x -> x.name == name, source)
+    if required && isnothing(ind)
+        throw(ValidationException("required parameter \"$name\" missing"))
+    elseif isnothing(ind)
+        return nothing
+    else
+        return source[ind]
+    end
+end
+
+
 function to_param_type(::Type{T}, strval::String) where {T <: Number}
     parse(T, strval)
 end
@@ -45,6 +57,7 @@ to_param_type(::Type{T}, val::T) where {T} = val
 to_param_type(::Type{T}, ::Nothing) where {T} = nothing
 to_param_type(::Type{String}, val::Vector{UInt8}) = String(copy(val))
 to_param_type(::Type{Vector{UInt8}}, val::String) = convert(Vector{UInt8}, copy(codeunits(val)))
+to_param_type(::Type{Vector{T}}, val::Vector{T}, _collection_format::Union{String,Nothing}) where {T} = val
 
 function to_param_type(::Type{T}, strval::String) where {T <: APIModel}
     from_json(T, JSON.parse(strval))
@@ -80,9 +93,25 @@ function to_param(T, source::Dict, name::String; required::Bool=false, collectio
     end
 end
 
+function to_param(T, source::Vector{HTTP.Forms.Multipart}, name::String; required::Bool=false, collection_format::Union{String,Nothing}=",", multipart::Bool=false, isfile::Bool=false)
+    param = get_param(source, name, required)
+    if param === nothing
+        return nothing
+    end
+    if multipart
+        # param is a Multipart
+        param = isfile ? take!(param.data) : String(take!(param.data))
+    end
+    if T <: Vector
+        return to_param_type(T, param, collection_format)
+    else
+        return to_param_type(T, param)
+    end
+end
+
 server_response(resp::HTTP.Response) = resp
 server_response(::Nothing) = server_response("")
 server_response(ret) = server_response(to_json(ret))
 server_response(resp::String) = HTTP.Response(200, resp)
 
-end # module Servers
+end # module Servers

test/client/petstore_v2/petstore/src/apis/api_PetApi.jl

@@ -239,7 +239,7 @@ function _oacinternal_upload_file(_api::PetApi, pet_id::Int64; additional_metada
     _ctx = OpenAPI.Clients.Ctx(_api.client, "POST", _returntypes_upload_file_PetApi, "/pet/{petId}/uploadImage", ["petstore_auth", ])
     OpenAPI.Clients.set_param(_ctx.path, "petId", pet_id)  # type Int64
     OpenAPI.Clients.set_param(_ctx.form, "additionalMetadata", additional_metadata)  # type String
-    OpenAPI.Clients.set_param(_ctx.file, "file", file)  # type String
+    OpenAPI.Clients.set_param(_ctx.file, "file", file)  # type Vector{UInt8}
     OpenAPI.Clients.set_header_accept(_ctx, ["application/json", ])
     OpenAPI.Clients.set_header_content_type(_ctx, (_mediaType === nothing) ? ["multipart/form-data", ] : [_mediaType])
     return _ctx

test/client/petstore_v3/petstore/src/apis/api_PetApi.jl

@@ -238,7 +238,7 @@ function _oacinternal_upload_file(_api::PetApi, pet_id::Int64; additional_metada
     _ctx = OpenAPI.Clients.Ctx(_api.client, "POST", _returntypes_upload_file_PetApi, "/pet/{petId}/uploadImage", ["petstore_auth", ])
     OpenAPI.Clients.set_param(_ctx.path, "petId", pet_id)  # type Int64
     OpenAPI.Clients.set_param(_ctx.form, "additionalMetadata", additional_metadata)  # type String
-    OpenAPI.Clients.set_param(_ctx.file, "file", file)  # type String
+    OpenAPI.Clients.set_param(_ctx.file, "file", file)  # type Vector{UInt8}
     OpenAPI.Clients.set_header_accept(_ctx, ["application/json", ])
     OpenAPI.Clients.set_header_content_type(_ctx, (_mediaType === nothing) ? ["multipart/form-data", ] : [_mediaType])
     return _ctx

test/client/petstore_v3/petstore_test_petapi.jl

@@ -6,7 +6,7 @@ using OpenAPI
 using OpenAPI.Clients
 import OpenAPI.Clients: Client
 
-function test(uri)
+function test(uri; test_file_upload=false)
     @info("PetApi")
     client = Client(uri)
     api = PetApi(client)
@@ -53,6 +53,16 @@ function test(uri)
     @test api_return === nothing
     @test http_resp.status == 200
 
+    if test_file_upload
+        @info("PetApi - upload_file")
+        api_return, http_resp = upload_file(api, 1; additional_metadata="my metadata", file=@__FILE__)
+        @test isa(api_return, ApiResponse)
+        @test api_return.code == 1
+        @test api_return.type == "pet"
+        @test api_return.message == "file uploaded"
+        @test http_resp.status == 200
+    end
+
     # does not work yet. issue: https://github.com/JuliaWeb/Requests.jl/issues/139
     #@info("PetApi - upload_file")
     #img = joinpath(dirname(@__FILE__), "cat.png")

test/client/petstore_v3/runtests.jl

@@ -20,19 +20,19 @@ function test_stress()
     TestUserApi.test_parallel(server)
 end
 
-function petstore_tests()
+function petstore_tests(; test_file_upload=false)
     TestUserApi.test(server)
     TestStoreApi.test(server)
-    TestPetApi.test(server)
+    TestPetApi.test(server; test_file_upload=test_file_upload)
 end
 
-function runtests()
+function runtests(; test_file_upload=false)
     @testset "petstore v3" begin
         @testset "miscellaneous" begin
             test_misc()
         end
         @testset "petstore apis" begin
-            petstore_tests()
+            petstore_tests(; test_file_upload=test_file_upload)
         end
         if get(ENV, "STRESS_PETSTORE", "false") == "true"
             @testset "stress" begin

test/client/runtests.jl

@@ -8,7 +8,7 @@ include("utilstests.jl")
 include("petstore_v3/runtests.jl")
 include("petstore_v2/runtests.jl")
 
-function runtests(; skip_petstore=false)
+function runtests(; skip_petstore=false, test_file_upload=false)
     @testset "Client" begin
         @testset "Utils" begin
             test_longpoll_exception_check()
@@ -25,7 +25,7 @@ function runtests(; skip_petstore=false)
                 if get(ENV, "RUNNER_OS", "") == "Linux"
                     @testset "V3" begin
                         @info("Running petstore v3 tests")
-                        PetStoreV3Tests.runtests()
+                        PetStoreV3Tests.runtests(; test_file_upload=test_file_upload)
                     end
                     @testset "V2" begin
                     @info("Running petstore v2 tests")

test/forms/FormsClient/.openapi-generator-ignore

@@ -0,0 +1,23 @@
+# OpenAPI Generator Ignore
+# Generated by openapi-generator https://github.com/openapitools/openapi-generator
+
+# Use this file to prevent files from being overwritten by the generator.
+# The patterns follow closely to .gitignore or .dockerignore.
+
+# As an example, the C# client generator defines ApiClient.cs.
+# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line:
+#ApiClient.cs
+
+# You can match any string of characters against a directory, file or extension with a single asterisk (*):
+#foo/*/qux
+# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux
+
+# You can recursively match patterns against a directory, file or extension with a double asterisk (**):
+#foo/**/qux
+# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux
+
+# You can also negate patterns with an exclamation (!).
+# For example, you can ignore all files in a docs folder with the file extension .md:
+#docs/*.md
+# Then explicitly reverse the ignore rule for a single file:
+#!docs/README.md

test/forms/FormsClient/.openapi-generator/FILES

@@ -0,0 +1,7 @@
+README.md
+docs/DefaultApi.md
+docs/TestResponse.md
+src/FormsClient.jl
+src/apis/api_DefaultApi.jl
+src/modelincludes.jl
+src/models/model_TestResponse.jl

test/forms/FormsClient/.openapi-generator/VERSION

@@ -0,0 +1 @@
+7.0.0-SNAPSHOT

test/forms/FormsClient/README.md

@@ -0,0 +1,41 @@
+# Julia API client for FormsClient
+
+Tests for different types of POST operations with forms and file uploads
+
+## Overview
+This API client was generated by the [OpenAPI Generator](https://openapi-generator.tech) project.  By using the [openapi-spec](https://openapis.org) from a remote server, you can easily generate an API client.
+
+- API version: 0.1.0
+- Build package: org.openapitools.codegen.languages.JuliaClientCodegen
+
+
+## Installation
+Place the Julia files generated under the `src` folder in your Julia project. Include FormsClient.jl in the project code.
+It would include the module named FormsClient.
+
+Documentation is generated as markdown files under the `docs` folder. You can include them in your project documentation.
+Documentation is also embedded in Julia which can be used with a Julia specific documentation generator.
+
+## API Endpoints
+
+Class | Method
+------------ | -------------
+*DefaultApi* | [**post_urlencoded_form**](docs/DefaultApi.md#post_urlencoded_form)<br/>**POST** /test/{form_id}/post_urlencoded_form_data<br/>posts a urlencoded form, with file contents and additional metadata, both of which are strings
+*DefaultApi* | [**upload_binary_file**](docs/DefaultApi.md#upload_binary_file)<br/>**POST** /test/{file_id}/upload_binary_file<br/>uploads a binary file given its path, along with some metadata
+*DefaultApi* | [**upload_text_file**](docs/DefaultApi.md#upload_text_file)<br/>**POST** /test/{file_id}/upload_text_file<br/>uploads text file contents along with some metadata
+
+
+## Models
+
+ - [TestResponse](docs/TestResponse.md)
+
+
+<a id="authorization"></a>
+## Authorization
+Endpoints do not require authorization.
+
+
+## Author
+
+
+