[#9] Always return map with :headers, :body and :status (#10)

Author: borkdude

README.md

@@ -18,29 +18,23 @@ Simple `GET` request:
 
 ``` clojure
 (curl/get "https://httpstat.us/200")
-;;=> "200 OK"
-```
-
-Simple `GET` request returned as a map with `:body`, `:status` and `:headers`:
-
-``` clojure
-(curl/get "https://httpstat.us/200" {:response true})
-;;=> {:status 200 :body "200 OK" :headers { ... }}
+;;=> {:status 200, :body "200 OK", :headers { ... }}
 ```
 
 Passing headers:
 
 ``` clojure
 (require '[cheshire.core :as json])
 (def resp (curl/get "https://httpstat.us/200" {:headers {"Accept" "application/json"}}))
-(json/parse-string resp) ;;=> {"code" 200, "description" "OK"}
+(json/parse-string (:body resp)) ;;=> {"code" 200, "description" "OK"}
 ```
 
 Query parameters:
 
 ``` clojure
 (->
   (curl/get "https://postman-echo.com/get" {:query-params {"q" "clojure"}})
+  :body
   (json/parse-string true)
   :args)
 ;;=> {:q "clojure"}
@@ -50,43 +44,48 @@ A `POST` request with a `:body`:
 
 ``` clojure
 (def resp (curl/post "https://postman-echo.com/post" {:body "From Clojure"}))
-(json/parse-string resp) ;;=> {"args" {}, "data" "", ...}
+(json/parse-string (:body resp)) ;;=> {"args" {}, "data" "", ...}
 ```
 
 Posting a file as a `POST` body:
 
 ``` clojure
-(curl/post "https://postman-echo.com/post" {:body (io/file "README.md")})
+(require '[clojure.java.io :as io])
+(:status (curl/post "https://postman-echo.com/post" {:body (io/file "README.md")}))
+;; => 100
 ```
 
 Posting form params:
 
 ``` clojure
-(curl/post "https://postman-echo.com/post" {:form-params {"name" "Michiel"}})
+(:status (curl/post "https://postman-echo.com/post" {:form-params {"name" "Michiel"}}))
+;; => 100
 ```
 
 Basic auth:
 
 ``` clojure
-(curl/get "https://postman-echo.com/basic-auth" {:basic-auth ["postman" "password"]})
+(:body (curl/get "https://postman-echo.com/basic-auth" {:basic-auth ["postman" "password"]}))
+;; => "{\"authenticated\":true}"
 ```
 
 Download a binary file as a stream:
 
 ``` clojure
-(require '[clojure.java.io :as io])
 (io/copy
-  (curl/get "https://github.com/borkdude/babashka/raw/master/logo/icon.png"
-    {:as :stream})
+  (:body (curl/get "https://github.com/borkdude/babashka/raw/master/logo/icon.png"
+    {:as :stream}))
   (io/file "icon.png"))
+(.length (io/file "icon.png"))
+;;=> 7748
 ```
 
 Passing raw arguments to `curl` can be done with `:raw-args`:
 
 ``` clojure
-(require '[clojure.string :as str])
-(def resp (curl/get "https://www.clojure.org" {:raw-args ["-D" "-"]}))
-(-> (str/split resp #"\n") first) ;;=> "HTTP/1.1 200 OK\r"
+(:status (curl/get "http://www.clojure.org" {:raw-args ["--max-redirs" "0"]}))
+curl: (47) Maximum (0) redirects followed
+301
 ```
 
 Talking to a UNIX socket:
@@ -95,10 +94,11 @@ Talking to a UNIX socket:
 (-> (curl/get "http://localhost/images/json"
               {:raw-args ["--unix-socket"
                           "/var/run/docker.sock"]})
+    :body
     (json/parse-string true)
     first
     :RepoTags)
-;;=> ["borkdude/babashka:0.0.73-SNAPSHOT"]
+;;=> ["borkdude/babashka:0.0.79-SNAPSHOT"]
 ```
 
 Using the low-level API for fine grained(and safer) URL construction:
@@ -109,6 +109,7 @@ Using the low-level API for fine grained(and safer) URL construction:
                          :port   443
                          :path   "/get"
                          :query  "q=test"}})
+    :body
     (json/parse-string true))
 ;;=>
 {:args {:q "test"},
@@ -125,7 +126,7 @@ Using the low-level API for fine grained(and safer) URL construction:
 ## Test
 
 ``` clojure
-$ clj -A:test
+$ clojure -A:test
 ```
 
 ## License

src/babashka/curl.clj

@@ -19,31 +19,18 @@
   return it.
   `:throw?`: Unless `false`, exits script when the shell-command has a
   non-zero exit code, unless `throw?` is set to false."
-  ([args] (shell-command args nil))
-  ([args {:keys [:throw? :as-stream?]
-          :or {throw? true}}]
-   (let [pb (let [pb (ProcessBuilder. ^java.util.List args)]
-              (doto pb
-                (.redirectInput ProcessBuilder$Redirect/INHERIT)
-                (.redirectError ProcessBuilder$Redirect/INHERIT)))
-         proc (.start pb)
-         out
-         (if as-stream? (.getInputStream proc)
-             (let [sw (java.io.StringWriter.)]
-               (with-open [w (io/reader (.getInputStream proc))]
-                 (io/copy w sw))
-               (str sw)))
-         exit-code (when-not as-stream? (.waitFor proc))]
-     (when (and throw?
-                (not as-stream?)
-                (not (zero? exit-code)))
-       (throw (ex-info "Got non-zero exit code" {:status exit-code})))
-     {:out out
-      :exit exit-code
-      :err ""})))
+  [args]
+  (let [pb (let [pb (ProcessBuilder. ^java.util.List args)]
+             (doto pb
+               (.redirectInput ProcessBuilder$Redirect/INHERIT)
+               (.redirectError ProcessBuilder$Redirect/INHERIT)))
+        proc (.start pb)
+        out (.getInputStream proc)]
+    {:out out
+     :proc proc}))
 
 (defn- exec-curl [args opts]
-  (let [res (shell-command args opts)
+  (let [res (shell-command args)
         exit (:exit res)
         out (:out res)]
     ;; TODO: handle non-zero exit with exception?  TODO: should we return a map
@@ -124,7 +111,7 @@
                      basic-auth)
         basic-auth (when basic-auth
                      ["--user" basic-auth])]
-    (conj (reduce into ["curl" "--silent" "--show-error" "--location"]
+    (conj (reduce into ["curl" "--silent" "--show-error" "--location" "--include"]
                   [method headers accept-header data-raw in-file basic-auth
                    form-params (:raw-args opts)])
           (str url
@@ -150,17 +137,26 @@
               (recur)))))
     (str sb)))
 
+(defn- read-headers [^PushbackInputStream is]
+  (loop [headers []]
+    (let [next-line (read-line is)]
+      (if (str/blank? next-line)
+        headers
+        (recur (conj headers next-line))))))
+
 (defn- curl-response->map
   "Parses a curl response input stream into a map"
   [^java.io.InputStream input-stream opts]
   (let [input-stream (PushbackInputStream. input-stream)]
     (loop [redirects []]
-      (let [[status-line & header-lines]
-            (take-while #(not (str/blank? %)) (repeatedly #(read-line input-stream)))
+      (let [headers (read-headers input-stream)
+            [status-line & header-lines] headers
             status   (Integer/parseInt (second (str/split status-line  #" ")))
             headers  (reduce (fn [acc header-line]
                                (let [[k v] (str/split header-line #":" 2)]
-                                 (assoc acc (str/lower-case k) (str/trim v))))
+                                 (if (and k v)
+                                   (assoc acc (str/lower-case k) (str/trim v))
+                                   acc)))
                              {}
                              header-lines)
             response {:status  status
@@ -181,13 +177,11 @@
 ;;;; End Response Parsing
 
 (defn request [opts]
-  (let [args (curl-command opts)
-        stream? (identical? :stream (:as opts))]
+  (let [args (curl-command opts)]
     (when (:debug? opts)
       (println (str/join " " (map pr-str args))))
-    (if (:response opts)
-      (curl-response->map (exec-curl (conj args "--include") (assoc opts :as-stream? true)) opts)
-      (exec-curl args (assoc opts :as-stream? stream?)))))
+    (-> (exec-curl args opts)
+        (curl-response->map opts))))
 
 (defn head
   ([url] (head url nil))

test/babashka/curl_test.clj

@@ -6,85 +6,84 @@
             [clojure.test :refer [deftest is testing are]]))
 
 (deftest get-test
-  (is (str/includes? (curl/get "https://httpstat.us/200")
+  (is (str/includes? (:body (curl/get "https://httpstat.us/200"))
                      "200"))
   (is (= 200
          (-> (curl/get "https://httpstat.us/200"
                        {:headers {"Accept" "application/json"}})
+             :body
              (json/parse-string true)
              :code)))
   (testing "query params"
     (is (= {:foo1 "bar1", :foo2 "bar2"}
            (-> (curl/get "https://postman-echo.com/get" {:query-params {"foo1" "bar1" "foo2" "bar2"}})
+               :body
                (json/parse-string true)
                :args)))))
 
 (deftest head-test
-  (is (str/includes? (curl/head "https://postman-echo.com/head")
-                     "200 OK")))
+  (is (= 200 (:status (curl/head "https://postman-echo.com/head")))))
 
 (deftest post-test
-  (is (subs (curl/post "https://postman-echo.com/post")
+  (is (subs (:body (curl/post "https://postman-echo.com/post"))
             0 10))
   (is (str/includes?
-       (curl/post "https://postman-echo.com/post"
-                  {:body "From Clojure"})
+       (:body (curl/post "https://postman-echo.com/post"
+                        {:body "From Clojure"}))
        "From Clojure"))
   (testing "file-body"
     (is (str/includes?
-         (curl/post "https://postman-echo.com/post"
-                    {:body (io/file "README.md")})
+         (:body (curl/post "https://postman-echo.com/post"
+                          {:body (io/file "README.md")}))
          "babashka.curl")))
   (testing "form-params"
     (is (str/includes?
-         (curl/post "https://postman-echo.com/post"
-                    {:form-params {"name" "michiel"}})
+         (:body (curl/post "https://postman-echo.com/post"
+                          {:form-params {"name" "michiel"}}))
          "michiel"))))
 
 (deftest patch-test
   (is (str/includes?
-       (curl/patch "https://postman-echo.com/patch"
-                   {:body "hello"})
+       (:body (curl/patch "https://postman-echo.com/patch"
+                         {:body "hello"}))
        "hello")))
 
 (deftest basic-auth-test
   (is (re-find #"authenticated.*true"
-       (curl/get "https://postman-echo.com/basic-auth" {:basic-auth ["postman" "password"]}))))
+               (:body
+                (curl/get "https://postman-echo.com/basic-auth"
+                          {:basic-auth ["postman" "password"]})))))
 
 (deftest raw-args-test
-  (is (str/includes?
-       (curl/post "https://postman-echo.com/post"
-                  {:body "From Clojure"
-                   :raw-args ["-D" "-"]})
-       "200 OK")))
+  (is (= 200 (:status (curl/post "https://postman-echo.com/post"
+                                 {:body "From Clojure"
+                                  :raw-args ["-D" "-"]})))))
 
 (deftest get-response-object-test
-  (let [response (curl/get "https://httpstat.us/200" {:response true})]
+  (let [response (curl/get "https://httpstat.us/200")]
     (is (map? response))
     (is (= 200 (:status response)))
     (is (= "200 OK" (:body response)))
     (is (= "Microsoft-IIS/10.0" (get-in response [:headers "server"]))))
 
   (testing "response object as stream"
-    (let [response (curl/get "https://httpstat.us/200" {:response true :as :stream})]
+    (let [response (curl/get "https://httpstat.us/200" {:as :stream})]
       (is (map? response))
       (is (= 200 (:status response)))
       (is (instance? java.io.InputStream (:body response)))
       (is (= "200 OK" (slurp (:body response))))))
 
   (testing "response object with following redirect"
     (let [response (curl/get "https://httpbin.org/redirect-to?url=https://www.httpbin.org"
-                             {:raw-args ["-L"]
-                              :response true})]
+                             {:raw-args ["-L"]})]
       (is (map? response))
       (is (= 200 (:status response)))
       (is (= 302 (-> response :redirects first :status)))
       (is (= "https://www.httpbin.org" (get-in response [:redirects 0 :headers "location"])))))
 
   (testing "response object without fully following redirects"
     (let [response (curl/get "https://httpbin.org/redirect-to?url=https://www.httpbin.org"
-                             {:response true
-                              :raw-args ["--max-redirs" "0"]})]
+                             {:raw-args ["--max-redirs" "0"]})]
       (is (map? response))
       (is (= 302 (:status response)))
       (is (= "" (:body response)))
@@ -95,12 +94,14 @@
   (is (= 200
          (-> (curl/get "https://httpstat.us/200"
                        {:accept :json})
+             :body
              (json/parse-string true)
              :code))))
 
 (deftest url-encode-query-params-test
   (is (= {"my query param?" "hello there"}
          (-> (curl/get "https://postman-echo.com/get" {:query-params {"my query param?" "hello there"}})
+             :body
              (json/parse-string)
              (get "args")))))
 
@@ -110,6 +111,7 @@
                                           :port   443
                                           :path   "/get"
                                           :query  "q=test"}})
+                     :body
                      (json/parse-string true))]
     (is (= {:q "test"} (:args response)))
     (is (= "httpbin.org" (get-in response [:headers :Host])))))
@@ -118,15 +120,14 @@
   (testing "download image"
     (let [tmp-file (java.io.File/createTempFile "icon" ".png")]
       (.deleteOnExit tmp-file)
-      (io/copy (curl/get "https://github.com/borkdude/babashka/raw/master/logo/icon.png" {:as :stream})
+      (io/copy (:body (curl/get "https://github.com/borkdude/babashka/raw/master/logo/icon.png" {:as :stream}))
                tmp-file)
       (is (= (.length (io/file "test" "icon.png"))
              (.length tmp-file)))))
   (testing "download image with response headers"
     (let [tmp-file (java.io.File/createTempFile "icon" ".png")]
       (.deleteOnExit tmp-file)
-      (let [resp (curl/get "https://github.com/borkdude/babashka/raw/master/logo/icon.png" {:as :stream
-                                                                                            :response true})]
+      (let [resp (curl/get "https://github.com/borkdude/babashka/raw/master/logo/icon.png" {:as :stream})]
         (is (= 200 (:status resp)))
         (io/copy (:body resp) tmp-file))
       (is (= (.length (io/file "test" "icon.png"))