Get a file from a repo, refactor iterating pages (#7)

* Get a file from a repo, refactor iterating pages

* Documentation.

* Don't use concat in recursive function, flatten pages instead.

* Use the first iteration function (better than mine). Refactor.

* Use and test the IReduceInit interface

* Documentation

* Avoid horizontal scrollbar in readme.

* Again

* Bump version.

Author: eamonnsullivan

.circleci/config.yml

@@ -23,6 +23,7 @@ commands:
       - save_cache:
           paths:
             - ~/.m2
+            - .cpcache
           key: v1-dependencies-{{ checksum "deps.edn" }}
 
   acceptance-tests:

README.md

@@ -103,6 +103,46 @@ The EDN returned will contain basic information about the repos found. For examp
 ]
 ```
 
+### Getting files in repos
+
+Retrieve information about a file in a repository, on a particular branch. You can use "HEAD" for the branch to retrieve a file from the default branch. The information returns includes `:byteSize` and `:text`.
+
+```clojure
+(require '[eamonnsullivan.github-api-lib.files :as files])
+(files/get-file token "eamonnsullivan" "github-api-lib" "HEAD" "README.md")
+
+{:commitResourcePath
+ "/eamonnsullivan/github-api-lib/commit/0805f4b95f5e01275e5962e0f8ed23def5129419",
+ :byteSize 4296,
+ :filepath "README.md",
+ :abbreviatedOid "0805f4b",
+ :isBinary false,
+ :oid "0805f4b95f5e01275e5962e0f8ed23def5129419",
+ :commitUrl
+ "https://github.com/eamonnsullivan/github-api-lib/commit/...",
+ :isTruncated false,
+ :text
+ "# github-api-lib\n\nA small, very simple..."}
+```
+
+You can also try several files and the first one found is returned.
+```clojure
+(files/get-first-file token "eamonnsullivan" "github-api-lib" "HEAD"
+["build.sbt" ".nvmrc" "deps.edn" "project.edn"])
+
+{:commitResourcePath
+ "/eamonnsullivan/github-api-lib/commit/74c3092ef552681a7fa5c1a96b3a11479b4f0a28",
+ :byteSize 1257,
+ :filepath "deps.edn",
+ :abbreviatedOid "74c3092",
+ :isBinary false,
+ :oid "74c3092ef552681a7fa5c1a96b3a11479b4f0a28",
+ :commitUrl
+ "https://github.com/eamonnsullivan/github-api-lib/commit/...",
+ :isTruncated false,
+ :text
+ "{:paths [\"src\" \"resources\"]\n :deps ..."}
+```
 ## Development Notes
 
 To run the project's tests:

pom.xml

@@ -3,7 +3,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>eamonnsullivan</groupId>
   <artifactId>github-api-lib</artifactId>
-  <version>0.1.16-SNAPSHOT</version>
+  <version>0.1.16</version>
   <name>github-api-lib</name>
   <description>Library of Github API calls that I happen to need.</description>
   <url>https://github.com/eamonnsullivan/github-api-lib</url>

resources/graphql/get-file-text-query.graphql

@@ -0,0 +1,16 @@
+query getRepoFile($owner: String!, $name: String!, $file: String!) {
+  repository(owner: $owner, name: $name) {
+    object(expression: $file) {
+      ... on Blob {
+        abbreviatedOid
+        byteSize
+        commitResourcePath
+        commitUrl
+        isBinary
+        isTruncated
+        oid
+        text
+      }
+    }
+  }
+}

src/eamonnsullivan/github_api_lib/core.clj

@@ -1,6 +1,7 @@
 (ns eamonnsullivan.github-api-lib.core
   (:require [clj-http.client :as client]
-            [clojure.data.json :as json]))
+            [clojure.data.json :as json]
+            [clojure.java.io :as io]))
 
 (def github-url "https://api.github.com/graphql")
 
@@ -19,6 +20,11 @@
   [access-token url opts]
   (client/get url (merge {:username access-token} opts)))
 
+(defn get-graphql
+  "Retrieve the GraphQL as a text blob"
+  [name]
+  (slurp (io/resource (format "graphql/%s.graphql" name))))
+
 (defn make-graphql-post
   "Make a GraphQL request to Github using the provided query/mutation
   and variables. If there are any errors, throw a RuntimeException,
@@ -64,3 +70,67 @@
       {:pullRequestUrl (format "https://github.com/%s/%s/pull/%s" owner name number)
        :issueComment comment}
       (throw (ex-info (format "Could not parse comment from url: %s" comment-url) {})))))
+
+(defn iteration
+  "Taken from https://clojure.atlassian.net/browse/CLJ-2555.
+   This function can just be removed when we start using 1.11 of Clojure.
+
+   creates a seqable/reducible given step!,
+   a function of some (opaque continuation data) k
+
+   step! - fn of k/nil to (opaque) 'ret'
+
+   :some? - fn of ret -> truthy, indicating there is a value
+            will not call vf/kf nor continue when false
+   :vf - fn of ret -> v, the values produced by the iteration
+   :kf - fn of ret -> next-k or nil (will not continue)
+   :initk - the first value passed to step!
+
+   vf, kf default to identity, some? defaults to some?, initk defaults to nil
+
+   it is presumed that step! with non-initk is unreproducible/non-idempotent
+   if step! with initk is unreproducible, it is on the consumer to not consume twice"
+  [step! & {:keys [vf kf some? initk]
+            :or {vf identity
+                 kf identity
+                 some? some?
+                 initk nil}}]
+  (reify
+   clojure.lang.Seqable
+   (seq [_]
+        ((fn next [ret]
+           (when (some? ret)
+             (cons (vf ret)
+                   (when-some [k (kf ret)]
+                     (lazy-seq (next (step! k)))))))
+         (step! initk)))
+   clojure.lang.IReduceInit
+   (reduce [_ rf init]
+           (loop [acc init
+                  ret (step! initk)]
+             (if (some? ret)
+               (let [acc (rf acc (vf ret))]
+                 (if (reduced? acc)
+                   @acc
+                   (if-some [k (kf ret)]
+                     (recur acc (step! k))
+                     acc)))
+               acc)))))
+
+(defn get-all-pages
+  "Convenience function for getting all of the results from a paged search.
+
+   getter: function that returns a single page, given a cursor string.
+   results?: function that returns a boolean indicating whether the current page contains values.
+   valuesfn: function to extract the values from a page.
+
+   Returns a flattened, realised sequence with all of the result. Don't
+   use this on an infinite or very big sequence."
+  [getter results? valuesfn]
+  (let [get-next (fn [ret] (if (-> ret :data :search :pageInfo :hasNextPage)
+                             (-> ret :data :search :pageInfo :endCursor)
+                             nil))]
+    (vec (reduce
+          (fn [acc page] (concat acc page))
+          []
+          (iteration getter :vf valuesfn :kf get-next :some? results?)))))

src/eamonnsullivan/github_api_lib/files.clj

@@ -0,0 +1,33 @@
+(ns eamonnsullivan.github-api-lib.files
+  (:require [eamonnsullivan.github-api-lib.core :as core]))
+
+
+(defn get-file
+  "Get information and properties on a file in a repo, or nil if the
+   file doesn't exist.
+
+   You can use \"HEAD\" if you want a file on the default branch, but
+   you aren't sure of its name (e.g. \"main\" or \"master\")."
+  [access-token owner repo branch filepath]
+  (let [variables {:owner owner :name repo :file (format "%s:%s" branch filepath)}
+        response (core/make-graphql-post
+                  access-token
+                  (core/get-graphql "get-file-text-query")
+                  variables)
+        object (-> response :data :repository :object)]
+    (when object
+      (merge {:filepath filepath} object))))
+
+(defn get-first-file
+  "Get the first matching file in a repo. We try each of the files specified
+  and return the first one that exists or nil if none of them do."
+  [access-token owner repo branch files]
+  (loop [files files
+         result nil]
+    (if-not (seq files)
+      result
+      (let [result (get-file access-token owner repo branch (first files))]
+        (if (:oid result)
+          result
+          (recur (rest files)
+                 nil))))))