Replace github library with graphql (#33)

* Replace github library with graphql Signed-off-by: Anna Jung (VMware) <[email protected]> * Add support for rate limit and other known external responses Signed-off-by: Anna Jung (VMware) <[email protected]>
vmware-archive · Feb 22, 2022 · b134e8c · b134e8c
1 parent 41779a7
commit b134e8c
Show file tree

Hide file tree

Showing 14 changed files with 1,112 additions and 125 deletions.
diff --git a/mcat/__init__.py b/mcat/__init__.py
@@ -0,0 +1,3 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
diff --git a/mcat/githubDataExtraction.py b/mcat/githubDataExtraction.py
diff --git a/mcat/queries/pull_requests_with_reactions.graphql b/mcat/queries/pull_requests_with_reactions.graphql
@@ -0,0 +1,69 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+query PullRequests($owner: String!, $repo: String!, $cursor: String) {
+    repository(owner: $owner, name: $repo) {
+        name
+        pullRequests(first: 100, after: $cursor) {
+            nodes {
+                number
+                title
+                author {
+                    login
+                }
+                url
+                state
+                body
+                additions
+                deletions
+                comments(first: 100) {
+                    totalCount
+                    nodes {
+                        author {
+                            login
+                        }
+                        createdAt
+                        body
+                        updatedAt
+                        reactions(first: 100) {
+                            nodes {
+                                content
+                                createdAt
+                                user {
+                                    login
+                                }
+                            }
+                        }
+                    }
+                }
+                commits {
+                    totalCount
+                }
+                createdAt
+                closedAt
+                merged
+                mergedAt
+                mergedBy {
+                    login
+                }
+                reviews(first: 100) {
+                    totalCount
+                    nodes {
+                        author {
+                            login
+                        }
+                        createdAt
+                        body
+                        updatedAt
+                    }
+                }
+                updatedAt
+            }
+            pageInfo {
+                endCursor
+                hasNextPage
+            }
+        }
+    }
+}
+
diff --git a/mcat/queries/pull_requests_without_reactions.graphql b/mcat/queries/pull_requests_without_reactions.graphql
@@ -0,0 +1,60 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+query PullRequests($owner: String!, $repo: String!, $cursor: String) {
+    repository(owner: $owner, name: $repo) {
+        name
+        pullRequests(first: 100, after: $cursor) {
+            nodes {
+                number
+                title
+                author {
+                    login
+                }
+                url
+                state
+                body
+                additions
+                deletions
+                comments(first: 100) {
+                    totalCount
+                    nodes {
+                        author {
+                            login
+                        }
+                        createdAt
+                        body
+                        updatedAt
+                    }
+                }
+                commits {
+                    totalCount
+                }
+                createdAt
+                closedAt
+                merged
+                mergedAt
+                mergedBy {
+                    login
+                }
+                reviews(first: 100) {
+                    totalCount
+                    nodes {
+                        author {
+                            login
+                        }
+                        createdAt
+                        body
+                        updatedAt
+                    }
+                }
+                updatedAt
+            }
+            pageInfo {
+                endCursor
+                hasNextPage
+            }
+        }
+    }
+}
+
diff --git a/mcat/queries/repos.graphql b/mcat/queries/repos.graphql
@@ -0,0 +1,17 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+query ($owner: String!, $cursor: String) {
+    organization(login: $owner) {
+        repositories(first: 100, after: $cursor) {
+            nodes {
+                name
+            }
+            pageInfo {
+                endCursor
+                hasNextPage
+            }
+        }
+    }
+}
+
diff --git a/mcat/utils.py b/mcat/utils.py
@@ -38,7 +38,5 @@ def construct_file_name(name, component_a, component_b, separator="_"):
 
 def string_to_dict(string):
     """Function to convert json strings to dictionary"""
-    string = ast.literal_eval(string)
-    for i in range(len(string)):
-        string[i] = ast.literal_eval(string[i])
-    return string
+    return ast.literal_eval(string)
+
diff --git a/requirements.txt b/requirements.txt
@@ -1,6 +1,5 @@
 requests>=2.26.0
 pandas>=1.3.2
-PyGithub>=1.55
 argparse>=1.4.0
 numpy>=1.19.5
 nltk>=3.6.2

diff --git a/tests/__init__.py b/tests/__init__.py
@@ -0,0 +1,3 @@
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
diff --git a/tests/resources/pr_post_processing_with_reactions_example.json b/tests/resources/pr_post_processing_with_reactions_example.json
@@ -0,0 +1,49 @@
+{
+  "Number": 1,
+  "Title": "test-title",
+  "User": "test-author-login",
+  "URL": "test-url",
+  "State": "MERGED",
+  "Body": "test-body",
+  "Additions": 2,
+  "Deletions": 3,
+  "Comments_Num": 1,
+  "Commits_Num": 1,
+  "Created_At": "2000-01-01T01:01:01Z",
+  "Closed_At": "2000-01-01T01:01:01Z",
+  "Merged": "True",
+  "Merged_At": "2000-01-01T01:01:01Z",
+  "Merged_By": "test-merged-by",
+  "Review_Comments_Num": 4,
+  "Updated_At": "2000-01-01T01:01:01Z",
+  "Comments": [
+    {
+      "Created_At": "2000-01-01T01:01:01Z",
+      "User": "test-comment-user",
+      "Body": "test-comment",
+      "Updated_At": "2000-01-01T01:01:01Z",
+      "Reactions": [
+        {
+          "Content": "test-content",
+          "Created_At": "2000-01-01T01:01:01Z",
+          "User": "test-reaction-user"
+        }
+      ]
+    }
+  ],
+  "Review_Comments": [
+    {
+      "Created_At": "2000-01-01T01:01:01Z",
+      "User": "test-review-user",
+      "Body": "test-body",
+      "Updated_At": "2000-01-01T01:01:01Z",
+      "Reactions": [
+        {
+          "Content": "test-content",
+          "Created_At": "2000-01-01T01:01:01Z",
+          "User": "test-reaction-user"
+        }
+      ]
+    }
+  ]
+}
diff --git a/tests/resources/pr_post_processing_without_reactions_example.json b/tests/resources/pr_post_processing_without_reactions_example.json
@@ -0,0 +1,35 @@
+{
+  "Number": 1,
+  "Title": "test-title",
+  "User": "test-author-login",
+  "URL": "test-url",
+  "State": "MERGED",
+  "Body": "test-body",
+  "Additions": 2,
+  "Deletions": 3,
+  "Comments_Num": 1,
+  "Commits_Num": 1,
+  "Created_At": "2000-01-01T01:01:01Z",
+  "Closed_At": "2000-01-01T01:01:01Z",
+  "Merged": "True",
+  "Merged_At": "2000-01-01T01:01:01Z",
+  "Merged_By": "test-merged-by",
+  "Review_Comments_Num": 4,
+  "Updated_At": "2000-01-01T01:01:01Z",
+  "Comments": [
+    {
+      "Created_At": "2000-01-01T01:01:01Z",
+      "User": "test-comment-user",
+      "Body": "test-comment",
+      "Updated_At": "2000-01-01T01:01:01Z"
+    }
+  ],
+  "Review_Comments": [
+    {
+      "Created_At": "2000-01-01T01:01:01Z",
+      "User": "test-review-user",
+      "Body": "test-body",
+      "Updated_At": "2000-01-01T01:01:01Z"
+    }
+  ]
+}
diff --git a/tests/resources/pr_raw_data_example_one.json b/tests/resources/pr_raw_data_example_one.json
@@ -0,0 +1,71 @@
+{
+  "number": 1,
+  "title": "test-title",
+  "author": {
+    "login": "test-author-login"
+  },
+  "url": "test-url",
+  "state": "MERGED",
+  "body": "test-body",
+  "additions": 2,
+  "deletions": 3,
+  "comments": {
+    "totalCount": 1,
+    "nodes": [
+      {
+        "author": {
+          "login": "test-comment-user"
+        },
+        "createdAt": "2000-01-01T01:01:01Z",
+        "body": "test-comment",
+        "updatedAt": "2000-01-01T01:01:01Z",
+        "reactions": {
+          "nodes": [
+            {
+              "content": "test-content",
+              "createdAt": "2000-01-01T01:01:01Z",
+              "user": {
+                "login": "test-reaction-user"
+              }
+            }
+          ]
+        }
+      }
+    ]
+  },
+  "commits": {
+    "totalCount": 1
+  },
+  "createdAt": "2000-01-01T01:01:01Z",
+  "closedAt": "2000-01-01T01:01:01Z",
+  "merged": "True",
+  "mergedAt": "2000-01-01T01:01:01Z",
+  "mergedBy": {
+    "login": "test-merged-by"
+  },
+  "reviews": {
+    "totalCount": 4,
+    "nodes": [
+      {
+        "author": {
+          "login": "test-review-user"
+        },
+        "createdAt": "2000-01-01T01:01:01Z",
+        "body": "test-body",
+        "updatedAt": "2000-01-01T01:01:01Z",
+        "reactions": {
+          "nodes": [
+            {
+              "content": "test-content",
+              "createdAt": "2000-01-01T01:01:01Z",
+              "user": {
+                "login": "test-reaction-user"
+              }
+            }
+          ]
+        }
+      }
+    ]
+  },
+  "updatedAt": "2000-01-01T01:01:01Z"
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# Copyright 2021 VMware, Inc.
		# SPDX-License-Identifier: Apache-2.0