From c4d5ec60faaf5c9e8204f53aed987730da4e0db1 Mon Sep 17 00:00:00 2001
From: Scott Shawcroft <scott@tannewt.org>
Date: Fri, 16 Aug 2024 12:39:29 -0700
Subject: [PATCH 1/2] Remove redis dep. JSON is much simpler
---
.github/workflows/bundle_cron.yml | 15 +++++++--------
adabot/circuitpython_bundle.py | 30 +++++++++++++++---------------
requirements.txt | 1 -
3 files changed, 22 insertions(+), 24 deletions(-)
diff --git a/.github/workflows/bundle_cron.yml b/.github/workflows/bundle_cron.yml
index 43a0a2e..9210a31 100644
--- a/.github/workflows/bundle_cron.yml
+++ b/.github/workflows/bundle_cron.yml
@@ -26,17 +26,16 @@ jobs:
# Its necessary to do this here, since 'schedule' events cannot (currently)
# be limited (they run on all forks' default branches).
if: startswith(github.repository, 'adafruit/')
- services:
- redis:
- image: redis
- ports:
- - 6379/tcp
- options: --entrypoint redis-server
steps:
- - name: Set up Python 3.9
+ - name: Set up Python 3.12
uses: actions/setup-python@v4
with:
- python-version: 3.9
+ python-version: 3.12
+ - name: Load contributor cache
+ uses: actions/cache@v4
+ with:
+ key: "contributor-cache"
+ path: "contributors.json"
- name: Versions
run: |
python3 --version
diff --git a/adabot/circuitpython_bundle.py b/adabot/circuitpython_bundle.py
index b1789d2..ed31398 100644
--- a/adabot/circuitpython_bundle.py
+++ b/adabot/circuitpython_bundle.py
@@ -9,12 +9,12 @@
from datetime import date
from io import StringIO
+import json
import os
+import pathlib
import shlex
import subprocess
-import redis as redis_py
-
import sh
from sh.contrib import git
@@ -22,11 +22,6 @@
from adabot.lib import common_funcs
from adabot import circuitpython_library_download_stats as dl_stats
-REDIS = None
-if "GITHUB_WORKSPACE" in os.environ:
- REDIS = redis_py.StrictRedis(port=os.environ["REDIS_PORT"])
-else:
- REDIS = redis_py.StrictRedis()
BUNDLES = ["Adafruit_CircuitPython_Bundle", "CircuitPython_Community_Bundle"]
@@ -380,20 +375,17 @@ def get_contributors(repo, commit_range):
return contributors
for log_line in output.split("\n"):
sha, author_email, committer_email = log_line.split(",")
- author = REDIS.get("github_username:" + author_email)
- committer = REDIS.get("github_username:" + committer_email)
+ author = CONTRIBUTOR_CACHE.get("github_username:" + author_email, None)
+ committer = CONTRIBUTOR_CACHE.get("github_username:" + committer_email, None)
if not author or not committer:
github_commit_info = gh_reqs.get("/repos/" + repo + "/commits/" + sha)
github_commit_info = github_commit_info.json()
if github_commit_info["author"]:
author = github_commit_info["author"]["login"]
- REDIS.set("github_username:" + author_email, author)
+ CONTRIBUTOR_CACHE["github_username:" + author_email] = author
if github_commit_info["committer"]:
committer = github_commit_info["committer"]["login"]
- REDIS.set("github_username:" + committer_email, committer)
- else:
- author = author.decode("utf-8")
- committer = committer.decode("utf-8")
+ CONTRIBUTOR_CACHE["github_username:" + committer_email] = committer
if committer_email == "noreply@github.com":
committer = None
@@ -506,7 +498,7 @@ def new_release(bundle, bundle_path):
release_description.append(
"The libraries in each release are compiled for all recent major versions of CircuitPython."
" Please download the one that matches the major version of your CircuitPython. For example"
- ", if you are running 8.2.6 you should download the `8.x` bundle.\n"
+ ", if you are running 9.1.1 you should download the `9.x` bundle.\n"
)
release_description.append(
@@ -554,6 +546,12 @@ def new_release(bundle, bundle_path):
if __name__ == "__main__":
+ contributor_cache_fn = pathlib.Path("contributors.json").resolve()
+ if contributor_cache_fn.exists():
+ CONTRIBUTOR_CACHE = json.loads(contributor_cache_fn.read_text())
+ else:
+ CONTRIBUTOR_CACHE = {}
+
bundles_dir = os.path.abspath(".bundles")
if "GITHUB_WORKSPACE" in os.environ:
git.config("--global", "user.name", "adabot")
@@ -570,3 +568,5 @@ def new_release(bundle, bundle_path):
except RuntimeError as e:
print("Failed to update and release:", cp_bundle)
print(e)
+ finally:
+ contributor_cache_fn.write_text(json.dumps(CONTRIBUTOR_CACHE))
diff --git a/requirements.txt b/requirements.txt
index 2d25e9b..ee4ca5d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,7 +7,6 @@ packaging==22.0
pylint==2.11.1
pytest
pyyaml>=5.4.1
-redis==4.5.4
requests==2.32.0
sh==1.12.14
requests-cache==0.5.2
From 03f581ede4ed9fc2146a389f2768fd0248389f14 Mon Sep 17 00:00:00 2001
From: Scott Shawcroft <scott@tannewt.org>
Date: Tue, 1 Oct 2024 11:07:18 -0700
Subject: [PATCH 2/2] Assume ok when remaining rate limit is missing
Otherwise, we try to pick out the time left until rate limit reset
and crash.
---
.github/workflows/bundle_cron.yml | 2 +-
.github/workflows/pre-commit.yml | 4 ++--
.github/workflows/test.yml | 11 ++---------
adabot/arduino_libraries.py | 4 ++--
adabot/circuitpython_bundle.py | 4 ++--
adabot/github_requests.py | 4 +++-
6 files changed, 12 insertions(+), 17 deletions(-)
diff --git a/.github/workflows/bundle_cron.yml b/.github/workflows/bundle_cron.yml
index 9210a31..2acc3b0 100644
--- a/.github/workflows/bundle_cron.yml
+++ b/.github/workflows/bundle_cron.yml
@@ -7,6 +7,7 @@ name: Update Bundles
on:
schedule:
- cron: 0 5 * * *
+ workflow_dispatch:
jobs:
check-repo-owner:
@@ -50,7 +51,6 @@ jobs:
ADABOT_EMAIL: ${{ secrets.ADABOT_EMAIL }}
ADABOT_GITHUB_USER: ${{ secrets.ADABOT_GITHUB_USER }}
ADABOT_GITHUB_ACCESS_TOKEN: ${{ secrets.ADABOT_GITHUB_ACCESS_TOKEN }}
- REDIS_PORT: ${{ job.services.redis.ports[6379] }}
BIGQUERY_PRIVATE_KEY: ${{ secrets.BIGQUERY_PRIVATE_KEY }}
BIGQUERY_CLIENT_EMAIL: ${{ secrets.BIGQUERY_CLIENT_EMAIL }}
run: |
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
index 6e21223..e222023 100644
--- a/.github/workflows/pre-commit.yml
+++ b/.github/workflows/pre-commit.yml
@@ -10,10 +10,10 @@ jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- - name: Set up Python 3.9
+ - name: Set up Python 3
uses: actions/setup-python@v4
with:
- python-version: 3.9
+ python-version: 3
- name: Versions
run: |
python3 --version
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e6b85bc..ed6d3ff 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -24,17 +24,11 @@ jobs:
# Its necessary to do this here, since 'schedule' events cannot (currently)
# be limited (they run on all forks' default branches).
if: startswith(github.repository, 'adafruit/')
- services:
- redis:
- image: redis
- ports:
- - 6379/tcp
- options: --entrypoint redis-server
steps:
- - name: Set up Python 3.9
+ - name: Set up Python
uses: actions/setup-python@v4
with:
- python-version: 3.9
+ python-version: 3
- name: Versions
run: |
python3 --version
@@ -50,6 +44,5 @@ jobs:
ADABOT_GITHUB_USER: ${{ secrets.ADABOT_GITHUB_USER }}
ADABOT_GITHUB_ACCESS_TOKEN: ${{ secrets.ADABOT_GITHUB_ACCESS_TOKEN }}
RTD_TOKEN: ${{ secrets.RTD_TOKEN }}
- REDIS_PORT: ${{ job.services.redis.ports[6379] }}
run: |
python3 -u -m pytest
diff --git a/adabot/arduino_libraries.py b/adabot/arduino_libraries.py
index 1c1d9d3..adbeb3b 100644
--- a/adabot/arduino_libraries.py
+++ b/adabot/arduino_libraries.py
@@ -322,8 +322,8 @@ def main(verbosity=1, output_file=None): # pylint: disable=missing-function-doc
run_arduino_lib_checks()
except:
_, exc_val, exc_tb = sys.exc_info()
- logger.error("Exception Occurred!", quiet=True)
- logger.error(("-" * 60), quiet=True)
+ logger.error("Exception Occurred!")
+ logger.error(("-" * 60))
logger.error("Traceback (most recent call last):")
trace = traceback.format_tb(exc_tb)
for line in trace:
diff --git a/adabot/circuitpython_bundle.py b/adabot/circuitpython_bundle.py
index ed31398..c7dfd71 100644
--- a/adabot/circuitpython_bundle.py
+++ b/adabot/circuitpython_bundle.py
@@ -25,6 +25,8 @@
BUNDLES = ["Adafruit_CircuitPython_Bundle", "CircuitPython_Community_Bundle"]
+CONTRIBUTOR_CACHE = {}
+
def fetch_bundle(bundle, bundle_path):
"""Clones `bundle` to `bundle_path`"""
@@ -549,8 +551,6 @@ def new_release(bundle, bundle_path):
contributor_cache_fn = pathlib.Path("contributors.json").resolve()
if contributor_cache_fn.exists():
CONTRIBUTOR_CACHE = json.loads(contributor_cache_fn.read_text())
- else:
- CONTRIBUTOR_CACHE = {}
bundles_dir = os.path.abspath(".bundles")
if "GITHUB_WORKSPACE" in os.environ:
diff --git a/adabot/github_requests.py b/adabot/github_requests.py
index 7cb2535..aa40c63 100644
--- a/adabot/github_requests.py
+++ b/adabot/github_requests.py
@@ -64,7 +64,9 @@ def request(method, url, **kwargs):
_fix_url(url), timeout=TIMEOUT, **_fix_kwargs(kwargs)
)
from_cache = getattr(response, "from_cache", False)
- remaining = int(response.headers.get("X-RateLimit-Remaining", 0))
+ # If rate limit remaining is missing, then assume we're fine. Use a million to signify this
+ # case. GitHub will be in the single thousands.
+ remaining = int(response.headers.get("X-RateLimit-Remaining", 1000000))
logging.debug(
"GET %s %s status=%s",
url,