From 153c8299ffcccf55d7acfdb1c42c56defd9e56bf Mon Sep 17 00:00:00 2001 From: Phil Elson Date: Tue, 10 Sep 2024 16:55:33 +0200 Subject: [PATCH] Proof-of-concept for using simple-repository instead of the non-standards based JSON API --- README.rst | 12 ++- pypi_timemachine/__main__.py | 3 +- pypi_timemachine/core.py | 139 ++++++++++++++++++++++------------- setup.cfg | 6 +- 4 files changed, 105 insertions(+), 55 deletions(-) diff --git a/README.rst b/README.rst index 5233d7c..c28cbd2 100644 --- a/README.rst +++ b/README.rst @@ -31,11 +31,21 @@ This will start up a Flask app, and will print out a line such as:: You can then call pip with:: - pip install --index-url http://127.0.0.1:5000/ astropy + pip install --index-url http://127.0.0.1:5000/simple/ astropy and this will then install the requested packages and all dependencies, ignoring any releases after the cutoff date specified above. +How it works +~~~~~~~~~~~~ + +`pypi-timemachine` builds upon the simple-repository stack, and uses the +standards based PEP-503 repository. In order to filter by time, the upstream +repository PyPI must provide PEP-700 metadata (which PyPI does). +The results are filtered by pypi-timemachine, and then served as HTML or JSON +via the standard PEP-503 interface. + + Caveats/warnings ~~~~~~~~~~~~~~~~ diff --git a/pypi_timemachine/__main__.py b/pypi_timemachine/__main__.py index e21e0b2..45544fb 100644 --- a/pypi_timemachine/__main__.py +++ b/pypi_timemachine/__main__.py @@ -1,4 +1,5 @@ +from .core import main + if __name__ == '__main__': - from pypi_timemachine.core import main main() diff --git a/pypi_timemachine/core.py b/pypi_timemachine/core.py index e11767c..194d709 100644 --- a/pypi_timemachine/core.py +++ b/pypi_timemachine/core.py @@ -1,37 +1,97 @@ -import socket +from contextlib import asynccontextmanager +import dataclasses from datetime import datetime +import socket +import sys +import typing import click -import requests +import fastapi +import httpx +from simple_repository.components.core import RepositoryContainer, SimpleRepository +from simple_repository.components.http import HttpRepository +from simple_repository_server.routers import simple +from simple_repository import model +import uvicorn + + +if sys.version_info >= (3, 12): + from typing import override +else: + override = lambda fn: fn -from tornado.ioloop import IOLoop -from tornado.web import RequestHandler, Application -from tornado.routing import PathMatches MAIN_PYPI = 'https://pypi.org/simple/' -JSON_URL = 'https://pypi.org/pypi/{package}/json' - -PACKAGE_HTML = """ - - - - Links for {package} - - -

Links for {package}

-{links} - - -""" - - -def parse_iso(dt): + + +def parse_iso(dt) -> datetime: try: return datetime.strptime(dt, '%Y-%m-%d') except: return datetime.strptime(dt, '%Y-%m-%dT%H:%M:%S') +def create_app(repo: SimpleRepository) -> fastapi.FastAPI: + @asynccontextmanager + async def lifespan(app: fastapi.FastAPI) -> typing.AsyncIterator[None]: + async with httpx.AsyncClient() as http_client: + app.include_router(simple.build_router(repo, http_client), prefix="") + yield + + app = fastapi.FastAPI( + openapi_url=None, # Disables automatic OpenAPI documentation (Swagger & Redoc) + lifespan=lifespan, + ) + return app + + +class DateFilteredReleases(RepositoryContainer): + """ + A component used to remove released projects from the source + repository if they were released after the configured date. + + This component can be used only if the source repository exposes the upload + date according to PEP-700: https://peps.python.org/pep-0700/. + + """ + def __init__( + self, + source: SimpleRepository, + cutoff_date: datetime, + ) -> None: + self._cutoff_date = cutoff_date + super().__init__(source) + + @override + async def get_project_page( + self, + project_name: str, + *, + request_context: model.RequestContext = model.RequestContext.DEFAULT, + ) -> model.ProjectDetail: + project_page = await super().get_project_page( + project_name, + request_context=request_context, + ) + + return self._exclude_recent_distributions( + project_page=project_page, + now=datetime.now(), + ) + + def _exclude_recent_distributions( + self, + project_page: model.ProjectDetail, + now: datetime, + ) -> model.ProjectDetail: + filtered_files = tuple( + file for file in project_page.files + if not file.upload_time or + (file.upload_time <= self._cutoff_date) + ) + return dataclasses.replace(project_page, files=filtered_files) + + @click.command() @click.argument('cutoff_date') @click.option('--port', default=None) @@ -40,30 +100,12 @@ def main(cutoff_date, port, quiet): CUTOFF = parse_iso(cutoff_date) - INDEX = requests.get(MAIN_PYPI).content + repo = DateFilteredReleases( + HttpRepository(MAIN_PYPI), + cutoff_date=CUTOFF, + ) - class MainIndexHandler(RequestHandler): - - async def get(self): - return self.write(INDEX) - - class PackageIndexHandler(RequestHandler): - - async def get(self, package): - - package_index = requests.get(JSON_URL.format(package=package)).json() - release_links = "" - for release in package_index['releases'].values(): - for file in release: - release_date = parse_iso(file['upload_time']) - if release_date < CUTOFF: - if file['requires_python'] is None: - release_links += ' {filename}
\n'.format(url=file['url'], sha256=file['digests']['sha256'], filename=file['filename']) - else: - rp = file['requires_python'].replace('>', '>') - release_links += ' {filename}
\n'.format(url=file['url'], sha256=file['digests']['sha256'], rp=rp, filename=file['filename']) - - self.write(PACKAGE_HTML.format(package=package, links=release_links)) + app = create_app(repo) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('localhost', 0)) @@ -71,12 +113,7 @@ async def get(self, package): port = sock.getsockname()[1] sock.close() - app = Application([(r"/", MainIndexHandler), - (PathMatches(r"/(?P\S+)\//?"), PackageIndexHandler)]) - - app.listen(port=port) - if not quiet: print(f'Starting pypi-timemachine server at http://localhost:{port}') - IOLoop.instance().start() + uvicorn.run(app=app, port=int(port)) diff --git a/setup.cfg b/setup.cfg index 3ddd0f7..c00842a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,8 +12,10 @@ packages = find: setup_requires = setuptools_scm install_requires = click - requests - tornado + fastapi + httpx + simple-repository + simple-repository-server [options.entry_points] console_scripts =