diff --git a/dev/environment b/dev/environment index faca00af98b7..33f931944358 100644 --- a/dev/environment +++ b/dev/environment @@ -19,6 +19,7 @@ CAMO_KEY=insecurecamokey DOCS_URL="https://pythonhosted.org/{project}/" FILES_BACKEND=warehouse.packaging.services.LocalFileStorage path=/var/opt/warehouse/packages/ url=http://localhost:9001/packages/{path} +SIMPLE_BACKEND=warehouse.packaging.services.LocalSimpleStorage path=/var/opt/warehouse/simple/ url=http://localhost:9001/simple/{path} DOCS_BACKEND=warehouse.packaging.services.LocalDocsStorage path=/var/opt/warehouse/docs/ SPONSORLOGOS_BACKEND=warehouse.admin.services.LocalSponsorLogoStorage path=/var/opt/warehouse/sponsorlogos/ diff --git a/docker-compose.yml b/docker-compose.yml index 1af7ef5e164e..b4f2e0c0e084 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ version: '3' volumes: + simple: packages: sponsorlogos: vault: @@ -86,6 +87,7 @@ services: - .coveragerc:/opt/warehouse/src/.coveragerc:z - packages:/var/opt/warehouse/packages - sponsorlogos:/var/opt/warehouse/sponsorlogos + - simple:/var/opt/warehouse/simple - ./bin:/opt/warehouse/src/bin:z ports: - "80:8000" @@ -98,6 +100,7 @@ services: volumes: - packages:/var/opt/warehouse/packages - sponsorlogos:/var/opt/warehouse/sponsorlogos + - simple:/var/opt/warehouse/simple ports: - "9001:9001" @@ -113,6 +116,7 @@ services: environment: C_FORCE_ROOT: "1" FILES_BACKEND: "warehouse.packaging.services.LocalFileStorage path=/var/opt/warehouse/packages/ url=http://files:9001/packages/{path}" + SIMPLE_BACKEND: "warehouse.packaging.services.LocalSimpleStorage path=/var/opt/warehouse/simple/ url=http://files:9001/simple/{path}" static: build: diff --git a/tests/conftest.py b/tests/conftest.py index a37044a1ee05..dcab1d2b025c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,13 +25,17 @@ import pytest import webtest as _webtest +from jinja2 import Environment, FileSystemLoader from psycopg2.errors import InvalidCatalogName from pyramid.i18n import TranslationString from pyramid.static import ManifestCacheBuster +from pyramid_jinja2 import IJinja2Environment from pytest_postgresql.config import get_config from pytest_postgresql.janitor import DatabaseJanitor from sqlalchemy import event +import warehouse + from warehouse import admin, config, static from warehouse.accounts import services as account_services from warehouse.macaroons import services as macaroon_services @@ -76,6 +80,22 @@ def metrics(): ) +@pytest.fixture +def jinja(): + dir_name = os.path.join(os.path.dirname(warehouse.__file__)) + + env = Environment( + loader=FileSystemLoader(dir_name), + extensions=[ + "jinja2.ext.i18n", + "warehouse.utils.html.ClientSideIncludeExtension", + ], + cache_size=0, + ) + + return env + + class _Services: def __init__(self): self._services = defaultdict(lambda: defaultdict(dict)) @@ -98,11 +118,13 @@ def pyramid_services(metrics): @pytest.fixture -def pyramid_request(pyramid_services): +def pyramid_request(pyramid_services, jinja): dummy_request = pyramid.testing.DummyRequest() dummy_request.find_service = pyramid_services.find_service dummy_request.remote_addr = "1.2.3.4" + dummy_request.registry.registerUtility(jinja, IJinja2Environment, name=".jinja2") + def localize(message, **kwargs): ts = TranslationString(message, **kwargs) return ts.interpolate() @@ -184,7 +206,8 @@ def app_config(database): "ratelimit.url": "memory://", "elasticsearch.url": "https://localhost/warehouse", "files.backend": "warehouse.packaging.services.LocalFileStorage", - "docs.backend": "warehouse.packaging.services.LocalFileStorage", + "simple.backend": "warehouse.packaging.services.LocalSimpleStorage", + "docs.backend": "warehouse.packaging.services.LocalDocsStorage", "sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage", "mail.backend": "warehouse.email.services.SMTPEmailSender", "malware_check.backend": ( diff --git a/tests/unit/packaging/test_init.py b/tests/unit/packaging/test_init.py index 3c7038f0e6ed..ac4856f18201 100644 --- a/tests/unit/packaging/test_init.py +++ b/tests/unit/packaging/test_init.py @@ -18,7 +18,7 @@ from warehouse import packaging from warehouse.accounts.models import Email, User from warehouse.manage.tasks import update_role_invitation_status -from warehouse.packaging.interfaces import IDocsStorage, IFileStorage +from warehouse.packaging.interfaces import IDocsStorage, IFileStorage, ISimpleStorage from warehouse.packaging.models import File, Project, Release, Role from warehouse.packaging.tasks import ( # sync_bigquery_release_files, compute_trending, @@ -51,7 +51,11 @@ def key_factory(keystring, iterate_on=None): lambda factory, iface, name=None: None ), registry=pretend.stub( - settings={"files.backend": "foo.bar", "docs.backend": "wu.tang"} + settings={ + "files.backend": "foo.bar", + "simple.backend": "bread.butter", + "docs.backend": "wu.tang", + } ), register_origin_cache_keys=pretend.call_recorder(lambda c, **kw: None), get_settings=lambda: settings, @@ -62,6 +66,7 @@ def key_factory(keystring, iterate_on=None): assert config.register_service_factory.calls == [ pretend.call(storage_class.create_service, IFileStorage), + pretend.call(storage_class.create_service, ISimpleStorage), pretend.call(storage_class.create_service, IDocsStorage), ] assert config.register_origin_cache_keys.calls == [ diff --git a/tests/unit/packaging/test_services.py b/tests/unit/packaging/test_services.py index 739380b6e61b..ef7f2e6c69f1 100644 --- a/tests/unit/packaging/test_services.py +++ b/tests/unit/packaging/test_services.py @@ -22,11 +22,14 @@ import warehouse.packaging.services -from warehouse.packaging.interfaces import IDocsStorage, IFileStorage +from warehouse.packaging.interfaces import IDocsStorage, IFileStorage, ISimpleStorage from warehouse.packaging.services import ( GCSFileStorage, + GCSSimpleStorage, + GenericLocalBlobStorage, LocalDocsStorage, LocalFileStorage, + LocalSimpleStorage, S3DocsStorage, S3FileStorage, ) @@ -137,6 +140,67 @@ def test_delete_already_gone(self, tmpdir): assert response is None +class TestLocalSimpleStorage: + def test_verify_service(self): + assert verifyClass(ISimpleStorage, LocalSimpleStorage) + + def test_basic_init(self): + storage = LocalSimpleStorage("/foo/bar/") + assert storage.base == "/foo/bar/" + + def test_create_service(self): + request = pretend.stub( + registry=pretend.stub(settings={"simple.path": "/simple/one/two/"}) + ) + storage = LocalSimpleStorage.create_service(None, request) + assert storage.base == "/simple/one/two/" + + def test_gets_file(self, tmpdir): + with open(str(tmpdir.join("file.txt")), "wb") as fp: + fp.write(b"my test file contents") + + storage = LocalSimpleStorage(str(tmpdir)) + file_object = storage.get("file.txt") + assert file_object.read() == b"my test file contents" + + def test_raises_when_file_non_existent(self, tmpdir): + storage = LocalSimpleStorage(str(tmpdir)) + with pytest.raises(FileNotFoundError): + storage.get("file.txt") + + def test_stores_file(self, tmpdir): + filename = str(tmpdir.join("testfile.txt")) + with open(filename, "wb") as fp: + fp.write(b"Test File!") + + storage_dir = str(tmpdir.join("storage")) + storage = LocalSimpleStorage(storage_dir) + storage.store("foo/bar.txt", filename) + + with open(os.path.join(storage_dir, "foo/bar.txt"), "rb") as fp: + assert fp.read() == b"Test File!" + + def test_stores_two_files(self, tmpdir): + filename1 = str(tmpdir.join("testfile1.txt")) + with open(filename1, "wb") as fp: + fp.write(b"First Test File!") + + filename2 = str(tmpdir.join("testfile2.txt")) + with open(filename2, "wb") as fp: + fp.write(b"Second Test File!") + + storage_dir = str(tmpdir.join("storage")) + storage = LocalSimpleStorage(storage_dir) + storage.store("foo/first.txt", filename1) + storage.store("foo/second.txt", filename2) + + with open(os.path.join(storage_dir, "foo/first.txt"), "rb") as fp: + assert fp.read() == b"First Test File!" + + with open(os.path.join(storage_dir, "foo/second.txt"), "rb") as fp: + assert fp.read() == b"Second Test File!" + + class TestS3FileStorage: def test_verify_service(self): assert verifyClass(IFileStorage, S3FileStorage) @@ -479,3 +543,98 @@ def test_delete_by_prefix_with_storage_prefix(self): }, ), ] + + +class TestGCSSimpleStorage: + def test_verify_service(self): + assert verifyClass(ISimpleStorage, GCSSimpleStorage) + + def test_basic_init(self): + bucket = pretend.stub() + storage = GCSSimpleStorage(bucket) + assert storage.bucket is bucket + + def test_create_service(self): + service = pretend.stub( + get_bucket=pretend.call_recorder(lambda bucket_name: pretend.stub()) + ) + request = pretend.stub( + find_service=pretend.call_recorder(lambda name: service), + registry=pretend.stub(settings={"simple.bucket": "froblob"}), + ) + GCSSimpleStorage.create_service(None, request) + + assert request.find_service.calls == [pretend.call(name="gcloud.gcs")] + assert service.get_bucket.calls == [pretend.call("froblob")] + + def test_gets_file_raises(self): + storage = GCSSimpleStorage(pretend.stub()) + + with pytest.raises(NotImplementedError): + storage.get("file.txt") + + def test_stores_file(self, tmpdir): + filename = str(tmpdir.join("testfile.txt")) + with open(filename, "wb") as fp: + fp.write(b"Test File!") + + blob = pretend.stub( + upload_from_filename=pretend.call_recorder(lambda file_path: None), + exists=lambda: False, + ) + bucket = pretend.stub(blob=pretend.call_recorder(lambda path: blob)) + storage = GCSSimpleStorage(bucket) + storage.store("foo/bar.txt", filename) + + assert bucket.blob.calls == [pretend.call("foo/bar.txt")] + assert blob.upload_from_filename.calls == [pretend.call(filename)] + + def test_stores_two_files(self, tmpdir): + filename1 = str(tmpdir.join("testfile1.txt")) + with open(filename1, "wb") as fp: + fp.write(b"First Test File!") + + filename2 = str(tmpdir.join("testfile2.txt")) + with open(filename2, "wb") as fp: + fp.write(b"Second Test File!") + + blob = pretend.stub( + upload_from_filename=pretend.call_recorder(lambda file_path: None), + exists=lambda: False, + ) + bucket = pretend.stub(blob=pretend.call_recorder(lambda path: blob)) + storage = GCSSimpleStorage(bucket) + storage.store("foo/first.txt", filename1) + storage.store("foo/second.txt", filename2) + + assert bucket.blob.calls == [ + pretend.call("foo/first.txt"), + pretend.call("foo/second.txt"), + ] + assert blob.upload_from_filename.calls == [ + pretend.call(filename1), + pretend.call(filename2), + ] + + def test_stores_metadata(self, tmpdir): + filename = str(tmpdir.join("testfile.txt")) + with open(filename, "wb") as fp: + fp.write(b"Test File!") + + blob = pretend.stub( + upload_from_filename=pretend.call_recorder(lambda file_path: None), + patch=pretend.call_recorder(lambda: None), + exists=lambda: False, + ) + bucket = pretend.stub(blob=pretend.call_recorder(lambda path: blob)) + storage = GCSSimpleStorage(bucket) + meta = {"foo": "bar"} + storage.store("foo/bar.txt", filename, meta=meta) + + assert blob.metadata == meta + + +class TestGenericLocalBlobStorage: + def test_notimplementederror(self): + with pytest.raises(NotImplementedError): + GenericLocalBlobStorage.create_service(pretend.stub(), pretend.stub()) diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py new file mode 100644 index 000000000000..d3ea82bf63bd --- /dev/null +++ b/tests/unit/packaging/test_utils.py @@ -0,0 +1,133 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import tempfile + +import pretend + +from warehouse.packaging.interfaces import ISimpleStorage +from warehouse.packaging.utils import _simple_detail, render_simple_detail + +from ...common.db.packaging import ProjectFactory + + +def test_render_simple_detail(db_request, monkeypatch, jinja): + project = ProjectFactory.create() + + fake_hasher = pretend.stub( + update=pretend.call_recorder(lambda x: None), + hexdigest=pretend.call_recorder(lambda: "deadbeefdeadbeefdeadbeefdeadbeef"), + ) + fakeblake2b = pretend.call_recorder(lambda *a, **kw: fake_hasher) + monkeypatch.setattr(hashlib, "blake2b", fakeblake2b) + + template = jinja.get_template("templates/legacy/api/simple/detail.html") + expected_content = template.render( + **_simple_detail(project, db_request), request=db_request + ).encode("utf-8") + + content_hash, path = render_simple_detail(project, db_request) + + assert fakeblake2b.calls == [pretend.call(digest_size=32)] + assert fake_hasher.update.calls == [pretend.call(expected_content)] + assert fake_hasher.hexdigest.calls == [pretend.call()] + + assert content_hash == "deadbeefdeadbeefdeadbeefdeadbeef" + assert path == ( + f"{project.normalized_name}/deadbeefdeadbeefdeadbeefdeadbeef" + + f".{project.normalized_name}.html" + ) + + +def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): + project = ProjectFactory.create() + + storage_service = pretend.stub( + store=pretend.call_recorder( + lambda path, file_path, *, meta=None: f"http://files/sponsorlogos/{path}" + ) + ) + db_request.find_service = pretend.call_recorder( + lambda svc, name=None, context=None: { + ISimpleStorage: storage_service, + }.get(svc) + ) + + fake_hasher = pretend.stub( + update=pretend.call_recorder(lambda x: None), + hexdigest=pretend.call_recorder(lambda: "deadbeefdeadbeefdeadbeefdeadbeef"), + ) + fakeblake2b = pretend.call_recorder(lambda *a, **kw: fake_hasher) + monkeypatch.setattr(hashlib, "blake2b", fakeblake2b) + + fake_named_temporary_file = pretend.stub( + name="/tmp/wutang", + write=pretend.call_recorder(lambda data: None), + flush=pretend.call_recorder(lambda: None), + ) + + class FakeNamedTemporaryFile: + def __init__(self): + return None + + def __enter__(self): + return fake_named_temporary_file + + def __exit__(self, type, value, traceback): + pass + + monkeypatch.setattr(tempfile, "NamedTemporaryFile", FakeNamedTemporaryFile) + + template = jinja.get_template("templates/legacy/api/simple/detail.html") + expected_content = template.render( + **_simple_detail(project, db_request), request=db_request + ).encode("utf-8") + + content_hash, path = render_simple_detail(project, db_request, store=True) + + assert fake_named_temporary_file.write.calls == [pretend.call(expected_content)] + assert fake_named_temporary_file.flush.calls == [pretend.call()] + + assert fakeblake2b.calls == [pretend.call(digest_size=32)] + assert fake_hasher.update.calls == [pretend.call(expected_content)] + assert fake_hasher.hexdigest.calls == [pretend.call()] + + assert storage_service.store.calls == [ + pretend.call( + ( + f"{project.normalized_name}/deadbeefdeadbeefdeadbeefdeadbeef" + + f".{project.normalized_name}.html" + ), + "/tmp/wutang", + meta={ + "project": project.normalized_name, + "pypi-last-serial": project.last_serial, + "hash": "deadbeefdeadbeefdeadbeefdeadbeef", + }, + ), + pretend.call( + f"{project.normalized_name}/index.html", + "/tmp/wutang", + meta={ + "project": project.normalized_name, + "pypi-last-serial": project.last_serial, + "hash": "deadbeefdeadbeefdeadbeefdeadbeef", + }, + ), + ] + + assert content_hash == "deadbeefdeadbeefdeadbeefdeadbeef" + assert path == ( + f"{project.normalized_name}/deadbeefdeadbeefdeadbeefdeadbeef" + + f".{project.normalized_name}.html" + ) diff --git a/warehouse/config.py b/warehouse/config.py index 672819ed8189..879145189874 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -225,6 +225,7 @@ def configure(settings=None): default=21600, # 6 hours ) maybe_set_compound(settings, "files", "backend", "FILES_BACKEND") + maybe_set_compound(settings, "simple", "backend", "SIMPLE_BACKEND") maybe_set_compound(settings, "docs", "backend", "DOCS_BACKEND") maybe_set_compound(settings, "sponsorlogos", "backend", "SPONSORLOGOS_BACKEND") maybe_set_compound(settings, "origin_cache", "backend", "ORIGIN_CACHE") diff --git a/warehouse/legacy/api/simple.py b/warehouse/legacy/api/simple.py index a3bf32ae5e27..57178187feac 100644 --- a/warehouse/legacy/api/simple.py +++ b/warehouse/legacy/api/simple.py @@ -11,15 +11,14 @@ # limitations under the License. -from packaging.version import parse from pyramid.httpexceptions import HTTPMovedPermanently from pyramid.view import view_config from sqlalchemy import func -from sqlalchemy.orm import joinedload from warehouse.cache.http import cache_control from warehouse.cache.origin import origin_cache -from warehouse.packaging.models import File, JournalEntry, Project, Release +from warehouse.packaging.models import JournalEntry, Project +from warehouse.packaging.utils import _simple_detail @view_config( @@ -74,14 +73,4 @@ def simple_detail(project, request): # Get the latest serial number for this project. request.response.headers["X-PyPI-Last-Serial"] = str(project.last_serial) - # Get all of the files for this project. - files = sorted( - request.db.query(File) - .options(joinedload(File.release)) - .join(Release) - .filter(Release.project == project) - .all(), - key=lambda f: (parse(f.release.version), f.filename), - ) - - return {"project": project, "files": files} + return _simple_detail(project, request) diff --git a/warehouse/packaging/__init__.py b/warehouse/packaging/__init__.py index 3615f213e422..d720b801a459 100644 --- a/warehouse/packaging/__init__.py +++ b/warehouse/packaging/__init__.py @@ -17,7 +17,7 @@ from warehouse.accounts.models import Email, User from warehouse.cache.origin import key_factory, receive_set from warehouse.manage.tasks import update_role_invitation_status -from warehouse.packaging.interfaces import IDocsStorage, IFileStorage +from warehouse.packaging.interfaces import IDocsStorage, IFileStorage, ISimpleStorage from warehouse.packaging.models import File, Project, Release, Role from warehouse.packaging.tasks import ( # sync_bigquery_release_files, compute_trending, @@ -43,6 +43,11 @@ def includeme(config): files_storage_class = config.maybe_dotted(config.registry.settings["files.backend"]) config.register_service_factory(files_storage_class.create_service, IFileStorage) + simple_storage_class = config.maybe_dotted( + config.registry.settings["simple.backend"] + ) + config.register_service_factory(simple_storage_class.create_service, ISimpleStorage) + docs_storage_class = config.maybe_dotted(config.registry.settings["docs.backend"]) config.register_service_factory(docs_storage_class.create_service, IDocsStorage) diff --git a/warehouse/packaging/interfaces.py b/warehouse/packaging/interfaces.py index 3d747075ca92..e71d2c4fbf80 100644 --- a/warehouse/packaging/interfaces.py +++ b/warehouse/packaging/interfaces.py @@ -13,7 +13,7 @@ from zope.interface import Interface -class IFileStorage(Interface): +class IGenericFileStorage(Interface): def create_service(context, request): """ Create the service, given the context and request for which it is being @@ -34,6 +34,14 @@ def store(path, file_path, *, meta=None): """ +class IFileStorage(IGenericFileStorage): + pass + + +class ISimpleStorage(IGenericFileStorage): + pass + + class IDocsStorage(Interface): def create_service(context, request): """ diff --git a/warehouse/packaging/services.py b/warehouse/packaging/services.py index 054948786688..ea26141da41e 100644 --- a/warehouse/packaging/services.py +++ b/warehouse/packaging/services.py @@ -21,15 +21,14 @@ from zope.interface import implementer -from warehouse.packaging.interfaces import IDocsStorage, IFileStorage +from warehouse.packaging.interfaces import IDocsStorage, IFileStorage, ISimpleStorage class InsecureStorageWarning(UserWarning): pass -@implementer(IFileStorage) -class LocalFileStorage: +class GenericLocalBlobStorage: def __init__(self, base): # This class should not be used in production, it's trivial for it to # be used to read arbitrary files from the disk. It is intended ONLY @@ -46,7 +45,7 @@ def __init__(self, base): @classmethod def create_service(cls, context, request): - return cls(request.registry.settings["files.path"]) + raise NotImplementedError def get(self, path): return open(os.path.join(self.base, path), "rb") @@ -59,6 +58,20 @@ def store(self, path, file_path, *, meta=None): dest_fp.write(src_fp.read()) +@implementer(IFileStorage) +class LocalFileStorage(GenericLocalBlobStorage): + @classmethod + def create_service(cls, context, request): + return cls(request.registry.settings["files.path"]) + + +@implementer(ISimpleStorage) +class LocalSimpleStorage(GenericLocalBlobStorage): + @classmethod + def create_service(cls, context, request): + return cls(request.registry.settings["simple.path"]) + + @implementer(IDocsStorage) class LocalDocsStorage: def __init__(self, base): @@ -87,7 +100,7 @@ def remove_by_prefix(self, prefix): pass -class GenericFileStorage: +class GenericBlobStorage: def __init__(self, bucket, *, prefix=None): self.bucket = bucket self.prefix = prefix @@ -107,16 +120,7 @@ def _get_path(self, path): return path -@implementer(IFileStorage) -class S3FileStorage(GenericFileStorage): - @classmethod - def create_service(cls, context, request): - session = request.find_service(name="aws.session") - s3 = session.resource("s3") - bucket = s3.Bucket(request.registry.settings["files.bucket"]) - prefix = request.registry.settings.get("files.prefix") - return cls(bucket, prefix=prefix) - +class GenericS3BlobStorage(GenericBlobStorage): def get(self, path): # Note: this is not actually used in production, instead our CDN is # configured to connect directly to our storage bucket. See: @@ -138,6 +142,17 @@ def store(self, path, file_path, *, meta=None): self.bucket.upload_file(file_path, path, ExtraArgs=extra_args) +@implementer(IFileStorage) +class S3FileStorage(GenericS3BlobStorage): + @classmethod + def create_service(cls, context, request): + session = request.find_service(name="aws.session") + s3 = session.resource("s3") + bucket = s3.Bucket(request.registry.settings["files.bucket"]) + prefix = request.registry.settings.get("files.prefix") + return cls(bucket, prefix=prefix) + + @implementer(IDocsStorage) class S3DocsStorage: def __init__(self, s3_client, bucket_name, *, prefix=None): @@ -171,22 +186,7 @@ def remove_by_prefix(self, prefix): ) -@implementer(IFileStorage) -class GCSFileStorage(GenericFileStorage): - @classmethod - @google.api_core.retry.Retry( - predicate=google.api_core.retry.if_exception_type( - google.api_core.exceptions.ServiceUnavailable - ) - ) - def create_service(cls, context, request): - storage_client = request.find_service(name="gcloud.gcs") - bucket_name = request.registry.settings["files.bucket"] - bucket = storage_client.get_bucket(bucket_name) - prefix = request.registry.settings.get("files.prefix") - - return cls(bucket, prefix=prefix) - +class GenericGCSBlobStorage(GenericBlobStorage): def get(self, path): # Note: this is not actually used in production, instead our CDN is # configured to connect directly to our storage bucket. See: @@ -219,3 +219,37 @@ def store(self, path, file_path, *, meta=None): blob.upload_from_filename(file_path) else: sentry_sdk.capture_message(f"Skipped uploading duplicate file: {file_path}") + + +@implementer(IFileStorage) +class GCSFileStorage(GenericGCSBlobStorage): + @classmethod + @google.api_core.retry.Retry( + predicate=google.api_core.retry.if_exception_type( + google.api_core.exceptions.ServiceUnavailable + ) + ) + def create_service(cls, context, request): + storage_client = request.find_service(name="gcloud.gcs") + bucket_name = request.registry.settings["files.bucket"] + bucket = storage_client.get_bucket(bucket_name) + prefix = request.registry.settings.get("files.prefix") + + return cls(bucket, prefix=prefix) + + +@implementer(ISimpleStorage) +class GCSSimpleStorage(GenericGCSBlobStorage): + @classmethod + @google.api_core.retry.Retry( + predicate=google.api_core.retry.if_exception_type( + google.api_core.exceptions.ServiceUnavailable + ) + ) + def create_service(cls, context, request): + storage_client = request.find_service(name="gcloud.gcs") + bucket_name = request.registry.settings["simple.bucket"] + bucket = storage_client.get_bucket(bucket_name) + prefix = request.registry.settings.get("simple.prefix") + + return cls(bucket, prefix=prefix) diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py new file mode 100644 index 000000000000..f321217f546f --- /dev/null +++ b/warehouse/packaging/utils.py @@ -0,0 +1,79 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import os.path +import tempfile + +from packaging.version import parse +from pyramid_jinja2 import IJinja2Environment +from sqlalchemy.orm import joinedload + +from warehouse.packaging.interfaces import ISimpleStorage +from warehouse.packaging.models import File, Release + + +def _simple_detail(project, request): + # Get all of the files for this project. + files = sorted( + request.db.query(File) + .options(joinedload(File.release)) + .join(Release) + .filter(Release.project == project) + .all(), + key=lambda f: (parse(f.release.version), f.filename), + ) + + return {"project": project, "files": files} + + +def render_simple_detail(project, request, store=False): + context = _simple_detail(project, request) + + env = request.registry.queryUtility(IJinja2Environment, name=".jinja2") + template = env.get_template("templates/legacy/api/simple/detail.html") + content = template.render(**context, request=request) + + content_hasher = hashlib.blake2b(digest_size=256 // 8) + content_hasher.update(content.encode("utf-8")) + content_hash = content_hasher.hexdigest().lower() + + simple_detail_path = ( + f"{project.normalized_name}/{content_hash}.{project.normalized_name}.html" + ) + + if store: + storage = request.find_service(ISimpleStorage) + with tempfile.NamedTemporaryFile() as f: + f.write(content.encode("utf-8")) + f.flush() + + storage.store( + simple_detail_path, + f.name, + meta={ + "project": project.normalized_name, + "pypi-last-serial": project.last_serial, + "hash": content_hash, + }, + ) + storage.store( + os.path.join(project.normalized_name, "index.html"), + f.name, + meta={ + "project": project.normalized_name, + "pypi-last-serial": project.last_serial, + "hash": content_hash, + }, + ) + + return (content_hash, simple_detail_path)