From ebec4e3c50998b806ef82c7ebc4d08b2a30ef0a5 Mon Sep 17 00:00:00 2001 From: Syed Date: Wed, 8 May 2024 15:15:57 -0400 Subject: [PATCH] storage: Add akamai storage provider (PROJQUAY-7238) Adds Akamai as another S3 backed storage provider for CDN redundency --- requirements.txt | 1 + storage/__init__.py | 2 + storage/akamaistorage.py | 87 ++++++++++++++++++++++++++++++++++++++ storage/multicdnstorage.py | 2 + 4 files changed, 92 insertions(+) create mode 100644 storage/akamaistorage.py diff --git a/requirements.txt b/requirements.txt index 9043451fa1..a16149728c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +akamai-edgeauth==0.3.2 alembic==1.3.3 aniso8601 @ git+https://github.com/DevTable/aniso8601-fake.git@bd7762c7dea0498706d3f57db60cd8a8af44ba90 APScheduler==3.10.4 diff --git a/storage/__init__.py b/storage/__init__.py index 2320a936bb..c0542010eb 100644 --- a/storage/__init__.py +++ b/storage/__init__.py @@ -1,3 +1,4 @@ +from storage.akamaistorage import AkamaiS3Storage from storage.azurestorage import AzureStorage from storage.cloud import ( CloudFrontedS3Storage, @@ -33,6 +34,7 @@ "MultiCDNStorage": MultiCDNStorage, "IBMCloudStorage": IBMCloudStorage, "STSS3Storage": STSS3Storage, + "AkamaiS3Storage": AkamaiS3Storage, } diff --git a/storage/akamaistorage.py b/storage/akamaistorage.py new file mode 100644 index 0000000000..c57a7dc8e7 --- /dev/null +++ b/storage/akamaistorage.py @@ -0,0 +1,87 @@ +import logging +import urllib.parse +from akamai.edgeauth import EdgeAuth, EdgeAuthError + +logger = logging.getLogger(__name__) + +from storage.cloud import S3Storage + +DEFAULT_SIGNED_URL_EXPIRY_SECONDS = 900 # 15 mins +TOKEN_QUERY_STRING = "akamai_signature" + + +class AkamaiS3Storage(S3Storage): + """ + Akamai CDN backed by S3 storage + """ + + def __init__( + self, + context, + akamai_domain, + akamai_shared_secret, + storage_path, + s3_bucket, + s3_region, + *args, + **kwargs, + ): + super(AkamaiS3Storage, self).__init__( + context, storage_path, s3_bucket, s3_region=s3_region, *args, **kwargs + ) + + self.akamai_domain = akamai_domain + self.akamai_shared_secret = akamai_shared_secret + self.region = s3_region + self.et = EdgeAuth( + token_name=TOKEN_QUERY_STRING, + key=self.akamai_shared_secret, + window_seconds=DEFAULT_SIGNED_URL_EXPIRY_SECONDS, + escape_early=True, + ) + + def get_direct_download_url( + self, path, request_ip=None, expires_in=60, requires_cors=False, head=False, **kwargs + ): + # If CloudFront could not be loaded, fall back to normal S3. + s3_presigned_url = super(AkamaiS3Storage, self).get_direct_download_url( + path, request_ip, expires_in, requires_cors, head + ) + s3_url_parsed = urllib.parse.urlparse(s3_presigned_url) + + # replace s3 location with Akamai domain + akamai_url_parsed = s3_url_parsed._replace(netloc=self.akamai_domain) + + # add akamai signed token + try: + + # add region to the query string + akamai_url_parsed = akamai_url_parsed._replace( + query=f"{akamai_url_parsed.query}®ion={self.region}" + ) + + # add additional params to the query string for metrics + additional_params = ["namespace", "username", "repo_name"] + for param in additional_params: + if param in kwargs and kwargs[param] is not None: + akamai_url_parsed = akamai_url_parsed._replace( + query=f"{akamai_url_parsed.query}&{param}={kwargs[param]}" + ) + + to_sign = f"{akamai_url_parsed.path}" + akamai_url_parsed = akamai_url_parsed._replace( + query=f"{akamai_url_parsed.query}&{TOKEN_QUERY_STRING}={self.et.generate_url_token(to_sign)}" + ) + + + except EdgeAuthError as e: + logger.error(f"Failed to generate Akamai token: {e}") + return s3_presigned_url + + logger.debug( + 'Returning Akamai URL for path "%s" with IP "%s": %s', + path, + request_ip, + akamai_url_parsed, + ) + return akamai_url_parsed.geturl() diff --git a/storage/multicdnstorage.py b/storage/multicdnstorage.py index ba6897e36a..41bd32d14a 100644 --- a/storage/multicdnstorage.py +++ b/storage/multicdnstorage.py @@ -2,6 +2,7 @@ from flask import has_request_context, request +from storage import AkamaiS3Storage from storage.basestorage import BaseStorageV2, InvalidStorageConfigurationException from storage.cloud import CloudFrontedS3Storage from storage.cloudflarestorage import CloudFlareS3Storage @@ -16,6 +17,7 @@ MULTICDN_STORAGE_PROVIDER_CLASSES = { "CloudFrontedS3Storage": CloudFrontedS3Storage, "CloudFlareStorage": CloudFlareS3Storage, + "AkamaiS3Storage": AkamaiS3Storage, }