Skip to content

Commit

Permalink
s3: collect HTTP response stats for unstable operations
Browse files Browse the repository at this point in the history
In some situations, inconsistent responses have been observed while
reading/iterating over files using the S3 protocol.

Record the HTTP status codes received during this operations as stats.
  • Loading branch information
nicois committed Nov 27, 2024
1 parent 81186ca commit 6a19820
Showing 1 changed file with 8 additions and 0 deletions.
8 changes: 8 additions & 0 deletions rohmu/object_storage/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,9 @@ def iter_key(
args["ContinuationToken"] = continuation_token
self.stats.operation(StorageOperation.iter_key)
response = self.get_client().list_objects_v2(**args)
status_code = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
if status_code:
self.stats.increase(metric="rohmu.s3.iter_key_response", tags={"status_code": str(status_code)})

for item in response["Contents"]:
if with_metadata:
Expand Down Expand Up @@ -405,17 +408,22 @@ def _get_object_stream(self, key: str, byte_range: Optional[tuple[int, int]]) ->
kwargs: dict[str, Any] = {}
if byte_range:
kwargs["Range"] = f"bytes={byte_range[0]}-{byte_range[1]}"
status_code : int|None = None
try:
# Actual usage is accounted for in
# _read_object_to_fileobj, although that omits the initial
# get_object call if it fails.
response = self.get_client().get_object(Bucket=self.bucket_name, Key=path, **kwargs)
status_code = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
except botocore.exceptions.ClientError as ex:
status_code = ex.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
if status_code == 404:
raise FileNotFoundFromStorageError(path)
else:
raise StorageError(f"Fetching the remote object {path} failed") from ex
finally:
if status_code:
self.stats.increase(metric="rohmu.s3.get_object_stream_response", tags={"status_code": str(status_code)})
return response["Body"], response["ContentLength"], response["Metadata"]

def _read_object_to_fileobj(
Expand Down

0 comments on commit 6a19820

Please sign in to comment.