Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

smart_open/s3: Initial implementations of str and repr #359

Merged
merged 7 commits into from
Oct 6, 2019
55 changes: 55 additions & 0 deletions smart_open/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,11 @@ class SeekableBufferedInputBase(BufferedInputBase):

def __init__(self, bucket, key, version_id=None, buffer_size=DEFAULT_BUFFER_SIZE,
line_terminator=BINARY_NEWLINE, session=None, resource_kwargs=None):

self._buffer_size = buffer_size
self._session = session
self._resource_kwargs = resource_kwargs

if session is None:
session = boto3.Session()
if resource_kwargs is None:
Expand Down Expand Up @@ -412,6 +417,29 @@ def truncate(self, size=None):
"""Unsupported."""
raise io.UnsupportedOperation

def __str__(self):
return "smart_open.s3.SeekableBufferedInputBase(%r, %r)" % (self._object.bucket_name, self._object.key)

def __repr__(self):
return (
"smart_open.s3.SeekableBufferedInputBase("
"bucket=%r, "
"key=%r, "
"version_id=%r, "
"buffer_size=%r, "
"line_terminator=%r, "
"session=%r, "
"resource_kwargs=%r)"
) % (
self._object.bucket_name,
self._object.key,
self._version_id,
self._buffer_size,
self._line_terminator,
self._session,
self._resource_kwargs,
)


class BufferedOutputBase(io.BufferedIOBase):
"""Writes bytes to S3.
Expand All @@ -427,6 +455,11 @@ def __init__(
resource_kwargs=None,
multipart_upload_kwargs=None,
):

self._session = session
self._resource_kwargs = resource_kwargs
self._multipart_upload_kwargs = multipart_upload_kwargs

if min_part_size < MIN_MIN_PART_SIZE:
logger.warning("S3 requires minimum part size >= 5MB; \
multipart upload may fail")
Expand All @@ -451,6 +484,7 @@ def __init__(
self._total_parts = 0
self._parts = []


#
# This member is part of the io.BufferedIOBase interface.
#
Expand Down Expand Up @@ -553,6 +587,27 @@ def __exit__(self, exc_type, exc_val, exc_tb):
else:
self.close()

def __str__(self):
return "smart_open.s3.BufferedOutputBase(%r, %r)" % (self._object.bucket_name, self._object.key)

def __repr__(self):
return (
"smart_open.s3.BufferedOutputBase("
"bucket=%r, "
"key=%r, "
"min_part_size=%r, "
"session=%r, "
"resource_kwargs=%r, "
"multipart_upload_kwargs=%r)"
) % (
self._object.bucket_name,
self._object.key,
self._min_part_size,
self._session,
self._resource_kwargs,
self._multipart_upload_kwargs,
)


def iter_bucket(bucket_name, prefix='', accept_key=None,
key_limit=None, workers=16, retries=3):
Expand Down