tests.system.test_transfer_manager: test_download_chunks_concurrently failed

[flaky-bot]

This test failed!

To configure my behavior, see the Flaky Bot documentation.

If I'm commenting on this issue too often, add the flakybot: quiet label and
I will stop commenting.

---

commit: a3a1159
buildURL: Build Status, Sponge
status: failed

Test output

self = 
file_obj = <_io.BufferedReader name='/tmpfs/src/github/python-storage/tests/data/five-point-one-mb-file.zip'>
rewind = False, size = 5253120, content_type = 'application/zip'
num_retries = None, client = None, predefined_acl = None
if_generation_match = None, if_generation_not_match = None
if_metageneration_match = None, if_metageneration_not_match = None, timeout = 60
checksum = None
retry = 
command = None
def _prep_and_do_upload(
    self,
    file_obj,
    rewind=False,
    size=None,
    content_type=None,
    num_retries=None,
    client=None,
    predefined_acl=None,
    if_generation_match=None,
    if_generation_not_match=None,
    if_metageneration_match=None,
    if_metageneration_not_match=None,
    timeout=_DEFAULT_TIMEOUT,
    checksum=None,
    retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,
    command=None,
):
    """Upload the contents of this blob from a file-like object.

    The content type of the upload will be determined in order
    of precedence:

    - The value passed in to this method (if not :data:`None`)
    - The value stored on the current blob
    - The default value ('application/octet-stream')

    .. note::
       The effect of uploading to an existing blob depends on the
       "versioning" and "lifecycle" policies defined on the blob's
       bucket.  In the absence of those policies, upload will
       overwrite any existing contents.

       See the [`object versioning`](https://cloud.google.com/storage/docs/object-versioning)
       and [`lifecycle`](https://cloud.google.com/storage/docs/lifecycle)
       API documents for details.

    If the size of the data to be uploaded exceeds 8 MB a resumable media
    request will be used, otherwise the content and the metadata will be
    uploaded in a single multipart upload request.

    For more fine-grained over the upload process, check out
    [`google-resumable-media`](https://googleapis.dev/python/google-resumable-media/latest/index.html).

    If :attr:`user_project` is set on the bucket, bills the API request
    to that project.

    :type file_obj: file
    :param file_obj: A file handle opened in binary mode for reading.

    :type rewind: bool
    :param rewind:
        If True, seek to the beginning of the file handle before writing
        the file to Cloud Storage.

    :type size: int
    :param size:
        The number of bytes to be uploaded (which will be read from
        ``file_obj``). If not provided, the upload will be concluded once
        ``file_obj`` is exhausted.

    :type content_type: str
    :param content_type: (Optional) Type of content being uploaded.

    :type num_retries: int
    :param num_retries:
        Number of upload retries. By default, only uploads with
        if_generation_match set will be retried, as uploads without the
        argument are not guaranteed to be idempotent. Setting num_retries
        will override this default behavior and guarantee retries even when
        if_generation_match is not set.  (Deprecated: This argument
        will be removed in a future release.)

    :type client: :class:`~google.cloud.storage.client.Client`
    :param client:
        (Optional) The client to use.  If not passed, falls back to the
        ``client`` stored on the blob's bucket.

    :type predefined_acl: str
    :param predefined_acl: (Optional) Predefined access control list

    :type if_generation_match: long
    :param if_generation_match:
        (Optional) See :ref:`using-if-generation-match`

    :type if_generation_not_match: long
    :param if_generation_not_match:
        (Optional) See :ref:`using-if-generation-not-match`

    :type if_metageneration_match: long
    :param if_metageneration_match:
        (Optional) See :ref:`using-if-metageneration-match`

    :type if_metageneration_not_match: long
    :param if_metageneration_not_match:
        (Optional) See :ref:`using-if-metageneration-not-match`

    :type timeout: float or tuple
    :param timeout:
        (Optional) The amount of time, in seconds, to wait
        for the server response.  See: :ref:`configuring_timeouts`

    :type checksum: str
    :param checksum:
        (Optional) The type of checksum to compute to verify
        the integrity of the object. If the upload is completed in a single
        request, the checksum will be entirely precomputed and the remote
        server will handle verification and error handling. If the upload
        is too large and must be transmitted in multiple requests, the
        checksum will be incrementally computed and the client will handle
        verification and error handling, raising
        google.resumable_media.common.DataCorruption on a mismatch and
        attempting to delete the corrupted file. Supported values are
        "md5", "crc32c" and None. The default is None.

    :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
    :param retry: (Optional) How to retry the RPC. A None value will disable
        retries. A google.api_core.retry.Retry value will enable retries,
        and the object will define retriable response codes and errors and
        configure backoff and timeout options.

        A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a
        Retry object and activates it only if certain conditions are met.
        This class exists to provide safe defaults for RPC calls that are
        not technically safe to retry normally (due to potential data
        duplication or other side-effects) but become safe to retry if a
        condition such as if_generation_match is set.

        See the retry.py source code and docstrings in this package
        (google.cloud.storage.retry) for information on retry types and how
        to configure them.

        Media operations (downloads and uploads) do not support non-default
        predicates in a Retry object. The default will always be used. Other
        configuration changes for Retry objects such as delays and deadlines
        are respected.

    :type command: str
    :param command:
        (Optional) Information about which interface for upload was used,
        to be included in the X-Goog-API-Client header. Please leave as None
        unless otherwise directed.

    :raises: :class:`~google.cloud.exceptions.GoogleCloudError`
             if the upload response returns an error status.
    """
    if num_retries is not None:
        warnings.warn(_NUM_RETRIES_MESSAGE, DeprecationWarning, stacklevel=2)
        # num_retries and retry are mutually exclusive. If num_retries is
        # set and retry is exactly the default, then nullify retry for
        # backwards compatibility.
        if retry is DEFAULT_RETRY_IF_GENERATION_SPECIFIED:
            retry = None

    _maybe_rewind(file_obj, rewind=rewind)
    predefined_acl = ACL.validate_predefined(predefined_acl)

    try:


      created_json = self._do_upload(


            client,
            file_obj,
            content_type,
            size,
            num_retries,
            predefined_acl,
            if_generation_match,
            if_generation_not_match,
            if_metageneration_match,
            if_metageneration_not_match,
            timeout=timeout,
            checksum=checksum,
            retry=retry,
            command=command,
        )

google/cloud/storage/blob.py:2602:

google/cloud/storage/blob.py:2408: in _do_upload

response = self._do_multipart_upload(

google/cloud/storage/blob.py:1921: in _do_multipart_upload

response = upload.transmit(

.nox/system-3-8/lib/python3.8/site-packages/google/resumable_media/requests/upload.py:153: in transmit

return _request_helpers.wait_and_retry(

.nox/system-3-8/lib/python3.8/site-packages/google/resumable_media/requests/_request_helpers.py:178: in wait_and_retry

raise error

.nox/system-3-8/lib/python3.8/site-packages/google/resumable_media/requests/_request_helpers.py:155: in wait_and_retry

response = func()

.nox/system-3-8/lib/python3.8/site-packages/google/resumable_media/requests/upload.py:149: in retriable_request

self._process_response(result)

.nox/system-3-8/lib/python3.8/site-packages/google/resumable_media/_upload.py:125: in _process_response

_helpers.require_status_code(response, (http.client.OK,), self._get_status_code)

response = <Response [503]>, status_codes = (<HTTPStatus.OK: 200>,)

get_status_code = <function RequestsMixin._get_status_code at 0x7f1d97565040>

callback = <function do_nothing at 0x7f1d97547a60>
def require_status_code(response, status_codes, get_status_code, callback=do_nothing):
    """Require a response has a status code among a list.

    Args:
        response (object): The HTTP response object.
        status_codes (tuple): The acceptable status codes.
        get_status_code (Callable[Any, int]): Helper to get a status code
            from a response.
        callback (Optional[Callable]): A callback that takes no arguments,
            to be executed when an exception is being raised.

    Returns:
        int: The status code.

    Raises:
        ~google.resumable_media.common.InvalidResponse: If the status code
            is not one of the values in ``status_codes``.
    """
    status_code = get_status_code(response)
    if status_code not in status_codes:
        if status_code not in common.RETRYABLE:
            callback()


      raise common.InvalidResponse(


            response,
            "Request failed with status code",
            status_code,
            "Expected one of",
            *status_codes
        )

E           google.resumable_media.common.InvalidResponse: ('Request failed with status code', 503, 'Expected one of', <HTTPStatus.OK: 200>)
.nox/system-3-8/lib/python3.8/site-packages/google/resumable_media/_helpers.py:108: InvalidResponse
During handling of the above exception, another exception occurred:
shared_bucket = <Bucket: gcp-systest-1696869300933>

file_data = {'big': {'hash': b'cEome4a+NYd7YIXzXQnR5Q==', 'path': '/tmpfs/src/github/python-storage/tests/data/five-point-one-mb-f...g'}, 'simple': {'hash': b'3Hkwjv2WvCnKjNR6Z3CboA==', 'path': '/tmpfs/src/github/python-storage/tests/data/simple.txt'}}
def test_download_chunks_concurrently(shared_bucket, file_data):
    # Upload a big file
    source_file = file_data["big"]
    upload_blob = shared_bucket.blob("chunky_file")


  upload_blob.upload_from_filename(source_file["path"])


tests/system/test_transfer_manager.py:145:

google/cloud/storage/blob.py:2924: in upload_from_filename

self._handle_filename_and_upload(

google/cloud/storage/blob.py:2794: in _handle_filename_and_upload

self._prep_and_do_upload(

google/cloud/storage/blob.py:2620: in _prep_and_do_upload

_raise_from_invalid_response(exc)

error = InvalidResponse('Request failed with status code', 503, 'Expected one of', <HTTPStatus.OK: 200>)
def _raise_from_invalid_response(error):
    """Re-wrap and raise an ``InvalidResponse`` exception.

    :type error: :exc:`google.resumable_media.InvalidResponse`
    :param error: A caught exception from the ``google-resumable-media``
                  library.

    :raises: :class:`~google.cloud.exceptions.GoogleCloudError` corresponding
             to the failed status code
    """
    response = error.response

    # The 'response.text' gives the actual reason of error, where 'error' gives
    # the message of expected status code.
    if response.text:
        error_message = response.text + ": " + str(error)
    else:
        error_message = str(error)

    message = f"{response.request.method} {response.request.url}: {error_message}"


  raise exceptions.from_http_status(response.status_code, message, response=response)


E       google.api_core.exceptions.ServiceUnavailable: 503 POST https://storage-preprod-test-unified.googleusercontent.com/upload/storage/v1_preprod/b/gcp-systest-1696869300933/o?uploadType=multipart: Service Unavailable: ('Request failed with status code', 503, 'Expected one of', <HTTPStatus.OK: 200>)
google/cloud/storage/blob.py:4774: ServiceUnavailable

[flaky-bot]

Test passed for commit 3a0f551 (Build Status, Sponge)! Closing this issue.