diff --git a/_gcloud_vendor/__init__.py b/_gcloud_vendor/__init__.py new file mode 100644 index 000000000000..9ee34b0c867b --- /dev/null +++ b/_gcloud_vendor/__init__.py @@ -0,0 +1,8 @@ +"""Dependencies "vendored in", due to dependencies, Python versions, etc. + +Current set +----------- + +``apitools`` (pending release to PyPI, plus acceptable Python version + support for its dependencies). Review before M2. +""" diff --git a/_gcloud_vendor/apitools/__init__.py b/_gcloud_vendor/apitools/__init__.py new file mode 100644 index 000000000000..9870b5e53b94 --- /dev/null +++ b/_gcloud_vendor/apitools/__init__.py @@ -0,0 +1 @@ +"""Package stub.""" diff --git a/_gcloud_vendor/apitools/base/__init__.py b/_gcloud_vendor/apitools/base/__init__.py new file mode 100644 index 000000000000..9870b5e53b94 --- /dev/null +++ b/_gcloud_vendor/apitools/base/__init__.py @@ -0,0 +1 @@ +"""Package stub.""" diff --git a/_gcloud_vendor/apitools/base/py/__init__.py b/_gcloud_vendor/apitools/base/py/__init__.py new file mode 100644 index 000000000000..9870b5e53b94 --- /dev/null +++ b/_gcloud_vendor/apitools/base/py/__init__.py @@ -0,0 +1 @@ +"""Package stub.""" diff --git a/_gcloud_vendor/apitools/base/py/exceptions.py b/_gcloud_vendor/apitools/base/py/exceptions.py new file mode 100644 index 000000000000..55faa4970ebb --- /dev/null +++ b/_gcloud_vendor/apitools/base/py/exceptions.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +"""Exceptions for generated client libraries.""" + + +class Error(Exception): + """Base class for all exceptions.""" + + +class TypecheckError(Error, TypeError): + """An object of an incorrect type is provided.""" + + +class NotFoundError(Error): + """A specified resource could not be found.""" + + +class UserError(Error): + """Base class for errors related to user input.""" + + +class InvalidDataError(Error): + """Base class for any invalid data error.""" + + +class CommunicationError(Error): + """Any communication error talking to an API server.""" + + +class HttpError(CommunicationError): + """Error making a request. Soon to be HttpError.""" + + def __init__(self, response, content, url): + super(HttpError, self).__init__() + self.response = response + self.content = content + self.url = url + + def __str__(self): + content = self.content.decode('ascii', 'replace') + return 'HttpError accessing <%s>: response: <%s>, content <%s>' % ( + self.url, self.response, content) + + @property + def status_code(self): + # TODO(craigcitro): Turn this into something better than a + # KeyError if there is no status. + return int(self.response['status']) + + @classmethod + def FromResponse(cls, http_response): + return cls(http_response.info, http_response.content, + http_response.request_url) + + +class InvalidUserInputError(InvalidDataError): + """User-provided input is invalid.""" + + +class InvalidDataFromServerError(InvalidDataError, CommunicationError): + """Data received from the server is malformed.""" + + +class BatchError(Error): + """Error generated while constructing a batch request.""" + + +class ConfigurationError(Error): + """Base class for configuration errors.""" + + +class GeneratedClientError(Error): + """The generated client configuration is invalid.""" + + +class ConfigurationValueError(UserError): + """Some part of the user-specified client configuration is invalid.""" + + +class ResourceUnavailableError(Error): + """User requested an unavailable resource.""" + + +class CredentialsError(Error): + """Errors related to invalid credentials.""" + + +class TransferError(CommunicationError): + """Errors related to transfers.""" + + +class TransferInvalidError(TransferError): + """The given transfer is invalid.""" + + +class NotYetImplementedError(GeneratedClientError): + """This functionality is not yet implemented.""" + + +class StreamExhausted(Error): + """Attempted to read more bytes from a stream than were available.""" diff --git a/_gcloud_vendor/apitools/base/py/http_wrapper.py b/_gcloud_vendor/apitools/base/py/http_wrapper.py new file mode 100644 index 000000000000..80454f495752 --- /dev/null +++ b/_gcloud_vendor/apitools/base/py/http_wrapper.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python +"""HTTP wrapper for apitools. + +This library wraps the underlying http library we use, which is +currently httplib2. +""" + +import collections +import httplib +import logging +import socket +import time +import urlparse + +import httplib2 + +from _gcloud_vendor.apitools.base.py import exceptions +from _gcloud_vendor.apitools.base.py import util + +__all__ = [ + 'GetHttp', + 'MakeRequest', + 'Request', +] + + +# 308 and 429 don't have names in httplib. +RESUME_INCOMPLETE = 308 +TOO_MANY_REQUESTS = 429 +_REDIRECT_STATUS_CODES = ( + httplib.MOVED_PERMANENTLY, + httplib.FOUND, + httplib.SEE_OTHER, + httplib.TEMPORARY_REDIRECT, + RESUME_INCOMPLETE, +) + + +class Request(object): + """Class encapsulating the data for an HTTP request.""" + + def __init__(self, url='', http_method='GET', headers=None, body=''): + self.url = url + self.http_method = http_method + self.headers = headers or {} + self.__body = None + self.body = body + + @property + def body(self): + return self.__body + + @body.setter + def body(self, value): + self.__body = value + if value is not None: + self.headers['content-length'] = str(len(self.__body)) + else: + self.headers.pop('content-length', None) + + +# Note: currently the order of fields here is important, since we want +# to be able to pass in the result from httplib2.request. +class Response(collections.namedtuple( + 'HttpResponse', ['info', 'content', 'request_url'])): + """Class encapsulating data for an HTTP response.""" + __slots__ = () + + def __len__(self): + def ProcessContentRange(content_range): + _, _, range_spec = content_range.partition(' ') + byte_range, _, _ = range_spec.partition('/') + start, _, end = byte_range.partition('-') + return int(end) - int(start) + 1 + + if '-content-encoding' in self.info and 'content-range' in self.info: + # httplib2 rewrites content-length in the case of a compressed + # transfer; we can't trust the content-length header in that + # case, but we *can* trust content-range, if it's present. + return ProcessContentRange(self.info['content-range']) + elif 'content-length' in self.info: + return int(self.info.get('content-length')) + elif 'content-range' in self.info: + return ProcessContentRange(self.info['content-range']) + return len(self.content) + + @property + def status_code(self): + return int(self.info['status']) + + @property + def retry_after(self): + if 'retry-after' in self.info: + return int(self.info['retry-after']) + + @property + def is_redirect(self): + return (self.status_code in _REDIRECT_STATUS_CODES and + 'location' in self.info) + + +def MakeRequest(http, http_request, retries=5, redirections=5): + """Send http_request via the given http. + + This wrapper exists to handle translation between the plain httplib2 + request/response types and the Request and Response types above. + This will also be the hook for error/retry handling. + + Args: + http: An httplib2.Http instance, or a http multiplexer that delegates to + an underlying http, for example, HTTPMultiplexer. + http_request: A Request to send. + retries: (int, default 5) Number of retries to attempt on 5XX replies. + redirections: (int, default 5) Number of redirects to follow. + + Returns: + A Response object. + + Raises: + InvalidDataFromServerError: if there is no response after retries. + """ + response = None + exc = None + connection_type = None + # Handle overrides for connection types. This is used if the caller + # wants control over the underlying connection for managing callbacks + # or hash digestion. + if getattr(http, 'connections', None): + url_scheme = urlparse.urlsplit(http_request.url).scheme + if url_scheme and url_scheme in http.connections: + connection_type = http.connections[url_scheme] + for retry in xrange(retries + 1): + # Note that the str() calls here are important for working around + # some funny business with message construction and unicode in + # httplib itself. See, eg, + # http://bugs.python.org/issue11898 + info = None + try: + info, content = http.request( + str(http_request.url), method=str(http_request.http_method), + body=http_request.body, headers=http_request.headers, + redirections=redirections, connection_type=connection_type) + except httplib.BadStatusLine as e: + logging.error('Caught BadStatusLine from httplib, retrying: %s', e) + exc = e + except socket.error as e: + if http_request.http_method != 'GET': + raise + logging.error('Caught socket error, retrying: %s', e) + exc = e + except httplib.IncompleteRead as e: + if http_request.http_method != 'GET': + raise + logging.error('Caught IncompleteRead error, retrying: %s', e) + exc = e + if info is not None: + response = Response(info, content, http_request.url) + if (response.status_code < 500 and + response.status_code != TOO_MANY_REQUESTS and + not response.retry_after): + break + logging.info('Retrying request to url <%s> after status code %s.', + response.request_url, response.status_code) + elif isinstance(exc, httplib.IncompleteRead): + logging.info('Retrying request to url <%s> after incomplete read.', + str(http_request.url)) + else: + logging.info('Retrying request to url <%s> after connection break.', + str(http_request.url)) + # TODO(craigcitro): Make this timeout configurable. + if response: + time.sleep(response.retry_after or util.CalculateWaitForRetry(retry)) + else: + time.sleep(util.CalculateWaitForRetry(retry)) + if response is None: + raise exceptions.InvalidDataFromServerError( + 'HTTP error on final retry: %s' % exc) + return response + + +def GetHttp(): + return httplib2.Http() diff --git a/_gcloud_vendor/apitools/base/py/transfer.py b/_gcloud_vendor/apitools/base/py/transfer.py new file mode 100644 index 000000000000..46dbc7f22c82 --- /dev/null +++ b/_gcloud_vendor/apitools/base/py/transfer.py @@ -0,0 +1,716 @@ +#!/usr/bin/env python +"""Upload and download support for apitools.""" + +import email.generator as email_generator +import email.mime.multipart as mime_multipart +import email.mime.nonmultipart as mime_nonmultipart +import httplib +import io +import json +import mimetypes +import os +import StringIO +import threading + +from _gcloud_vendor.apitools.base.py import exceptions +from _gcloud_vendor.apitools.base.py import http_wrapper +from _gcloud_vendor.apitools.base.py import util + +__all__ = [ + 'Download', + 'Upload', +] + +_RESUMABLE_UPLOAD_THRESHOLD = 5 << 20 +_SIMPLE_UPLOAD = 'simple' +_RESUMABLE_UPLOAD = 'resumable' + + +class _Transfer(object): + """Generic bits common to Uploads and Downloads.""" + + def __init__(self, stream, close_stream=False, chunksize=None, + auto_transfer=True, http=None): + self.__bytes_http = None + self.__close_stream = close_stream + self.__http = http + self.__stream = stream + self.__url = None + + self.auto_transfer = auto_transfer + self.chunksize = chunksize or 1048576L + + def __repr__(self): + return str(self) + + @property + def close_stream(self): + return self.__close_stream + + @property + def http(self): + return self.__http + + @property + def bytes_http(self): + return self.__bytes_http or self.http + + @bytes_http.setter + def bytes_http(self, value): + self.__bytes_http = value + + @property + def stream(self): + return self.__stream + + @property + def url(self): + return self.__url + + def _Initialize(self, http, url): + """Initialize this download by setting self.http and self.url. + + We want the user to be able to override self.http by having set + the value in the constructor; in that case, we ignore the provided + http. + + Args: + http: An httplib2.Http instance or None. + url: The url for this transfer. + + Returns: + None. Initializes self. + """ + self.EnsureUninitialized() + if self.http is None: + self.__http = http or http_wrapper.GetHttp() + self.__url = url + + @property + def initialized(self): + return self.url is not None and self.http is not None + + @property + def _type_name(self): + return type(self).__name__ + + def EnsureInitialized(self): + if not self.initialized: + raise exceptions.TransferInvalidError( + 'Cannot use uninitialized %s', self._type_name) + + def EnsureUninitialized(self): + if self.initialized: + raise exceptions.TransferInvalidError( + 'Cannot re-initialize %s', self._type_name) + + def __del__(self): + if self.__close_stream: + self.__stream.close() + + def _ExecuteCallback(self, callback, response): + # TODO(craigcitro): Push these into a queue. + if callback is not None: + threading.Thread(target=callback, args=(response, self)).start() + + +class Download(_Transfer): + """Data for a single download. + + Public attributes: + chunksize: default chunksize to use for transfers. + """ + _ACCEPTABLE_STATUSES = set(( + httplib.OK, + httplib.NO_CONTENT, + httplib.PARTIAL_CONTENT, + httplib.REQUESTED_RANGE_NOT_SATISFIABLE, + )) + _REQUIRED_SERIALIZATION_KEYS = set(( + 'auto_transfer', 'progress', 'total_size', 'url')) + + def __init__(self, *args, **kwds): + super(Download, self).__init__(*args, **kwds) + self.__initial_response = None + self.__progress = 0 + self.__total_size = None + + @property + def progress(self): + return self.__progress + + @classmethod + def FromFile(cls, filename, overwrite=False, auto_transfer=True): + """Create a new download object from a filename.""" + path = os.path.expanduser(filename) + if os.path.exists(path) and not overwrite: + raise exceptions.InvalidUserInputError( + 'File %s exists and overwrite not specified' % path) + return cls(open(path, 'wb'), close_stream=True, auto_transfer=auto_transfer) + + @classmethod + def FromStream(cls, stream, auto_transfer=True): + """Create a new Download object from a stream.""" + return cls(stream, auto_transfer=auto_transfer) + + @classmethod + def FromData(cls, stream, json_data, http=None, auto_transfer=None): + """Create a new Download object from a stream and serialized data.""" + info = json.loads(json_data) + missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) + if missing_keys: + raise exceptions.InvalidDataError( + 'Invalid serialization data, missing keys: %s' % ( + ', '.join(missing_keys))) + download = cls.FromStream(stream) + if auto_transfer is not None: + download.auto_transfer = auto_transfer + else: + download.auto_transfer = info['auto_transfer'] + setattr(download, '_Download__progress', info['progress']) + setattr(download, '_Download__total_size', info['total_size']) + download._Initialize(http, info['url']) # pylint: disable=protected-access + return download + + @property + def serialization_data(self): + self.EnsureInitialized() + return { + 'auto_transfer': self.auto_transfer, + 'progress': self.progress, + 'total_size': self.total_size, + 'url': self.url, + } + + @property + def total_size(self): + return self.__total_size + + def __str__(self): + if not self.initialized: + return 'Download (uninitialized)' + else: + return 'Download with %d/%s bytes transferred from url %s' % ( + self.progress, self.total_size, self.url) + + def ConfigureRequest(self, http_request, url_builder): + url_builder.query_params['alt'] = 'media' + http_request.headers['Range'] = 'bytes=0-%d' % (self.chunksize - 1,) + + def __SetTotal(self, info): + if 'content-range' in info: + _, _, total = info['content-range'].rpartition('/') + if total != '*': + self.__total_size = int(total) + # Note "total_size is None" means we don't know it; if no size + # info was returned on our initial range request, that means we + # have a 0-byte file. (That last statement has been verified + # empirically, but is not clearly documented anywhere.) + if self.total_size is None: + self.__total_size = 0 + + def InitializeDownload(self, http_request, http=None, client=None): + """Initialize this download by making a request. + + Args: + http_request: The HttpRequest to use to initialize this download. + http: The httplib2.Http instance for this request. + client: If provided, let this client process the final URL before + sending any additional requests. If client is provided and + http is not, client.http will be used instead. + """ + self.EnsureUninitialized() + if http is None and client is None: + raise exceptions.UserError('Must provide client or http.') + http = http or client.http + if client is not None: + http_request.url = client.FinalizeTransferUrl(http_request.url) + response = http_wrapper.MakeRequest(self.bytes_http or http, http_request) + if response.status_code not in self._ACCEPTABLE_STATUSES: + raise exceptions.HttpError.FromResponse(response) + self.__initial_response = response + self.__SetTotal(response.info) + url = response.info.get('content-location', response.request_url) + if client is not None: + url = client.FinalizeTransferUrl(url) + self._Initialize(http, url) + # Unless the user has requested otherwise, we want to just + # go ahead and pump the bytes now. + if self.auto_transfer: + self.StreamInChunks() + + @staticmethod + def _ArgPrinter(response, unused_download): + if 'content-range' in response.info: + print 'Received %s' % response.info['content-range'] + else: + print 'Received %d bytes' % len(response) + + @staticmethod + def _CompletePrinter(*unused_args): + print 'Download complete' + + def __NormalizeStartEnd(self, start, end=None): + if end is not None: + if start < 0: + raise exceptions.TransferInvalidError( + 'Cannot have end index with negative start index') + elif start >= self.total_size: + raise exceptions.TransferInvalidError( + 'Cannot have start index greater than total size') + end = min(end, self.total_size - 1) + if end < start: + raise exceptions.TransferInvalidError( + 'Range requested with end[%s] < start[%s]' % (end, start)) + return start, end + else: + if start < 0: + start = max(0, start + self.total_size) + return start, self.total_size + + def __SetRangeHeader(self, request, start, end=None): + if start < 0: + request.headers['range'] = 'bytes=%d' % start + elif end is None: + request.headers['range'] = 'bytes=%d-' % start + else: + request.headers['range'] = 'bytes=%d-%d' % (start, end) + + def __GetChunk(self, start, end=None, additional_headers=None): + """Retrieve a chunk, and return the full response.""" + self.EnsureInitialized() + end_byte = min(end or start + self.chunksize, self.total_size) + request = http_wrapper.Request(url=self.url) + self.__SetRangeHeader(request, start, end=end_byte) + if additional_headers is not None: + request.headers.update(additional_headers) + return http_wrapper.MakeRequest(self.bytes_http, request) + + def __ProcessResponse(self, response): + """Process this response (by updating self and writing to self.stream).""" + if response.status_code not in self._ACCEPTABLE_STATUSES: + raise exceptions.TransferInvalidError(response.content) + if response.status_code in (httplib.OK, httplib.PARTIAL_CONTENT): + self.stream.write(response.content) + self.__progress += len(response) + elif response.status_code == httplib.NO_CONTENT: + # It's important to write something to the stream for the case + # of a 0-byte download to a file, as otherwise python won't + # create the file. + self.stream.write('') + return response + + def GetRange(self, start, end=None, additional_headers=None): + """Retrieve a given byte range from this download, inclusive. + + Range must be of one of these three forms: + * 0 <= start, end = None: Fetch from start to the end of the file. + * 0 <= start <= end: Fetch the bytes from start to end. + * start < 0, end = None: Fetch the last -start bytes of the file. + + (These variations correspond to those described in the HTTP 1.1 + protocol for range headers in RFC 2616, sec. 14.35.1.) + + Args: + start: (int) Where to start fetching bytes. (See above.) + end: (int, optional) Where to stop fetching bytes. (See above.) + additional_headers: (bool, optional) Any additional headers to + pass with the request. + + Returns: + None. Streams bytes into self.stream. + """ + self.EnsureInitialized() + progress, end = self.__NormalizeStartEnd(start, end) + while progress < end: + chunk_end = min(progress + self.chunksize, end) + response = self.__GetChunk(progress, end=chunk_end, + additional_headers=additional_headers) + response = self.__ProcessResponse(response) + progress += len(response) + if not response: + raise exceptions.TransferInvalidError( + 'Zero bytes unexpectedly returned in download response') + + def StreamInChunks(self, callback=None, finish_callback=None, + additional_headers=None): + """Stream the entire download.""" + callback = callback or self._ArgPrinter + finish_callback = finish_callback or self._CompletePrinter + + self.EnsureInitialized() + while True: + if self.__initial_response is not None: + response = self.__initial_response + self.__initial_response = None + else: + response = self.__GetChunk(self.progress, + additional_headers=additional_headers) + response = self.__ProcessResponse(response) + self._ExecuteCallback(callback, response) + if (response.status_code == httplib.OK or + self.progress >= self.total_size): + break + self._ExecuteCallback(finish_callback, response) + + +class Upload(_Transfer): + """Data for a single Upload. + + Fields: + stream: The stream to upload. + mime_type: MIME type of the upload. + total_size: (optional) Total upload size for the stream. + close_stream: (default: False) Whether or not we should close the + stream when finished with the upload. + auto_transfer: (default: True) If True, stream all bytes as soon as + the upload is created. + """ + _REQUIRED_SERIALIZATION_KEYS = set(( + 'auto_transfer', 'mime_type', 'total_size', 'url')) + + def __init__(self, stream, mime_type, total_size=None, http=None, + close_stream=False, chunksize=None, auto_transfer=True): + super(Upload, self).__init__( + stream, close_stream=close_stream, chunksize=chunksize, + auto_transfer=auto_transfer, http=http) + self.__complete = False + self.__mime_type = mime_type + self.__progress = 0 + self.__server_chunk_granularity = None + self.__strategy = None + + self.total_size = total_size + + @property + def progress(self): + return self.__progress + + @classmethod + def FromFile(cls, filename, mime_type=None, auto_transfer=True): + """Create a new Upload object from a filename.""" + path = os.path.expanduser(filename) + if not os.path.exists(path): + raise exceptions.NotFoundError('Could not find file %s' % path) + if not mime_type: + mime_type, _ = mimetypes.guess_type(path) + if mime_type is None: + raise exceptions.InvalidUserInputError( + 'Could not guess mime type for %s' % path) + size = os.stat(path).st_size + return cls(open(path, 'rb'), mime_type, total_size=size, close_stream=True, + auto_transfer=auto_transfer) + + @classmethod + def FromStream(cls, stream, mime_type, total_size=None, auto_transfer=True): + """Create a new Upload object from a stream.""" + if mime_type is None: + raise exceptions.InvalidUserInputError( + 'No mime_type specified for stream') + return cls(stream, mime_type, total_size=total_size, close_stream=False, + auto_transfer=auto_transfer) + + @classmethod + def FromData(cls, stream, json_data, http, auto_transfer=None): + """Create a new Upload of stream from serialized json_data using http.""" + info = json.loads(json_data) + missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) + if missing_keys: + raise exceptions.InvalidDataError( + 'Invalid serialization data, missing keys: %s' % ( + ', '.join(missing_keys))) + upload = cls.FromStream(stream, info['mime_type'], + total_size=info.get('total_size')) + if isinstance(stream, io.IOBase) and not stream.seekable(): + raise exceptions.InvalidUserInputError( + 'Cannot restart resumable upload on non-seekable stream') + if auto_transfer is not None: + upload.auto_transfer = auto_transfer + else: + upload.auto_transfer = info['auto_transfer'] + upload.strategy = _RESUMABLE_UPLOAD + upload._Initialize(http, info['url']) # pylint: disable=protected-access + upload._RefreshResumableUploadState() # pylint: disable=protected-access + upload.EnsureInitialized() + if upload.auto_transfer: + upload.StreamInChunks() + return upload + + @property + def serialization_data(self): + self.EnsureInitialized() + if self.strategy != _RESUMABLE_UPLOAD: + raise exceptions.InvalidDataError( + 'Serialization only supported for resumable uploads') + return { + 'auto_transfer': self.auto_transfer, + 'mime_type': self.mime_type, + 'total_size': self.total_size, + 'url': self.url, + } + + @property + def complete(self): + return self.__complete + + @property + def mime_type(self): + return self.__mime_type + + def __str__(self): + if not self.initialized: + return 'Upload (uninitialized)' + else: + return 'Upload with %d/%s bytes transferred for url %s' % ( + self.progress, self.total_size or '???', self.url) + + @property + def strategy(self): + return self.__strategy + + @strategy.setter + def strategy(self, value): + if value not in (_SIMPLE_UPLOAD, _RESUMABLE_UPLOAD): + raise exceptions.UserError(( + 'Invalid value "%s" for upload strategy, must be one of ' + '"simple" or "resumable".') % value) + self.__strategy = value + + @property + def total_size(self): + return self.__total_size + + @total_size.setter + def total_size(self, value): + self.EnsureUninitialized() + self.__total_size = value + + def __SetDefaultUploadStrategy(self, upload_config, http_request): + """Determine and set the default upload strategy for this upload. + + We generally prefer simple or multipart, unless we're forced to + use resumable. This happens when any of (1) the upload is too + large, (2) the simple endpoint doesn't support multipart requests + and we have metadata, or (3) there is no simple upload endpoint. + + Args: + upload_config: Configuration for the upload endpoint. + http_request: The associated http request. + + Returns: + None. + """ + if self.strategy is not None: + return + strategy = _SIMPLE_UPLOAD + if (self.total_size is not None and + self.total_size > _RESUMABLE_UPLOAD_THRESHOLD): + strategy = _RESUMABLE_UPLOAD + if http_request.body and not upload_config.simple_multipart: + strategy = _RESUMABLE_UPLOAD + if not upload_config.simple_path: + strategy = _RESUMABLE_UPLOAD + self.strategy = strategy + + def ConfigureRequest(self, upload_config, http_request, url_builder): + """Configure the request and url for this upload.""" + # Validate total_size vs. max_size + if (self.total_size and upload_config.max_size and + self.total_size > upload_config.max_size): + raise exceptions.InvalidUserInputError( + 'Upload too big: %s larger than max size %s' % ( + self.total_size, upload_config.max_size)) + # Validate mime type + if not util.AcceptableMimeType(upload_config.accept, self.mime_type): + raise exceptions.InvalidUserInputError( + 'MIME type %s does not match any accepted MIME ranges %s' % ( + self.mime_type, upload_config.accept)) + + self.__SetDefaultUploadStrategy(upload_config, http_request) + if self.strategy == _SIMPLE_UPLOAD: + url_builder.relative_path = upload_config.simple_path + if http_request.body: + url_builder.query_params['uploadType'] = 'multipart' + self.__ConfigureMultipartRequest(http_request) + else: + url_builder.query_params['uploadType'] = 'media' + self.__ConfigureMediaRequest(http_request) + else: + url_builder.relative_path = upload_config.resumable_path + url_builder.query_params['uploadType'] = 'resumable' + self.__ConfigureResumableRequest(http_request) + + def __ConfigureMediaRequest(self, http_request): + """Configure http_request as a simple request for this upload.""" + http_request.headers['content-type'] = self.mime_type + http_request.body = self.stream.read() + + def __ConfigureMultipartRequest(self, http_request): + """Configure http_request as a multipart request for this upload.""" + # This is a multipart/related upload. + msg_root = mime_multipart.MIMEMultipart('related') + # msg_root should not write out its own headers + setattr(msg_root, '_write_headers', lambda self: None) + + # attach the body as one part + msg = mime_nonmultipart.MIMENonMultipart( + *http_request.headers['content-type'].split('/')) + msg.set_payload(http_request.body) + msg_root.attach(msg) + + # attach the media as the second part + msg = mime_nonmultipart.MIMENonMultipart(*self.mime_type.split('/')) + msg['Content-Transfer-Encoding'] = 'binary' + msg.set_payload(self.stream.read()) + msg_root.attach(msg) + + # encode the body: note that we can't use `as_string`, because + # it plays games with `From ` lines. + fp = StringIO.StringIO() + g = email_generator.Generator(fp, mangle_from_=False) + g.flatten(msg_root, unixfrom=False) + http_request.body = fp.getvalue() + + multipart_boundary = msg_root.get_boundary() + http_request.headers['content-type'] = ( + 'multipart/related; boundary=%r' % multipart_boundary) + + def __ConfigureResumableRequest(self, http_request): + http_request.headers['X-Upload-Content-Type'] = self.mime_type + if self.total_size is not None: + http_request.headers['X-Upload-Content-Length'] = str(self.total_size) + + def _RefreshResumableUploadState(self): + """Talk to the server and refresh the state of this resumable upload.""" + if self.strategy != _RESUMABLE_UPLOAD: + return + self.EnsureInitialized() + refresh_request = http_wrapper.Request( + url=self.url, http_method='PUT', headers={'Content-Range': 'bytes */*'}) + refresh_response = http_wrapper.MakeRequest( + self.http, refresh_request, redirections=0) + range_header = refresh_response.info.get( + 'Range', refresh_response.info.get('range')) + if refresh_response.status_code in (httplib.OK, httplib.CREATED): + self.__complete = True + elif refresh_response.status_code == http_wrapper.RESUME_INCOMPLETE: + if range_header is None: + self.__progress = 0 + else: + self.__progress = self.__GetLastByte(range_header) + 1 + self.stream.seek(self.progress) + else: + raise exceptions.HttpError.FromResponse(refresh_response) + + def InitializeUpload(self, http_request, http=None, client=None): + """Initialize this upload from the given http_request.""" + if self.strategy is None: + raise exceptions.UserError( + 'No upload strategy set; did you call ConfigureRequest?') + if http is None and client is None: + raise exceptions.UserError('Must provide client or http.') + if self.strategy != _RESUMABLE_UPLOAD: + return + if self.total_size is None: + raise exceptions.InvalidUserInputError( + 'Cannot stream upload without total size') + http = http or client.http + if client is not None: + http_request.url = client.FinalizeTransferUrl(http_request.url) + self.EnsureUninitialized() + http_response = http_wrapper.MakeRequest(http, http_request) + if http_response.status_code != httplib.OK: + raise exceptions.HttpError.FromResponse(http_response) + + self.__server_chunk_granularity = http_response.info.get( + 'X-Goog-Upload-Chunk-Granularity') + self.__ValidateChunksize() + url = http_response.info['location'] + if client is not None: + url = client.FinalizeTransferUrl(url) + self._Initialize(http, url) + + # Unless the user has requested otherwise, we want to just + # go ahead and pump the bytes now. + if self.auto_transfer: + return self.StreamInChunks() + + def __GetLastByte(self, range_header): + _, _, end = range_header.partition('-') + # TODO(craigcitro): Validate start == 0? + return int(end) + + def __ValidateChunksize(self, chunksize=None): + if self.__server_chunk_granularity is None: + return + chunksize = chunksize or self.chunksize + if chunksize % self.__server_chunk_granularity: + raise exceptions.ConfigurationValueError( + 'Server requires chunksize to be a multiple of %d', + self.__server_chunk_granularity) + + @staticmethod + def _ArgPrinter(response, unused_upload): + print 'Sent %s' % response.info['range'] + + @staticmethod + def _CompletePrinter(*unused_args): + print 'Upload complete' + + def StreamInChunks(self, callback=None, finish_callback=None, + additional_headers=None): + """Send this (resumable) upload in chunks.""" + if self.strategy != _RESUMABLE_UPLOAD: + raise exceptions.InvalidUserInputError( + 'Cannot stream non-resumable upload') + if self.total_size is None: + raise exceptions.InvalidUserInputError( + 'Cannot stream upload without total size') + callback = callback or self._ArgPrinter + finish_callback = finish_callback or self._CompletePrinter + response = None + self.__ValidateChunksize(self.chunksize) + self.EnsureInitialized() + while not self.complete: + response = self.__SendChunk(self.stream.tell(), + additional_headers=additional_headers) + if response.status_code in (httplib.OK, httplib.CREATED): + self.__complete = True + break + self.__progress = self.__GetLastByte(response.info['range']) + if self.progress + 1 != self.stream.tell(): + # TODO(craigcitro): Add a better way to recover here. + raise exceptions.CommunicationError( + 'Failed to transfer all bytes in chunk, upload paused at byte ' + '%d' % self.progress) + self._ExecuteCallback(callback, response) + self._ExecuteCallback(finish_callback, response) + return response + + def __SendChunk(self, start, additional_headers=None, data=None): + """Send the specified chunk.""" + self.EnsureInitialized() + if data is None: + data = self.stream.read(self.chunksize) + end = start + len(data) + + request = http_wrapper.Request(url=self.url, http_method='PUT', body=data) + request.headers['Content-Type'] = self.mime_type + if data: + request.headers['Content-Range'] = 'bytes %s-%s/%s' % ( + start, end - 1, self.total_size) + if additional_headers: + request.headers.update(additional_headers) + + response = http_wrapper.MakeRequest(self.bytes_http, request) + if response.status_code not in (httplib.OK, httplib.CREATED, + http_wrapper.RESUME_INCOMPLETE): + raise exceptions.HttpError.FromResponse(response) + if response.status_code in (httplib.OK, httplib.CREATED): + return response + # TODO(craigcitro): Add retries on no progress? + last_byte = self.__GetLastByte(response.info['range']) + if last_byte + 1 != end: + new_start = last_byte + 1 - start + response = self.__SendChunk(last_byte + 1, data=data[new_start:]) + return response diff --git a/_gcloud_vendor/apitools/base/py/util.py b/_gcloud_vendor/apitools/base/py/util.py new file mode 100644 index 000000000000..4d64bedf538e --- /dev/null +++ b/_gcloud_vendor/apitools/base/py/util.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python +"""Assorted utilities shared between parts of apitools.""" + +import collections +import httplib +import os +import random +import types +import urllib +import urllib2 + +from _gcloud_vendor.apitools.base.py import exceptions + +__all__ = [ + 'DetectGae', + 'DetectGce', +] + +_RESERVED_URI_CHARS = r":/?#[]@!$&'()*+,;=" + + +def DetectGae(): + """Determine whether or not we're running on GAE. + + This is based on: + https://developers.google.com/appengine/docs/python/#The_Environment + + Returns: + True iff we're running on GAE. + """ + server_software = os.environ.get('SERVER_SOFTWARE', '') + return (server_software.startswith('Development/') or + server_software.startswith('Google App Engine/')) + + +def DetectGce(): + """Determine whether or not we're running on GCE. + + This is based on: + https://cloud.google.com/compute/docs/metadata#runninggce + + Returns: + True iff we're running on a GCE instance. + """ + try: + o = urllib2.urlopen('http://metadata.google.internal') + except urllib2.URLError: + return False + return (o.getcode() == httplib.OK and + o.headers.get('metadata-flavor') == 'Google') + + +def NormalizeScopes(scope_spec): + """Normalize scope_spec to a set of strings.""" + if isinstance(scope_spec, types.StringTypes): + return set(scope_spec.split(' ')) + elif isinstance(scope_spec, collections.Iterable): + return set(scope_spec) + raise exceptions.TypecheckError( + 'NormalizeScopes expected string or iterable, found %s' % ( + type(scope_spec),)) + + +def Typecheck(arg, arg_type, msg=None): + if not isinstance(arg, arg_type): + if msg is None: + if isinstance(arg_type, tuple): + msg = 'Type of arg is "%s", not one of %r' % (type(arg), arg_type) + else: + msg = 'Type of arg is "%s", not "%s"' % (type(arg), arg_type) + raise exceptions.TypecheckError(msg) + return arg + + +def ExpandRelativePath(method_config, params, relative_path=None): + """Determine the relative path for request.""" + path = relative_path or method_config.relative_path or '' + + for param in method_config.path_params: + param_template = '{%s}' % param + # For more details about "reserved word expansion", see: + # http://tools.ietf.org/html/rfc6570#section-3.2.2 + reserved_chars = '' + reserved_template = '{+%s}' % param + if reserved_template in path: + reserved_chars = _RESERVED_URI_CHARS + path = path.replace(reserved_template, param_template) + if param_template not in path: + raise exceptions.InvalidUserInputError( + 'Missing path parameter %s' % param) + try: + # TODO(craigcitro): Do we want to support some sophisticated + # mapping here? + value = params[param] + except KeyError: + raise exceptions.InvalidUserInputError( + 'Request missing required parameter %s' % param) + if value is None: + raise exceptions.InvalidUserInputError( + 'Request missing required parameter %s' % param) + try: + if not isinstance(value, basestring): + value = str(value) + path = path.replace(param_template, + urllib.quote(value.encode('utf_8'), reserved_chars)) + except TypeError as e: + raise exceptions.InvalidUserInputError( + 'Error setting required parameter %s to value %s: %s' % ( + param, value, e)) + return path + + +def CalculateWaitForRetry(retry_attempt, max_wait=60): + """Calculates amount of time to wait before a retry attempt. + + Wait time grows exponentially with the number of attempts. + A random amount of jitter is added to spread out retry attempts from different + clients. + + Args: + retry_attempt: Retry attempt counter. + max_wait: Upper bound for wait time. + + Returns: + Amount of time to wait before retrying request. + """ + + wait_time = 2 ** retry_attempt + # randrange requires a nonzero interval, so we want to drop it if + # the range is too small for jitter. + if retry_attempt: + max_jitter = (2 ** retry_attempt) / 2 + wait_time += random.randrange(-max_jitter, max_jitter) + return min(wait_time, max_wait) + + +def AcceptableMimeType(accept_patterns, mime_type): + """Return True iff mime_type is acceptable for one of accept_patterns. + + Note that this function assumes that all patterns in accept_patterns + will be simple types of the form "type/subtype", where one or both + of these can be "*". We do not support parameters (i.e. "; q=") in + patterns. + + Args: + accept_patterns: list of acceptable MIME types. + mime_type: the mime type we would like to match. + + Returns: + Whether or not mime_type matches (at least) one of these patterns. + """ + unsupported_patterns = [p for p in accept_patterns if ';' in p] + if unsupported_patterns: + raise exceptions.GeneratedClientError( + 'MIME patterns with parameter unsupported: "%s"' % ', '.join( + unsupported_patterns)) + def MimeTypeMatches(pattern, mime_type): + """Return True iff mime_type is acceptable for pattern.""" + # Some systems use a single '*' instead of '*/*'. + if pattern == '*': + pattern = '*/*' + return all(accept in ('*', provided) for accept, provided + in zip(pattern.split('/'), mime_type.split('/'))) + + return any(MimeTypeMatches(pattern, mime_type) for pattern in accept_patterns) diff --git a/run_pylint.py b/run_pylint.py index fa3c7f7cd1a8..a6b6684d0594 100644 --- a/run_pylint.py +++ b/run_pylint.py @@ -14,6 +14,9 @@ import sys +IGNORED_DIRECTORIES = [ + '_gcloud_vendor/', +] IGNORED_FILES = [ 'gcloud/datastore/datastore_v1_pb2.py', 'docs/conf.py', @@ -73,6 +76,9 @@ def make_test_rc(base_rc_filename, additions_dict, target_filename): def valid_filename(filename): """Checks if a file is a Python file and is not ignored.""" + for directory in IGNORED_DIRECTORIES: + if filename.startswith(directory): + return False return (filename.endswith('.py') and filename not in IGNORED_FILES) diff --git a/tox.ini b/tox.ini index 9e6c1c65421d..437b711b4626 100644 --- a/tox.ini +++ b/tox.ini @@ -49,7 +49,7 @@ deps = Sphinx [pep8] -exclude = gcloud/datastore/datastore_v1_pb2.py,docs/conf.py,*.egg/,.*/ +exclude = gcloud/datastore/datastore_v1_pb2.py,docs/conf.py,*.egg/,.*/,_gcloud_vendor/ verbose = 1 [testenv:lint]