From 185161038920d96008b11a76f3be2c8ec85dae5b Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Thu, 8 Nov 2018 16:04:34 -0800 Subject: [PATCH 01/11] Humble beginnings --- storage/signed_urls/generate_signed_url.py | 129 +++++++++++++++++++++ storage/signed_urls/requirements.txt | 2 + 2 files changed, 131 insertions(+) create mode 100644 storage/signed_urls/generate_signed_url.py create mode 100644 storage/signed_urls/requirements.txt diff --git a/storage/signed_urls/generate_signed_url.py b/storage/signed_urls/generate_signed_url.py new file mode 100644 index 000000000000..8a26fe0a4f13 --- /dev/null +++ b/storage/signed_urls/generate_signed_url.py @@ -0,0 +1,129 @@ +import argparse + +# [START storage_signed_url_dependencies] +import collections +import datetime +import hashlib +import json +import sys + +# pip install six +from six.moves.urllib.parse import quote + +# pip install google-auth +from google.oauth2 import service_account +# [END storage_signed_url_dependencies] + +def create_signed_url(service_account_file, bucket_name, object_name, + expiration, http_method='GET', query_parameters=None, headers=None): + + if expiration > 604800: + print('Expiration time can\'t be longer than a week') + sys.exit(1) + + # [START storage_signed_url_datestamp] + datetime_now = datetime.datetime.utcnow() + goog_date = datetime_now.strftime("%Y%m%dT%H%M%SZ") + datestamp = datetime_now.strftime("%Y%m%d") + # [START storage_signed_url_datestamp] + + # [START storage_signed_url_canonical_uri] + escaped_object_name = quote(object_name, safe='') + canonical_uri = "/{}/{}".format(bucket_name, escaped_object_name) + # [END storage_signed_url_canonical_uri] + + # [START storage_signed_url_canonical_request] + # [START storage_signed_url_string_to_sign] + credentials = service_account.Credentials.from_service_account_file(service_account_file) + client_email = credentials.service_account_email + credential_scope = "{}/auto/gcs/goog4_request".format(datestamp) + credential = "{}/{}".format(client_email, credential_scope) + # [END storage_signed_url_string_to_sign] + # [END storage_signed_url_canonical_request] + + if headers is None: + headers = dict() + # [START storage_signed_url_canonical_headers] + # Preapre headers + headers['host'] = "storage.googleapis.com" + + canonical_headers = "" + signed_headers = "" + ordered_headers = collections.OrderedDict(sorted(headers.items())) + for k,v in ordered_headers.items(): + lower_k = str(k).lower() + strip_v = str(v).lower() + canonical_headers += "{}:{}\n".format(lower_k, strip_v) + signed_headers += "{};".format(lower_k) + signed_headers = signed_headers[:-1] # remove trailing ';' + # [END storage_signed_url_canonical_headers] + + if query_parameters is None: + query_parameters = dict() + # [START storage_signed_url_canonical_query_parameters] + query_parameters["X-Goog-Algorithm"] = "GOOG4-RSA-SHA256" + query_parameters["X-Goog-Credential"] = credential + query_parameters["X-Goog-Date"] = goog_date + query_parameters["X-Goog-Expires"] = expiration + query_parameters["X-Goog-SignedHeaders"] = signed_headers + + canonical_query_string = "" + ordered_query_parameters = collections.OrderedDict(sorted(query_parameters.items())) + for k,v in ordered_query_parameters.items(): + encoded_k = quote(str(k), safe='') + encoded_v = quote(str(v), safe='') + canonical_query_string += "{}={}&".format(encoded_k, encoded_v) + canonical_query_string = canonical_query_string[:-1] # remove trailing ';' + # [END storage_signed_url_canonical_query_parameters] + + # [START storage_signed_url_canonical_request] + canonical_request = "\n".join([http_method, + canonical_uri, + canonical_query_string, + canonical_headers, + signed_headers, + "UNSIGNED-PAYLOAD"]) + # [END storage_signed_url_canonical_request] + + # [START storage_signed_url_hash] + canonical_request_hash = hashlib.sha256(canonical_request.encode()).hexdigest() + # [END storage_signed_url_hash] + + # [START storage_signed_url_string_to_sign] + string_to_sign = "\n".join(["GOOG4-RSA-SHA256", + goog_date, + credential_scope, + canonical_request_hash]) + # [END storage_signed_url_string_to_sign] + + # [START storage_signed_url_signer] + signature = credentials.signer.sign(string_to_sign).hex() + # [END storage_signed_url_signer] + + # [START storage_signed_url_construction] + signed_url = "https://storage.googleapis.com{}?{}&x-goog-signature={}".format( + canonical_uri, canonical_query_string, signature) + # [END storage_signed_url_construction] + return signed_url + +if __name__ == '__main__': + # parser = argparse.ArgumentParser( + # description=__doc__, + # formatter_class=argparse.RawDescriptionHelpFormatter) + # parser.add_argument('service_account_file', help='Path to your Google service account.') + # parser.add_argument('request_method', help='A request method, e.g GET, POST.') + # parser.add_argument('bucket_name', help='Your Cloud Storage bucket name.') + # parser.add_argument('object_name', help='Your Cloud Storage object name.') + # parser.add_argument('expiration', help='Expiration Time.') + + # args = parser.parse_args() + # signed_url = create_signed_url( + # service_account_file=args.service_account_file, request_method='GET', + # bucket_name=args.bucket_name, object_name=args.object_name, + # expiration=args.expiration) + + signed_url = create_signed_url( + service_account_file="spec-test-ruby-samples-3fb0d39a74ab.json", + bucket_name="jessefrank2", object_name="test_lifecycle.json", + expiration=3600) + print(signed_url) diff --git a/storage/signed_urls/requirements.txt b/storage/signed_urls/requirements.txt new file mode 100644 index 000000000000..cd94c743a7de --- /dev/null +++ b/storage/signed_urls/requirements.txt @@ -0,0 +1,2 @@ +google-auth==1.5.1 +six==1.11.0 \ No newline at end of file From a82daa7a54d6e320ebf768a3f1ce8513404386bf Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Thu, 8 Nov 2018 16:35:12 -0800 Subject: [PATCH 02/11] Update --- storage/signed_urls/generate_signed_url.py | 34 ++++++++++++---------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/storage/signed_urls/generate_signed_url.py b/storage/signed_urls/generate_signed_url.py index 8a26fe0a4f13..cc1c36ebdfec 100644 --- a/storage/signed_urls/generate_signed_url.py +++ b/storage/signed_urls/generate_signed_url.py @@ -21,42 +21,44 @@ def create_signed_url(service_account_file, bucket_name, object_name, print('Expiration time can\'t be longer than a week') sys.exit(1) - # [START storage_signed_url_datestamp] - datetime_now = datetime.datetime.utcnow() - goog_date = datetime_now.strftime("%Y%m%dT%H%M%SZ") - datestamp = datetime_now.strftime("%Y%m%d") - # [START storage_signed_url_datestamp] - # [START storage_signed_url_canonical_uri] escaped_object_name = quote(object_name, safe='') canonical_uri = "/{}/{}".format(bucket_name, escaped_object_name) # [END storage_signed_url_canonical_uri] - # [START storage_signed_url_canonical_request] - # [START storage_signed_url_string_to_sign] - credentials = service_account.Credentials.from_service_account_file(service_account_file) - client_email = credentials.service_account_email + # [START storage_signed_url_canonical_datetime] + datetime_now = datetime.datetime.utcnow() + goog_date = datetime_now.strftime("%Y%m%dT%H%M%SZ") + datestamp = datetime_now.strftime("%Y%m%d") + # [END storage_signed_url_canonical_datetime] + + # [START storage_signed_url_credentials] + google_credentials = service_account.Credentials.from_service_account_file(service_account_file) + client_email = google_credentials.service_account_email credential_scope = "{}/auto/gcs/goog4_request".format(datestamp) credential = "{}/{}".format(client_email, credential_scope) - # [END storage_signed_url_string_to_sign] - # [END storage_signed_url_canonical_request] + # [END storage_signed_url_credentials] if headers is None: headers = dict() # [START storage_signed_url_canonical_headers] - # Preapre headers headers['host'] = "storage.googleapis.com" canonical_headers = "" - signed_headers = "" ordered_headers = collections.OrderedDict(sorted(headers.items())) for k,v in ordered_headers.items(): lower_k = str(k).lower() strip_v = str(v).lower() canonical_headers += "{}:{}\n".format(lower_k, strip_v) + # [END storage_signed_url_canonical_headers] + + # [START storage_signed_url_signed_headers] + signed_headers = "" + for k,_ in ordered_headers.items(): + lower_k = str(k).lower() signed_headers += "{};".format(lower_k) signed_headers = signed_headers[:-1] # remove trailing ';' - # [END storage_signed_url_canonical_headers] + # [END storage_signed_url_signed_headers] if query_parameters is None: query_parameters = dict() @@ -97,7 +99,7 @@ def create_signed_url(service_account_file, bucket_name, object_name, # [END storage_signed_url_string_to_sign] # [START storage_signed_url_signer] - signature = credentials.signer.sign(string_to_sign).hex() + signature = google_credentials.signer.sign(string_to_sign).hex() # [END storage_signed_url_signer] # [START storage_signed_url_construction] From 8d68bee153277aaaab6417eca1503a22c004f510 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Thu, 8 Nov 2018 16:55:43 -0800 Subject: [PATCH 03/11] Update --- storage/signed_urls/generate_signed_url.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/signed_urls/generate_signed_url.py b/storage/signed_urls/generate_signed_url.py index cc1c36ebdfec..d71827798151 100644 --- a/storage/signed_urls/generate_signed_url.py +++ b/storage/signed_urls/generate_signed_url.py @@ -28,7 +28,7 @@ def create_signed_url(service_account_file, bucket_name, object_name, # [START storage_signed_url_canonical_datetime] datetime_now = datetime.datetime.utcnow() - goog_date = datetime_now.strftime("%Y%m%dT%H%M%SZ") + request_date = datetime_now.strftime("%Y%m%dT%H%M%SZ") datestamp = datetime_now.strftime("%Y%m%d") # [END storage_signed_url_canonical_datetime] @@ -65,7 +65,7 @@ def create_signed_url(service_account_file, bucket_name, object_name, # [START storage_signed_url_canonical_query_parameters] query_parameters["X-Goog-Algorithm"] = "GOOG4-RSA-SHA256" query_parameters["X-Goog-Credential"] = credential - query_parameters["X-Goog-Date"] = goog_date + query_parameters["X-Goog-Date"] = request_date query_parameters["X-Goog-Expires"] = expiration query_parameters["X-Goog-SignedHeaders"] = signed_headers @@ -93,7 +93,7 @@ def create_signed_url(service_account_file, bucket_name, object_name, # [START storage_signed_url_string_to_sign] string_to_sign = "\n".join(["GOOG4-RSA-SHA256", - goog_date, + request_date, credential_scope, canonical_request_hash]) # [END storage_signed_url_string_to_sign] From 6cadfbd59b6b8545df52ab334e9b5050c449b8ad Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Thu, 8 Nov 2018 16:57:46 -0800 Subject: [PATCH 04/11] Update --- storage/signed_urls/generate_signed_url.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/signed_urls/generate_signed_url.py b/storage/signed_urls/generate_signed_url.py index d71827798151..0fb792e3a08d 100644 --- a/storage/signed_urls/generate_signed_url.py +++ b/storage/signed_urls/generate_signed_url.py @@ -28,7 +28,7 @@ def create_signed_url(service_account_file, bucket_name, object_name, # [START storage_signed_url_canonical_datetime] datetime_now = datetime.datetime.utcnow() - request_date = datetime_now.strftime("%Y%m%dT%H%M%SZ") + request_timestamp = datetime_now.strftime("%Y%m%dT%H%M%SZ") datestamp = datetime_now.strftime("%Y%m%d") # [END storage_signed_url_canonical_datetime] @@ -65,7 +65,7 @@ def create_signed_url(service_account_file, bucket_name, object_name, # [START storage_signed_url_canonical_query_parameters] query_parameters["X-Goog-Algorithm"] = "GOOG4-RSA-SHA256" query_parameters["X-Goog-Credential"] = credential - query_parameters["X-Goog-Date"] = request_date + query_parameters["X-Goog-Date"] = request_timestamp query_parameters["X-Goog-Expires"] = expiration query_parameters["X-Goog-SignedHeaders"] = signed_headers @@ -93,7 +93,7 @@ def create_signed_url(service_account_file, bucket_name, object_name, # [START storage_signed_url_string_to_sign] string_to_sign = "\n".join(["GOOG4-RSA-SHA256", - request_date, + request_timestamp, credential_scope, canonical_request_hash]) # [END storage_signed_url_string_to_sign] From 1d1c12a05fdf068f5dbcbda6519de6d02ceed5a5 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Tue, 13 Nov 2018 15:07:59 -0800 Subject: [PATCH 05/11] Add test and README for generate_signed_urls.py --- storage/signed_urls/README.rst | 97 ++++++++++++++++ storage/signed_urls/README.rst.in | 22 ++++ ..._signed_url.py => generate_signed_urls.py} | 108 +++++++++++------- .../signed_urls/generate_signed_urls_test.py | 42 +++++++ storage/signed_urls/requirements.txt | 1 + 5 files changed, 229 insertions(+), 41 deletions(-) create mode 100644 storage/signed_urls/README.rst create mode 100644 storage/signed_urls/README.rst.in rename storage/signed_urls/{generate_signed_url.py => generate_signed_urls.py} (50%) create mode 100644 storage/signed_urls/generate_signed_urls_test.py diff --git a/storage/signed_urls/README.rst b/storage/signed_urls/README.rst new file mode 100644 index 000000000000..9f00f00fdf2d --- /dev/null +++ b/storage/signed_urls/README.rst @@ -0,0 +1,97 @@ +.. This file is automatically generated. Do not edit this file directly. + +Google Cloud Storage Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/signed_urls/README.rst + + +This directory contains samples for Google Cloud Storage. `Google Cloud Storage`_ allows world-wide storage and retrieval of any amount of data at any time. + + + + +.. _Google Cloud Storage: https://cloud.google.com/storage/docs + +Setup +------------------------------------------------------------------------------- + + +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started + +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ + +Samples +------------------------------------------------------------------------------- + +Generate Signed URLs in Python ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/signed_urls/generate_signed_urls.py,storage/signed_urls/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python generate_signed_urls.py + + usage: generate_signed_urls.py [-h] + service_account_file request_method bucket_name + object_name expiration + + positional arguments: + service_account_file Path to your Google service account. + request_method A request method, e.g GET, POST. + bucket_name Your Cloud Storage bucket name. + object_name Your Cloud Storage object name. + expiration Expiration Time. + + optional arguments: + -h, --help show this help message and exit + + + + + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/storage/signed_urls/README.rst.in b/storage/signed_urls/README.rst.in new file mode 100644 index 000000000000..fdc427f66174 --- /dev/null +++ b/storage/signed_urls/README.rst.in @@ -0,0 +1,22 @@ +# This file is used to generate README.rst + +product: + name: Google Cloud Storage + short_name: Cloud Storage + url: https://cloud.google.com/storage/docs + description: > + `Google Cloud Storage`_ allows world-wide storage and retrieval of any + amount of data at any time. + +setup: +- auth +- install_deps + +samples: +- name: Generate Signed URLs in Python + file: generate_signed_urls.py + show_help: true + +cloud_client_library: false + +folder: storage/signed_urls \ No newline at end of file diff --git a/storage/signed_urls/generate_signed_url.py b/storage/signed_urls/generate_signed_urls.py similarity index 50% rename from storage/signed_urls/generate_signed_url.py rename to storage/signed_urls/generate_signed_urls.py index 0fb792e3a08d..4ae7a4e15087 100644 --- a/storage/signed_urls/generate_signed_url.py +++ b/storage/signed_urls/generate_signed_urls.py @@ -1,10 +1,31 @@ +# Copyright 2018 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import argparse +"""This application demonstrates how to construct a Signed URL for objects in + Google Cloud Storage. + +For more information, see the README.md under /storage and the documentation +at https://cloud.google.com/storage/docs/access-control/signing-urls-manually. +""" + +# [START storage_signed_url_all] # [START storage_signed_url_dependencies] import collections import datetime import hashlib -import json import sys # pip install six @@ -14,8 +35,10 @@ from google.oauth2 import service_account # [END storage_signed_url_dependencies] -def create_signed_url(service_account_file, bucket_name, object_name, - expiration, http_method='GET', query_parameters=None, headers=None): + +def generate_signed_url(service_account_file, bucket_name, object_name, + expiration, http_method='GET', query_parameters=None, + headers=None): if expiration > 604800: print('Expiration time can\'t be longer than a week') @@ -33,7 +56,8 @@ def create_signed_url(service_account_file, bucket_name, object_name, # [END storage_signed_url_canonical_datetime] # [START storage_signed_url_credentials] - google_credentials = service_account.Credentials.from_service_account_file(service_account_file) + google_credentials = service_account.Credentials.from_service_account_file( + service_account_file) client_email = google_credentials.service_account_email credential_scope = "{}/auto/gcs/goog4_request".format(datestamp) credential = "{}/{}".format(client_email, credential_scope) @@ -46,7 +70,7 @@ def create_signed_url(service_account_file, bucket_name, object_name, canonical_headers = "" ordered_headers = collections.OrderedDict(sorted(headers.items())) - for k,v in ordered_headers.items(): + for k, v in ordered_headers.items(): lower_k = str(k).lower() strip_v = str(v).lower() canonical_headers += "{}:{}\n".format(lower_k, strip_v) @@ -54,10 +78,10 @@ def create_signed_url(service_account_file, bucket_name, object_name, # [START storage_signed_url_signed_headers] signed_headers = "" - for k,_ in ordered_headers.items(): + for k, _ in ordered_headers.items(): lower_k = str(k).lower() signed_headers += "{};".format(lower_k) - signed_headers = signed_headers[:-1] # remove trailing ';' + signed_headers = signed_headers[:-1] # remove trailing ';' # [END storage_signed_url_signed_headers] if query_parameters is None: @@ -70,32 +94,34 @@ def create_signed_url(service_account_file, bucket_name, object_name, query_parameters["X-Goog-SignedHeaders"] = signed_headers canonical_query_string = "" - ordered_query_parameters = collections.OrderedDict(sorted(query_parameters.items())) - for k,v in ordered_query_parameters.items(): + ordered_query_parameters = collections.OrderedDict( + sorted(query_parameters.items())) + for k, v in ordered_query_parameters.items(): encoded_k = quote(str(k), safe='') encoded_v = quote(str(v), safe='') canonical_query_string += "{}={}&".format(encoded_k, encoded_v) - canonical_query_string = canonical_query_string[:-1] # remove trailing ';' + canonical_query_string = canonical_query_string[:-1] # remove trailing ';' # [END storage_signed_url_canonical_query_parameters] # [START storage_signed_url_canonical_request] canonical_request = "\n".join([http_method, - canonical_uri, - canonical_query_string, - canonical_headers, - signed_headers, - "UNSIGNED-PAYLOAD"]) + canonical_uri, + canonical_query_string, + canonical_headers, + signed_headers, + "UNSIGNED-PAYLOAD"]) # [END storage_signed_url_canonical_request] # [START storage_signed_url_hash] - canonical_request_hash = hashlib.sha256(canonical_request.encode()).hexdigest() + canonical_request_hash = hashlib.sha256( + canonical_request.encode()).hexdigest() # [END storage_signed_url_hash] # [START storage_signed_url_string_to_sign] string_to_sign = "\n".join(["GOOG4-RSA-SHA256", - request_timestamp, - credential_scope, - canonical_request_hash]) + request_timestamp, + credential_scope, + canonical_request_hash]) # [END storage_signed_url_string_to_sign] # [START storage_signed_url_signer] @@ -103,29 +129,29 @@ def create_signed_url(service_account_file, bucket_name, object_name, # [END storage_signed_url_signer] # [START storage_signed_url_construction] - signed_url = "https://storage.googleapis.com{}?{}&x-goog-signature={}".format( - canonical_uri, canonical_query_string, signature) + host_name = "https://storage.googleapis.com" + signed_url = "{}{}?{}&x-goog-signature={}".format(host_name, canonical_uri, + canonical_query_string, + signature) # [END storage_signed_url_construction] return signed_url +# [END storage_signed_url_all] + if __name__ == '__main__': - # parser = argparse.ArgumentParser( - # description=__doc__, - # formatter_class=argparse.RawDescriptionHelpFormatter) - # parser.add_argument('service_account_file', help='Path to your Google service account.') - # parser.add_argument('request_method', help='A request method, e.g GET, POST.') - # parser.add_argument('bucket_name', help='Your Cloud Storage bucket name.') - # parser.add_argument('object_name', help='Your Cloud Storage object name.') - # parser.add_argument('expiration', help='Expiration Time.') - - # args = parser.parse_args() - # signed_url = create_signed_url( - # service_account_file=args.service_account_file, request_method='GET', - # bucket_name=args.bucket_name, object_name=args.object_name, - # expiration=args.expiration) - - signed_url = create_signed_url( - service_account_file="spec-test-ruby-samples-3fb0d39a74ab.json", - bucket_name="jessefrank2", object_name="test_lifecycle.json", - expiration=3600) - print(signed_url) + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('service_account_file', + help='Path to your Google service account.') + parser.add_argument( + 'request_method', help='A request method, e.g GET, POST.') + parser.add_argument('bucket_name', help='Your Cloud Storage bucket name.') + parser.add_argument('object_name', help='Your Cloud Storage object name.') + parser.add_argument('expiration', help='Expiration Time.') + + args = parser.parse_args() + signed_url = generate_signed_url( + service_account_file=args.service_account_file, + http_method=args.request_method, bucket_name=args.bucket_name, + object_name=args.object_name, expiration=args.expiration) diff --git a/storage/signed_urls/generate_signed_urls_test.py b/storage/signed_urls/generate_signed_urls_test.py new file mode 100644 index 000000000000..2f3c426a6f6c --- /dev/null +++ b/storage/signed_urls/generate_signed_urls_test.py @@ -0,0 +1,42 @@ +# Copyright 2018 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.cloud import storage +import pytest +import requests + +import generate_signed_urls + +BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] +GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] + + +@pytest.fixture +def test_blob(): + """Provides a pre-existing blob in the test bucket.""" + bucket = storage.Client().bucket(BUCKET) + blob = bucket.blob('storage_snippets_test_sigil') + blob.upload_from_string('Hello, is it me you\'re looking for?') + return blob + + +def test_generate_get_signed_url(test_blob, capsys): + get_signed_url = generate_signed_urls.generate_signed_url( + service_account_file=GOOGLE_APPLICATION_CREDENTIALS, + bucket_name=BUCKET, object_name=test_blob.name, + expiration=60) + response = requests.get(get_signed_url) + assert response.ok diff --git a/storage/signed_urls/requirements.txt b/storage/signed_urls/requirements.txt index cd94c743a7de..27198cf6e790 100644 --- a/storage/signed_urls/requirements.txt +++ b/storage/signed_urls/requirements.txt @@ -1,2 +1,3 @@ +google-cloud-storage==1.13.0 google-auth==1.5.1 six==1.11.0 \ No newline at end of file From e8a479153740aa72adce098a7f30426376708194 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Tue, 13 Nov 2018 15:12:01 -0800 Subject: [PATCH 06/11] Update error message --- storage/signed_urls/generate_signed_urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/signed_urls/generate_signed_urls.py b/storage/signed_urls/generate_signed_urls.py index 4ae7a4e15087..a25ff3a1ed75 100644 --- a/storage/signed_urls/generate_signed_urls.py +++ b/storage/signed_urls/generate_signed_urls.py @@ -41,7 +41,7 @@ def generate_signed_url(service_account_file, bucket_name, object_name, headers=None): if expiration > 604800: - print('Expiration time can\'t be longer than a week') + print('Expiration Time can\'t be longer than 604800 seconds (7 days).') sys.exit(1) # [START storage_signed_url_canonical_uri] From 430483bada4758f8a55397e22914e9cac198ba31 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Wed, 14 Nov 2018 16:47:55 -0800 Subject: [PATCH 07/11] Fix issues in CLI --- storage/signed_urls/generate_signed_urls.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/signed_urls/generate_signed_urls.py b/storage/signed_urls/generate_signed_urls.py index a25ff3a1ed75..02a0a55725ff 100644 --- a/storage/signed_urls/generate_signed_urls.py +++ b/storage/signed_urls/generate_signed_urls.py @@ -154,4 +154,6 @@ def generate_signed_url(service_account_file, bucket_name, object_name, signed_url = generate_signed_url( service_account_file=args.service_account_file, http_method=args.request_method, bucket_name=args.bucket_name, - object_name=args.object_name, expiration=args.expiration) + object_name=args.object_name, expiration=int(args.expiration)) + + print(signed_url) From 154cbbfa5827d0eca317138a28848558141687e5 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Sun, 18 Nov 2018 23:33:53 -0700 Subject: [PATCH 08/11] Fix region tag and address feedback. --- storage/signed_urls/generate_signed_urls.py | 48 +++++++++++---------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/storage/signed_urls/generate_signed_urls.py b/storage/signed_urls/generate_signed_urls.py index 02a0a55725ff..561ae26cd86b 100644 --- a/storage/signed_urls/generate_signed_urls.py +++ b/storage/signed_urls/generate_signed_urls.py @@ -31,8 +31,10 @@ # pip install six from six.moves.urllib.parse import quote +# [START storage_signed_url_signer] # pip install google-auth from google.oauth2 import service_account +# [END storage_signed_url_signer] # [END storage_signed_url_dependencies] @@ -46,70 +48,72 @@ def generate_signed_url(service_account_file, bucket_name, object_name, # [START storage_signed_url_canonical_uri] escaped_object_name = quote(object_name, safe='') - canonical_uri = "/{}/{}".format(bucket_name, escaped_object_name) + canonical_uri = '/{}/{}'.format(bucket_name, escaped_object_name) # [END storage_signed_url_canonical_uri] # [START storage_signed_url_canonical_datetime] datetime_now = datetime.datetime.utcnow() - request_timestamp = datetime_now.strftime("%Y%m%dT%H%M%SZ") - datestamp = datetime_now.strftime("%Y%m%d") + request_timestamp = datetime_now.strftime('%Y%m%dT%H%M%SZ') + datestamp = datetime_now.strftime('%Y%m%d') # [END storage_signed_url_canonical_datetime] # [START storage_signed_url_credentials] + # [START storage_signed_url_signer] google_credentials = service_account.Credentials.from_service_account_file( service_account_file) + # [END storage_signed_url_signer] client_email = google_credentials.service_account_email - credential_scope = "{}/auto/gcs/goog4_request".format(datestamp) - credential = "{}/{}".format(client_email, credential_scope) + credential_scope = '{}/auto/gcs/goog4_request'.format(datestamp) + credential = '{}/{}'.format(client_email, credential_scope) # [END storage_signed_url_credentials] if headers is None: headers = dict() # [START storage_signed_url_canonical_headers] - headers['host'] = "storage.googleapis.com" + headers['host'] = 'storage.googleapis.com' - canonical_headers = "" + canonical_headers = '' ordered_headers = collections.OrderedDict(sorted(headers.items())) for k, v in ordered_headers.items(): lower_k = str(k).lower() strip_v = str(v).lower() - canonical_headers += "{}:{}\n".format(lower_k, strip_v) + canonical_headers += '{}:{}\n'.format(lower_k, strip_v) # [END storage_signed_url_canonical_headers] # [START storage_signed_url_signed_headers] - signed_headers = "" + signed_headers = '' for k, _ in ordered_headers.items(): lower_k = str(k).lower() - signed_headers += "{};".format(lower_k) + signed_headers += '{};'.format(lower_k) signed_headers = signed_headers[:-1] # remove trailing ';' # [END storage_signed_url_signed_headers] if query_parameters is None: query_parameters = dict() # [START storage_signed_url_canonical_query_parameters] - query_parameters["X-Goog-Algorithm"] = "GOOG4-RSA-SHA256" - query_parameters["X-Goog-Credential"] = credential - query_parameters["X-Goog-Date"] = request_timestamp - query_parameters["X-Goog-Expires"] = expiration - query_parameters["X-Goog-SignedHeaders"] = signed_headers + query_parameters['X-Goog-Algorithm'] = 'GOOG4-RSA-SHA256' + query_parameters['X-Goog-Credential'] = credential + query_parameters['X-Goog-Date'] = request_timestamp + query_parameters['X-Goog-Expires'] = expiration + query_parameters['X-Goog-SignedHeaders'] = signed_headers - canonical_query_string = "" + canonical_query_string = '' ordered_query_parameters = collections.OrderedDict( sorted(query_parameters.items())) for k, v in ordered_query_parameters.items(): encoded_k = quote(str(k), safe='') encoded_v = quote(str(v), safe='') - canonical_query_string += "{}={}&".format(encoded_k, encoded_v) + canonical_query_string += '{}={}&'.format(encoded_k, encoded_v) canonical_query_string = canonical_query_string[:-1] # remove trailing ';' # [END storage_signed_url_canonical_query_parameters] # [START storage_signed_url_canonical_request] - canonical_request = "\n".join([http_method, + canonical_request = '\n'.join([http_method, canonical_uri, canonical_query_string, canonical_headers, signed_headers, - "UNSIGNED-PAYLOAD"]) + 'UNSIGNED-PAYLOAD']) # [END storage_signed_url_canonical_request] # [START storage_signed_url_hash] @@ -118,7 +122,7 @@ def generate_signed_url(service_account_file, bucket_name, object_name, # [END storage_signed_url_hash] # [START storage_signed_url_string_to_sign] - string_to_sign = "\n".join(["GOOG4-RSA-SHA256", + string_to_sign = '\n'.join(['GOOG4-RSA-SHA256', request_timestamp, credential_scope, canonical_request_hash]) @@ -129,8 +133,8 @@ def generate_signed_url(service_account_file, bucket_name, object_name, # [END storage_signed_url_signer] # [START storage_signed_url_construction] - host_name = "https://storage.googleapis.com" - signed_url = "{}{}?{}&x-goog-signature={}".format(host_name, canonical_uri, + host_name = 'https://storage.googleapis.com' + signed_url = '{}{}?{}&x-goog-signature={}'.format(host_name, canonical_uri, canonical_query_string, signature) # [END storage_signed_url_construction] From 37043d4b0b443b4d7a87e0b620eda9be254af9aa Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Mon, 19 Nov 2018 17:14:48 -0700 Subject: [PATCH 09/11] Add spacing --- storage/signed_urls/generate_signed_urls.py | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/signed_urls/generate_signed_urls.py b/storage/signed_urls/generate_signed_urls.py index 561ae26cd86b..6b5df414b3be 100644 --- a/storage/signed_urls/generate_signed_urls.py +++ b/storage/signed_urls/generate_signed_urls.py @@ -34,6 +34,7 @@ # [START storage_signed_url_signer] # pip install google-auth from google.oauth2 import service_account + # [END storage_signed_url_signer] # [END storage_signed_url_dependencies] From 6c613b7b6f17fcdcc79f7f14f58f68f09428cc80 Mon Sep 17 00:00:00 2001 From: Charles Engelke Date: Tue, 20 Nov 2018 14:49:21 -0800 Subject: [PATCH 10/11] Python 2 and 3 compatible string to hex used --- storage/signed_urls/generate_signed_urls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/signed_urls/generate_signed_urls.py b/storage/signed_urls/generate_signed_urls.py index 6b5df414b3be..05c5980625cc 100644 --- a/storage/signed_urls/generate_signed_urls.py +++ b/storage/signed_urls/generate_signed_urls.py @@ -23,6 +23,7 @@ # [START storage_signed_url_all] # [START storage_signed_url_dependencies] +import binascii import collections import datetime import hashlib @@ -130,7 +131,7 @@ def generate_signed_url(service_account_file, bucket_name, object_name, # [END storage_signed_url_string_to_sign] # [START storage_signed_url_signer] - signature = google_credentials.signer.sign(string_to_sign).hex() + signature = binascii.hexlify(google_credentials.signer.sign(string_to_sign)).decode() # [END storage_signed_url_signer] # [START storage_signed_url_construction] From 39a822e342f14188b6c56b675d2a87b711842982 Mon Sep 17 00:00:00 2001 From: Charles Engelke Date: Tue, 20 Nov 2018 14:59:50 -0800 Subject: [PATCH 11/11] Fixed too long line --- storage/signed_urls/generate_signed_urls.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/signed_urls/generate_signed_urls.py b/storage/signed_urls/generate_signed_urls.py index 05c5980625cc..132c997b9454 100644 --- a/storage/signed_urls/generate_signed_urls.py +++ b/storage/signed_urls/generate_signed_urls.py @@ -131,7 +131,9 @@ def generate_signed_url(service_account_file, bucket_name, object_name, # [END storage_signed_url_string_to_sign] # [START storage_signed_url_signer] - signature = binascii.hexlify(google_credentials.signer.sign(string_to_sign)).decode() + signature = binascii.hexlify( + google_credentials.signer.sign(string_to_sign) + ).decode() # [END storage_signed_url_signer] # [START storage_signed_url_construction]