From 37e746a3e9ca6bc311fd3b4b3cde895aba75522b Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 23 Jul 2015 16:50:19 -0400 Subject: [PATCH 1/4] Add examples for copying / exporting table data. --- docs/bigquery-usage.rst | 125 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 124 insertions(+), 1 deletion(-) diff --git a/docs/bigquery-usage.rst b/docs/bigquery-usage.rst index 9bf03710f8ea..7318799822db 100644 --- a/docs/bigquery-usage.rst +++ b/docs/bigquery-usage.rst @@ -227,6 +227,20 @@ Update all writable metadata for a table ... SchemaField(name='age', type='int', mode='required)] >>> table.update() # API request +Get rows from a table's data: + +.. doctest:: + + >>> from gcloud import bigquery + >>> client = bigquery.Client() + >>> dataset = client.dataset('dataset_name') + >>> table = dataset.table(name='person_ages') + >>> rows, next_page_token = table.data(max_results=100) # API request + >>> rows.csv.headers + ('full_name', 'age') + >>> list(rows.csv) + [('Abel Adamson', 27), ('Beverly Bowman', 33)] + Delete a table: .. doctest:: @@ -307,7 +321,7 @@ Background a query, loading the results into a table: >>> job.job_id 'e3344fba-09df-4ae0-8337-fddee34b3840' >>> job.type - 'load' + 'query' >>> job.created None >>> job.state @@ -429,3 +443,112 @@ Poll until the job is complete: 'done' >>> job.ended datetime.datetime(2015, 7, 23, 9, 30, 21, 334792, tzinfo=) + +Exporting data (async) +~~~~~~~~~~~~~~~~~~~~~~ + +Start a job exporting a table's data asynchronously to a set of CSV files, +located on GCloud Storage. First, create the job locally: + +.. doctest:: + + >>> from gcloud import bigquery + >>> client = bigquery.Client() + >>> table = dataset.table(name='person_ages') + >>> job = table.export_to_storage(bucket_name='bucket-name', + ... object_name='export-prefix*.csv', + ... destination_format='CSV', + ... print_header=1, + ... write_disposition='truncate') + >>> job.job_id + 'e3344fba-09df-4ae0-8337-fddee34b3840' + >>> job.type + 'load' + >>> job.created + None + >>> job.state + None + +.. note:: + + - ``gcloud.bigquery`` generates a UUID for each job. + - The ``created`` and ``state`` fields are not set until the job + is submitted to the BigQuery back-end. + +Then, begin executing the job on the server: + +.. doctest:: + + >>> job.submit() # API call + >>> job.created + datetime.datetime(2015, 7, 23, 9, 30, 20, 268260, tzinfo=) + >>> job.state + 'running' + +Poll until the job is complete: + +.. doctest:: + + >>> import time + >>> retry_count = 100 + >>> while retry_count > 0 and job.state == 'running': + ... retry_count -= 1 + ... time.sleep(10) + ... job.reload() # API call + >>> job.state + 'done' + >>> job.ended + datetime.datetime(2015, 7, 23, 9, 30, 21, 334792, tzinfo=) + + +Copy tables (async) +~~~~~~~~~~~~~~~~~~~ + +First, create the job locally: + +.. doctest:: + + >>> from gcloud import bigquery + >>> client = bigquery.Client() + >>> source_table = dataset.table(name='person_ages') + >>> destination_table = dataset.table(name='person_ages_copy') + >>> job = source_table.copy_to(destination_table) # API request + >>> job.job_id + 'e3344fba-09df-4ae0-8337-fddee34b3840' + >>> job.type + 'copy' + >>> job.created + None + >>> job.state + None + +.. note:: + + - ``gcloud.bigquery`` generates a UUID for each job. + - The ``created`` and ``state`` fields are not set until the job + is submitted to the BigQuery back-end. + +Then, begin executing the job on the server: + +.. doctest:: + + >>> job.submit() # API call + >>> job.created + datetime.datetime(2015, 7, 23, 9, 30, 20, 268260, tzinfo=) + >>> job.state + 'running' + +Poll until the job is complete: + +.. doctest:: + + >>> import time + >>> retry_count = 100 + >>> while retry_count > 0 and job.state == 'running': + ... retry_count -= 1 + ... time.sleep(10) + ... job.reload() # API call + >>> job.state + 'done' + >>> job.ended + datetime.datetime(2015, 7, 23, 9, 30, 21, 334792, tzinfo=) From 443fa7d99e8de149e0ec567db42a12955bb20183 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 27 Jul 2015 13:10:49 -0400 Subject: [PATCH 2/4] Prefer official branding: 'GCloud' -> 'Google Cloud'. [ci skip] --- docs/bigquery-usage.rst | 6 +++--- gcloud/bigquery/__init__.py | 2 +- gcloud/pubsub/__init__.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/bigquery-usage.rst b/docs/bigquery-usage.rst index 7318799822db..b61da0c9934a 100644 --- a/docs/bigquery-usage.rst +++ b/docs/bigquery-usage.rst @@ -391,8 +391,8 @@ Inserting data (asynchronous) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Start a job loading data asynchronously from a set of CSV files, located on -GCloud Storage, appending rows into an existing table. First, create the job -locally: +Google Cloud Storage, appending rows into an existing table. First, create +the job locally: .. doctest:: @@ -448,7 +448,7 @@ Exporting data (async) ~~~~~~~~~~~~~~~~~~~~~~ Start a job exporting a table's data asynchronously to a set of CSV files, -located on GCloud Storage. First, create the job locally: +located on Google Cloud Storage. First, create the job locally: .. doctest:: diff --git a/gcloud/bigquery/__init__.py b/gcloud/bigquery/__init__.py index 330233bbc8a8..ea17dcb19a56 100644 --- a/gcloud/bigquery/__init__.py +++ b/gcloud/bigquery/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""GCloud BigQuery API wrapper. +"""Google Cloud BigQuery API wrapper. The main concepts with this API are: diff --git a/gcloud/pubsub/__init__.py b/gcloud/pubsub/__init__.py index 155454c30ab0..c39eb343ee6c 100644 --- a/gcloud/pubsub/__init__.py +++ b/gcloud/pubsub/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""GCloud Pubsub API wrapper. +"""Google Cloud Pubsub API wrapper. The main concepts with this API are: From 5baabbf9c90a00f2e1909a94d9304f332de4fb36 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 27 Jul 2015 13:13:26 -0400 Subject: [PATCH 3/4] Track updated paramter name from 906e120. [ci skip] --- docs/bigquery-usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/bigquery-usage.rst b/docs/bigquery-usage.rst index b61da0c9934a..deb04e56523d 100644 --- a/docs/bigquery-usage.rst +++ b/docs/bigquery-usage.rst @@ -456,7 +456,7 @@ located on Google Cloud Storage. First, create the job locally: >>> client = bigquery.Client() >>> table = dataset.table(name='person_ages') >>> job = table.export_to_storage(bucket_name='bucket-name', - ... object_name='export-prefix*.csv', + ... object_name_glob='export-prefix*.csv', ... destination_format='CSV', ... print_header=1, ... write_disposition='truncate') From 2624822682ef4a91d1fab203b3e9987c1a77b424 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 28 Jul 2015 13:07:38 -0400 Subject: [PATCH 4/4] Add example showing interation w/ result rows / schema. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1016#issuecomment-125677495 --- docs/bigquery-usage.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/bigquery-usage.rst b/docs/bigquery-usage.rst index deb04e56523d..92032a4f72dd 100644 --- a/docs/bigquery-usage.rst +++ b/docs/bigquery-usage.rst @@ -240,6 +240,9 @@ Get rows from a table's data: ('full_name', 'age') >>> list(rows.csv) [('Abel Adamson', 27), ('Beverly Bowman', 33)] + >>> for row in rows: + ... for field, value in zip(table.schema, row): + ... do_something(field, value) Delete a table: