From 0e69c1ecc72d27463233c8f5b4b75e49dbd432e2 Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Thu, 19 Nov 2020 13:19:22 +0530 Subject: [PATCH 1/3] fix: timestamp precision in insert_rows --- google/cloud/bigquery/_helpers.py | 1 + tests/unit/test__helpers.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py index b59b3d794..9058e4527 100644 --- a/google/cloud/bigquery/_helpers.py +++ b/google/cloud/bigquery/_helpers.py @@ -319,6 +319,7 @@ def _timestamp_to_json_row(value): """ if isinstance(value, datetime.datetime): value = _microseconds_from_datetime(value) * 1e-6 + value = round(value, 6) return value diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py index 16c4fb8a5..e340a83a7 100644 --- a/tests/unit/test__helpers.py +++ b/tests/unit/test__helpers.py @@ -733,6 +733,13 @@ def test_w_datetime(self): when = datetime.datetime(2016, 12, 20, 15, 58, 27, 339328) self.assertEqual(self._call_fut(when), _microseconds_from_datetime(when) / 1e6) + def test_w_datetime_w_utc_zone(self): + from google.cloud._helpers import _microseconds_from_datetime + from google.cloud._helpers import UTC + + when = datetime.datetime(2020, 11, 17, 1, 6, 52, 353795, tzinfo=UTC) + self.assertEqual(self._call_fut(when), _microseconds_from_datetime(when) / 1e6) + class Test_datetime_to_json(unittest.TestCase): def _call_fut(self, value): From b139c078ed88c83cd2bb0c335ec1b4e04cff1926 Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Fri, 20 Nov 2020 16:47:03 +0530 Subject: [PATCH 2/3] fix: remove floating point coversion and add datetime format --- google/cloud/bigquery/_helpers.py | 10 +++------- tests/unit/test__helpers.py | 8 ++++---- tests/unit/test_client.py | 25 +++++++++++++++++-------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py index 9058e4527..35129d844 100644 --- a/google/cloud/bigquery/_helpers.py +++ b/google/cloud/bigquery/_helpers.py @@ -23,7 +23,7 @@ from google.cloud._helpers import UTC from google.cloud._helpers import _date_from_iso8601_date from google.cloud._helpers import _datetime_from_microseconds -from google.cloud._helpers import _microseconds_from_datetime +from google.cloud._helpers import _RFC3339_MICROS from google.cloud._helpers import _RFC3339_NO_FRACTION from google.cloud._helpers import _to_bytes @@ -313,13 +313,9 @@ def _timestamp_to_json_parameter(value): def _timestamp_to_json_row(value): - """Coerce 'value' to an JSON-compatible representation. - - This version returns floating-point seconds value used in row data. - """ + """Coerce 'value' to an JSON-compatible representation.""" if isinstance(value, datetime.datetime): - value = _microseconds_from_datetime(value) * 1e-6 - value = round(value, 6) + value = value.strftime(_RFC3339_MICROS) return value diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py index e340a83a7..ea364a9aa 100644 --- a/tests/unit/test__helpers.py +++ b/tests/unit/test__helpers.py @@ -728,17 +728,17 @@ def test_w_string(self): self.assertEqual(self._call_fut(ZULU), ZULU) def test_w_datetime(self): - from google.cloud._helpers import _microseconds_from_datetime + from google.cloud._helpers import _RFC3339_MICROS when = datetime.datetime(2016, 12, 20, 15, 58, 27, 339328) - self.assertEqual(self._call_fut(when), _microseconds_from_datetime(when) / 1e6) + self.assertEqual(self._call_fut(when), when.strftime(_RFC3339_MICROS)) def test_w_datetime_w_utc_zone(self): - from google.cloud._helpers import _microseconds_from_datetime + from google.cloud._helpers import _RFC3339_MICROS from google.cloud._helpers import UTC when = datetime.datetime(2020, 11, 17, 1, 6, 52, 353795, tzinfo=UTC) - self.assertEqual(self._call_fut(when), _microseconds_from_datetime(when) / 1e6) + self.assertEqual(self._call_fut(when), when.strftime(_RFC3339_MICROS)) class Test_datetime_to_json(unittest.TestCase): diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index dd57ee798..b154286bc 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -5804,7 +5804,7 @@ def test_insert_rows_w_schema(self): import datetime from google.cloud._helpers import UTC from google.cloud._helpers import _datetime_to_rfc3339 - from google.cloud._helpers import _microseconds_from_datetime + from google.cloud._helpers import _RFC3339_MICROS from google.cloud.bigquery.schema import SchemaField WHEN_TS = 1437767599.006 @@ -5834,7 +5834,7 @@ def _row_data(row): result = {"full_name": row[0], "age": str(row[1])} joined = row[2] if isinstance(joined, datetime.datetime): - joined = _microseconds_from_datetime(joined) * 1e-6 + joined = joined.strftime(_RFC3339_MICROS) if joined is not None: result["joined"] = joined return result @@ -5864,7 +5864,7 @@ def test_insert_rows_w_list_of_dictionaries(self): import datetime from google.cloud._helpers import UTC from google.cloud._helpers import _datetime_to_rfc3339 - from google.cloud._helpers import _microseconds_from_datetime + from google.cloud._helpers import _RFC3339_MICROS from google.cloud.bigquery.schema import SchemaField from google.cloud.bigquery.table import Table @@ -5910,7 +5910,7 @@ def _row_data(row): row = copy.deepcopy(row) del row["joined"] elif isinstance(joined, datetime.datetime): - row["joined"] = _microseconds_from_datetime(joined) * 1e-6 + row["joined"] = joined.strftime(_RFC3339_MICROS) row["age"] = str(row["age"]) return row @@ -6052,6 +6052,7 @@ def _row_data(row): ) def test_insert_rows_w_repeated_fields(self): + from google.cloud._helpers import _RFC3339_MICROS from google.cloud.bigquery.schema import SchemaField from google.cloud.bigquery.table import Table @@ -6109,16 +6110,24 @@ def test_insert_rows_w_repeated_fields(self): { "score": "12", "times": [ - 1543665600.0, # 2018-12-01 12:00 UTC - 1543669200.0, # 2018-12-01 13:00 UTC + datetime.datetime( + 2018, 12, 1, 12, 0, 0, tzinfo=pytz.utc + ).strftime(_RFC3339_MICROS), + datetime.datetime( + 2018, 12, 1, 13, 0, 0, tzinfo=pytz.utc + ).strftime(_RFC3339_MICROS), ], "distances": [1.25, 2.5], }, { "score": "13", "times": [ - 1543752000.0, # 2018-12-02 12:00 UTC - 1543755600.0, # 2018-12-02 13:00 UTC + datetime.datetime( + 2018, 12, 2, 12, 0, 0, tzinfo=pytz.utc + ).strftime(_RFC3339_MICROS), + datetime.datetime( + 2018, 12, 2, 13, 0, 0, tzinfo=pytz.utc + ).strftime(_RFC3339_MICROS), ], "distances": [-1.25, -2.5], }, From c675232c6e100af3531844bbd9ccdfe5014e191b Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Sat, 21 Nov 2020 12:22:02 +0530 Subject: [PATCH 3/3] fix: add formatted string in unit tests --- tests/unit/test__helpers.py | 7 ++----- tests/unit/test_client.py | 17 ++++------------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py index ea364a9aa..a52581501 100644 --- a/tests/unit/test__helpers.py +++ b/tests/unit/test__helpers.py @@ -728,17 +728,14 @@ def test_w_string(self): self.assertEqual(self._call_fut(ZULU), ZULU) def test_w_datetime(self): - from google.cloud._helpers import _RFC3339_MICROS - when = datetime.datetime(2016, 12, 20, 15, 58, 27, 339328) - self.assertEqual(self._call_fut(when), when.strftime(_RFC3339_MICROS)) + self.assertEqual(self._call_fut(when), "2016-12-20T15:58:27.339328Z") def test_w_datetime_w_utc_zone(self): - from google.cloud._helpers import _RFC3339_MICROS from google.cloud._helpers import UTC when = datetime.datetime(2020, 11, 17, 1, 6, 52, 353795, tzinfo=UTC) - self.assertEqual(self._call_fut(when), when.strftime(_RFC3339_MICROS)) + self.assertEqual(self._call_fut(when), "2020-11-17T01:06:52.353795Z") class Test_datetime_to_json(unittest.TestCase): diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index b154286bc..4fba1150c 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -6052,7 +6052,6 @@ def _row_data(row): ) def test_insert_rows_w_repeated_fields(self): - from google.cloud._helpers import _RFC3339_MICROS from google.cloud.bigquery.schema import SchemaField from google.cloud.bigquery.table import Table @@ -6110,24 +6109,16 @@ def test_insert_rows_w_repeated_fields(self): { "score": "12", "times": [ - datetime.datetime( - 2018, 12, 1, 12, 0, 0, tzinfo=pytz.utc - ).strftime(_RFC3339_MICROS), - datetime.datetime( - 2018, 12, 1, 13, 0, 0, tzinfo=pytz.utc - ).strftime(_RFC3339_MICROS), + "2018-12-01T12:00:00.000000Z", + "2018-12-01T13:00:00.000000Z", ], "distances": [1.25, 2.5], }, { "score": "13", "times": [ - datetime.datetime( - 2018, 12, 2, 12, 0, 0, tzinfo=pytz.utc - ).strftime(_RFC3339_MICROS), - datetime.datetime( - 2018, 12, 2, 13, 0, 0, tzinfo=pytz.utc - ).strftime(_RFC3339_MICROS), + "2018-12-02T12:00:00.000000Z", + "2018-12-02T13:00:00.000000Z", ], "distances": [-1.25, -2.5], },