feat: auto convert numbers (#436)

googleapis · Apr 8, 2024 · f0505e7 · f0505e7
1 parent cf51780
commit f0505e7
Show file tree

Hide file tree

Showing 6 changed files with 156 additions and 11 deletions.
diff --git a/protos/protos.d.ts b/protos/protos.d.ts
diff --git a/protos/protos.js b/protos/protos.js
diff --git a/protos/protos.json b/protos/protos.json
diff --git a/samples/append_rows_table_to_proto2.js b/samples/append_rows_table_to_proto2.js
@@ -151,7 +151,7 @@ function main(
       // Row 10
       row = {
         row_num: 10,
-        numeric_col: '123456',
+        numeric_col: 123456,
         bignumeric_col: '99999999999999999999999999999.999999999',
       };
       rows.push(row);

diff --git a/src/managedwriter/encoder.ts b/src/managedwriter/encoder.ts
@@ -94,11 +94,11 @@ export class JSONEncoder {
       if (value === null) {
         continue;
       }
+      const pfield = this._type.fields[key];
+      if (!pfield) {
+        continue;
+      }
       if (value instanceof Date) {
-        const pfield = this._type.fields[key];
-        if (!pfield) {
-          continue;
-        }
         switch (pfield.type) {
           case 'int32': // DATE
             // The value is the number of days since the Unix epoch (1970-01-01)
@@ -114,6 +114,15 @@ export class JSONEncoder {
         }
         continue;
       }
+      // NUMERIC and BIGNUMERIC integer
+      if (typeof value === 'number' || typeof value === 'bigint') {
+        switch (pfield.type) {
+          case 'string':
+            row[key] = value.toString(10);
+            break;
+        }
+        continue;
+      }
       if (Array.isArray(value)) {
         row[key] = value.map(v => {
           if (!this.isPlainObject(v)) {

diff --git a/system-test/managed_writer_client_test.ts b/system-test/managed_writer_client_test.ts
@@ -437,6 +437,66 @@ describe('managedwriter.WriterClient', () => {
         customer_updated_at: 1673236186564000,
       });
     });
+
+    it('should automatically convert numeric/bignumeric to expect BigQuery format', () => {
+      const updatedSchema = {
+        fields: [
+          ...(schema.fields || []),
+          {
+            name: 'customer_points',
+            type: 'NUMERIC',
+          },
+          {
+            name: 'customer_funds',
+            type: 'BIGNUMERIC',
+          },
+        ],
+      };
+      const storageSchema =
+        adapt.convertBigQuerySchemaToStorageTableSchema(updatedSchema);
+      const protoDescriptor: DescriptorProto =
+        adapt.convertStorageSchemaToProto2Descriptor(storageSchema, 'root');
+      const encoder = new JSONEncoder({
+        protoDescriptor,
+      });
+
+      // accept plain integers and bigint
+      const row1 = {
+        customer_name: 'Ada Lovelace',
+        row_num: 1,
+        customer_points: 1234,
+        customer_funds: BigInt(123456789),
+      };
+
+      // accept floats
+      const row2 = {
+        customer_name: 'Alan Turing',
+        row_num: 2,
+        customer_points: 1234.56,
+        customer_funds: '123456789.001234', // still accept in string
+      };
+
+      const Proto = Type.fromDescriptor(protoDescriptor);
+      const encoded = encoder.encodeRows([row1, row2]);
+
+      const encodedRow1 = encoded[0];
+      const decodedRow1 = Proto.decode(encodedRow1).toJSON();
+      assert.deepEqual(decodedRow1, {
+        customer_name: 'Ada Lovelace',
+        row_num: 1,
+        customer_points: '1234',
+        customer_funds: '123456789',
+      });
+
+      const encodedRow2 = encoded[1];
+      const decodedRow2 = Proto.decode(encodedRow2).toJSON();
+      assert.deepEqual(decodedRow2, {
+        customer_name: 'Alan Turing',
+        row_num: 2,
+        customer_points: '1234.56',
+        customer_funds: '123456789.001234',
+      });
+    });
   });
 
   describe('JSONWriter', () => {