Skip to content

Commit

Permalink
#7727 source postgres: add several types to integration tests (#7913)
Browse files Browse the repository at this point in the history
  • Loading branch information
@yurii-bidiuk
yurii-bidiuk authored Dec 2, 2021
1 parent 9a54c4d commit 8726d39
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 11 deletions.
21 changes: 19 additions & 2 deletions ...byte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,8 +321,8 @@ protected void initTests() {
TestDataHolder.builder()
.sourceType("numeric")
.airbyteType(JsonSchemaPrimitive.NUMBER)
.addInsertValues("'99999'", "'NAN'", null)
.addExpectedValues("99999", "NAN", null)
.addInsertValues("'99999'", "'NAN'", "10000000000000000000000000000000000000", null)
.addExpectedValues("99999", "NAN", "10000000000000000000000000000000000000", null)
.build());

addDataTypeTestData(
Expand Down Expand Up @@ -507,6 +507,23 @@ protected void initTests() {
"'((0,0),(999999999999999999999999,0))'", "null")
.addExpectedValues("((3.0,7.0),(15.0,18.0))", "((0.0,0.0),(0.0,0.0))", "((0.0,0.0),(1.0E24,0.0))", null)
.build());

addDataTypeTestData(
TestDataHolder.builder()
.sourceType("real")
.airbyteType(JsonSchemaPrimitive.STRING)
.addInsertValues("'123'", "'1234567890.1234567'", "null")
.addExpectedValues("123.0", "1.23456794E9", null)
.build());

addDataTypeTestData(
TestDataHolder.builder()
.sourceType("tsvector")
.airbyteType(JsonSchemaPrimitive.STRING)
.addInsertValues("to_tsvector('The quick brown fox jumped over the lazy dog.')")
.addExpectedValues("'brown':3 'dog':9 'fox':4 'jumped':5 'lazy':8 'over':6 'quick':2 'the':1,7")
.build());

}

}
60 changes: 58 additions & 2 deletions .../airbyte/integrations/io/airbyte/integration_tests/sources/PostresSourceDatatypeTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,8 @@ protected void initTests() {
.addExpectedValues("true", "true", "true", "false", "false", "false", null)
.build());

// BUG: The represented value is encoded by Base64
// https://github.com/airbytehq/airbyte/issues/7905
addDataTypeTestData(
TestDataHolder.builder()
.sourceType("bytea")
Expand All @@ -179,6 +181,15 @@ protected void initTests() {
.addExpectedValues("{asb123}", "{asb12} ")
.build());

addDataTypeTestData(
TestDataHolder.builder()
.sourceType("character_varying")
.fullSourceDataType("character varying(8)")
.airbyteType(JsonSchemaPrimitive.STRING)
.addInsertValues("'{asb123}'", "'{asb12}'")
.addExpectedValues("{asb123}", "{asb12}")
.build());

addDataTypeTestData(
TestDataHolder.builder()
.sourceType("varchar")
Expand Down Expand Up @@ -223,6 +234,7 @@ protected void initTests() {
// Values "'-Infinity'", "'Infinity'", "'Nan'" will not be parsed due to:
// JdbcUtils -> setJsonField contains:
// case FLOAT, DOUBLE -> o.put(columnName, nullIfInvalid(() -> r.getDouble(i), Double::isFinite));
// https://github.com/airbytehq/airbyte/issues/7871
addDataTypeTestData(
TestDataHolder.builder()
.sourceType("float8")
Expand All @@ -231,9 +243,19 @@ protected void initTests() {
.addExpectedValues("123.0", "1.2345678901234567E9", null)
.build());

addDataTypeTestData(
TestDataHolder.builder()
.sourceType("double_precision")
.fullSourceDataType("double precision")
.airbyteType(JsonSchemaPrimitive.NUMBER)
.addInsertValues("'123'", "'1234567890.1234567'", "null")
.addExpectedValues("123.0", "1.2345678901234567E9", null)
.build());

// Values "'-Infinity'", "'Infinity'", "'Nan'" will not be parsed due to:
// JdbcUtils -> setJsonField contains:
// case FLOAT, DOUBLE -> o.put(columnName, nullIfInvalid(() -> r.getDouble(i), Double::isFinite));
// https://github.com/airbytehq/airbyte/issues/7871
addDataTypeTestData(
TestDataHolder.builder()
.sourceType("float")
Expand All @@ -258,6 +280,14 @@ protected void initTests() {
.addExpectedValues(null, "-2147483648", "2147483647")
.build());

addDataTypeTestData(
TestDataHolder.builder()
.sourceType("integer")
.airbyteType(JsonSchemaPrimitive.NUMBER)
.addInsertValues("null", "-2147483648", "2147483647")
.addExpectedValues(null, "-2147483648", "2147483647")
.build());

addDataTypeTestData(
TestDataHolder.builder()
.sourceType("interval")
Expand Down Expand Up @@ -323,17 +353,35 @@ protected void initTests() {
// The numeric type in Postres may contain 'Nan' type, but in JdbcUtils-> rowToJson
// we try to map it like this, so it fails
// case NUMERIC, DECIMAL -> o.put(columnName, nullIfInvalid(() -> r.getBigDecimal(i)));
// https://github.com/airbytehq/airbyte/issues/7871
addDataTypeTestData(
TestDataHolder.builder()
.sourceType("numeric")
.airbyteType(JsonSchemaPrimitive.NUMBER)
.addInsertValues("'99999'", "null")
.addExpectedValues("99999", null)
.addInsertValues("'99999'", "999999999999.9999999999", "10000000000000000000000000000000000000", "null")
.addExpectedValues("99999", "1.0E12", "1.0E37", null)
.build());

addDataTypeTestData(
TestDataHolder.builder()
.sourceType("real")
.airbyteType(JsonSchemaPrimitive.STRING)
.addInsertValues("'123'", "'1234567890.1234567'", "null")
.addExpectedValues("123.0", "1.23456794E9", null)
.build());

addDataTypeTestData(
TestDataHolder.builder()
.sourceType("pg_lsn")
.airbyteType(JsonSchemaPrimitive.STRING)
.addInsertValues("'7/A25801C8'::pg_lsn", "'0/0'::pg_lsn", "null")
.addExpectedValues("7/A25801C8", "0/0",null)
.build());

// The numeric type in Postres may contain 'Nan' type, but in JdbcUtils-> rowToJson
// we try to map it like this, so it fails
// case NUMERIC, DECIMAL -> o.put(columnName, nullIfInvalid(() -> r.getBigDecimal(i)));
// https://github.com/airbytehq/airbyte/issues/7871
addDataTypeTestData(
TestDataHolder.builder()
.sourceType("decimal")
Expand Down Expand Up @@ -404,6 +452,14 @@ protected void initTests() {
.addExpectedValues("'brown':3 'dog':9 'fox':4 'jump':5 'lazi':8 'quick':2")
.build());

addDataTypeTestData(
TestDataHolder.builder()
.sourceType("tsquery")
.airbyteType(JsonSchemaPrimitive.STRING)
.addInsertValues("to_tsquery('fat & rat')", "to_tsquery('Fat:ab & Cats')", "null")
.addExpectedValues("'fat' & 'rat'", "'fat':AB & 'cat'", null)
.build());

addDataTypeTestData(
TestDataHolder.builder()
.sourceType("uuid")
Expand Down
15 changes: 8 additions & 7 deletions docs/integrations/sources/postgres.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,21 +209,21 @@ Postgres data types are mapped to the following data types when synchronizing da
| :--- | :--- | :--- |
| `bigint` | number | |
| `bigserial` | number | |
| `bit` | boolean | |
| `bit` | boolean | bit is mapped to boolean type and `read` is failed for n-bit values |
| `blob` | boolean | |
| `boolean` | boolean | |
| `box` | string | |
| `bytea` | object | |
| `bytea` | object | parsed value is encoded by Base64 [#7905](https://github.com/airbytehq/airbyte/issues/7905)|
| `character` | string | |
| `character varying` | string | |
| `cidr` | string | |
| `circle` | string | |
| `citext` | string | |
| `date` | string | |
| `double precision` | string | |
| `double precision` | string | Values `-Infinity`, `Infinity`, `Nan` will not be parsed correctly. Parsed values for all of them are null [#7871](https://github.com/airbytehq/airbyte/issues/7871) |
| `enum` | number | |
| `float` | number | |
| `float8` | number | |
| `float` | number | Values `-Infinity`, `Infinity`, `Nan` will not be parsed correctly. Parsed values for all of them are null [#7871](https://github.com/airbytehq/airbyte/issues/7871) |
| `float8` | number | Values `-Infinity`, `Infinity`, `Nan` will not be parsed correctly. Parsed values for all of them are null [#7871](https://github.com/airbytehq/airbyte/issues/7871) |
| `hstore` | object | may be de-nested depending on the destination you are syncing into |
| `inet` | string | |
| `int` | number | |
Expand All @@ -235,13 +235,13 @@ Postgres data types are mapped to the following data types when synchronizing da
| `lseg` | string | |
| `macaddr` | string | |
| `macaddr8` | string | |
| `money` | string | When running logical replication (CDC), `money` values larger than 999999999999999 (15 nines) or smaller than -999999999999999 (15 nines) are transmitted as null; |
| `money` | string | When running logical replication (CDC), `money` values larger than 999999999999999 (15 nines) or smaller than -999999999999999 (15 nines) are transmitted as null; When running default mode `money` value fail when amount is > 1000 [#7870](https://github.com/airbytehq/airbyte/issues/7870) |
| `mood` | string | |
| `numeric` | number | |
| `path` | string | |
| `point` | number | |
| `polygon` | number | |
| `real` | number | |
| `real` | number | Values `-Infinity`, `Infinity`, `Nan` will not be parsed correctly. Parsed values for all of them are null [#7871](https://github.com/airbytehq/airbyte/issues/7871) |
| `serial` | number | |
| `smallint` | number | |
| `smallserial` | number | |
Expand All @@ -255,6 +255,7 @@ Postgres data types are mapped to the following data types when synchronizing da
| `timestamp without timezone` | string | may be written as a native date type depending on the destination |
| `tsrange` | string | |
| `tsvector` | string | |
| `tsquery` | string | is not supported with CDC node. Parsed value is null [#7911](https://github.com/airbytehq/airbyte/issues/7911) |
| `uuid` | string | |
| `varchar` | string | |
| `xml` | string | |
Expand Down

0 comments on commit 8726d39

Please sign in to comment.