diff --git a/csharp/src/Drivers/BigQuery/AssemblyInfo.cs b/csharp/src/Drivers/BigQuery/AssemblyInfo.cs new file mode 100644 index 0000000000..f322fe8c6a --- /dev/null +++ b/csharp/src/Drivers/BigQuery/AssemblyInfo.cs @@ -0,0 +1,20 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Apache.Arrow.Adbc.Tests.Drivers.BigQuery, PublicKey=0024000004800000940000000602000000240000525341310004000001000100e504183f6d470d6b67b6d19212be3e1f598f70c246a120194bc38130101d0c1853e4a0f2232cb12e37a7a90e707aabd38511dac4f25fcb0d691b2aa265900bf42de7f70468fc997551a40e1e0679b605aa2088a4a69e07c117e988f5b1738c570ee66997fba02485e7856a49eca5fd0706d09899b8312577cbb9034599fc92d4")] diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index 398365314f..1915263997 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -47,7 +47,7 @@ public class BigQueryConnection : AdbcConnection const string infoVendorName = "BigQuery"; const string infoDriverArrowVersion = "1.0.0"; - readonly AdbcInfoCode[] infoSupportedCodes = new [] { + readonly AdbcInfoCode[] infoSupportedCodes = new[] { AdbcInfoCode.DriverName, AdbcInfoCode.DriverVersion, AdbcInfoCode.DriverArrowVersion, @@ -447,7 +447,7 @@ private StructArray GetTableSchemas( nullBitmapBuffer.Append(true); length++; - if (includeConstraints) + if (depth == GetObjectsDepth.All && includeConstraints) { tableConstraintsValues.Add(GetConstraintSchema( depth, catalog, dbSchema, GetValue(row["table_name"]), columnNamePattern)); @@ -530,11 +530,15 @@ private StructArray GetColumnSchema( ordinalPositionBuilder.Append((int)(long)row["ordinal_position"]); remarksBuilder.Append(""); - string dataType = ToTypeName(GetValue(row["data_type"])); + string dataType = ToTypeName(GetValue(row["data_type"]), out string suffix); - if (dataType.StartsWith("NUMERIC") || dataType.StartsWith("DECIMAL") || dataType.StartsWith("BIGNUMERIC") || dataType.StartsWith("BIGDECIMAL")) + if ((dataType.StartsWith("NUMERIC") || + dataType.StartsWith("DECIMAL") || + dataType.StartsWith("BIGNUMERIC") || + dataType.StartsWith("BIGDECIMAL")) + && !string.IsNullOrEmpty(suffix)) { - ParsedDecimalValues values = ParsePrecisionAndScale(dataType); + ParsedDecimalValues values = ParsePrecisionAndScale(suffix); xdbcColumnSizeBuilder.Append(values.Precision); xdbcDecimalDigitsBuilder.Append(Convert.ToInt16(values.Scale)); } @@ -652,6 +656,7 @@ private StructArray GetConstraintSchema( constraintColumnNamesValues.BuildListArrayForType(StringType.Default), constraintColumnUsageValues.BuildListArrayForType(new StructType(StandardSchemas.UsageSchema)) }; + StandardSchemas.ConstraintSchema.Validate(dataArrays); return new StructArray( @@ -752,10 +757,19 @@ private string PatternToRegEx(string? pattern) return builder.ToString(); } - private string ToTypeName(string type) + private string ToTypeName(string type, out string suffix) { - int index = Math.Min(type.IndexOf("("), type.IndexOf("<")); + suffix = string.Empty; + + int index = type.IndexOf("("); + if (index == -1) + index = type.IndexOf("<"); + string dataType = index == -1 ? type : type.Substring(0, index); + + if (index > -1) + suffix = type.Substring(dataType.Length); + return dataType; } @@ -914,7 +928,7 @@ private Field.Builder GetFieldBuilder(string name, string type, string dataType, case "TIME": return fieldBuilder.DataType(Time64Type.Default); case "DATE": - return fieldBuilder.DataType(Date64Type.Default); + return fieldBuilder.DataType(Date32Type.Default); case "RECORD" or "STRUCT": string fieldRecords = type.Substring(index + 1); fieldRecords = fieldRecords.Remove(fieldRecords.Length - 1); @@ -965,7 +979,7 @@ private ParsedDecimalValues ParsePrecisionAndScale(string type) public override IArrowArrayStream GetTableTypes() { StringArray.Builder tableTypesBuilder = new StringArray.Builder(); - tableTypesBuilder.AppendRange(new string[] { "BASE TABLE", "VIEW" }); + tableTypesBuilder.AppendRange(BigQueryTableTypes.TableTypes); IArrowArray[] dataArrays = new IArrowArray[] { diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index ab7764b032..96b6f371a4 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -161,7 +161,7 @@ private IArrowType TranslateType(TableFieldSchema field) case "TIME": return GetType(field, Time64Type.Default); case "DATE": - return GetType(field, Date64Type.Default); + return GetType(field, Date32Type.Default); case "RECORD" or "STRUCT": // its a json string return GetType(field, StringType.Default); diff --git a/csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs b/csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs new file mode 100644 index 0000000000..e2d47fc7a9 --- /dev/null +++ b/csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs @@ -0,0 +1,25 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +using System.Collections.Generic; + +namespace Apache.Arrow.Adbc.Drivers.BigQuery +{ + internal static class BigQueryTableTypes + { + public static readonly string[] TableTypes = new string[]{ "BASE TABLE", "VIEW", "CLONE", "SNAPSHOT" }; + } +} diff --git a/csharp/src/Drivers/BigQuery/readme.md b/csharp/src/Drivers/BigQuery/readme.md index d78ab4c891..92af3ef1e5 100644 --- a/csharp/src/Drivers/BigQuery/readme.md +++ b/csharp/src/Drivers/BigQuery/readme.md @@ -87,7 +87,7 @@ The following table depicts how the BigQuery ADBC driver converts a BigQuery typ | BIGNUMERIC | Decimal256 | string | BOOL | Boolean | bool | BYTES | Binary | byte[] -| DATE | Date64 | DateTime +| DATE | Date32 | DateTime | DATETIME | Timestamp | DateTime | FLOAT64 | Double | double | GEOGRAPHY | String | string diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs b/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs index 6b10f4cde3..8d5dc25b60 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs @@ -156,9 +156,14 @@ public static List ParseCatalog(RecordBatch recordBatch, string? sc { if (constraintsArray == null) return null; + // constraint details may not be loaded correctly if the depth wasn't Columns + int fieldCount = constraintsArray?.Fields?.Count ?? 0; + if (fieldCount == 0) + return null; + List constraints = new List(); - StringArray name = (StringArray)constraintsArray.Fields[StandardSchemas.ConstraintSchema.FindIndexOrThrow("constraint_name")]; // constraint_name | utf8 + StringArray name = (StringArray)constraintsArray!.Fields[StandardSchemas.ConstraintSchema.FindIndexOrThrow("constraint_name")]; // constraint_name | utf8 StringArray type = (StringArray)constraintsArray.Fields[StandardSchemas.ConstraintSchema.FindIndexOrThrow("constraint_type")]; // constraint_type | utf8 not null ListArray columnNames = (ListArray)constraintsArray.Fields[StandardSchemas.ConstraintSchema.FindIndexOrThrow("constraint_column_names")]; // constraint_column_names | list not null ListArray columnUsages = (ListArray)constraintsArray.Fields[StandardSchemas.ConstraintSchema.FindIndexOrThrow("constraint_column_usage")]; // constraint_column_usage | list diff --git a/csharp/test/Drivers/BigQuery/BigQueryData.cs b/csharp/test/Drivers/BigQuery/BigQueryData.cs index d5959f837a..aa58ca0226 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryData.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryData.cs @@ -72,7 +72,7 @@ public static SampleDataBuilder GetSampleData() new ColumnNetTypeArrowTypeValue("is_active", typeof(bool), typeof(BooleanType), true), new ColumnNetTypeArrowTypeValue("name", typeof(string), typeof(StringType), "John Doe"), new ColumnNetTypeArrowTypeValue("data", typeof(byte[]), typeof(BinaryType), UTF8Encoding.UTF8.GetBytes("abc123")), - new ColumnNetTypeArrowTypeValue("date", typeof(DateTime), typeof(Date64Type), new DateTime(2023, 9, 8)), + new ColumnNetTypeArrowTypeValue("date", typeof(DateTime), typeof(Date32Type), new DateTime(2023, 9, 8)), #if NET6_0_OR_GREATER new ColumnNetTypeArrowTypeValue("time", typeof(TimeOnly), typeof(Time64Type), new TimeOnly(12, 34, 56)), //'12:34:56' #else @@ -134,7 +134,7 @@ public static SampleDataBuilder GetSampleData() new ColumnNetTypeArrowTypeValue("is_active", typeof(bool), typeof(BooleanType), null), new ColumnNetTypeArrowTypeValue("name", typeof(string), typeof(StringType), null), new ColumnNetTypeArrowTypeValue("data", typeof(byte[]), typeof(BinaryType), null), - new ColumnNetTypeArrowTypeValue("date", typeof(DateTime), typeof(Date64Type), null), + new ColumnNetTypeArrowTypeValue("date", typeof(DateTime), typeof(Date32Type), null), #if NET6_0_OR_GREATER new ColumnNetTypeArrowTypeValue("time", typeof(TimeOnly), typeof(Time64Type), null), #else diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs b/csharp/test/Drivers/BigQuery/DriverTests.cs index 9c587c1ebd..579a2a83cd 100644 --- a/csharp/test/Drivers/BigQuery/DriverTests.cs +++ b/csharp/test/Drivers/BigQuery/DriverTests.cs @@ -18,6 +18,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Apache.Arrow.Adbc.Drivers.BigQuery; using Apache.Arrow.Adbc.Tests.Metadata; using Apache.Arrow.Adbc.Tests.Xunit; using Apache.Arrow.Ipc; @@ -116,7 +117,7 @@ public void CanGetObjects() catalogPattern: catalogName, dbSchemaPattern: schemaName, tableNamePattern: tableName, - tableTypes: new List { "BASE TABLE", "VIEW" }, + tableTypes: BigQueryTableTypes.TableTypes, columnNamePattern: columnName); RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; @@ -134,6 +135,40 @@ public void CanGetObjects() Assert.Equal(_testConfiguration.Metadata.ExpectedColumnCount, columns?.Count); } + [SkippableFact, Order(3)] + public void CanGetObjectsTables() + { + string? catalogName = _testConfiguration.Metadata.Catalog; + string? schemaName = _testConfiguration.Metadata.Schema; + string? tableName = _testConfiguration.Metadata.Table; + + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); + + IArrowArrayStream stream = adbcConnection.GetObjects( + depth: AdbcConnection.GetObjectsDepth.Tables, + catalogPattern: catalogName, + dbSchemaPattern: schemaName, + tableNamePattern: null, + tableTypes: BigQueryTableTypes.TableTypes, + columnNamePattern: null); + + RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; + + List catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName); + + List? tables = catalogs + .Where(c => string.Equals(c.Name, catalogName)) + .Select(c => c.DbSchemas) + .FirstOrDefault() + ?.Where(s => string.Equals(s.Name, schemaName)) + .Select(s => s.Tables) + .FirstOrDefault(); + + AdbcTable? table = tables?.Where((table) => string.Equals(table.Name, tableName)).FirstOrDefault(); + Assert.True(table != null, "table should not be null"); + Assert.Equal("BASE TABLE", table.Type); + } + /// /// Validates if the driver can call GetTableSchema. /// @@ -167,10 +202,7 @@ public void CanGetTableTypes() StringArray stringArray = (StringArray)recordBatch.Column("table_type"); - List known_types = new List - { - "BASE TABLE", "VIEW" - }; + List known_types = BigQueryTableTypes.TableTypes.ToList(); int results = 0;