diff --git a/go/adbc/driver/bigquery/connection.go b/go/adbc/driver/bigquery/connection.go index e5ab3651ac..47921ff398 100644 --- a/go/adbc/driver/bigquery/connection.go +++ b/go/adbc/driver/bigquery/connection.go @@ -208,7 +208,7 @@ func (c *connectionImpl) GetTablesForDBSchema(ctx context.Context, catalog strin if err != nil { return nil, err } - xdbcDataType := driverbase.ToXdbcDataType(field.Type) + xdbcDataType := internal.ToXdbcDataType(field.Type) columns = append(columns, driverbase.ColumnInfo{ ColumnName: fieldschema.Name, diff --git a/go/adbc/driver/flightsql/flightsql_adbc_server_test.go b/go/adbc/driver/flightsql/flightsql_adbc_server_test.go index 07e0181b20..e3c1321838 100644 --- a/go/adbc/driver/flightsql/flightsql_adbc_server_test.go +++ b/go/adbc/driver/flightsql/flightsql_adbc_server_test.go @@ -38,7 +38,6 @@ import ( "github.com/apache/arrow-adbc/go/adbc" driver "github.com/apache/arrow-adbc/go/adbc/driver/flightsql" "github.com/apache/arrow-adbc/go/adbc/driver/internal" - "github.com/apache/arrow-adbc/go/adbc/validation" "github.com/apache/arrow-go/v18/arrow" "github.com/apache/arrow-go/v18/arrow/array" "github.com/apache/arrow-go/v18/arrow/flight" @@ -142,6 +141,10 @@ func TestSessionOptions(t *testing.T) { suite.Run(t, &SessionOptionTests{}) } +func TestGetObjects(t *testing.T) { + suite.Run(t, &GetObjectsTests{}) +} + // ---- AuthN Tests -------------------- type AuthnTestServer struct { @@ -1874,106 +1877,278 @@ func (suite *SessionOptionTests) TestGetSetStringList() { suite.Equal(`[]`, val) } -type GetObjectsTests struct { - suite.Suite +// ---- GetObjects Tests -------------------- - Driver adbc.Driver - Quirks validation.DriverQuirks - Cnxn adbc.Connection - ctx context.Context - DB adbc.Database +type GetObjectsTestServer struct { + flightsql.BaseServer + catalogName string + schemaName string + tableName string + testData map[string][]string } -func (suite *GetObjectsTests) SetupSuite() { - var err error - suite.Driver = suite.Quirks.SetupDriver(suite.T()) - suite.DB, err = suite.Driver.NewDatabase(suite.Quirks.DatabaseOptions()) - suite.NoError(err) +func (srv *GetObjectsTestServer) GetFlightInfoCatalogs(ctx context.Context, desc *flight.FlightDescriptor) (*flight.FlightInfo, error) { + return srv.flightInfoForSchema(schema_ref.Catalogs, desc), nil +} - suite.ctx = context.Background() - suite.Cnxn, err = suite.DB.Open(suite.ctx) - suite.Require().NoError(err) +func (srv *GetObjectsTestServer) GetFlightInfoSchemas(ctx context.Context, cmd flightsql.GetDBSchemas, desc *flight.FlightDescriptor) (*flight.FlightInfo, error) { + return srv.flightInfoForSchema(schema_ref.DBSchemas, desc), nil } -func (suite *GetObjectsTests) TestMetadataGetObjectsColumnsXdbc() { +func (srv *GetObjectsTestServer) GetFlightInfoTables(ctx context.Context, cmd flightsql.GetTables, desc *flight.FlightDescriptor) (*flight.FlightInfo, error) { + return srv.flightInfoForSchema(schema_ref.TablesWithIncludedSchema, desc), nil +} - suite.Require().NoError(suite.Quirks.DropTable(suite.Cnxn, "bulk_ingest")) - - mdInts := make(map[string]string) - mdInts["TYPE_NAME"] = "NUMERIC" - mdInts["ORDINAL_POSITION"] = "1" - mdInts["XDBC_DATA_TYPE"] = strconv.Itoa(int(arrow.PrimitiveTypes.Int64.ID())) - mdInts["XDBC_TYPE_NAME"] = "NUMERIC" - mdInts["XDBC_SQL_DATA_TYPE"] = strconv.Itoa(int(internal.XdbcDataType_XDBC_BIGINT)) - mdInts["XDBC_NULLABLE"] = strconv.FormatBool(true) - mdInts["XDBC_IS_NULLABLE"] = "YES" - mdInts["XDBC_PRECISION"] = strconv.Itoa(38) - mdInts["XDBC_SCALE"] = strconv.Itoa(0) - mdInts["XDBC_NUM_PREC_RADIX"] = strconv.Itoa(10) - - mdStrings := make(map[string]string) - mdStrings["TYPE_NAME"] = "TEXT" - mdStrings["ORDINAL_POSITION"] = "2" - mdStrings["XDBC_DATA_TYPE"] = strconv.Itoa(int(arrow.BinaryTypes.String.ID())) - mdStrings["XDBC_TYPE_NAME"] = "TEXT" - mdStrings["XDBC_SQL_DATA_TYPE"] = strconv.Itoa(int(internal.XdbcDataType_XDBC_VARCHAR)) - mdStrings["XDBC_IS_NULLABLE"] = "YES" - mdStrings["CHARACTER_MAXIMUM_LENGTH"] = strconv.Itoa(16777216) - mdStrings["XDBC_CHAR_OCTET_LENGTH"] = strconv.Itoa(16777216) - - rec, _, err := array.RecordFromJSON(suite.Quirks.Alloc(), arrow.NewSchema( - []arrow.Field{ - {Name: "int64s", Type: arrow.PrimitiveTypes.Int64, Nullable: true, Metadata: arrow.MetadataFrom(mdInts)}, - {Name: "strings", Type: arrow.BinaryTypes.String, Nullable: true, Metadata: arrow.MetadataFrom(mdStrings)}, - }, nil), strings.NewReader(`[ - {"int64s": 42, "strings": "foo"}, - {"int64s": -42, "strings": null}, - {"int64s": null, "strings": ""} - ]`)) - suite.Require().NoError(err) - defer rec.Release() +func (srv *GetObjectsTestServer) flightInfoForSchema(sc *arrow.Schema, desc *flight.FlightDescriptor) *flight.FlightInfo { + return &flight.FlightInfo{ + Endpoint: []*flight.FlightEndpoint{{Ticket: &flight.Ticket{Ticket: desc.Cmd}}}, + FlightDescriptor: desc, + Schema: flight.SerializeSchema(sc, srv.Alloc), + TotalRecords: -1, + TotalBytes: -1, + } +} + +func (srv *GetObjectsTestServer) DoGetCatalogs(ctx context.Context) (*arrow.Schema, <-chan flight.StreamChunk, error) { + // no catalogs + schema := schema_ref.Catalogs + ch := make(chan flight.StreamChunk, 1) + defer close(ch) + return schema, ch, nil +} + +func (srv *GetObjectsTestServer) DoGetTables(ctx context.Context, cmd flightsql.GetTables) (*arrow.Schema, <-chan flight.StreamChunk, error) { + schemaBldr := array.NewBinaryBuilder(srv.Alloc, arrow.BinaryTypes.Binary) + + columnFields := make([]arrow.Field, 0) + for key, val := range srv.testData { + + bldr := flightsql.NewColumnMetadataBuilder() + defer bldr.Clear() + + bldr.CatalogName(srv.catalogName) + bldr.SchemaName(srv.schemaName) + bldr.TableName(srv.tableName) + bldr.TypeName(val[0]) + if val, err := strconv.ParseInt(val[2], 10, 32); err != nil { + panic(err) + } else { + bldr.Precision(int32(val)) + } + if val, err := strconv.ParseInt(val[3], 10, 32); err != nil { + panic(err) + } else { + bldr.Scale(int32(val)) + } + bldr.IsAutoIncrement(val[4] == "true") + bldr.IsCaseSensitive(val[5] == "true") + bldr.IsReadOnly(val[6] == "true") + bldr.IsSearchable(val[7] == "true") + + colType, err := strconv.ParseInt(val[1], 10, 32) + if err != nil { + panic(err) + } + var fieldType arrow.DataType + switch colType { + case int64(arrow.PrimitiveTypes.Int32.ID()): + fieldType = arrow.PrimitiveTypes.Int32 + case int64(arrow.PrimitiveTypes.Float32.ID()): + fieldType = arrow.PrimitiveTypes.Float32 + case int64(arrow.PrimitiveTypes.Float64.ID()): + fieldType = arrow.PrimitiveTypes.Float64 + default: + panic(fmt.Errorf("unknown column type %d", colType)) + } + + columnFields = append(columnFields, arrow.Field{ + Name: key, + Type: fieldType, + Nullable: false, + Metadata: bldr.Metadata(), + }) + + } + + schemaBldr.Append(flight.SerializeSchema(arrow.NewSchema(columnFields, nil), srv.Alloc)) + schemaCol := schemaBldr.NewArray() + defer schemaCol.Release() - suite.Require().NoError(suite.Quirks.CreateSampleTable("bulk_ingest", rec)) + jsonStr := fmt.Sprintf(`[{"catalog_name": "%s", "db_schema_name": "%s", "table_name": "%s", "table_type": "TABLE"}]`, + srv.catalogName, // variable for catalog_name + srv.schemaName, // variable for db_schema_name + srv.tableName) // variable for table_type + tablesRecord, _, _ := array.RecordFromJSON(srv.Alloc, schema_ref.Tables, strings.NewReader(jsonStr)) + defer tablesRecord.Release() + tablesRecordWithSchema := array.NewRecord(schema_ref.TablesWithIncludedSchema, append(tablesRecord.Columns(), schemaCol), tablesRecord.NumRows()) + defer tablesRecordWithSchema.Release() + + ch := make(chan flight.StreamChunk) + + rdr, err := array.NewRecordReader(schema_ref.TablesWithIncludedSchema, []arrow.Record{tablesRecordWithSchema}) + go flight.StreamChunksFromReader(rdr, ch) + return schema_ref.TablesWithIncludedSchema, ch, err +} + +func (srv *GetObjectsTestServer) DoGetDBSchemas(ctx context.Context, cmd flightsql.GetDBSchemas) (*arrow.Schema, <-chan flight.StreamChunk, error) { + schema := schema_ref.DBSchemas + ch := make(chan flight.StreamChunk, 1) + // Not really a proper match, but good enough + catalogs, _, err := array.FromJSON(srv.Alloc, arrow.BinaryTypes.String, strings.NewReader(fmt.Sprintf(`["%s"]`, srv.catalogName))) + if err != nil { + return nil, nil, err + } + defer catalogs.Release() + + dbSchemas, _, err := array.FromJSON(srv.Alloc, arrow.BinaryTypes.String, strings.NewReader(fmt.Sprintf(`["%s"]`, srv.schemaName))) + if err != nil { + return nil, nil, err + } + defer dbSchemas.Release() + + batch := array.NewRecord(schema, []arrow.Array{catalogs, dbSchemas}, 1) + ch <- flight.StreamChunk{Data: batch} + close(ch) + return schema, ch, nil +} + +type GetObjectsTests struct { + ServerBasedTests + + catalogName string + schemaName string + tableName string +} + +func (suite *GetObjectsTests) SetupSuite() { + srv := &GetObjectsTestServer{} + suite.catalogName = "" + suite.schemaName = "test_schema" + suite.tableName = "test_table" + srv.catalogName = suite.catalogName + srv.schemaName = suite.schemaName + srv.tableName = suite.tableName + srv.testData = map[string][]string{ + "intcols": { + arrow.PrimitiveTypes.Int32.Name(), // TYPE_NAME + strconv.Itoa(int(arrow.PrimitiveTypes.Int32.ID())), // FieldType + strconv.Itoa(10), // PRECISION + strconv.Itoa(15), // SCALE + strconv.FormatBool(true), // IS_AUTO_INCREMENT + strconv.FormatBool(false), // IS_CASE_SENSITIVE + strconv.FormatBool(true), // IS_READ_ONLY + strconv.FormatBool(true), // IS_SEARCHABLE + }, + "floatcols": { + arrow.PrimitiveTypes.Float32.Name(), // TYPE_NAME + strconv.Itoa(int(arrow.PrimitiveTypes.Float32.ID())), // FieldType + strconv.Itoa(15), // PRECISION + strconv.Itoa(15), // SCALE + strconv.FormatBool(false), // IS_AUTO_INCREMENT + strconv.FormatBool(false), // IS_CASE_SENSITIVE + strconv.FormatBool(false), // IS_READ_ONLY + strconv.FormatBool(false), // IS_SEARCHABLE + }, + "currencycol": { + "CURRENCY", // TYPE_NAME + strconv.Itoa(int(arrow.PrimitiveTypes.Float64.ID())), // FieldType + strconv.Itoa(15), // PRECISION + strconv.Itoa(15), // SCALE + strconv.FormatBool(false), // IS_AUTO_INCREMENT + strconv.FormatBool(false), // IS_CASE_SENSITIVE + strconv.FormatBool(false), // IS_READ_ONLY + strconv.FormatBool(false), // IS_SEARCHABLE + }, + } + srv.Alloc = memory.NewCheckedAllocator(memory.DefaultAllocator) + suite.DoSetupSuite(srv, nil, nil) +} + +// Testing metadata from flight driver is converted to xdbc metadata. +// Ordering is being ignored to avoid flakiness as the order of the columns is not guaranteed. +func (suite *GetObjectsTests) TestMetadataGetObjectsColumnsXdbc() { tests := []struct { - name string - colnames []string - positions []string - dataTypes []string - comments []string - xdbcDataType []string - xdbcTypeName []string - xdbcSqlDataType []string - xdbcNullable []string - xdbcIsNullable []string - xdbcScale []string - xdbcNumPrecRadix []string - xdbcCharMaxLen []string - xdbcCharOctetLen []string - xdbcDateTimeSub []string + name string + columnName []string + //ordinalPosition []string + remarks []string + xdbcDataType []string + xdbcTypeName []string + xdbcColumnSize []string + xdbcDecimalDigits []string + xdbcNumPrecRadix []string + xdbcNullable []string + xdbcColumnDef []string + xdbcSqlDataType []string + xdbcDatetimeSub []string + xdbcCharOctetLength []string + xdbcIsNullable []string + xdbcScopeCatalog []string + xdbcScopeSchema []string + xdbcScopeTable []string + xdbcIsAutoincrement []string + xdbcIsAutogeneratedColumn []string }{ { - "BASIC", // name - []string{"int64s", "strings"}, // colNames - []string{"1", "2"}, // positions - []string{"NUMBER", "TEXT"}, // dataTypes - []string{"", ""}, // comments - []string{"9", "13"}, // xdbcDataType - []string{"NUMBER", "TEXT"}, // xdbcTypeName - []string{"-5", "12"}, // xdbcSqlDataType - []string{"1", "1"}, // xdbcNullable - []string{"YES", "YES"}, // xdbcIsNullable - []string{"0", "0"}, // xdbcScale - []string{"10", "0"}, // xdbcNumPrecRadix - []string{"38", "16777216"}, // xdbcCharMaxLen (xdbcPrecision) - []string{"0", "16777216"}, // xdbcCharOctetLen - []string{"-5", "12", "0"}, // xdbcDateTimeSub + fmt.Sprintf("%s.%s.%s", suite.catalogName, suite.schemaName, suite.tableName), + []string{"currencycol", "floatcols", "intcols"}, //columnName + //[]string{"1", "2", "3"}, //ordinalPosition + []string{"currencycol_", "floatcols_", "intcols_"}, //remarks + []string{ //xdbcDataType + "currencycol_" + strconv.Itoa(int(internal.ToXdbcDataType(arrow.PrimitiveTypes.Float64))), + "floatcols_" + strconv.Itoa(int(internal.ToXdbcDataType(arrow.PrimitiveTypes.Float32))), + "intcols_" + strconv.Itoa(int(internal.ToXdbcDataType(arrow.PrimitiveTypes.Int32))), + }, + []string{ //xdbcTypeName + "currencycol_CURRENCY", + "floatcols_" + arrow.PrimitiveTypes.Float32.Name(), + "intcols_" + arrow.PrimitiveTypes.Int32.Name(), + }, + []string{"currencycol_0", "floatcols_0", "intcols_0"}, //xdbcColumnSize + []string{"currencycol_0", "floatcols_0", "intcols_0"}, //xdbcDecimalDigits + []string{"currencycol_0", "floatcols_0", "intcols_0"}, //xdbcNumPrecRadix + []string{"currencycol_0", "floatcols_0", "intcols_0"}, //xdbcNullable + []string{"currencycol_", "floatcols_", "intcols_"}, //xdbcColumnDef + []string{ //xdbcSqlDataType + "currencycol_" + strconv.Itoa(int(internal.ToXdbcDataType(arrow.PrimitiveTypes.Float64))), + "floatcols_" + strconv.Itoa(int(internal.ToXdbcDataType(arrow.PrimitiveTypes.Float32))), + "intcols_" + strconv.Itoa(int(internal.ToXdbcDataType(arrow.PrimitiveTypes.Int32))), + }, + []string{"currencycol_0", "floatcols_0", "intcols_0"}, //xdbcDatetimeSub + []string{"currencycol_0", "floatcols_0", "intcols_0"}, //xdbcCharOctetLength + []string{"currencycol_", "floatcols_", "intcols_"}, //xdbcIsNullable + []string{ //xdbcScopeCatalog + "currencycol_" + suite.catalogName, + "floatcols_" + suite.catalogName, + "intcols_" + suite.catalogName, + }, + []string{ //xdbcScopeSchema + "currencycol_" + suite.schemaName, + "floatcols_" + suite.schemaName, + "intcols_" + suite.schemaName, + }, + []string{ //xdbcScopeTable + "currencycol_" + suite.tableName, + "floatcols_" + suite.tableName, + "intcols_" + suite.tableName, + }, + []string{ //xdbcIsAutoincrement + "currencycol_false", + "floatcols_false", + "intcols_true", + }, + []string{ //xdbcIsAutogeneratedColumn + "currencycol_false", + "floatcols_false", + "intcols_false", + }, }, } for _, tt := range tests { suite.Run(tt.name, func() { - rdr, err := suite.Cnxn.GetObjects(suite.ctx, adbc.ObjectDepthColumns, nil, nil, nil, nil, nil) + rdr, err := suite.cnxn.GetObjects(context.Background(), adbc.ObjectDepthColumns, nil, nil, nil, nil, nil) suite.Require().NoError(err) defer rdr.Release() @@ -1991,20 +2166,25 @@ func (suite *GetObjectsTests) TestMetadataGetObjectsColumnsXdbc() { tableColumnsList = dbSchemaTables.Field(2).(*array.List) tableColumns = tableColumnsList.ListValues().(*array.Struct) - colnames = make([]string, 0) - positions = make([]string, 0) - comments = make([]string, 0) - xdbcDataTypes = make([]string, 0) - dataTypes = make([]string, 0) - xdbcTypeNames = make([]string, 0) - xdbcCharMaxLens = make([]string, 0) - xdbcScales = make([]string, 0) - xdbcNumPrecRadixs = make([]string, 0) - xdbcNullables = make([]string, 0) - xdbcSqlDataTypes = make([]string, 0) - xdbcDateTimeSub = make([]string, 0) - xdbcCharOctetLen = make([]string, 0) - xdbcIsNullables = make([]string, 0) + columnName = make([]string, 0) + //ordinalPosition = make([]string, 0) + remarks = make([]string, 0) + xdbcDataType = make([]string, 0) + xdbcTypeName = make([]string, 0) + xdbcColumnSize = make([]string, 0) + xdbcDecimalDigits = make([]string, 0) + xdbcNumPrecRadix = make([]string, 0) + xdbcNullable = make([]string, 0) + xdbcColumnDef = make([]string, 0) + xdbcSqlDataType = make([]string, 0) + xdbcDatetimeSub = make([]string, 0) + xdbcCharOctetLength = make([]string, 0) + xdbcIsNullable = make([]string, 0) + xdbcScopeCatalog = make([]string, 0) + xdbcScopeSchema = make([]string, 0) + xdbcScopeTable = make([]string, 0) + xdbcIsAutoincrement = make([]string, 0) + xdbcIsAutogeneratedColumn = make([]string, 0) ) for row := 0; row < int(rec.NumRows()); row++ { dbSchemaIdxStart, dbSchemaIdxEnd := catalogDbSchemasList.ValueOffsets(row) @@ -2014,49 +2194,67 @@ func (suite *GetObjectsTests) TestMetadataGetObjectsColumnsXdbc() { for tblIdx := tblIdxStart; tblIdx < tblIdxEnd; tblIdx++ { tableName := dbSchemaTables.Field(0).(*array.String).Value(int(tblIdx)) - if strings.EqualFold(schemaName, suite.Quirks.DBSchema()) && strings.EqualFold("bulk_ingest", tableName) { + if strings.EqualFold(schemaName, suite.schemaName) && strings.EqualFold(suite.tableName, tableName) { foundExpected = true colIdxStart, colIdxEnd := tableColumnsList.ValueOffsets(int(tblIdx)) for colIdx := colIdxStart; colIdx < colIdxEnd; colIdx++ { name := tableColumns.Field(0).(*array.String).Value(int(colIdx)) - colnames = append(colnames, strings.ToLower(name)) + columnName = append(columnName, name) - pos := tableColumns.Field(1).(*array.Int32).Value(int(colIdx)) - positions = append(positions, strconv.Itoa(int(pos))) + // pos := tableColumns.Field(1).(*array.Int32).Value(int(colIdx)) + // ordinalPosition = append(ordinalPosition, strconv.Itoa(int(pos))) - comments = append(comments, tableColumns.Field(2).(*array.String).Value(int(colIdx))) + rm := tableColumns.Field(2).(*array.String).Value(int(colIdx)) + remarks = append(remarks, name+"_"+rm) xdt := tableColumns.Field(3).(*array.Int16).Value(int(colIdx)) - xdbcDataTypes = append(xdbcDataTypes, strconv.Itoa(int(xdt))) + xdbcDataType = append(xdbcDataType, name+"_"+strconv.Itoa(int(xdt))) dataType := tableColumns.Field(4).(*array.String).Value(int(colIdx)) - dataTypes = append(dataTypes, dataType) - xdbcTypeNames = append(xdbcTypeNames, dataType) + xdbcTypeName = append(xdbcTypeName, name+"_"+dataType) + + columnSize := tableColumns.Field(5).(*array.Int32).Value(int(colIdx)) + xdbcColumnSize = append(xdbcColumnSize, name+"_"+strconv.Itoa(int(columnSize))) - // these are column size attributes used for either precision for numbers OR the length for text - maxLenOrPrecision := tableColumns.Field(5).(*array.Int32).Value(int(colIdx)) - xdbcCharMaxLens = append(xdbcCharMaxLens, strconv.Itoa(int(maxLenOrPrecision))) + decimalDigits := tableColumns.Field(6).(*array.Int16).Value(int(colIdx)) + xdbcDecimalDigits = append(xdbcDecimalDigits, name+"_"+strconv.Itoa(int(decimalDigits))) - scale := tableColumns.Field(6).(*array.Int16).Value(int(colIdx)) - xdbcScales = append(xdbcScales, strconv.Itoa(int(scale))) + numPrecRadix := tableColumns.Field(7).(*array.Int16).Value(int(colIdx)) + xdbcNumPrecRadix = append(xdbcNumPrecRadix, name+"_"+strconv.Itoa(int(numPrecRadix))) - radix := tableColumns.Field(7).(*array.Int16).Value(int(colIdx)) - xdbcNumPrecRadixs = append(xdbcNumPrecRadixs, strconv.Itoa(int(radix))) + nullable := tableColumns.Field(8).(*array.Int16).Value(int(colIdx)) + xdbcNullable = append(xdbcNullable, name+"_"+strconv.Itoa(int(nullable))) - isnull := tableColumns.Field(8).(*array.Int16).Value(int(colIdx)) - xdbcNullables = append(xdbcNullables, strconv.Itoa(int(isnull))) + columnDef := tableColumns.Field(9).(*array.String).Value(int(colIdx)) + xdbcColumnDef = append(xdbcColumnDef, name+"_"+columnDef) sqlType := tableColumns.Field(10).(*array.Int16).Value(int(colIdx)) - xdbcSqlDataTypes = append(xdbcSqlDataTypes, strconv.Itoa(int(sqlType))) + xdbcSqlDataType = append(xdbcSqlDataType, name+"_"+strconv.Itoa(int(sqlType))) dtPrec := tableColumns.Field(11).(*array.Int16).Value(int(colIdx)) - xdbcDateTimeSub = append(xdbcSqlDataTypes, strconv.Itoa(int(dtPrec))) + xdbcDatetimeSub = append(xdbcDatetimeSub, name+"_"+strconv.Itoa(int(dtPrec))) charOctetLen := tableColumns.Field(12).(*array.Int32).Value(int(colIdx)) - xdbcCharOctetLen = append(xdbcCharOctetLen, strconv.Itoa(int(charOctetLen))) + xdbcCharOctetLength = append(xdbcCharOctetLength, name+"_"+strconv.Itoa(int(charOctetLen))) + + isNullable := tableColumns.Field(13).(*array.String).Value(int(colIdx)) + xdbcIsNullable = append(xdbcIsNullable, name+"_"+isNullable) + + scopeCatalog := tableColumns.Field(14).(*array.String).Value(int(colIdx)) + xdbcScopeCatalog = append(xdbcScopeCatalog, name+"_"+scopeCatalog) + + scopeSchema := tableColumns.Field(15).(*array.String).Value(int(colIdx)) + xdbcScopeSchema = append(xdbcScopeSchema, name+"_"+scopeSchema) + + scopeTable := tableColumns.Field(16).(*array.String).Value(int(colIdx)) + xdbcScopeTable = append(xdbcScopeTable, name+"_"+scopeTable) + + isAutoIncrement := tableColumns.Field(17).(*array.Boolean).Value(int(colIdx)) + xdbcIsAutoincrement = append(xdbcIsAutoincrement, name+"_"+strconv.FormatBool(isAutoIncrement)) - xdbcIsNullables = append(xdbcIsNullables, tableColumns.Field(13).(*array.String).Value(int(colIdx))) + isAutoGenerated := tableColumns.Field(18).(*array.Boolean).Value(int(colIdx)) + xdbcIsAutogeneratedColumn = append(xdbcIsAutogeneratedColumn, name+"_"+strconv.FormatBool(isAutoGenerated)) } } } @@ -2065,21 +2263,26 @@ func (suite *GetObjectsTests) TestMetadataGetObjectsColumnsXdbc() { suite.False(rdr.Next()) suite.True(foundExpected) - suite.Equal(tt.colnames, colnames) // colNames - suite.Equal(tt.positions, positions) // positions - suite.Equal(tt.comments, comments) // comments - suite.Equal(tt.xdbcDataType, xdbcDataTypes) // xdbcDataType - suite.Equal(tt.dataTypes, dataTypes) // dataTypes - suite.Equal(tt.xdbcTypeName, xdbcTypeNames) // xdbcTypeName - suite.Equal(tt.xdbcCharMaxLen, xdbcCharMaxLens) // xdbcCharMaxLen - suite.Equal(tt.xdbcScale, xdbcScales) // xdbcScale - suite.Equal(tt.xdbcNumPrecRadix, xdbcNumPrecRadixs) // xdbcNumPrecRadix - suite.Equal(tt.xdbcNullable, xdbcNullables) // xdbcNullable - suite.Equal(tt.xdbcSqlDataType, xdbcSqlDataTypes) // xdbcSqlDataType - suite.Equal(tt.xdbcDateTimeSub, xdbcDateTimeSub) // xdbcDateTimeSub - suite.Equal(tt.xdbcCharOctetLen, xdbcCharOctetLen) // xdbcCharOctetLen - suite.Equal(tt.xdbcIsNullable, xdbcIsNullables) // xdbcIsNullable + suite.ElementsMatch(tt.columnName, columnName, "columnName") + //suite.Equal(tt.ordinalPosition, ordinalPosition, "ordinalPosition") + suite.ElementsMatch(tt.remarks, remarks, "remarks") + suite.ElementsMatch(tt.xdbcDataType, xdbcDataType, "xdbcDataType") + suite.ElementsMatch(tt.xdbcTypeName, xdbcTypeName, "xdbcTypeName") + suite.ElementsMatch(tt.xdbcColumnSize, xdbcColumnSize, "xdbcColumnSize") + suite.ElementsMatch(tt.xdbcDecimalDigits, xdbcDecimalDigits, "xdbcDecimalDigits") + suite.ElementsMatch(tt.xdbcNumPrecRadix, xdbcNumPrecRadix, "xdbcNumPrecRadix") + suite.ElementsMatch(tt.xdbcNullable, xdbcNullable, "xdbcNullable") + suite.ElementsMatch(tt.xdbcColumnDef, xdbcColumnDef, "xdbcColumnDef") + suite.ElementsMatch(tt.xdbcSqlDataType, xdbcSqlDataType, "xdbcSqlDataType") + suite.ElementsMatch(tt.xdbcDatetimeSub, xdbcDatetimeSub, "xdbcDatetimeSub") + suite.ElementsMatch(tt.xdbcCharOctetLength, xdbcCharOctetLength, "xdbcCharOctetLength") + suite.ElementsMatch(tt.xdbcIsNullable, xdbcIsNullable, "xdbcIsNullable") + suite.ElementsMatch(tt.xdbcScopeCatalog, xdbcScopeCatalog, "xdbcScopeCatalog") + suite.ElementsMatch(tt.xdbcScopeSchema, xdbcScopeSchema, "xdbcScopeSchema") + suite.ElementsMatch(tt.xdbcScopeTable, xdbcScopeTable, "xdbcScopeTable") + suite.ElementsMatch(tt.xdbcIsAutoincrement, xdbcIsAutoincrement, "xdbcIsAutoincrement") + suite.ElementsMatch(tt.xdbcIsAutogeneratedColumn, xdbcIsAutogeneratedColumn, "xdbcIsAutogeneratedColumn") }) } } diff --git a/go/adbc/driver/flightsql/flightsql_connection.go b/go/adbc/driver/flightsql/flightsql_connection.go index 39c10231ae..99c0ccda05 100644 --- a/go/adbc/driver/flightsql/flightsql_connection.go +++ b/go/adbc/driver/flightsql/flightsql_connection.go @@ -57,10 +57,63 @@ type connectionImpl struct { supportInfo support } +type flightSqlMetadata struct { + internal.DefaultXdbcMetadataBuilder + columnMetadata *flightsql.ColumnMetadata +} + +func (md *flightSqlMetadata) SetMetadata(metadata arrow.Metadata) { + md.columnMetadata = &flightsql.ColumnMetadata{Data: &metadata} +} + +func (md *flightSqlMetadata) SetXdbcScopeCatalog(b *array.StringBuilder) { + if v, ok := md.columnMetadata.CatalogName(); ok { + b.Append(v) + } else { + md.DefaultXdbcMetadataBuilder.SetXdbcScopeCatalog(b) + } +} + +func (md *flightSqlMetadata) SetXdbcScopeSchema(b *array.StringBuilder) { + if v, ok := md.columnMetadata.SchemaName(); ok { + b.Append(v) + } else { + md.DefaultXdbcMetadataBuilder.SetXdbcScopeSchema(b) + } +} + +func (md *flightSqlMetadata) SetXdbcScopeTable(b *array.StringBuilder) { + if v, ok := md.columnMetadata.TableName(); ok { + b.Append(v) + } else { + md.DefaultXdbcMetadataBuilder.SetXdbcScopeTable(b) + } +} + +func (md *flightSqlMetadata) SetXdbcSqlDataType(columnType arrow.DataType, b *array.Int16Builder) { + b.Append(int16(internal.ToXdbcDataType(columnType))) +} + +func (md *flightSqlMetadata) SetXdbcTypeName(b *array.StringBuilder) { + if v, ok := md.columnMetadata.TypeName(); ok { + b.Append(v) + } else { + md.DefaultXdbcMetadataBuilder.SetXdbcTypeName(b) + } +} + +func (md *flightSqlMetadata) SetXdbcIsAutoincrement(builder *array.BooleanBuilder) { + if v, ok := md.columnMetadata.IsAutoIncrement(); ok { + builder.Append(v) + } else { + md.DefaultXdbcMetadataBuilder.SetXdbcIsAutoincrement(builder) + } +} + func (c *connectionImpl) GetObjects(ctx context.Context, depth adbc.ObjectDepth, catalog *string, dbSchema *string, tableName *string, columnName *string, tableType []string) (array.RecordReader, error) { // To avoid an N+1 query problem, we assume result sets here will fit in memory and build up a single response. g := internal.GetObjects{Ctx: ctx, Depth: depth, Catalog: catalog, DbSchema: dbSchema, TableName: tableName, ColumnName: columnName, TableType: tableType} - if err := g.Init(c.Base().Alloc, c.GetObjectsDbSchemas, c.GetObjectsTables); err != nil { + if err := g.Init(c.Base().Alloc, c.GetObjectsDbSchemas, c.GetObjectsTables, &flightSqlMetadata{}); err != nil { return nil, err } defer g.Release() @@ -705,7 +758,7 @@ func (c *connectionImpl) GetObjectsCatalogs(ctx context.Context, catalog *string } // Helper function to build up a map of catalogs to DB schemas -func (c *connectionImpl) GetObjectsDbSchemas(ctx context.Context, depth adbc.ObjectDepth, catalog *string, dbSchema *string, metadataRecords []internal.Metadata) (result map[string][]string, err error) { +func (c *connectionImpl) GetObjectsDbSchemas(ctx context.Context, depth adbc.ObjectDepth, catalog *string, dbSchema *string) (result map[string][]string, err error) { if depth == adbc.ObjectDepthCatalogs { return } @@ -747,7 +800,7 @@ func (c *connectionImpl) GetObjectsDbSchemas(ctx context.Context, depth adbc.Obj return } -func (c *connectionImpl) GetObjectsTables(ctx context.Context, depth adbc.ObjectDepth, catalog *string, dbSchema *string, tableName *string, columnName *string, tableType []string, metadataRecords []internal.Metadata) (result internal.SchemaToTableInfo, err error) { +func (c *connectionImpl) GetObjectsTables(ctx context.Context, depth adbc.ObjectDepth, catalog *string, dbSchema *string, tableName *string, columnName *string, tableType []string) (result internal.SchemaToTableInfo, err error) { if depth == adbc.ObjectDepthCatalogs || depth == adbc.ObjectDepthDBSchemas { return } diff --git a/go/adbc/driver/internal/driverbase/connection.go b/go/adbc/driver/internal/driverbase/connection.go index e7aac4d15e..716bd763a9 100644 --- a/go/adbc/driver/internal/driverbase/connection.go +++ b/go/adbc/driver/internal/driverbase/connection.go @@ -26,7 +26,6 @@ import ( "log/slog" "github.com/apache/arrow-adbc/go/adbc" - "github.com/apache/arrow-adbc/go/adbc/driver/internal" "github.com/apache/arrow-go/v18/arrow" "github.com/apache/arrow-go/v18/arrow/array" "github.com/apache/arrow-go/v18/arrow/memory" @@ -699,51 +698,6 @@ func PatternToNamedArg(name string, pattern *string) sql.NamedArg { return sql.Named(name, *pattern) } -func ToXdbcDataType(dt arrow.DataType) (xdbcType internal.XdbcDataType) { - switch dt.ID() { - case arrow.EXTENSION: - return ToXdbcDataType(dt.(arrow.ExtensionType).StorageType()) - case arrow.DICTIONARY: - return ToXdbcDataType(dt.(*arrow.DictionaryType).ValueType) - case arrow.RUN_END_ENCODED: - return ToXdbcDataType(dt.(*arrow.RunEndEncodedType).Encoded()) - case arrow.INT8, arrow.UINT8: - return internal.XdbcDataType_XDBC_TINYINT - case arrow.INT16, arrow.UINT16: - return internal.XdbcDataType_XDBC_SMALLINT - case arrow.INT32, arrow.UINT32: - return internal.XdbcDataType_XDBC_SMALLINT - case arrow.INT64, arrow.UINT64: - return internal.XdbcDataType_XDBC_BIGINT - case arrow.FLOAT32, arrow.FLOAT16, arrow.FLOAT64: - return internal.XdbcDataType_XDBC_FLOAT - case arrow.DECIMAL, arrow.DECIMAL256: - return internal.XdbcDataType_XDBC_DECIMAL - case arrow.STRING, arrow.LARGE_STRING: - return internal.XdbcDataType_XDBC_VARCHAR - case arrow.BINARY, arrow.LARGE_BINARY: - return internal.XdbcDataType_XDBC_BINARY - case arrow.FIXED_SIZE_BINARY: - return internal.XdbcDataType_XDBC_BINARY - case arrow.BOOL: - return internal.XdbcDataType_XDBC_BIT - case arrow.TIME32, arrow.TIME64: - return internal.XdbcDataType_XDBC_TIME - case arrow.DATE32, arrow.DATE64: - return internal.XdbcDataType_XDBC_DATE - case arrow.TIMESTAMP: - return internal.XdbcDataType_XDBC_TIMESTAMP - case arrow.DENSE_UNION, arrow.SPARSE_UNION: - return internal.XdbcDataType_XDBC_VARBINARY - case arrow.LIST, arrow.LARGE_LIST, arrow.FIXED_SIZE_LIST: - return internal.XdbcDataType_XDBC_VARBINARY - case arrow.STRUCT, arrow.MAP: - return internal.XdbcDataType_XDBC_VARBINARY - default: - return internal.XdbcDataType_XDBC_UNKNOWN_TYPE - } -} - // Nullable wraps a value and returns a pointer to the value, which is // how nullable values are represented for purposes of JSON serialization. func Nullable[T any](val T) *T { diff --git a/go/adbc/driver/internal/shared_utils.go b/go/adbc/driver/internal/shared_utils.go index 8143da8faf..f8c51c636e 100644 --- a/go/adbc/driver/internal/shared_utils.go +++ b/go/adbc/driver/internal/shared_utils.go @@ -19,11 +19,9 @@ package internal import ( "context" - "database/sql" "regexp" "strconv" "strings" - "time" "github.com/apache/arrow-adbc/go/adbc" "github.com/apache/arrow-go/v18/arrow" @@ -58,15 +56,6 @@ type TableInfo struct { Schema *arrow.Schema } -type Metadata struct { - Created time.Time - ColName, DataType, Dbname, Kind, Schema, TblName, TblType, IdentGen, IdentIncrement, Comment, ConstraintName, ConstraintType sql.NullString - OrdinalPos int - NumericPrec, NumericPrecRadix, NumericScale, DatetimePrec sql.NullInt16 - IsNullable, IsIdent bool - CharMaxLength, CharOctetLength sql.NullInt32 -} - type UsageSchema struct { ForeignKeyCatalog, ForeignKeyDbSchema, ForeignKeyTable, ForeignKeyColName string } @@ -77,9 +66,12 @@ type ConstraintSchema struct { ConstraintColumnUsages []UsageSchema } -type GetObjDBSchemasFn func(ctx context.Context, depth adbc.ObjectDepth, catalog *string, schema *string, metadataRecords []Metadata) (map[string][]string, error) -type GetObjTablesFn func(ctx context.Context, depth adbc.ObjectDepth, catalog *string, schema *string, tableName *string, columnName *string, tableType []string, metadataRecords []Metadata) (map[CatalogAndSchema][]TableInfo, error) +type GetObjDBSchemasFn func(ctx context.Context, depth adbc.ObjectDepth, catalog *string, schema *string) (map[string][]string, error) +type GetObjTablesFn func(ctx context.Context, depth adbc.ObjectDepth, catalog *string, schema *string, tableName *string, columnName *string, tableType []string) (map[CatalogAndSchema][]TableInfo, error) type SchemaToTableInfo = map[CatalogAndSchema][]TableInfo +type MetadataToBuilders = map[string]array.Builder +type MetadataHandlers = func(md arrow.Field, builder array.Builder) +type MetadataToHandlers = map[string]MetadataHandlers // Helper function that compiles a SQL-style pattern (%, _) to a regex func PatternToRegexp(pattern *string) (*regexp.Regexp, error) { @@ -127,7 +119,6 @@ type GetObjects struct { schemaLookup map[string][]string tableLookup map[CatalogAndSchema][]TableInfo ConstraintLookup map[CatalogSchemaTable][]ConstraintSchema - MetadataRecords []Metadata catalogPattern *regexp.Regexp columnNamePattern *regexp.Regexp @@ -172,16 +163,18 @@ type GetObjects struct { columnUsageSchemaBuilder *array.StringBuilder columnUsageTableBuilder *array.StringBuilder columnUsageColumnBuilder *array.StringBuilder + + metadataHandler XdbcMetadataBuilder } -func (g *GetObjects) Init(mem memory.Allocator, getObj GetObjDBSchemasFn, getTbls GetObjTablesFn) error { - if catalogToDbSchemas, err := getObj(g.Ctx, g.Depth, g.Catalog, g.DbSchema, g.MetadataRecords); err != nil { +func (g *GetObjects) Init(mem memory.Allocator, getObj GetObjDBSchemasFn, getTbls GetObjTablesFn, mdHandler XdbcMetadataBuilder) error { + if catalogToDbSchemas, err := getObj(g.Ctx, g.Depth, g.Catalog, g.DbSchema); err != nil { return err } else { g.schemaLookup = catalogToDbSchemas } - if tableLookup, err := getTbls(g.Ctx, g.Depth, g.Catalog, g.DbSchema, g.TableName, g.ColumnName, g.TableType, g.MetadataRecords); err != nil { + if tableLookup, err := getTbls(g.Ctx, g.Depth, g.Catalog, g.DbSchema, g.TableName, g.ColumnName, g.TableType); err != nil { return err } else { g.tableLookup = tableLookup @@ -247,6 +240,11 @@ func (g *GetObjects) Init(mem memory.Allocator, getObj GetObjDBSchemasFn, getTbl g.columnUsageTableBuilder = g.constraintColumnUsageItems.FieldBuilder(2).(*array.StringBuilder) g.columnUsageColumnBuilder = g.constraintColumnUsageItems.FieldBuilder(3).(*array.StringBuilder) + if mdHandler == nil { + mdHandler = &DefaultXdbcMetadataBuilder{} + } + g.metadataHandler = mdHandler + return nil } @@ -379,129 +377,291 @@ func (g *GetObjects) appendColumnsInfo(tableInfo TableInfo) { if g.columnNamePattern != nil && !g.columnNamePattern.MatchString(column.Name) { continue } + + g.metadataHandler.SetMetadata(column.Metadata) + + pos := int32(colIndex + 1) + g.columnNameBuilder.Append(column.Name) - if !column.HasMetadata() { - g.ordinalPositionBuilder.Append(int32(colIndex + 1)) - g.remarksBuilder.AppendNull() - - g.xdbcDataTypeBuilder.AppendNull() - g.xdbcTypeNameBuilder.AppendNull() - g.xdbcNullableBuilder.AppendNull() - g.xdbcIsNullableBuilder.AppendNull() - g.xdbcColumnSizeBuilder.AppendNull() - g.xdbcDecimalDigitsBuilder.AppendNull() - g.xdbcNumPrecRadixBuilder.AppendNull() - g.xdbcCharOctetLengthBuilder.AppendNull() - g.xdbcDatetimeSubBuilder.AppendNull() - g.xdbcSqlDataTypeBuilder.AppendNull() - } else { - if remark, ok := column.Metadata.GetValue("COMMENT"); ok { - g.remarksBuilder.Append(remark) - } else { - g.remarksBuilder.AppendNull() - } + g.metadataHandler.SetOrdinalPosition(pos, g.ordinalPositionBuilder) + g.metadataHandler.SetRemarks(g.remarksBuilder) + g.metadataHandler.SetXdbcDataType(column.Type, g.xdbcDataTypeBuilder) + g.metadataHandler.SetXdbcTypeName(g.xdbcTypeNameBuilder) + g.metadataHandler.SetXdbcColumnSize(g.xdbcColumnSizeBuilder) + g.metadataHandler.SetXdbcDecimalDigits(g.xdbcDecimalDigitsBuilder) + g.metadataHandler.SetXdbcNumPrecRadix(g.xdbcNumPrecRadixBuilder) + g.metadataHandler.SetXdbcNullable(g.xdbcNullableBuilder) + g.metadataHandler.SetXdbcColumnDef(g.xdbcColumnDefBuilder) + g.metadataHandler.SetXdbcSqlDataType(column.Type, g.xdbcSqlDataTypeBuilder) + g.metadataHandler.SetXdbcDatetimeSub(g.xdbcDatetimeSubBuilder) + g.metadataHandler.SetXdbcCharOctetLength(g.xdbcCharOctetLengthBuilder) + g.metadataHandler.SetXdbcIsNullable(g.xdbcIsNullableBuilder) + g.metadataHandler.SetXdbcScopeCatalog(g.xdbcScopeCatalogBuilder) + g.metadataHandler.SetXdbcScopeSchema(g.xdbcScopeSchemaBuilder) + g.metadataHandler.SetXdbcScopeTable(g.xdbcScopeTableBuilder) + g.metadataHandler.SetXdbcIsAutoincrement(g.xdbcIsAutoincrementBuilder) + g.metadataHandler.SetXdbcIsAutogeneratedColumn(g.xdbcIsGeneratedcolumnBuilder) - if typeName, ok := column.Metadata.GetValue("XDBC_TYPE_NAME"); ok { - g.xdbcTypeNameBuilder.Append(typeName) - } else { - g.xdbcTypeNameBuilder.AppendNull() - } + g.tableColumnsItems.Append(true) + } +} - if strNullable, ok := column.Metadata.GetValue("XDBC_NULLABLE"); ok { - nullable, _ := strconv.ParseBool(strNullable) - g.xdbcNullableBuilder.Append(boolToInt16(nullable)) - } else { - g.xdbcNullableBuilder.AppendNull() - } +type XdbcMetadataBuilder interface { + Metadata() *arrow.Metadata + SetMetadata(md arrow.Metadata) + SetOrdinalPosition(defaultPos int32, b *array.Int32Builder) + SetRemarks(b *array.StringBuilder) + SetXdbcDataType(defaultType arrow.DataType, b *array.Int16Builder) + SetXdbcTypeName(b *array.StringBuilder) + SetXdbcColumnSize(b *array.Int32Builder) + SetXdbcDecimalDigits(b *array.Int16Builder) + SetXdbcNumPrecRadix(b *array.Int16Builder) + SetXdbcNullable(b *array.Int16Builder) + SetXdbcColumnDef(b *array.StringBuilder) + SetXdbcSqlDataType(defaultType arrow.DataType, b *array.Int16Builder) + SetXdbcDatetimeSub(b *array.Int16Builder) + SetXdbcCharOctetLength(b *array.Int32Builder) + SetXdbcIsNullable(b *array.StringBuilder) + SetXdbcScopeCatalog(b *array.StringBuilder) + SetXdbcScopeSchema(b *array.StringBuilder) + SetXdbcScopeTable(b *array.StringBuilder) + SetXdbcIsAutoincrement(b *array.BooleanBuilder) + SetXdbcIsAutogeneratedColumn(b *array.BooleanBuilder) +} - if strIsNullable, ok := column.Metadata.GetValue("XDBC_IS_NULLABLE"); ok { - g.xdbcIsNullableBuilder.Append(strIsNullable) - } else { - g.xdbcIsNullableBuilder.AppendNull() - } +type DefaultXdbcMetadataBuilder struct { + Data *arrow.Metadata +} - if strDataTypeId, ok := column.Metadata.GetValue("XDBC_DATA_TYPE"); ok { - dataTypeId64, _ := strconv.ParseInt(strDataTypeId, 10, 16) - g.xdbcDataTypeBuilder.Append(int16(dataTypeId64)) +func (c *DefaultXdbcMetadataBuilder) Metadata() *arrow.Metadata { + return c.Data +} - } else { - g.xdbcDataTypeBuilder.AppendNull() - } +func (c *DefaultXdbcMetadataBuilder) SetMetadata(md arrow.Metadata) { + c.Data = &md +} - if strSqlDataTypeId, ok := column.Metadata.GetValue("XDBC_SQL_DATA_TYPE"); ok { - sqlDataTypeId64, _ := strconv.ParseInt(strSqlDataTypeId, 10, 16) - g.xdbcSqlDataTypeBuilder.Append(int16(sqlDataTypeId64)) - } else { - g.xdbcSqlDataTypeBuilder.AppendNull() - } +func (c *DefaultXdbcMetadataBuilder) findStrVal(key string) (string, bool) { + if c.Data == nil { + return "", false + } + idx := c.Data.FindKey(key) + if idx == -1 { + return "", false + } + return c.Data.Values()[idx], true +} - if strPrecision, ok := column.Metadata.GetValue("XDBC_PRECISION"); ok { // for numeric values - precision64, _ := strconv.ParseInt(strPrecision, 10, 32) - g.xdbcColumnSizeBuilder.Append(int32(precision64)) - } else if strCharLimit, ok := column.Metadata.GetValue("CHARACTER_MAXIMUM_LENGTH"); ok { // for text values - charLimit64, _ := strconv.ParseInt(strCharLimit, 10, 32) - g.xdbcColumnSizeBuilder.Append(int32(charLimit64)) - } else { - g.xdbcColumnSizeBuilder.AppendNull() - } +func (c *DefaultXdbcMetadataBuilder) findBoolVal(key string) (bool, bool) { + if c.Data == nil { + return false, false + } + idx := c.Data.FindKey(key) + if idx == -1 { + return false, false + } + v, err := strconv.ParseBool(c.Data.Values()[idx]) + if err != nil { + return false, false + } + return v, true +} - if strScale, ok := column.Metadata.GetValue("XDBC_SCALE"); ok { - scale64, _ := strconv.ParseInt(strScale, 10, 16) - g.xdbcDecimalDigitsBuilder.Append(int16(scale64)) - } else { - g.xdbcDecimalDigitsBuilder.AppendNull() - } +func (c *DefaultXdbcMetadataBuilder) findInt32Val(key string) (int32, bool) { + if c.Data == nil { + return 0, false + } + idx := c.Data.FindKey(key) + if idx == -1 { + return 0, false + } + v, err := strconv.ParseInt(c.Data.Values()[idx], 10, 32) + if err != nil { + return 0, false + } + return int32(v), true +} - if strPrecRadix, ok := column.Metadata.GetValue("XDBC_NUM_PREC_RADIX"); ok { - precRadix64, _ := strconv.ParseInt(strPrecRadix, 10, 16) - g.xdbcNumPrecRadixBuilder.Append(int16(precRadix64)) - } else { - g.xdbcNumPrecRadixBuilder.AppendNull() - } +func (c *DefaultXdbcMetadataBuilder) findInt16Val(key string) (int16, bool) { + if c.Data == nil { + return 0, false + } + idx := c.Data.FindKey(key) + if idx == -1 { + return 0, false + } + v, err := strconv.ParseInt(c.Data.Values()[idx], 10, 32) + if err != nil { + return 0, false + } + return int16(v), true +} - if strCharOctetLen, ok := column.Metadata.GetValue("XDBC_CHAR_OCTET_LENGTH"); ok { - charOctLen64, _ := strconv.ParseInt(strCharOctetLen, 10, 32) - g.xdbcCharOctetLengthBuilder.Append(int32(charOctLen64)) - } else { - g.xdbcCharOctetLengthBuilder.AppendNull() - } +func (c *DefaultXdbcMetadataBuilder) SetOrdinalPosition(defaultPos int32, b *array.Int32Builder) { + if v, ok := c.findInt32Val(ORDINAL_POSITION); ok { + b.Append(v) + } else { + b.Append(defaultPos) + } +} +func (b *DefaultXdbcMetadataBuilder) SetRemarks(builder *array.StringBuilder) { + if v, ok := b.findStrVal(REMARKS); ok { + builder.Append(v) + } else { + builder.AppendNull() + } +} - if strDateTimeSub, ok := column.Metadata.GetValue("XDBC_DATETIME_SUB"); ok { - dateTimeSub64, _ := strconv.ParseInt(strDateTimeSub, 10, 16) - g.xdbcDatetimeSubBuilder.Append(int16(dateTimeSub64)) - } else { - g.xdbcDatetimeSubBuilder.AppendNull() - } +func (b *DefaultXdbcMetadataBuilder) SetXdbcDataType(columnType arrow.DataType, builder *array.Int16Builder) { + if v, ok := b.findInt16Val(XDBC_DATA_TYPE); ok { + builder.Append(v) + } else { + builder.Append(int16(ToXdbcDataType(columnType))) + } +} - pos := int32(colIndex + 1) +func (b *DefaultXdbcMetadataBuilder) SetXdbcTypeName(builder *array.StringBuilder) { + if v, ok := b.findStrVal(XDBC_TYPE_NAME); ok { + builder.Append(v) + } else { + builder.AppendNull() + } +} - if ordinal, ok := column.Metadata.GetValue("ORDINAL_POSITION"); ok { - v, err := strconv.ParseInt(ordinal, 10, 32) - if err == nil { - pos = int32(v) - } - } - g.ordinalPositionBuilder.Append(pos) - } +func (b *DefaultXdbcMetadataBuilder) SetXdbcColumnSize(builder *array.Int32Builder) { + if v, ok := b.findInt32Val(XDBC_COLUMN_SIZE); ok { + builder.Append(v) + } else { + builder.AppendNull() + } +} + +func (b *DefaultXdbcMetadataBuilder) SetXdbcDecimalDigits(builder *array.Int16Builder) { + if v, ok := b.findInt16Val(XDBC_DECIMAL_DIGITS); ok { + builder.Append(v) + } else { + builder.AppendNull() + } +} - g.xdbcColumnDefBuilder.AppendNull() - g.xdbcScopeCatalogBuilder.AppendNull() - g.xdbcScopeSchemaBuilder.AppendNull() - g.xdbcScopeTableBuilder.AppendNull() - g.xdbcIsAutoincrementBuilder.AppendNull() - g.xdbcIsGeneratedcolumnBuilder.AppendNull() +func (b *DefaultXdbcMetadataBuilder) SetXdbcNumPrecRadix(builder *array.Int16Builder) { + if v, ok := b.findInt16Val(XDBC_NUM_PREC_RADIX); ok { + builder.Append(v) + } else { + builder.AppendNull() + } +} - g.tableColumnsItems.Append(true) +func (b *DefaultXdbcMetadataBuilder) SetXdbcNullable(builder *array.Int16Builder) { + if v, ok := b.findInt16Val(XDBC_NULLABLE); ok { + builder.Append(v) + } else { + builder.AppendNull() + } +} + +func (b *DefaultXdbcMetadataBuilder) SetXdbcColumnDef(builder *array.StringBuilder) { + if v, ok := b.findStrVal(XDBC_COLUMN_DEF); ok { + builder.Append(v) + } else { + builder.AppendNull() + } +} + +func (b *DefaultXdbcMetadataBuilder) SetXdbcSqlDataType(columnType arrow.DataType, builder *array.Int16Builder) { + if v, ok := b.findInt16Val(XDBC_SQL_DATA_TYPE); ok { + builder.Append(v) + } else { + builder.AppendNull() + } +} + +func (b *DefaultXdbcMetadataBuilder) SetXdbcDatetimeSub(builder *array.Int16Builder) { + if v, ok := b.findInt16Val(XDBC_DATETIME_SUB); ok { + builder.Append(v) + } else { + builder.AppendNull() + } +} + +func (b *DefaultXdbcMetadataBuilder) SetXdbcCharOctetLength(builder *array.Int32Builder) { + if v, ok := b.findInt32Val(XDBC_CHAR_OCTET_LENGTH); ok { + builder.Append(v) + } else { + builder.AppendNull() + } +} + +func (b *DefaultXdbcMetadataBuilder) SetXdbcIsNullable(builder *array.StringBuilder) { + if v, ok := b.findStrVal(XDBC_IS_NULLABLE); ok { + builder.Append(v) + } else { + builder.AppendNull() + } +} + +func (b *DefaultXdbcMetadataBuilder) SetXdbcScopeCatalog(builder *array.StringBuilder) { + if v, ok := b.findStrVal(XDBC_SCOPE_CATALOG); ok { + builder.Append(v) + } else { + builder.AppendNull() + } +} + +func (b *DefaultXdbcMetadataBuilder) SetXdbcScopeSchema(builder *array.StringBuilder) { + if v, ok := b.findStrVal(XDBC_SCOPE_SCHEMA); ok { + builder.Append(v) + } else { + builder.AppendNull() + } +} + +func (b *DefaultXdbcMetadataBuilder) SetXdbcScopeTable(builder *array.StringBuilder) { + if v, ok := b.findStrVal(XDBC_SCOPE_TABLE); ok { + builder.Append(v) + } else { + builder.AppendNull() + } +} + +func (b *DefaultXdbcMetadataBuilder) SetXdbcIsAutoincrement(builder *array.BooleanBuilder) { + if v, ok := b.findBoolVal(XDBC_IS_AUTOINCREMENT); ok { + builder.Append(v) + } else { + builder.AppendNull() } } -func boolToInt16(b bool) int16 { - if b { - return 1 +func (b *DefaultXdbcMetadataBuilder) SetXdbcIsAutogeneratedColumn(builder *array.BooleanBuilder) { + if v, ok := b.findBoolVal(XDBC_IS_AUTOGENERATEDCOLUMN); ok { + builder.Append(v) + } else { + builder.AppendNull() } - return 0 } +const ( + COLUMN_NAME = "COLUMN_NAME" + ORDINAL_POSITION = "ORDINAL_POSITION" + REMARKS = "REMARKS" + XDBC_DATA_TYPE = "XDBC_DATA_TYPE" + XDBC_TYPE_NAME = "XDBC_TYPE_NAME" + XDBC_COLUMN_SIZE = "XDBC_COLUMN_SIZE" + XDBC_DECIMAL_DIGITS = "XDBC_DECIMAL_DIGITS" + XDBC_NUM_PREC_RADIX = "XDBC_NUM_PREC_RADIX" + XDBC_NULLABLE = "XDBC_NULLABLE" + XDBC_COLUMN_DEF = "XDBC_COLUMN_DEF" + XDBC_SQL_DATA_TYPE = "XDBC_SQL_DATA_TYPE" + XDBC_DATETIME_SUB = "XDBC_DATETIME_SUB" + XDBC_CHAR_OCTET_LENGTH = "XDBC_CHAR_OCTET_LENGTH" + XDBC_IS_NULLABLE = "XDBC_IS_NULLABLE" + XDBC_SCOPE_CATALOG = "XDBC_SCOPE_CATALOG" + XDBC_SCOPE_SCHEMA = "XDBC_SCOPE_SCHEMA" + XDBC_SCOPE_TABLE = "XDBC_SCOPE_TABLE" + XDBC_IS_AUTOINCREMENT = "XDBC_IS_AUTOINCREMENT" + XDBC_IS_AUTOGENERATEDCOLUMN = "XDBC_IS_AUTOGENERATEDCOLUMN" +) + // The JDBC/ODBC-defined type of any object. // All the values here are the sames as in the JDBC and ODBC specs. type XdbcDataType int32 @@ -532,3 +692,48 @@ const ( XdbcDataType_XDBC_WCHAR XdbcDataType = -8 XdbcDataType_XDBC_WVARCHAR XdbcDataType = -9 ) + +func ToXdbcDataType(dt arrow.DataType) (xdbcType XdbcDataType) { + switch dt.ID() { + case arrow.EXTENSION: + return ToXdbcDataType(dt.(arrow.ExtensionType).StorageType()) + case arrow.DICTIONARY: + return ToXdbcDataType(dt.(*arrow.DictionaryType).ValueType) + case arrow.RUN_END_ENCODED: + return ToXdbcDataType(dt.(*arrow.RunEndEncodedType).Encoded()) + case arrow.INT8, arrow.UINT8: + return XdbcDataType_XDBC_TINYINT + case arrow.INT16, arrow.UINT16: + return XdbcDataType_XDBC_SMALLINT + case arrow.INT32, arrow.UINT32: + return XdbcDataType_XDBC_INTEGER + case arrow.INT64, arrow.UINT64: + return XdbcDataType_XDBC_BIGINT + case arrow.FLOAT32, arrow.FLOAT16, arrow.FLOAT64: + return XdbcDataType_XDBC_FLOAT + case arrow.DECIMAL, arrow.DECIMAL256: + return XdbcDataType_XDBC_DECIMAL + case arrow.STRING, arrow.LARGE_STRING: + return XdbcDataType_XDBC_VARCHAR + case arrow.BINARY, arrow.LARGE_BINARY: + return XdbcDataType_XDBC_BINARY + case arrow.FIXED_SIZE_BINARY: + return XdbcDataType_XDBC_BINARY + case arrow.BOOL: + return XdbcDataType_XDBC_BIT + case arrow.TIME32, arrow.TIME64: + return XdbcDataType_XDBC_TIME + case arrow.DATE32, arrow.DATE64: + return XdbcDataType_XDBC_DATE + case arrow.TIMESTAMP: + return XdbcDataType_XDBC_TIMESTAMP + case arrow.DENSE_UNION, arrow.SPARSE_UNION: + return XdbcDataType_XDBC_VARBINARY + case arrow.LIST, arrow.LARGE_LIST, arrow.FIXED_SIZE_LIST: + return XdbcDataType_XDBC_VARBINARY + case arrow.STRUCT, arrow.MAP: + return XdbcDataType_XDBC_VARBINARY + default: + return XdbcDataType_XDBC_UNKNOWN_TYPE + } +} diff --git a/go/adbc/driver/snowflake/connection.go b/go/adbc/driver/snowflake/connection.go index fe4ce4cc17..2658e6f09f 100644 --- a/go/adbc/driver/snowflake/connection.go +++ b/go/adbc/driver/snowflake/connection.go @@ -32,6 +32,7 @@ import ( "time" "github.com/apache/arrow-adbc/go/adbc" + "github.com/apache/arrow-adbc/go/adbc/driver/internal" "github.com/apache/arrow-adbc/go/adbc/driver/internal/driverbase" "github.com/apache/arrow-go/v18/arrow" "github.com/apache/arrow-go/v18/arrow/array" @@ -365,7 +366,7 @@ func (c *connectionImpl) GetObjects(ctx context.Context, depth adbc.ObjectDepth, for j, tab := range sch.DbSchemaTables { for k, col := range tab.TableColumns { field := c.toArrowField(col) - xdbcDataType := driverbase.ToXdbcDataType(field.Type) + xdbcDataType := internal.ToXdbcDataType(field.Type) getObjectsCatalog.CatalogDbSchemas[i].DbSchemaTables[j].TableColumns[k].XdbcDataType = driverbase.Nullable(int16(field.Type.ID())) getObjectsCatalog.CatalogDbSchemas[i].DbSchemaTables[j].TableColumns[k].XdbcSqlDataType = driverbase.Nullable(int16(xdbcDataType)) diff --git a/go/adbc/go.sum b/go/adbc/go.sum index bf78977781..d50d0f3662 100644 --- a/go/adbc/go.sum +++ b/go/adbc/go.sum @@ -40,8 +40,6 @@ github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvK github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/apache/arrow-go/v18 v18.1.0 h1:agLwJUiVuwXZdwPYVrlITfx7bndULJ/dggbnLFgDp/Y= -github.com/apache/arrow-go/v18 v18.1.0/go.mod h1:tigU/sIgKNXaesf5d7Y95jBBKS5KsxTqYBKXFsvKzo0= github.com/apache/arrow-go/v18 v18.1.1-0.20250116162745-f533d2066dee h1:LRDJtjipOzw1j1P1VedYDBIZvDVfz+lj1abQ998VGms= github.com/apache/arrow-go/v18 v18.1.1-0.20250116162745-f533d2066dee/go.mod h1:WbR+28APHo5LrJrHfwGPWRpWEtejDAUWRF3yoSLpSx4= github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE=