Skip to content

Commit

Permalink
Changed enumeration to use flags
Browse files Browse the repository at this point in the history
  • Loading branch information
@birschick-bq
birschick-bq committed Sep 19, 2024
1 parent a74cfe4 commit 0ba07c6
Show file tree
Hide file tree
Showing 10 changed files with 55 additions and 76 deletions.
2 changes: 1 addition & 1 deletion csharp/src/Drivers/Apache/Hive2/HiveServer2Connection.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ internal async Task OpenAsync()

internal TSessionHandle? SessionHandle { get; private set; }

protected internal IReadOnlyCollection<HiveServer2DataTypeConversion> DataTypeConversion { get; set; } = [HiveServer2DataTypeConversion.None];
protected internal HiveServer2DataTypeConversion DataTypeConversion { get; set; } = HiveServer2DataTypeConversion.None;

protected abstract Task<TTransport> CreateTransportAsync();

Expand Down
35 changes: 18 additions & 17 deletions csharp/src/Drivers/Apache/Hive2/HiveServer2Parameters.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* limitations under the License.
*/

using System.Collections.Generic;
using System;

namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
{
Expand All @@ -25,42 +25,43 @@ public static class HiveServer2DataTypeConversionConstants
public const string Scalar = "scalar";
public const string SupportedList = None + ", " + Scalar;

public static IReadOnlyCollection<HiveServer2DataTypeConversion> Parse(string? dataTypeConversion)
public static HiveServer2DataTypeConversion Parse(string? dataTypeConversion)
{
// Using a sorted set to make testing repeatable.
var result = new SortedSet<HiveServer2DataTypeConversion>();
if (dataTypeConversion == null) {
HiveServer2DataTypeConversion result = HiveServer2DataTypeConversion.Empty;

if (string.IsNullOrWhiteSpace(dataTypeConversion)) {
// Default
result.Add(HiveServer2DataTypeConversion.Scalar);
return result;
return HiveServer2DataTypeConversion.Scalar;
}

string[] conversions = dataTypeConversion.Split(',');
string[] conversions = dataTypeConversion!.Split(',');
foreach (string? conversion in conversions)
{
HiveServer2DataTypeConversion dataTypeConversionValue = (conversion?.Trim().ToLowerInvariant()) switch
result |= (conversion?.Trim().ToLowerInvariant()) switch
{
null or "" => HiveServer2DataTypeConversion.Empty,
None => HiveServer2DataTypeConversion.None,
Scalar => HiveServer2DataTypeConversion.Scalar,
_ => HiveServer2DataTypeConversion.Invalid,
_ => throw new ArgumentOutOfRangeException(nameof(dataTypeConversion), conversion, "Invalid or unsupported data type conversion"),
};

if (!result.Contains(dataTypeConversionValue) && dataTypeConversionValue != HiveServer2DataTypeConversion.Empty) result.Add(dataTypeConversionValue);
}

if (result.HasFlag(HiveServer2DataTypeConversion.None) && result.HasFlag(HiveServer2DataTypeConversion.Scalar))
{
throw new ArgumentOutOfRangeException(nameof(dataTypeConversion), dataTypeConversion, "Conflicting data type conversion options");
}
// Default
if (result.Count == 0) result.Add(HiveServer2DataTypeConversion.Scalar);
if (result == HiveServer2DataTypeConversion.Empty) result = HiveServer2DataTypeConversion.Scalar;

return result;
}
}

[Flags]
public enum HiveServer2DataTypeConversion
{
Invalid = 0,
None,
Scalar,
Empty = int.MaxValue,
Empty = 0,
None = 1,
Scalar = 2,
}
}
8 changes: 3 additions & 5 deletions csharp/src/Drivers/Apache/Hive2/HiveServer2Reader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,8 @@
using System.Collections.Generic;
using System.Data.SqlTypes;
using System.Globalization;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Apache.Arrow.Adbc.Drivers.Apache.Spark;
using Apache.Arrow.Ipc;
using Apache.Arrow.Types;
using Apache.Hive.Service.Rpc.Thrift;
Expand All @@ -33,7 +31,7 @@ internal class HiveServer2Reader : IArrowArrayStream
{
private HiveServer2Statement? _statement;
private readonly long _batchSize;
private readonly IReadOnlyCollection<HiveServer2DataTypeConversion> _dataTypeConversion;
private readonly HiveServer2DataTypeConversion _dataTypeConversion;
private static readonly IReadOnlyDictionary<ArrowTypeId, Func<StringArray, IArrowType, IArrowArray>> s_arrowStringConverters =
new Dictionary<ArrowTypeId, Func<StringArray, IArrowType, IArrowArray>>()
{
Expand All @@ -46,7 +44,7 @@ internal class HiveServer2Reader : IArrowArrayStream
public HiveServer2Reader(
HiveServer2Statement statement,
Schema schema,
IReadOnlyCollection<HiveServer2DataTypeConversion> dataTypeConversion,
HiveServer2DataTypeConversion dataTypeConversion,
long batchSize = HiveServer2Connection.BatchSizeDefault)
{
_statement = statement;
Expand All @@ -69,7 +67,7 @@ public HiveServer2Reader(

int columnCount = response.Results.Columns.Count;
IList<IArrowArray> columnData = [];
bool shouldConvertScalar = _dataTypeConversion.Contains(HiveServer2DataTypeConversion.Scalar);
bool shouldConvertScalar = _dataTypeConversion.HasFlag(HiveServer2DataTypeConversion.Scalar);
for (int i = 0; i < columnCount; i++)
{
IArrowType? expectedType = shouldConvertScalar ? Schema.FieldsList[i].DataType : null;
Expand Down
6 changes: 2 additions & 4 deletions csharp/src/Drivers/Apache/Hive2/HiveServer2SchemaParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,16 @@
*/

using System;
using System.Collections.Generic;
using System.Linq;
using Apache.Arrow.Types;
using Apache.Hive.Service.Rpc.Thrift;

namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
{
internal class HiveServer2SchemaParser : SchemaParser
{
public override IArrowType GetArrowType(TPrimitiveTypeEntry thriftType, IReadOnlyCollection<HiveServer2DataTypeConversion> dataTypeConversion)
public override IArrowType GetArrowType(TPrimitiveTypeEntry thriftType, HiveServer2DataTypeConversion dataTypeConversion)
{
bool convertScalar = dataTypeConversion.Contains(HiveServer2DataTypeConversion.Scalar);
bool convertScalar = dataTypeConversion.HasFlag(HiveServer2DataTypeConversion.Scalar);
return thriftType.Type switch
{
TTypeId.BIGINT_TYPE => Int64Type.Default,
Expand Down
11 changes: 1 addition & 10 deletions csharp/src/Drivers/Apache/Spark/SparkDatabricksConnection.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@
* limitations under the License.
*/

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Apache.Arrow.Adbc.Drivers.Apache.Hive2;
using Apache.Arrow.Ipc;
Expand Down Expand Up @@ -49,15 +47,8 @@ protected override TOpenSessionReq CreateSessionRequest()
protected override void ValidateOptions()
{
Properties.TryGetValue(SparkParameters.DataTypeConv, out string? dataTypeConv);
IReadOnlyCollection<HiveServer2DataTypeConversion> dataTypeConvValue = HiveServer2DataTypeConversionConstants.Parse(dataTypeConv);
// Note: In Databricks, scalar types are provided implicitly.
IReadOnlyCollection<HiveServer2DataTypeConversion> unsupportedConversions = dataTypeConvValue
.Except([HiveServer2DataTypeConversion.None, HiveServer2DataTypeConversion.Scalar, HiveServer2DataTypeConversion.Empty])
.ToList();
if (unsupportedConversions.Count > 0) {
throw new NotImplementedException($"Invalid or unsupported data type conversion option: '{dataTypeConv}'. Supported values: {HiveServer2DataTypeConversionConstants.SupportedList}");
}
DataTypeConversion = dataTypeConvValue;
DataTypeConversion = HiveServer2DataTypeConversionConstants.Parse(dataTypeConv);
}

protected override Task<TGetResultSetMetadataResp> GetResultSetMetadataAsync(TGetSchemasResp response) =>
Expand Down
3 changes: 1 addition & 2 deletions csharp/src/Drivers/Apache/Spark/SparkDatabricksSchemaParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
*/

using System;
using System.Collections.Generic;
using Apache.Arrow.Adbc.Drivers.Apache.Hive2;
using Apache.Arrow.Types;
using Apache.Hive.Service.Rpc.Thrift;
Expand All @@ -25,7 +24,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
{
internal class SparkDatabricksSchemaParser : SchemaParser
{
public override IArrowType GetArrowType(TPrimitiveTypeEntry thriftType, IReadOnlyCollection<HiveServer2DataTypeConversion> dataTypeConversion)
public override IArrowType GetArrowType(TPrimitiveTypeEntry thriftType, HiveServer2DataTypeConversion dataTypeConversion)
{
return thriftType.Type switch
{
Expand Down
11 changes: 1 addition & 10 deletions csharp/src/Drivers/Apache/Spark/SparkHttpConnection.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
Expand Down Expand Up @@ -116,15 +115,7 @@ protected override void ValidateConnection()
protected override void ValidateOptions()
{
Properties.TryGetValue(SparkParameters.DataTypeConv, out string? dataTypeConv);
IReadOnlyCollection<HiveServer2DataTypeConversion> dataTypeConversionValue = HiveServer2DataTypeConversionConstants.Parse(dataTypeConv);
IReadOnlyCollection<HiveServer2DataTypeConversion> unsupportedConversions = dataTypeConversionValue
.Except([HiveServer2DataTypeConversion.None, HiveServer2DataTypeConversion.Scalar, HiveServer2DataTypeConversion.Empty])
.ToList();
if (unsupportedConversions.Count > 0)
{
throw new NotImplementedException($"Invalid or unsupported data type conversion option: '{dataTypeConv}'. Supported values: {HiveServer2DataTypeConversionConstants.SupportedList}");
}
DataTypeConversion = dataTypeConversionValue;
DataTypeConversion = HiveServer2DataTypeConversionConstants.Parse(dataTypeConv);
}

internal override IArrowArrayStream NewReader<T>(T statement, Schema schema) => new HiveServer2Reader(statement, schema, dataTypeConversion: statement.Connection.DataTypeConversion);
Expand Down
8 changes: 3 additions & 5 deletions csharp/src/Drivers/Apache/Thrift/SchemaParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,15 @@
*/

using System;
using System.Collections.Generic;
using Apache.Arrow.Adbc.Drivers.Apache.Hive2;
using Apache.Arrow.Adbc.Drivers.Apache.Spark;
using Apache.Arrow.Types;
using Apache.Hive.Service.Rpc.Thrift;

namespace Apache.Arrow.Adbc.Drivers.Apache
{
internal abstract class SchemaParser
{
internal Schema GetArrowSchema(TTableSchema thriftSchema, IReadOnlyCollection<HiveServer2DataTypeConversion> dataTypeConversion)
internal Schema GetArrowSchema(TTableSchema thriftSchema, HiveServer2DataTypeConversion dataTypeConversion)
{
Field[] fields = new Field[thriftSchema.Columns.Count];
for (int i = 0; i < thriftSchema.Columns.Count; i++)
Expand All @@ -38,7 +36,7 @@ internal Schema GetArrowSchema(TTableSchema thriftSchema, IReadOnlyCollection<Hi
return new Schema(fields, null);
}

IArrowType GetArrowType(TTypeEntry thriftType, IReadOnlyCollection<HiveServer2DataTypeConversion> dataTypeConversion)
IArrowType GetArrowType(TTypeEntry thriftType, HiveServer2DataTypeConversion dataTypeConversion)
{
if (thriftType.PrimitiveEntry != null)
{
Expand All @@ -47,7 +45,7 @@ IArrowType GetArrowType(TTypeEntry thriftType, IReadOnlyCollection<HiveServer2Da
throw new InvalidOperationException();
}

public abstract IArrowType GetArrowType(TPrimitiveTypeEntry thriftType, IReadOnlyCollection<HiveServer2DataTypeConversion> dataTypeConversion);
public abstract IArrowType GetArrowType(TPrimitiveTypeEntry thriftType, HiveServer2DataTypeConversion dataTypeConversion);

protected static Decimal128Type NewDecima128Type(TPrimitiveTypeEntry thriftType) =>
new(thriftType.TypeQualifiers.Qualifiers["precision"].I32Value, thriftType.TypeQualifiers.Qualifiers["scale"].I32Value);
Expand Down
43 changes: 23 additions & 20 deletions csharp/test/Drivers/Apache/Hive2/HiveServer2ParametersTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,6 @@

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Apache.Arrow.Adbc.Drivers.Apache.Hive2;
using Xunit;

Expand All @@ -29,31 +26,37 @@ public class HiveServer2ParametersTest
{
[SkippableTheory]
[MemberData(nameof(GetParametersTestData))]
public void TestParametersParse(string? dataTypeConversion, IReadOnlyCollection<HiveServer2DataTypeConversion> expected)
public void TestParametersParse(string? dataTypeConversion, HiveServer2DataTypeConversion expected, Type? excptionType = default)
{
Assert.Equal(expected, HiveServer2DataTypeConversionConstants.Parse(dataTypeConversion));
if (excptionType == default)
Assert.Equal(expected, HiveServer2DataTypeConversionConstants.Parse(dataTypeConversion));
else
Assert.Throws(excptionType, () => HiveServer2DataTypeConversionConstants.Parse(dataTypeConversion));
}

public static IEnumerable<object?[]> GetParametersTestData()
{
// Default
yield return new object?[] { null, new List<HiveServer2DataTypeConversion>() { HiveServer2DataTypeConversion.Scalar } };
yield return new object?[] { "", new List<HiveServer2DataTypeConversion>() { HiveServer2DataTypeConversion.Scalar } };
yield return new object?[] { ",", new List<HiveServer2DataTypeConversion>() { HiveServer2DataTypeConversion.Scalar } };
yield return new object?[] { null, HiveServer2DataTypeConversion.Scalar };
yield return new object?[] { "", HiveServer2DataTypeConversion.Scalar };
yield return new object?[] { ",", HiveServer2DataTypeConversion.Scalar };
// Explicit
yield return new object?[] { $"scalar", new List<HiveServer2DataTypeConversion>() { HiveServer2DataTypeConversion.Scalar } };
yield return new object?[] { $"none", new List<HiveServer2DataTypeConversion>() { HiveServer2DataTypeConversion.None } };
// Ignore "empty"
yield return new object?[] { $"scalar,", new List<HiveServer2DataTypeConversion>() { HiveServer2DataTypeConversion.Scalar } };
yield return new object?[] { $",scalar,", new List<HiveServer2DataTypeConversion>() { HiveServer2DataTypeConversion.Scalar } };
// Combined, embedded space, mixed-case
yield return new object?[] { $"none,scalar", new List<HiveServer2DataTypeConversion>() { HiveServer2DataTypeConversion.None, HiveServer2DataTypeConversion.Scalar, } };
yield return new object?[] { $" nOnE, scAlAr ", new List<HiveServer2DataTypeConversion>() { HiveServer2DataTypeConversion.None, HiveServer2DataTypeConversion.Scalar, } };
yield return new object?[] { $", none, scalar, ", new List<HiveServer2DataTypeConversion>() { HiveServer2DataTypeConversion.None, HiveServer2DataTypeConversion.Scalar, } };
yield return new object?[] { $"scalar,none", new List<HiveServer2DataTypeConversion>() { HiveServer2DataTypeConversion.None, HiveServer2DataTypeConversion.Scalar, } };
yield return new object?[] { $"scalar", HiveServer2DataTypeConversion.Scalar };
yield return new object?[] { $"none", HiveServer2DataTypeConversion.None };
// Ignore "empty", embedded space, mixed-case
yield return new object?[] { $"scalar,", HiveServer2DataTypeConversion.Scalar };
yield return new object?[] { $",scalar,", HiveServer2DataTypeConversion.Scalar };
yield return new object?[] { $",scAlAr,", HiveServer2DataTypeConversion.Scalar };
yield return new object?[] { $"scAlAr", HiveServer2DataTypeConversion.Scalar };
yield return new object?[] { $" scalar ", HiveServer2DataTypeConversion.Scalar };
// Combined - conflicting
yield return new object?[] { $"none,scalar", HiveServer2DataTypeConversion.None | HiveServer2DataTypeConversion.Scalar, typeof(ArgumentOutOfRangeException) };
yield return new object?[] { $" nOnE, scAlAr ", HiveServer2DataTypeConversion.None | HiveServer2DataTypeConversion.Scalar, typeof(ArgumentOutOfRangeException) };
yield return new object?[] { $", none, scalar, ", HiveServer2DataTypeConversion.None | HiveServer2DataTypeConversion.Scalar , typeof(ArgumentOutOfRangeException) };
yield return new object?[] { $"scalar,none", HiveServer2DataTypeConversion.None | HiveServer2DataTypeConversion.Scalar , typeof(ArgumentOutOfRangeException) };
// Invalid options
yield return new object?[] { $"xxx", new List<HiveServer2DataTypeConversion>() { HiveServer2DataTypeConversion.Invalid, } };
yield return new object?[] { $"none,scalar,xxx", new List<HiveServer2DataTypeConversion>() { HiveServer2DataTypeConversion.Invalid, HiveServer2DataTypeConversion.None, HiveServer2DataTypeConversion.Scalar } };
yield return new object?[] { $"xxx", HiveServer2DataTypeConversion.Empty, typeof(ArgumentOutOfRangeException) };
yield return new object?[] { $"none,scalar,xxx", HiveServer2DataTypeConversion.None | HiveServer2DataTypeConversion.Scalar, typeof(ArgumentOutOfRangeException) };
}
}
}
4 changes: 2 additions & 2 deletions csharp/test/Drivers/Apache/Spark/SparkTestEnvironment.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@ public override string GetCreateTemporaryTableStatement(string tableName, string
}

public string? GetValueForProtocolVersion(string? hiveValue, string? databrickValue) =>
ServerType != SparkServerType.Databricks && ((HiveServer2Connection)Connection).DataTypeConversion.Contains(HiveServer2DataTypeConversion.None) ? hiveValue : databrickValue;
ServerType != SparkServerType.Databricks && ((HiveServer2Connection)Connection).DataTypeConversion.HasFlag(HiveServer2DataTypeConversion.None) ? hiveValue : databrickValue;

public object? GetValueForProtocolVersion(object? hiveValue, object? databrickValue) =>
ServerType != SparkServerType.Databricks && ((HiveServer2Connection)Connection).DataTypeConversion.Contains(HiveServer2DataTypeConversion.None) ? hiveValue : databrickValue;
ServerType != SparkServerType.Databricks && ((HiveServer2Connection)Connection).DataTypeConversion.HasFlag(HiveServer2DataTypeConversion.None) ? hiveValue : databrickValue;

public override string Delimiter => "`";

Expand Down

0 comments on commit 0ba07c6

Please sign in to comment.