Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GH-38316: [C#] Implement interval types #39043

Merged
merged 33 commits into from
Dec 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
f4778f5
MapType : Init type and array
Platob Apr 21, 2023
e159ca9
ListType : seal type
Platob Apr 21, 2023
d11d75b
MapType : Add IPC tests
Platob Apr 21, 2023
3c5b5b9
code : clean
Platob Apr 21, 2023
10ae223
code : clean
Platob Apr 21, 2023
220bd97
MapArray : Add builder test
Platob Apr 21, 2023
96b3965
MapArray : test netcore 3 1 constraint
Platob Apr 21, 2023
a6b927a
MapArray : inner struct not null array
Platob Apr 21, 2023
093a248
MapArray : GetKeyValuePairs with constructor
Platob Apr 21, 2023
acc2bc7
MapArray : fix KeyValues slice
Platob Apr 21, 2023
e403994
MapArray : GetTuple
Platob Apr 21, 2023
62013c1
code : clean
Platob Apr 21, 2023
571cb5e
MapArray : rename GetTuple to GetTuples
Platob Apr 21, 2023
c9b4694
MapArray : IPC get meta
Platob Apr 21, 2023
fb01fe5
rm #if NETCOREAPP3_1_OR_GREATER constraint
Platob Apr 22, 2023
d06c753
Merge from main
CurtHagenlocher Sep 25, 2023
eb0e3d0
Support C API for maps
CurtHagenlocher Sep 25, 2023
bc703d6
Finish implementing Map.
CurtHagenlocher Sep 26, 2023
af69e08
One fix plus one test
CurtHagenlocher Sep 26, 2023
396427c
Add missing copyright header
CurtHagenlocher Sep 26, 2023
87faaf6
Make "keysSorted" a little more optional.
CurtHagenlocher Sep 26, 2023
cef8911
Actually fix "KeysSorted" problem.
CurtHagenlocher Sep 26, 2023
addd081
Address code review feedback
CurtHagenlocher Oct 2, 2023
d2377d6
Merge branch 'Map' of https://github.com/CurtHagenlocher/arrow into Map
CurtHagenlocher Oct 2, 2023
4f7e548
Fixed comment
CurtHagenlocher Oct 2, 2023
cfaff68
Merge from main
CurtHagenlocher Oct 2, 2023
5316971
Restore changes lost to the merge
CurtHagenlocher Oct 2, 2023
66dca2f
Initial changes for Interval support
CurtHagenlocher Oct 16, 2023
5300f71
Merge from main
CurtHagenlocher Oct 16, 2023
fb77fe7
Merge branch 'main' of https://github.com/CurtHagenlocher/arrow into …
CurtHagenlocher Dec 2, 2023
4d66bb3
Finish implementing product changes for Interval
CurtHagenlocher Dec 2, 2023
20ca9b0
Add interval support to Json test infrastructure
CurtHagenlocher Dec 2, 2023
7d20143
Add tests
CurtHagenlocher Dec 10, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion csharp/src/Apache.Arrow/Arrays/ArrayDataTypeComparer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ internal sealed class ArrayDataTypeComparer :
IArrowTypeVisitor<FixedSizeListType>,
IArrowTypeVisitor<StructType>,
IArrowTypeVisitor<UnionType>,
IArrowTypeVisitor<MapType>
IArrowTypeVisitor<MapType>,
IArrowTypeVisitor<IntervalType>
{
private readonly IArrowType _expectedType;
private bool _dataTypeMatch;
Expand Down Expand Up @@ -133,6 +134,15 @@ public void Visit(MapType actualType)
}
}

public void Visit(IntervalType actualType)
{
if (_expectedType is IntervalType expectedType
&& expectedType.Unit == actualType.Unit)
{
_dataTypeMatch = true;
}
}

private static bool CompareNested(NestedType expectedType, NestedType actualType)
{
if (expectedType.Fields.Count != actualType.Fields.Count)
Expand Down
1 change: 1 addition & 0 deletions csharp/src/Apache.Arrow/Arrays/ArrowArrayFactory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ public static IArrowArray BuildArray(ArrayData data)
case ArrowTypeId.FixedSizeList:
return new FixedSizeListArray(data);
case ArrowTypeId.Interval:
return IntervalArray.Create(data);
default:
throw new NotSupportedException($"An ArrowArray cannot be built for type {data.DataType.TypeId}.");
}
Expand Down
140 changes: 140 additions & 0 deletions csharp/src/Apache.Arrow/Arrays/IntervalArray.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

using System;
using Apache.Arrow.Scalars;
using Apache.Arrow.Types;

namespace Apache.Arrow
{
internal static class IntervalArray
{
internal static IArrowArray Create(ArrayData data) => ((IntervalType)data.DataType).Unit switch
{
IntervalUnit.YearMonth => new YearMonthIntervalArray(data),
IntervalUnit.DayTime => new DayTimeIntervalArray(data),
IntervalUnit.MonthDayNanosecond => new MonthDayNanosecondIntervalArray(data),
_ => throw new InvalidOperationException($"Unsupported interval unit {((IntervalType)data.DataType).Unit}"),
};
}

public abstract class IntervalArray<T> : PrimitiveArray<T>
where T : struct
{
protected IntervalArray(ArrayData data)
: base(data)
{
data.EnsureBufferCount(2);
data.EnsureDataType(ArrowTypeId.Interval);
}

public IntervalType Type => (IntervalType)Data.DataType;

public IntervalUnit Unit => Type.Unit;

internal static IArrowArray Create(ArrayData data) => ((IntervalType)data.DataType).Unit switch
{
IntervalUnit.YearMonth => new YearMonthIntervalArray(data),
IntervalUnit.DayTime => new DayTimeIntervalArray(data),
IntervalUnit.MonthDayNanosecond => new MonthDayNanosecondIntervalArray(data),
_ => throw new InvalidOperationException($"Unsupported interval unit {((IntervalType)data.DataType).Unit}"),
};

internal static void ValidateUnit(IntervalUnit expected, IntervalUnit actual)
{
if (expected != actual)
{
throw new ArgumentException(
$"Specified interval unit <{actual}> does not match expected unit <{expected}>",
"Unit");
}
}
}

public sealed class YearMonthIntervalArray : IntervalArray<YearMonthInterval>
{
public class Builder : PrimitiveArrayBuilder<YearMonthInterval, YearMonthIntervalArray, Builder>
{
protected override YearMonthIntervalArray Build(
ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer,
int length, int nullCount, int offset) =>
new YearMonthIntervalArray(valueBuffer, nullBitmapBuffer, length, nullCount, offset);
}

public YearMonthIntervalArray(
ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer,
int length, int nullCount, int offset)
: this(new ArrayData(IntervalType.YearMonth, length, nullCount, offset,
new[] { nullBitmapBuffer, valueBuffer }))
{ }

public YearMonthIntervalArray(ArrayData data) : base(data)
{
ValidateUnit(IntervalUnit.YearMonth, Unit);
}

public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor);
}

public sealed class DayTimeIntervalArray : IntervalArray<DayTimeInterval>
{
public class Builder : PrimitiveArrayBuilder<DayTimeInterval, DayTimeIntervalArray, Builder>
{
protected override DayTimeIntervalArray Build(
ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer,
int length, int nullCount, int offset) =>
new DayTimeIntervalArray(valueBuffer, nullBitmapBuffer, length, nullCount, offset);
}

public DayTimeIntervalArray(
ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer,
int length, int nullCount, int offset)
: this(new ArrayData(IntervalType.DayTime, length, nullCount, offset,
new[] { nullBitmapBuffer, valueBuffer }))
{ }

public DayTimeIntervalArray(ArrayData data) : base(data)
{
ValidateUnit(IntervalUnit.DayTime, Unit);
}

public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor);
}

public sealed class MonthDayNanosecondIntervalArray : IntervalArray<MonthDayNanosecondInterval>
{
public class Builder : PrimitiveArrayBuilder<MonthDayNanosecondInterval, MonthDayNanosecondIntervalArray, Builder>
{
protected override MonthDayNanosecondIntervalArray Build(
ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer,
int length, int nullCount, int offset) =>
new MonthDayNanosecondIntervalArray(valueBuffer, nullBitmapBuffer, length, nullCount, offset);
}

public MonthDayNanosecondIntervalArray(
ArrowBuffer valueBuffer, ArrowBuffer nullBitmapBuffer,
int length, int nullCount, int offset)
: this(new ArrayData(IntervalType.MonthDayNanosecond, length, nullCount, offset,
new[] { nullBitmapBuffer, valueBuffer }))
{ }

public MonthDayNanosecondIntervalArray(ArrayData data) : base(data)
{
ValidateUnit(IntervalUnit.MonthDayNanosecond, Unit);
}

public override void Accept(IArrowArrayVisitor visitor) => Accept(this, visitor);
}
}
9 changes: 9 additions & 0 deletions csharp/src/Apache.Arrow/C/CArrowSchemaExporter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,15 @@ private static string GetFormat(IArrowType datatype)
// Timestamp
case TimestampType timestampType:
return String.Format("ts{0}:{1}", FormatTimeUnit(timestampType.Unit), timestampType.Timezone);
// Interval
case IntervalType intervalType:
return intervalType.Unit switch
{
IntervalUnit.YearMonth => "tiM",
IntervalUnit.DayTime => "tiD",
IntervalUnit.MonthDayNanosecond => "tin",
_ => throw new InvalidDataException($"Unsupported interval unit for export: {intervalType.Unit}"),
};
// Nested
case ListType _: return "+l";
case FixedSizeListType fixedListType:
Expand Down
2 changes: 1 addition & 1 deletion csharp/src/Apache.Arrow/C/CArrowSchemaImporter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,7 @@ public ArrowType GetAsType()
"tDn" => DurationType.Nanosecond,
"tiM" => IntervalType.YearMonth,
"tiD" => IntervalType.DayTime,
//"tin" => IntervalType.MonthDayNanosecond, // Not yet implemented
"tin" => IntervalType.MonthDayNanosecond,
_ => throw new NotSupportedException("Data type is not yet supported in import.")
};
}
Expand Down
2 changes: 2 additions & 0 deletions csharp/src/Apache.Arrow/Extensions/FlatbufExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ public static Types.IntervalUnit ToArrow(this Flatbuf.IntervalUnit unit)
return Types.IntervalUnit.DayTime;
case Flatbuf.IntervalUnit.YEAR_MONTH:
return Types.IntervalUnit.YearMonth;
case Flatbuf.IntervalUnit.MONTH_DAY_NANO:
return Types.IntervalUnit.MonthDayNanosecond;
default:
throw new ArgumentException($"Unexpected Flatbuf IntervalUnit", nameof(unit));
}
Expand Down
6 changes: 6 additions & 0 deletions csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ internal class ArrowRecordBatchFlatBufferBuilder :
IArrowArrayVisitor<Time32Array>,
IArrowArrayVisitor<Time64Array>,
IArrowArrayVisitor<DurationArray>,
IArrowArrayVisitor<YearMonthIntervalArray>,
IArrowArrayVisitor<DayTimeIntervalArray>,
IArrowArrayVisitor<MonthDayNanosecondIntervalArray>,
IArrowArrayVisitor<ListArray>,
IArrowArrayVisitor<FixedSizeListArray>,
IArrowArrayVisitor<StringArray>,
Expand Down Expand Up @@ -106,6 +109,9 @@ public ArrowRecordBatchFlatBufferBuilder()
public void Visit(Time32Array array) => CreateBuffers(array);
public void Visit(Time64Array array) => CreateBuffers(array);
public void Visit(DurationArray array) => CreateBuffers(array);
public void Visit(YearMonthIntervalArray array) => CreateBuffers(array);
public void Visit(DayTimeIntervalArray array) => CreateBuffers(array);
public void Visit(MonthDayNanosecondIntervalArray array) => CreateBuffers(array);

public void Visit(ListArray array)
{
Expand Down
19 changes: 19 additions & 0 deletions csharp/src/Apache.Arrow/Ipc/ArrowTypeFlatbufferBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ class TypeVisitor :
IArrowTypeVisitor<Time32Type>,
IArrowTypeVisitor<Time64Type>,
IArrowTypeVisitor<DurationType>,
IArrowTypeVisitor<IntervalType>,
IArrowTypeVisitor<BinaryType>,
IArrowTypeVisitor<TimestampType>,
IArrowTypeVisitor<ListType>,
Expand Down Expand Up @@ -196,6 +197,13 @@ public void Visit(DurationType type)
Flatbuf.Duration.CreateDuration(Builder, ToFlatBuffer(type.Unit)));
}

public void Visit(IntervalType type)
{
Result = FieldType.Build(
Flatbuf.Type.Interval,
Flatbuf.Interval.CreateInterval(Builder, ToFlatBuffer(type.Unit)));
}

public void Visit(StructType type)
{
Flatbuf.Struct_.StartStruct_(Builder);
Expand Down Expand Up @@ -307,5 +315,16 @@ private static Flatbuf.UnionMode ToFlatBuffer(Types.UnionMode mode)
_ => throw new ArgumentException($"unsupported union mode <{mode}>", nameof(mode)),
};
}

private static Flatbuf.IntervalUnit ToFlatBuffer(Types.IntervalUnit unit)
{
return unit switch
{
Types.IntervalUnit.YearMonth => Flatbuf.IntervalUnit.YEAR_MONTH,
Types.IntervalUnit.DayTime => Flatbuf.IntervalUnit.DAY_TIME,
Types.IntervalUnit.MonthDayNanosecond => Flatbuf.IntervalUnit.MONTH_DAY_NANO,
_ => throw new ArgumentException($"unsupported interval unit <{unit}>", nameof(unit))
}; ;
}
}
}
63 changes: 63 additions & 0 deletions csharp/src/Apache.Arrow/Scalars/DayTimeInterval.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

using System;
using System.Runtime.InteropServices;

namespace Apache.Arrow.Scalars
{
[StructLayout(LayoutKind.Explicit)]
public struct DayTimeInterval : IEquatable<DayTimeInterval>
{
[FieldOffset(0)]
public readonly int Days;

[FieldOffset(4)]
public readonly int Milliseconds;

public DayTimeInterval(int days, int milliseconds)
{
Days = days;
Milliseconds = milliseconds;
}

public override bool Equals(object obj) => obj switch
{
DayTimeInterval interval => Equals(interval),
_ => false,
};

public override int GetHashCode() => unchecked((17 + Days) * 23 + Milliseconds);

public bool Equals(DayTimeInterval interval)
{
return this.Days == interval.Days && this.Milliseconds == interval.Milliseconds;
}

public static DateTime operator +(DateTime dateTime, DayTimeInterval interval)
=> dateTime.AddDays(interval.Days).AddMilliseconds(interval.Milliseconds);
public static DateTime operator +(DayTimeInterval interval, DateTime dateTime)
=> dateTime.AddDays(interval.Days).AddMilliseconds(interval.Milliseconds);
public static DateTime operator -(DateTime dateTime, DayTimeInterval interval)
=> dateTime.AddDays(-interval.Days).AddMilliseconds(-interval.Milliseconds);

public static DateTimeOffset operator +(DateTimeOffset dateTime, DayTimeInterval interval)
=> dateTime.AddDays(interval.Days).AddMilliseconds(interval.Milliseconds);
public static DateTimeOffset operator +(DayTimeInterval interval, DateTimeOffset dateTime)
=> dateTime.AddDays(interval.Days).AddMilliseconds(interval.Milliseconds);
public static DateTimeOffset operator -(DateTimeOffset dateTime, DayTimeInterval interval)
=> dateTime.AddDays(-interval.Days).AddMilliseconds(-interval.Milliseconds);
}
}
67 changes: 67 additions & 0 deletions csharp/src/Apache.Arrow/Scalars/MonthDayNanosecondInterval.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

using System;
using System.Runtime.InteropServices;

namespace Apache.Arrow.Scalars
{
[StructLayout(LayoutKind.Explicit)]
public struct MonthDayNanosecondInterval : IEquatable<MonthDayNanosecondInterval>
{
[FieldOffset(0)]
public readonly int Months;

[FieldOffset(4)]
public readonly int Days;

[FieldOffset(8)]
public readonly long Nanoseconds;

public MonthDayNanosecondInterval(int months, int days, long nanoseconds)
{
Months = months;
Days = days;
Nanoseconds = nanoseconds;
}

public override bool Equals(object obj) => obj switch
{
MonthDayNanosecondInterval interval => Equals(interval),
_ => false,
};

public override int GetHashCode() => unchecked(((17 + Months) * 23 + Days) * 23 + (int)Nanoseconds);

public bool Equals(MonthDayNanosecondInterval interval)
{
return this.Months == interval.Months && this.Days == interval.Days && this.Nanoseconds == interval.Nanoseconds;
}

public static DateTime operator +(DateTime dateTime, MonthDayNanosecondInterval interval)
=> dateTime.AddMonths(interval.Months).AddDays(interval.Days).AddTicks(interval.Nanoseconds / 100);
public static DateTime operator +(MonthDayNanosecondInterval interval, DateTime dateTime)
=> dateTime.AddMonths(interval.Months).AddDays(interval.Days).AddTicks(interval.Nanoseconds / 100);
public static DateTime operator -(DateTime dateTime, MonthDayNanosecondInterval interval)
=> dateTime.AddMonths(-interval.Months).AddDays(-interval.Days).AddTicks(-interval.Nanoseconds / 100);

public static DateTimeOffset operator +(DateTimeOffset dateTime, MonthDayNanosecondInterval interval)
=> dateTime.AddMonths(interval.Months).AddDays(interval.Days).AddTicks(interval.Nanoseconds / 100);
public static DateTimeOffset operator +(MonthDayNanosecondInterval interval, DateTimeOffset dateTime)
=> dateTime.AddMonths(interval.Months).AddDays(interval.Days).AddTicks(interval.Nanoseconds / 100);
public static DateTimeOffset operator -(DateTimeOffset dateTime, MonthDayNanosecondInterval interval)
=> dateTime.AddMonths(-interval.Months).AddDays(-interval.Days).AddTicks(-interval.Nanoseconds / 100);
}
}
Loading
Loading