Change dictionary id assignment to conform to Java integration test e…

…xpectations. Support dictionaries in C data interface integration testing.
apache · Dec 9, 2023 · fd67722 · fd67722
1 parent 8ac6a76
commit fd67722
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 15 deletions.
diff --git a/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs b/csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
@@ -502,7 +502,7 @@ private protected virtual void FinishedWritingDictionary(long bodyLength, long m
         private protected void WriteDictionaries(DictionaryMemo dictionaryMemo)
         {
             int fieldCount = dictionaryMemo?.DictionaryCount ?? 0;
-            for (int i = 1; i <= fieldCount; i++)
+            for (int i = 0; i < fieldCount; i++)
             {
                 WriteDictionary(i, dictionaryMemo.GetDictionaryType(i), dictionaryMemo.GetDictionary(i));
             }
@@ -526,7 +526,7 @@ private protected void WriteDictionary(long id, IArrowType valueType, IArrowArra
         private protected async Task WriteDictionariesAsync(DictionaryMemo dictionaryMemo, CancellationToken cancellationToken)
         {
             int fieldCount = dictionaryMemo?.DictionaryCount ?? 0;
-            for (int i = 1; i <= fieldCount; i++)
+            for (int i = 0; i < fieldCount; i++)
             {
                 await WriteDictionaryAsync(i, dictionaryMemo.GetDictionaryType(i), dictionaryMemo.GetDictionary(i), cancellationToken).ConfigureAwait(false);
             }

diff --git a/csharp/src/Apache.Arrow/Ipc/DictionaryMemo.cs b/csharp/src/Apache.Arrow/Ipc/DictionaryMemo.cs
@@ -96,7 +96,7 @@ public long GetOrAssignId(Field field)
         {
             if (!_fieldToId.TryGetValue(field, out long id))
             {
-                id = _fieldToId.Count + 1;
+                id = _fieldToId.Count;
                 AddField(id, field);
             }
             return id;

diff --git a/csharp/test/Apache.Arrow.IntegrationTest/JsonFile.cs b/csharp/test/Apache.Arrow.IntegrationTest/JsonFile.cs
@@ -61,6 +61,22 @@ public Schema GetSchemaAndDictionaries(out Func<DictionaryType, IArrowArray> dic
             return schema;
         }
 
+        /// <summary>
+        /// Return both the schema and a specific batch number.
+        /// This method is used by C Data Interface integration testing.
+        /// </summary>
+        public Schema ToArrow(int batchNumber, out RecordBatch batch)
+        {
+            Schema schema = Schema.ToArrow(out Dictionary<DictionaryType, int> dictionaryIndexes);
+
+            Func<DictionaryType, IArrowArray> lookup = null;
+            lookup = type => Dictionaries.Single(d => d.Id == dictionaryIndexes[type]).Data.ToArrow(type.ValueType, lookup);
+
+            batch = Batches[batchNumber].ToArrow(schema, lookup);
+
+            return schema;
+        }
+
         private static JsonSerializerOptions GetJsonOptions()
         {
             JsonSerializerOptions options = new JsonSerializerOptions()
@@ -366,15 +382,6 @@ public RecordBatch ToArrow(Schema schema, Func<DictionaryType, IArrowArray> dict
             return CreateRecordBatch(schema, dictionaries, this);
         }
 
-        /// <summary>
-        /// Decode this JSON record batch as a RecordBatch instance without supporting dictionaries.
-        /// This method is used by C Data Interface integration testing.
-        /// </summary>
-        public RecordBatch ToArrow(Schema schema)
-        {
-            return CreateRecordBatch(schema, _ => throw new NotImplementedException(), this);
-        }
-
         public IArrowArray ToArrow(IArrowType arrowType, Func<DictionaryType, IArrowArray> dictionaries)
         {
             ArrayCreator creator = new ArrayCreator(this.Columns[0], dictionaries);

diff --git a/dev/archery/archery/integration/tester_csharp.py b/dev/archery/archery/integration/tester_csharp.py
@@ -78,9 +78,7 @@ def _pointer_to_int(self, c_ptr):
     def _read_batch_from_json(self, json_path, num_batch):
         from Apache.Arrow.IntegrationTest import CDataInterface
 
-        jf = CDataInterface.ParseJsonFile(json_path)
-        schema = jf.Schema.ToArrow()
-        return schema, jf.Batches[num_batch].ToArrow(schema)
+        return CDataInterface.ParseJsonFile(json_path).ToArrow(num_batch)
 
     def _run_gc(self):
         from Apache.Arrow.IntegrationTest import CDataInterface