From 16bd698c07c92008067cfbe3385f3f64add8026a Mon Sep 17 00:00:00 2001 From: to11mtm Date: Thu, 29 Apr 2021 13:18:25 -0400 Subject: [PATCH 01/13] Add Option to Force Parameter Usage on MultipleRowsCopy/MultipleRowsCopyAsyinc --- Source/LinqToDB/Data/BulkCopyOptions.cs | 5 ++ Source/LinqToDB/DataProvider/BasicBulkCopy.cs | 52 ++++++++++++++++--- .../DataProvider/MultipleRowsHelper.cs | 21 +++++--- Tests/Linq/Update/BulkCopyTests.cs | 17 ++++++ 4 files changed, 80 insertions(+), 15 deletions(-) diff --git a/Source/LinqToDB/Data/BulkCopyOptions.cs b/Source/LinqToDB/Data/BulkCopyOptions.cs index c0c54afaca..c41c0185a5 100644 --- a/Source/LinqToDB/Data/BulkCopyOptions.cs +++ b/Source/LinqToDB/Data/BulkCopyOptions.cs @@ -88,5 +88,10 @@ public BulkCopyOptions(BulkCopyOptions options) /// This callback will not be used if set to 0. /// public Action? RowsCopiedCallback { get; set; } + + /// + /// Gets or sets whether to Always use Parameters for MultipleRowsCopy. + /// + public bool UseParameters { get; set; } } } diff --git a/Source/LinqToDB/DataProvider/BasicBulkCopy.cs b/Source/LinqToDB/DataProvider/BasicBulkCopy.cs index fc9094ea5c..60d72f6bea 100644 --- a/Source/LinqToDB/DataProvider/BasicBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/BasicBulkCopy.cs @@ -344,20 +344,32 @@ protected static BulkCopyRowsCopied MultipleRowsCopyHelper( Action prepFunction, Action addFunction, Action finishFunction, - int maxParameters = 10000, + int maxParameters = 999, int maxSqlLength = 100000) { prepFunction(helper); foreach (var item in source) { + helper.LastRowParameterIndex = helper.ParameterIndex; + helper.LastRowStringIndex = helper.StringBuilder.Length; addFunction(helper, item!, from); - - if (helper.CurrentCount >= helper.BatchSize || helper.Parameters.Count > maxParameters || helper.StringBuilder.Length > maxSqlLength) + var needRemove = helper.Parameters.Count > maxParameters || + helper.StringBuilder.Length > maxSqlLength; + if (helper.CurrentCount >= helper.BatchSize || needRemove) { + if (needRemove) + { + helper.Parameters.RemoveRange(helper.LastRowParameterIndex, helper.ParameterIndex-helper.LastRowParameterIndex); + helper.StringBuilder.Length = helper.LastRowStringIndex; + } finishFunction(helper); if (!helper.Execute()) return helper.RowsCopied; + if (needRemove) + { + addFunction(helper, item!, from); + } } } @@ -378,20 +390,33 @@ protected static async Task MultipleRowsCopyHelperAsync( Action addFunction, Action finishFunction, CancellationToken cancellationToken, - int maxParameters = 10000, + int maxParameters = 999, int maxSqlLength = 100000) { prepFunction(helper); foreach (var item in source) { + helper.LastRowParameterIndex = helper.ParameterIndex; + helper.LastRowStringIndex = helper.LastRowStringIndex; addFunction(helper, item!, from); - if (helper.CurrentCount >= helper.BatchSize || helper.Parameters.Count > maxParameters || helper.StringBuilder.Length > maxSqlLength) + var needRemove = helper.Parameters.Count > maxParameters || + helper.StringBuilder.Length > maxSqlLength; + if (helper.CurrentCount >= helper.BatchSize || needRemove) { + if (needRemove) + { + helper.Parameters.RemoveRange(helper.LastRowParameterIndex, helper.ParameterIndex-helper.LastRowParameterIndex); + helper.StringBuilder.Length = helper.LastRowStringIndex; + } finishFunction(helper); if (!await helper.ExecuteAsync(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)) return helper.RowsCopied; + if (needRemove) + { + addFunction(helper, item!, from); + } } } @@ -413,20 +438,33 @@ protected static async Task MultipleRowsCopyHelperAsync( Action addFunction, Action finishFunction, CancellationToken cancellationToken, - int maxParameters = 10000, + int maxParameters = 999, int maxSqlLength = 100000) { prepFunction(helper); await foreach (var item in source.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext).WithCancellation(cancellationToken)) { + helper.LastRowParameterIndex = helper.ParameterIndex; + helper.LastRowStringIndex = helper.LastRowStringIndex; addFunction(helper, item!, from); - if (helper.CurrentCount >= helper.BatchSize || helper.Parameters.Count > maxParameters || helper.StringBuilder.Length > maxSqlLength) + var needRemove = helper.Parameters.Count > maxParameters || + helper.StringBuilder.Length > maxSqlLength; + if (helper.CurrentCount >= helper.BatchSize || needRemove) { + if (needRemove) + { + helper.Parameters.RemoveRange(helper.LastRowParameterIndex, helper.ParameterIndex-helper.LastRowParameterIndex); + helper.StringBuilder.Length = helper.LastRowStringIndex; + } finishFunction(helper); if (!await helper.ExecuteAsync(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)) return helper.RowsCopied; + if (needRemove) + { + addFunction(helper, item!, from); + } } } diff --git a/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs b/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs index 95fd91abf7..4439ecb4a7 100644 --- a/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs +++ b/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs @@ -38,7 +38,6 @@ protected MultipleRowsHelper(DataConnection dataConnection, BulkCopyOptions opti ParameterName = SqlBuilder.ConvertInline("p", ConvertType.NameToQueryParameter); BatchSize = Math.Max(10, Options.MaxBatchSize ?? 1000); } - public readonly ISqlBuilder SqlBuilder; public readonly DataConnection DataConnection; public readonly BulkCopyOptions Options; @@ -57,6 +56,8 @@ protected MultipleRowsHelper(DataConnection dataConnection, BulkCopyOptions opti public int ParameterIndex; public int HeaderSize; public int BatchSize; + public int LastRowStringIndex; + public int LastRowParameterIndex; public void SetHeader() { @@ -72,7 +73,7 @@ public virtual void BuildColumns(object item, Func? skip var column = Columns[i]; var value = column.GetValue(item); - if (skipConvert(column) || !ValueConverter.TryConvert(StringBuilder, ColumnTypes[i], value)) + if (Options.UseParameters || skipConvert(column) || !ValueConverter.TryConvert(StringBuilder, ColumnTypes[i], value)) { var name = ParameterName == "?" ? ParameterName : ParameterName + ++ParameterIndex; @@ -109,9 +110,11 @@ public bool Execute() } Parameters.Clear(); - ParameterIndex = 0; - CurrentCount = 0; - StringBuilder.Length = HeaderSize; + ParameterIndex = 0; + CurrentCount = 0; + LastRowParameterIndex = 0; + LastRowStringIndex = HeaderSize; + StringBuilder.Length = HeaderSize; return true; } @@ -130,9 +133,11 @@ await DataConnection.ExecuteAsync(StringBuilder.AppendLine().ToString(), cancell } Parameters.Clear(); - ParameterIndex = 0; - CurrentCount = 0; - StringBuilder.Length = HeaderSize; + ParameterIndex = 0; + CurrentCount = 0; + LastRowParameterIndex = 0; + LastRowStringIndex = HeaderSize; + StringBuilder.Length = HeaderSize; return true; } diff --git a/Tests/Linq/Update/BulkCopyTests.cs b/Tests/Linq/Update/BulkCopyTests.cs index 4c430d280d..65cd2b2c5c 100644 --- a/Tests/Linq/Update/BulkCopyTests.cs +++ b/Tests/Linq/Update/BulkCopyTests.cs @@ -276,5 +276,22 @@ public void ReuseOptionTest([DataSources(false, ProviderName.DB2)] string contex db.Child. BulkCopy(options, new[] { new Child { ParentID = 111001 } }); } } + + [Test] + public void UseParametersTest([DataSources(false, ProviderName.DB2)] string context) + { + using (var db = new TestDataConnection(context)) + using (db.BeginTransaction()) + { + var options = new BulkCopyOptions(){ UseParameters = true, MaxBatchSize = 50}; + var rowsToIns = Enumerable.Range(111001, 200) + .Select(r => new Parent() {ParentID = r}).ToList(); + db.Parent.BulkCopy(options, rowsToIns); + Assert.AreEqual(rowsToIns.Count, + db.Parent.Where(r => + r.ParentID >= 111001 && r.ParentID <= 111201).Count()); + + } + } } } From 9caf4b9ad5603c9abc346b35eb2ece0b2cd7ed16 Mon Sep 17 00:00:00 2001 From: to11mtm Date: Thu, 29 Apr 2021 15:43:07 -0400 Subject: [PATCH 02/13] Fixup bad Last string position index logic. Make sure to handle the case where a single row takes us over a limit; we should try anyway. Made MaxParameters and MaxSqlLength virtual properties so we can set it per-provider bulk copy. Set for Postgres, oracle, sqlite, sqlserver --- Source/LinqToDB/DataProvider/BasicBulkCopy.cs | 61 +++++++++++-------- .../DataProvider/Oracle/OracleBulkCopy.cs | 19 +++--- .../PostgreSQL/PostgreSQLBulkCopy.cs | 4 +- .../DataProvider/SQLite/SQLiteBulkCopy.cs | 3 + .../SqlServer/SqlServerBulkCopy.cs | 4 +- Tests/Linq/Update/BulkCopyTests.cs | 4 +- 6 files changed, 57 insertions(+), 38 deletions(-) diff --git a/Source/LinqToDB/DataProvider/BasicBulkCopy.cs b/Source/LinqToDB/DataProvider/BasicBulkCopy.cs index 60d72f6bea..8f9b4e55bc 100644 --- a/Source/LinqToDB/DataProvider/BasicBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/BasicBulkCopy.cs @@ -15,6 +15,9 @@ namespace LinqToDB.DataProvider public class BasicBulkCopy { + protected virtual int MaxParameters => 999; + protected virtual int MaxSqlLength => 100000; + public virtual BulkCopyRowsCopied BulkCopy(BulkCopyType bulkCopyType, ITable table, BulkCopyOptions options, IEnumerable source) where T : notnull { @@ -344,8 +347,8 @@ protected static BulkCopyRowsCopied MultipleRowsCopyHelper( Action prepFunction, Action addFunction, Action finishFunction, - int maxParameters = 999, - int maxSqlLength = 100000) + int maxParameters, + int maxSqlLength) { prepFunction(helper); @@ -356,17 +359,19 @@ protected static BulkCopyRowsCopied MultipleRowsCopyHelper( addFunction(helper, item!, from); var needRemove = helper.Parameters.Count > maxParameters || helper.StringBuilder.Length > maxSqlLength; + var isSingle = helper.CurrentCount == 1; if (helper.CurrentCount >= helper.BatchSize || needRemove) { - if (needRemove) + if (needRemove && !isSingle) { helper.Parameters.RemoveRange(helper.LastRowParameterIndex, helper.ParameterIndex-helper.LastRowParameterIndex); - helper.StringBuilder.Length = helper.LastRowStringIndex; + helper.StringBuilder.Length = helper.LastRowStringIndex; + helper.RowsCopied.RowsCopied--; } finishFunction(helper); if (!helper.Execute()) return helper.RowsCopied; - if (needRemove) + if (needRemove && !isSingle) { addFunction(helper, item!, from); } @@ -390,30 +395,32 @@ protected static async Task MultipleRowsCopyHelperAsync( Action addFunction, Action finishFunction, CancellationToken cancellationToken, - int maxParameters = 999, - int maxSqlLength = 100000) + int maxParameters, + int maxSqlLength) { prepFunction(helper); foreach (var item in source) { helper.LastRowParameterIndex = helper.ParameterIndex; - helper.LastRowStringIndex = helper.LastRowStringIndex; + helper.LastRowStringIndex = helper.StringBuilder.Length; addFunction(helper, item!, from); var needRemove = helper.Parameters.Count > maxParameters || helper.StringBuilder.Length > maxSqlLength; + var isSingle = helper.CurrentCount == 1; if (helper.CurrentCount >= helper.BatchSize || needRemove) { - if (needRemove) + if (needRemove && !isSingle) { helper.Parameters.RemoveRange(helper.LastRowParameterIndex, helper.ParameterIndex-helper.LastRowParameterIndex); - helper.StringBuilder.Length = helper.LastRowStringIndex; + helper.StringBuilder.Length = helper.LastRowStringIndex; + helper.RowsCopied.RowsCopied--; } finishFunction(helper); if (!await helper.ExecuteAsync(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)) return helper.RowsCopied; - if (needRemove) + if (needRemove && !isSingle) { addFunction(helper, item!, from); } @@ -438,30 +445,32 @@ protected static async Task MultipleRowsCopyHelperAsync( Action addFunction, Action finishFunction, CancellationToken cancellationToken, - int maxParameters = 999, - int maxSqlLength = 100000) + int maxParameters, + int maxSqlLength) { prepFunction(helper); await foreach (var item in source.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext).WithCancellation(cancellationToken)) { helper.LastRowParameterIndex = helper.ParameterIndex; - helper.LastRowStringIndex = helper.LastRowStringIndex; + helper.LastRowStringIndex = helper.StringBuilder.Length; addFunction(helper, item!, from); var needRemove = helper.Parameters.Count > maxParameters || helper.StringBuilder.Length > maxSqlLength; + var isSingle = helper.CurrentCount == 1; if (helper.CurrentCount >= helper.BatchSize || needRemove) { - if (needRemove) + if (needRemove && !isSingle) { helper.Parameters.RemoveRange(helper.LastRowParameterIndex, helper.ParameterIndex-helper.LastRowParameterIndex); - helper.StringBuilder.Length = helper.LastRowStringIndex; + helper.StringBuilder.Length = helper.LastRowStringIndex; + helper.RowsCopied.RowsCopied--; } finishFunction(helper); if (!await helper.ExecuteAsync(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)) return helper.RowsCopied; - if (needRemove) + if (needRemove && !isSingle) { addFunction(helper, item!, from); } @@ -483,14 +492,14 @@ protected BulkCopyRowsCopied MultipleRowsCopy1(ITable table, BulkCopyOptio => MultipleRowsCopy1(new MultipleRowsHelper(table, options), source); protected BulkCopyRowsCopied MultipleRowsCopy1(MultipleRowsHelper helper, IEnumerable source) - => MultipleRowsCopyHelper(helper, source, null, MultipleRowsCopy1Prep, MultipleRowsCopy1Add, MultipleRowsCopy1Finish); + => MultipleRowsCopyHelper(helper, source, null, MultipleRowsCopy1Prep, MultipleRowsCopy1Add, MultipleRowsCopy1Finish,MaxParameters, MaxSqlLength); protected Task MultipleRowsCopy1Async(ITable table, BulkCopyOptions options, IEnumerable source, CancellationToken cancellationToken) where T : notnull => MultipleRowsCopy1Async(new MultipleRowsHelper(table, options), source, cancellationToken); protected Task MultipleRowsCopy1Async(MultipleRowsHelper helper, IEnumerable source, CancellationToken cancellationToken) - => MultipleRowsCopyHelperAsync(helper, source, null, MultipleRowsCopy1Prep, MultipleRowsCopy1Add, MultipleRowsCopy1Finish, cancellationToken); + => MultipleRowsCopyHelperAsync(helper, source, null, MultipleRowsCopy1Prep, MultipleRowsCopy1Add, MultipleRowsCopy1Finish, cancellationToken, MaxParameters, MaxSqlLength); #if NATIVE_ASYNC protected Task MultipleRowsCopy1Async(ITable table, BulkCopyOptions options, IAsyncEnumerable source, CancellationToken cancellationToken) @@ -499,7 +508,7 @@ protected Task MultipleRowsCopy1Async(ITable table, Bu protected Task MultipleRowsCopy1Async(MultipleRowsHelper helper, IAsyncEnumerable source, CancellationToken cancellationToken) where T: notnull - => MultipleRowsCopyHelperAsync(helper, source, null, MultipleRowsCopy1Prep, MultipleRowsCopy1Add, MultipleRowsCopy1Finish, cancellationToken); + => MultipleRowsCopyHelperAsync(helper, source, null, MultipleRowsCopy1Prep, MultipleRowsCopy1Add, MultipleRowsCopy1Finish, cancellationToken, MaxParameters, MaxSqlLength); #endif private void MultipleRowsCopy1Prep(MultipleRowsHelper helper) @@ -551,14 +560,14 @@ protected BulkCopyRowsCopied MultipleRowsCopy2(ITable table, BulkCopyOptio => MultipleRowsCopy2(new MultipleRowsHelper(table, options), source, from); protected BulkCopyRowsCopied MultipleRowsCopy2(MultipleRowsHelper helper, IEnumerable source, string from) - => MultipleRowsCopyHelper(helper, source, from, MultipleRowsCopy2Prep, MultipleRowsCopy2Add, MultipleRowsCopy2Finish); + => MultipleRowsCopyHelper(helper, source, from, MultipleRowsCopy2Prep, MultipleRowsCopy2Add, MultipleRowsCopy2Finish, MaxParameters, MaxSqlLength); protected Task MultipleRowsCopy2Async(ITable table, BulkCopyOptions options, IEnumerable source, string from, CancellationToken cancellationToken) where T : notnull => MultipleRowsCopy2Async(new MultipleRowsHelper(table, options), source, from, cancellationToken); protected Task MultipleRowsCopy2Async(MultipleRowsHelper helper, IEnumerable source, string from, CancellationToken cancellationToken) - => MultipleRowsCopyHelperAsync(helper, source, from, MultipleRowsCopy2Prep, MultipleRowsCopy2Add, MultipleRowsCopy2Finish, cancellationToken); + => MultipleRowsCopyHelperAsync(helper, source, from, MultipleRowsCopy2Prep, MultipleRowsCopy2Add, MultipleRowsCopy2Finish, cancellationToken, MaxParameters, MaxSqlLength); #if NATIVE_ASYNC protected Task MultipleRowsCopy2Async(ITable table, BulkCopyOptions options, IAsyncEnumerable source, string from, CancellationToken cancellationToken) @@ -567,7 +576,7 @@ protected Task MultipleRowsCopy2Async(ITable table, Bu protected Task MultipleRowsCopy2Async(MultipleRowsHelper helper, IAsyncEnumerable source, string from, CancellationToken cancellationToken) where T: notnull - => MultipleRowsCopyHelperAsync(helper, source, from, MultipleRowsCopy2Prep, MultipleRowsCopy2Add, MultipleRowsCopy2Finish, cancellationToken); + => MultipleRowsCopyHelperAsync(helper, source, from, MultipleRowsCopy2Prep, MultipleRowsCopy2Add, MultipleRowsCopy2Finish, cancellationToken, MaxParameters, MaxSqlLength); #endif private void MultipleRowsCopy2Prep(MultipleRowsHelper helper) @@ -612,15 +621,15 @@ private void MultipleRowsCopy2Finish(MultipleRowsHelper helper) } protected BulkCopyRowsCopied MultipleRowsCopy3(MultipleRowsHelper helper, BulkCopyOptions options, IEnumerable source, string from) - => MultipleRowsCopyHelper(helper, source, from, MultipleRowsCopy3Prep, MultipleRowsCopy3Add, MultipleRowsCopy3Finish); + => MultipleRowsCopyHelper(helper, source, from, MultipleRowsCopy3Prep, MultipleRowsCopy3Add, MultipleRowsCopy3Finish, MaxParameters, MaxSqlLength); protected Task MultipleRowsCopy3Async(MultipleRowsHelper helper, BulkCopyOptions options, IEnumerable source, string from, CancellationToken cancellationToken) - => MultipleRowsCopyHelperAsync(helper, source, from, MultipleRowsCopy3Prep, MultipleRowsCopy3Add, MultipleRowsCopy3Finish, cancellationToken); + => MultipleRowsCopyHelperAsync(helper, source, from, MultipleRowsCopy3Prep, MultipleRowsCopy3Add, MultipleRowsCopy3Finish, cancellationToken, MaxParameters, MaxSqlLength); #if NATIVE_ASYNC protected Task MultipleRowsCopy3Async(MultipleRowsHelper helper, BulkCopyOptions options, IAsyncEnumerable source, string from, CancellationToken cancellationToken) where T: notnull - => MultipleRowsCopyHelperAsync(helper, source, from, MultipleRowsCopy3Prep, MultipleRowsCopy3Add, MultipleRowsCopy3Finish, cancellationToken); + => MultipleRowsCopyHelperAsync(helper, source, from, MultipleRowsCopy3Prep, MultipleRowsCopy3Add, MultipleRowsCopy3Finish, cancellationToken, MaxParameters, MaxSqlLength); #endif private void MultipleRowsCopy3Prep(MultipleRowsHelper helper) diff --git a/Source/LinqToDB/DataProvider/Oracle/OracleBulkCopy.cs b/Source/LinqToDB/DataProvider/Oracle/OracleBulkCopy.cs index 8295475353..07b16777f8 100644 --- a/Source/LinqToDB/DataProvider/Oracle/OracleBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/Oracle/OracleBulkCopy.cs @@ -14,8 +14,11 @@ namespace LinqToDB.DataProvider.Oracle class OracleBulkCopy : BasicBulkCopy { - private readonly OracleDataProvider _provider; - + private const int _maxParameters = 32766; + private const int _maxSqlLength = 327670; + protected override int MaxParameters => _maxParameters; + protected override int MaxSqlLength => _maxSqlLength; + private readonly OracleDataProvider _provider; public OracleBulkCopy(OracleDataProvider provider) { _provider = provider; @@ -212,14 +215,14 @@ static void OracleMultipleRowsCopy1Finish(MultipleRowsHelper helper) } static BulkCopyRowsCopied OracleMultipleRowsCopy1(MultipleRowsHelper helper, IEnumerable source) - => MultipleRowsCopyHelper(helper, source, null, OracleMultipleRowsCopy1Prep, OracleMultipleRowsCopy1Add, OracleMultipleRowsCopy1Finish); + => MultipleRowsCopyHelper(helper, source, null, OracleMultipleRowsCopy1Prep, OracleMultipleRowsCopy1Add, OracleMultipleRowsCopy1Finish, _maxParameters,_maxSqlLength); static Task OracleMultipleRowsCopy1Async(MultipleRowsHelper helper, IEnumerable source, CancellationToken cancellationToken) - => MultipleRowsCopyHelperAsync(helper, source, null, OracleMultipleRowsCopy1Prep, OracleMultipleRowsCopy1Add, OracleMultipleRowsCopy1Finish, cancellationToken); + => MultipleRowsCopyHelperAsync(helper, source, null, OracleMultipleRowsCopy1Prep, OracleMultipleRowsCopy1Add, OracleMultipleRowsCopy1Finish, cancellationToken, _maxParameters,_maxSqlLength); #if NATIVE_ASYNC static Task OracleMultipleRowsCopy1Async(MultipleRowsHelper helper, IAsyncEnumerable source, CancellationToken cancellationToken) - => MultipleRowsCopyHelperAsync(helper, source, null, OracleMultipleRowsCopy1Prep, OracleMultipleRowsCopy1Add, OracleMultipleRowsCopy1Finish, cancellationToken); + => MultipleRowsCopyHelperAsync(helper, source, null, OracleMultipleRowsCopy1Prep, OracleMultipleRowsCopy1Add, OracleMultipleRowsCopy1Finish, cancellationToken, _maxParameters,_maxSqlLength); #endif static List OracleMultipleRowsCopy2Prep(MultipleRowsHelper helper) @@ -415,14 +418,14 @@ static void OracleMultipleRowsCopy3Finish(MultipleRowsHelper helper) } static BulkCopyRowsCopied OracleMultipleRowsCopy3(MultipleRowsHelper helper, IEnumerable source) - => MultipleRowsCopyHelper(helper, source, null, OracleMultipleRowsCopy3Prep, OracleMultipleRowsCopy3Add, OracleMultipleRowsCopy3Finish); + => MultipleRowsCopyHelper(helper, source, null, OracleMultipleRowsCopy3Prep, OracleMultipleRowsCopy3Add, OracleMultipleRowsCopy3Finish, _maxParameters, _maxSqlLength); static Task OracleMultipleRowsCopy3Async(MultipleRowsHelper helper, IEnumerable source, CancellationToken cancellationToken) - => MultipleRowsCopyHelperAsync(helper, source, null, OracleMultipleRowsCopy3Prep, OracleMultipleRowsCopy3Add, OracleMultipleRowsCopy3Finish, cancellationToken); + => MultipleRowsCopyHelperAsync(helper, source, null, OracleMultipleRowsCopy3Prep, OracleMultipleRowsCopy3Add, OracleMultipleRowsCopy3Finish, cancellationToken, _maxParameters, _maxSqlLength); #if NATIVE_ASYNC static Task OracleMultipleRowsCopy3Async(MultipleRowsHelper helper, IAsyncEnumerable source, CancellationToken cancellationToken) - => MultipleRowsCopyHelperAsync(helper, source, null, OracleMultipleRowsCopy3Prep, OracleMultipleRowsCopy3Add, OracleMultipleRowsCopy3Finish, cancellationToken); + => MultipleRowsCopyHelperAsync(helper, source, null, OracleMultipleRowsCopy3Prep, OracleMultipleRowsCopy3Add, OracleMultipleRowsCopy3Finish, cancellationToken, _maxParameters, _maxSqlLength); #endif } } diff --git a/Source/LinqToDB/DataProvider/PostgreSQL/PostgreSQLBulkCopy.cs b/Source/LinqToDB/DataProvider/PostgreSQL/PostgreSQLBulkCopy.cs index 650b09cc36..cc5d4be280 100644 --- a/Source/LinqToDB/DataProvider/PostgreSQL/PostgreSQLBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/PostgreSQL/PostgreSQLBulkCopy.cs @@ -13,7 +13,9 @@ namespace LinqToDB.DataProvider.PostgreSQL { class PostgreSQLBulkCopy : BasicBulkCopy { - readonly PostgreSQLDataProvider _provider; + protected override int MaxParameters => 32767; + protected override int MaxSqlLength => 327670; + readonly PostgreSQLDataProvider _provider; public PostgreSQLBulkCopy(PostgreSQLDataProvider dataProvider) { diff --git a/Source/LinqToDB/DataProvider/SQLite/SQLiteBulkCopy.cs b/Source/LinqToDB/DataProvider/SQLite/SQLiteBulkCopy.cs index 2fc68a07ab..f21fec3040 100644 --- a/Source/LinqToDB/DataProvider/SQLite/SQLiteBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/SQLite/SQLiteBulkCopy.cs @@ -8,6 +8,9 @@ namespace LinqToDB.DataProvider.SQLite class SQLiteBulkCopy : BasicBulkCopy { + protected override int MaxParameters => 999; + protected override int MaxSqlLength => 1000000; + protected override BulkCopyRowsCopied MultipleRowsCopy( ITable table, BulkCopyOptions options, IEnumerable source) { diff --git a/Source/LinqToDB/DataProvider/SqlServer/SqlServerBulkCopy.cs b/Source/LinqToDB/DataProvider/SqlServer/SqlServerBulkCopy.cs index adabede828..d7b6c8bc39 100644 --- a/Source/LinqToDB/DataProvider/SqlServer/SqlServerBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/SqlServer/SqlServerBulkCopy.cs @@ -12,7 +12,9 @@ namespace LinqToDB.DataProvider.SqlServer class SqlServerBulkCopy : BasicBulkCopy { - private readonly SqlServerDataProvider _provider; + protected override int MaxParameters => 2099; + protected override int MaxSqlLength => 327670; + private readonly SqlServerDataProvider _provider; public SqlServerBulkCopy(SqlServerDataProvider provider) { diff --git a/Tests/Linq/Update/BulkCopyTests.cs b/Tests/Linq/Update/BulkCopyTests.cs index 65cd2b2c5c..509de77088 100644 --- a/Tests/Linq/Update/BulkCopyTests.cs +++ b/Tests/Linq/Update/BulkCopyTests.cs @@ -284,12 +284,12 @@ public void UseParametersTest([DataSources(false, ProviderName.DB2)] string cont using (db.BeginTransaction()) { var options = new BulkCopyOptions(){ UseParameters = true, MaxBatchSize = 50}; - var rowsToIns = Enumerable.Range(111001, 200) + var rowsToIns = Enumerable.Range(111001, 149) .Select(r => new Parent() {ParentID = r}).ToList(); db.Parent.BulkCopy(options, rowsToIns); Assert.AreEqual(rowsToIns.Count, db.Parent.Where(r => - r.ParentID >= 111001 && r.ParentID <= 111201).Count()); + r.ParentID >= rowsToIns.First().ParentID && r.ParentID <= rowsToIns.Last().ParentID).Count()); } } From cbec45a6616924a9b8109f4bcde76114abc41ca8 Mon Sep 17 00:00:00 2001 From: to11mtm Date: Sun, 2 May 2021 21:55:48 -0400 Subject: [PATCH 03/13] Add more DB and Param counts for DBs --- Source/LinqToDB/DataProvider/Access/AccessBulkCopy.cs | 2 ++ Source/LinqToDB/DataProvider/DB2/DB2BulkCopy.cs | 4 +++- Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs | 3 +++ Source/LinqToDB/DataProvider/Informix/InformixBulkCopy.cs | 3 ++- Source/LinqToDB/DataProvider/MySql/MySqlBulkCopy.cs | 7 ++++++- Source/LinqToDB/DataProvider/Sybase/SybaseBulkCopy.cs | 4 +++- 6 files changed, 19 insertions(+), 4 deletions(-) diff --git a/Source/LinqToDB/DataProvider/Access/AccessBulkCopy.cs b/Source/LinqToDB/DataProvider/Access/AccessBulkCopy.cs index c66180a906..70bc136ebe 100644 --- a/Source/LinqToDB/DataProvider/Access/AccessBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/Access/AccessBulkCopy.cs @@ -2,5 +2,7 @@ { class AccessBulkCopy : BasicBulkCopy { + protected override int MaxParameters => 767; + protected override int MaxSqlLength => 64000; } } diff --git a/Source/LinqToDB/DataProvider/DB2/DB2BulkCopy.cs b/Source/LinqToDB/DataProvider/DB2/DB2BulkCopy.cs index 72d1769c43..581540edcd 100644 --- a/Source/LinqToDB/DataProvider/DB2/DB2BulkCopy.cs +++ b/Source/LinqToDB/DataProvider/DB2/DB2BulkCopy.cs @@ -13,7 +13,9 @@ namespace LinqToDB.DataProvider.DB2 class DB2BulkCopy : BasicBulkCopy { - private readonly DB2DataProvider _provider; + protected override int MaxParameters => 1999; + protected override int MaxSqlLength => 327670; + private readonly DB2DataProvider _provider; public DB2BulkCopy(DB2DataProvider provider) { diff --git a/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs b/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs index fbc9c6a253..44020c079f 100644 --- a/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs @@ -8,6 +8,9 @@ namespace LinqToDB.DataProvider.Firebird class FirebirdBulkCopy : BasicBulkCopy { + // TODO: Firebird 2.5 has 64k limit, Firebird 3.0+ 10MB. Add Compat Switch + protected override int MaxSqlLength => 65535; + protected override BulkCopyRowsCopied MultipleRowsCopy( ITable table, BulkCopyOptions options, IEnumerable source) { diff --git a/Source/LinqToDB/DataProvider/Informix/InformixBulkCopy.cs b/Source/LinqToDB/DataProvider/Informix/InformixBulkCopy.cs index 40593c1cd4..553b05ee16 100644 --- a/Source/LinqToDB/DataProvider/Informix/InformixBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/Informix/InformixBulkCopy.cs @@ -13,7 +13,8 @@ namespace LinqToDB.DataProvider.Informix class InformixBulkCopy : BasicBulkCopy { - private readonly InformixDataProvider _provider; + protected override int MaxSqlLength => 32767; + private readonly InformixDataProvider _provider; public InformixBulkCopy(InformixDataProvider provider) { diff --git a/Source/LinqToDB/DataProvider/MySql/MySqlBulkCopy.cs b/Source/LinqToDB/DataProvider/MySql/MySqlBulkCopy.cs index baefd864df..35187a8030 100644 --- a/Source/LinqToDB/DataProvider/MySql/MySqlBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/MySql/MySqlBulkCopy.cs @@ -11,7 +11,12 @@ namespace LinqToDB.DataProvider.MySql class MySqlBulkCopy : BasicBulkCopy { - private readonly MySqlDataProvider _provider; + /// + /// MySQL supports more but realistically this might be too much already. + /// + protected override int MaxParameters => 32767; + protected override int MaxSqlLength => 327670; + private readonly MySqlDataProvider _provider; public MySqlBulkCopy(MySqlDataProvider provider) { diff --git a/Source/LinqToDB/DataProvider/Sybase/SybaseBulkCopy.cs b/Source/LinqToDB/DataProvider/Sybase/SybaseBulkCopy.cs index 4e2d8ebb68..0275495cbb 100644 --- a/Source/LinqToDB/DataProvider/Sybase/SybaseBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/Sybase/SybaseBulkCopy.cs @@ -14,7 +14,9 @@ namespace LinqToDB.DataProvider.Sybase // AseException : Incorrect syntax near ','. class SybaseBulkCopy : BasicBulkCopy { - private readonly SybaseDataProvider _provider; + protected override int MaxSqlLength => 327670; + protected override int MaxParameters => 1999; + private readonly SybaseDataProvider _provider; public SybaseBulkCopy(SybaseDataProvider provider) { From 74edd8c7b8c02841c23090db6ad041c11c26845d Mon Sep 17 00:00:00 2001 From: to11mtm Date: Fri, 28 May 2021 12:29:29 -0400 Subject: [PATCH 04/13] Adjust batch size in bulk copy methods --- Source/LinqToDB/DataProvider/BasicBulkCopy.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Source/LinqToDB/DataProvider/BasicBulkCopy.cs b/Source/LinqToDB/DataProvider/BasicBulkCopy.cs index 8f9b4e55bc..34d65e4484 100644 --- a/Source/LinqToDB/DataProvider/BasicBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/BasicBulkCopy.cs @@ -350,6 +350,7 @@ protected static BulkCopyRowsCopied MultipleRowsCopyHelper( int maxParameters, int maxSqlLength) { + var adjustedBatchSize = helper.Options.UseParameters ? Math.Min(helper.BatchSize, maxParameters / helper.Columns.Length): helper.BatchSize; prepFunction(helper); foreach (var item in source) @@ -360,7 +361,7 @@ protected static BulkCopyRowsCopied MultipleRowsCopyHelper( var needRemove = helper.Parameters.Count > maxParameters || helper.StringBuilder.Length > maxSqlLength; var isSingle = helper.CurrentCount == 1; - if (helper.CurrentCount >= helper.BatchSize || needRemove) + if (helper.CurrentCount >= adjustedBatchSize || needRemove) { if (needRemove && !isSingle) { @@ -398,6 +399,7 @@ protected static async Task MultipleRowsCopyHelperAsync( int maxParameters, int maxSqlLength) { + var adjustedBatchSize = helper.Options.UseParameters ? Math.Min(helper.BatchSize, maxParameters / helper.Columns.Length): helper.BatchSize; prepFunction(helper); foreach (var item in source) @@ -409,7 +411,7 @@ protected static async Task MultipleRowsCopyHelperAsync( var needRemove = helper.Parameters.Count > maxParameters || helper.StringBuilder.Length > maxSqlLength; var isSingle = helper.CurrentCount == 1; - if (helper.CurrentCount >= helper.BatchSize || needRemove) + if (helper.CurrentCount >= adjustedBatchSize || needRemove) { if (needRemove && !isSingle) { @@ -448,6 +450,7 @@ protected static async Task MultipleRowsCopyHelperAsync( int maxParameters, int maxSqlLength) { + var adjustedBatchSize = helper.Options.UseParameters ? Math.Min(helper.BatchSize, maxParameters / helper.Columns.Length): helper.BatchSize; prepFunction(helper); await foreach (var item in source.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext).WithCancellation(cancellationToken)) @@ -459,7 +462,7 @@ protected static async Task MultipleRowsCopyHelperAsync( var needRemove = helper.Parameters.Count > maxParameters || helper.StringBuilder.Length > maxSqlLength; var isSingle = helper.CurrentCount == 1; - if (helper.CurrentCount >= helper.BatchSize || needRemove) + if (helper.CurrentCount >= adjustedBatchSize || needRemove) { if (needRemove && !isSingle) { From f391ad99c9d929aa2313992078bf7dbbf497047e Mon Sep 17 00:00:00 2001 From: to11mtm Date: Sat, 29 May 2021 11:32:56 -0400 Subject: [PATCH 05/13] Add a little XMLDoc, Allow override of MaxParamters via BulkCopyOptions. --- Source/LinqToDB/Data/BulkCopyOptions.cs | 9 ++++++++- Source/LinqToDB/DataProvider/BasicBulkCopy.cs | 6 +++--- Tests/Linq/Update/BulkCopyTests.cs | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Source/LinqToDB/Data/BulkCopyOptions.cs b/Source/LinqToDB/Data/BulkCopyOptions.cs index c41c0185a5..8034b19763 100644 --- a/Source/LinqToDB/Data/BulkCopyOptions.cs +++ b/Source/LinqToDB/Data/BulkCopyOptions.cs @@ -90,8 +90,15 @@ public BulkCopyOptions(BulkCopyOptions options) public Action? RowsCopiedCallback { get; set; } /// - /// Gets or sets whether to Always use Parameters for MultipleRowsCopy. + /// Gets or sets whether to Always use Parameters for MultipleRowsCopy. Default is false. + /// If True, provider's override for will be used to determine the maximum number of rows per insert, + /// Unless overridden by . /// public bool UseParameters { get; set; } + + /// + /// If set, will override the Maximum parameters per batch statement from . + /// + public int? MaxParametersForBatch { get; set; } } } diff --git a/Source/LinqToDB/DataProvider/BasicBulkCopy.cs b/Source/LinqToDB/DataProvider/BasicBulkCopy.cs index 34d65e4484..2cec9b744c 100644 --- a/Source/LinqToDB/DataProvider/BasicBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/BasicBulkCopy.cs @@ -350,7 +350,7 @@ protected static BulkCopyRowsCopied MultipleRowsCopyHelper( int maxParameters, int maxSqlLength) { - var adjustedBatchSize = helper.Options.UseParameters ? Math.Min(helper.BatchSize, maxParameters / helper.Columns.Length): helper.BatchSize; + var adjustedBatchSize = helper.Options.UseParameters ? Math.Min(helper.BatchSize, helper.Options.MaxParametersForBatch.GetValueOrDefault(maxParameters) / helper.Columns.Length): helper.BatchSize; prepFunction(helper); foreach (var item in source) @@ -399,7 +399,7 @@ protected static async Task MultipleRowsCopyHelperAsync( int maxParameters, int maxSqlLength) { - var adjustedBatchSize = helper.Options.UseParameters ? Math.Min(helper.BatchSize, maxParameters / helper.Columns.Length): helper.BatchSize; + var adjustedBatchSize = helper.Options.UseParameters ? Math.Min(helper.BatchSize, helper.Options.MaxParametersForBatch.GetValueOrDefault(maxParameters) / helper.Columns.Length): helper.BatchSize; prepFunction(helper); foreach (var item in source) @@ -450,7 +450,7 @@ protected static async Task MultipleRowsCopyHelperAsync( int maxParameters, int maxSqlLength) { - var adjustedBatchSize = helper.Options.UseParameters ? Math.Min(helper.BatchSize, maxParameters / helper.Columns.Length): helper.BatchSize; + var adjustedBatchSize = helper.Options.UseParameters ? Math.Min(helper.BatchSize, helper.Options.MaxParametersForBatch.GetValueOrDefault(maxParameters) / helper.Columns.Length): helper.BatchSize; prepFunction(helper); await foreach (var item in source.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext).WithCancellation(cancellationToken)) diff --git a/Tests/Linq/Update/BulkCopyTests.cs b/Tests/Linq/Update/BulkCopyTests.cs index 509de77088..411eb5221b 100644 --- a/Tests/Linq/Update/BulkCopyTests.cs +++ b/Tests/Linq/Update/BulkCopyTests.cs @@ -278,7 +278,7 @@ public void ReuseOptionTest([DataSources(false, ProviderName.DB2)] string contex } [Test] - public void UseParametersTest([DataSources(false, ProviderName.DB2)] string context) + public void UseParametersTest([DataSources(false)] string context) { using (var db = new TestDataConnection(context)) using (db.BeginTransaction()) From 92ad5128e89ea8eded931ab4467021321db34f44 Mon Sep 17 00:00:00 2001 From: to11mtm Date: Sat, 29 May 2021 15:24:57 -0400 Subject: [PATCH 06/13] add references for max values. --- .../LinqToDB/DataProvider/Access/AccessBulkCopy.cs | 7 +++++++ Source/LinqToDB/DataProvider/DB2/DB2BulkCopy.cs | 8 ++++++++ .../DataProvider/Firebird/FirebirdBulkCopy.cs | 12 +++++++++++- Source/LinqToDB/DataProvider/MySql/MySqlBulkCopy.cs | 8 +++++++- .../LinqToDB/DataProvider/Oracle/OracleBulkCopy.cs | 10 +++++++++- .../DataProvider/PostgreSQL/PostgreSQLBulkCopy.cs | 10 +++++++++- .../LinqToDB/DataProvider/SQLite/SQLiteBulkCopy.cs | 11 ++++++++++- .../DataProvider/SqlServer/SqlServerBulkCopy.cs | 8 ++++++++ .../LinqToDB/DataProvider/Sybase/SybaseBulkCopy.cs | 10 +++++++++- 9 files changed, 78 insertions(+), 6 deletions(-) diff --git a/Source/LinqToDB/DataProvider/Access/AccessBulkCopy.cs b/Source/LinqToDB/DataProvider/Access/AccessBulkCopy.cs index 70bc136ebe..7532d79326 100644 --- a/Source/LinqToDB/DataProvider/Access/AccessBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/Access/AccessBulkCopy.cs @@ -2,7 +2,14 @@ { class AccessBulkCopy : BasicBulkCopy { + /// + /// Settings based on https://www.jooq.org/doc/3.12/manual/sql-building/dsl-context/custom-settings/settings-inline-threshold/ + /// We subtract 1 here to be safe since some ADO providers use parameter for command itself. + /// protected override int MaxParameters => 767; + /// + /// This max is based on https://support.microsoft.com/en-us/office/access-specifications-0cf3c66f-9cf2-4e32-9568-98c1025bb47c + /// protected override int MaxSqlLength => 64000; } } diff --git a/Source/LinqToDB/DataProvider/DB2/DB2BulkCopy.cs b/Source/LinqToDB/DataProvider/DB2/DB2BulkCopy.cs index 581540edcd..e2250b209a 100644 --- a/Source/LinqToDB/DataProvider/DB2/DB2BulkCopy.cs +++ b/Source/LinqToDB/DataProvider/DB2/DB2BulkCopy.cs @@ -13,7 +13,15 @@ namespace LinqToDB.DataProvider.DB2 class DB2BulkCopy : BasicBulkCopy { + /// + /// Settings based on https://www.ibm.com/docs/en/i/7.3?topic=reference-sql-limits + /// We subtract 1 here to be safe since some ADO providers use parameter for command itself. + /// protected override int MaxParameters => 1999; + /// + /// Setting based on https://www.ibm.com/docs/en/i/7.3?topic=reference-sql-limits + /// Max is actually 2MIB, but we keep a lower number here to avoid the cost of huge statements. + /// protected override int MaxSqlLength => 327670; private readonly DB2DataProvider _provider; diff --git a/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs b/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs index 44020c079f..6261d601e4 100644 --- a/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs @@ -8,9 +8,19 @@ namespace LinqToDB.DataProvider.Firebird class FirebirdBulkCopy : BasicBulkCopy { - // TODO: Firebird 2.5 has 64k limit, Firebird 3.0+ 10MB. Add Compat Switch + + /// + /// Number based on http://www.firebirdfaq.org/faq197/ + /// TODO: Add Compat Switch. Firebird 2.5 has 64k limit, Firebird 3.0+ 10MB. + /// protected override int MaxSqlLength => 65535; + /// + /// Based on https://github.com/FirebirdSQL/firebird/blob/799bca3ca5f9eb604433addc0f2b7cb3b6c07275/src/dsql/DsqlCompilerScratch.cpp#L528 + /// Max is 65536/2. We subtract one from that in case ADO provider uses parameter for statemnt. + /// + protected override int MaxParameters => 32767; + protected override BulkCopyRowsCopied MultipleRowsCopy( ITable table, BulkCopyOptions options, IEnumerable source) { diff --git a/Source/LinqToDB/DataProvider/MySql/MySqlBulkCopy.cs b/Source/LinqToDB/DataProvider/MySql/MySqlBulkCopy.cs index 35187a8030..c23309234a 100644 --- a/Source/LinqToDB/DataProvider/MySql/MySqlBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/MySql/MySqlBulkCopy.cs @@ -12,9 +12,15 @@ namespace LinqToDB.DataProvider.MySql class MySqlBulkCopy : BasicBulkCopy { /// - /// MySQL supports more but realistically this might be too much already. + /// Settings based on https://www.jooq.org/doc/3.12/manual/sql-building/dsl-context/custom-settings/settings-inline-threshold/ + /// MySQL supports more but realistically this might be too much already for practical cases. /// protected override int MaxParameters => 32767; + /// + /// MySQL can support much larger sizes, based on + /// https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_max_allowed_packet + /// But we keep a smaller number here to avoid choking the network. + /// protected override int MaxSqlLength => 327670; private readonly MySqlDataProvider _provider; diff --git a/Source/LinqToDB/DataProvider/Oracle/OracleBulkCopy.cs b/Source/LinqToDB/DataProvider/Oracle/OracleBulkCopy.cs index 07b16777f8..40819cab04 100644 --- a/Source/LinqToDB/DataProvider/Oracle/OracleBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/Oracle/OracleBulkCopy.cs @@ -14,8 +14,16 @@ namespace LinqToDB.DataProvider.Oracle class OracleBulkCopy : BasicBulkCopy { + /// + /// Settings based on https://www.jooq.org/doc/3.12/manual/sql-building/dsl-context/custom-settings/settings-inline-threshold/ + /// We subtract 1 based on possibility of provider using parameter for command. + /// private const int _maxParameters = 32766; - private const int _maxSqlLength = 327670; + /// + /// Setting is conservative, based on https://docs.oracle.com/cd/A58617_01/server.804/a58242/ch5.htm + /// Max is actually more arbitrary in later versions than Oracle 8. + /// + private const int _maxSqlLength = 65535; protected override int MaxParameters => _maxParameters; protected override int MaxSqlLength => _maxSqlLength; private readonly OracleDataProvider _provider; diff --git a/Source/LinqToDB/DataProvider/PostgreSQL/PostgreSQLBulkCopy.cs b/Source/LinqToDB/DataProvider/PostgreSQL/PostgreSQLBulkCopy.cs index cc5d4be280..86184bc571 100644 --- a/Source/LinqToDB/DataProvider/PostgreSQL/PostgreSQLBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/PostgreSQL/PostgreSQLBulkCopy.cs @@ -13,7 +13,15 @@ namespace LinqToDB.DataProvider.PostgreSQL { class PostgreSQLBulkCopy : BasicBulkCopy { - protected override int MaxParameters => 32767; + /// + /// Settings based on https://www.jooq.org/doc/3.12/manual/sql-building/dsl-context/custom-settings/settings-inline-threshold/ + /// We subtract 1 based on possibility of provider using parameter for command. + /// + protected override int MaxParameters => 32766; + /// + /// Setting based on https://stackoverflow.com/a/4937695/2937845 + /// Max is actually 2GiB, but we keep a lower number here to avoid the cost of huge statements. + /// protected override int MaxSqlLength => 327670; readonly PostgreSQLDataProvider _provider; diff --git a/Source/LinqToDB/DataProvider/SQLite/SQLiteBulkCopy.cs b/Source/LinqToDB/DataProvider/SQLite/SQLiteBulkCopy.cs index f21fec3040..0dcbe2ed80 100644 --- a/Source/LinqToDB/DataProvider/SQLite/SQLiteBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/SQLite/SQLiteBulkCopy.cs @@ -8,7 +8,16 @@ namespace LinqToDB.DataProvider.SQLite class SQLiteBulkCopy : BasicBulkCopy { - protected override int MaxParameters => 999; + + /// + /// Settings based on https://www.jooq.org/doc/3.12/manual/sql-building/dsl-context/custom-settings/settings-inline-threshold/ + /// We subtract 1 based on possibility of ADO Provider using parameter for command. + /// + protected override int MaxParameters => 998; + /// + /// Based on https://www.sqlite.org/limits.html. + /// Since SQLite is parsed locally by the lib, we aren't worried about network congestion and keep the max. + /// protected override int MaxSqlLength => 1000000; protected override BulkCopyRowsCopied MultipleRowsCopy( diff --git a/Source/LinqToDB/DataProvider/SqlServer/SqlServerBulkCopy.cs b/Source/LinqToDB/DataProvider/SqlServer/SqlServerBulkCopy.cs index d7b6c8bc39..a146bb9ed3 100644 --- a/Source/LinqToDB/DataProvider/SqlServer/SqlServerBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/SqlServer/SqlServerBulkCopy.cs @@ -12,7 +12,15 @@ namespace LinqToDB.DataProvider.SqlServer class SqlServerBulkCopy : BasicBulkCopy { + /// + /// Settings based on https://www.jooq.org/doc/3.12/manual/sql-building/dsl-context/custom-settings/settings-inline-threshold/ + /// We subtract 1 since SQL Server ADO Provider uses one parameter for command. + /// protected override int MaxParameters => 2099; + /// + /// Based on https://docs.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server?redirectedfrom=MSDN&view=sql-server-ver15 + /// Default Max is actually (4096*65536) = 256MIB, but we keep a lower number here to avoid the cost of huge statements. + /// protected override int MaxSqlLength => 327670; private readonly SqlServerDataProvider _provider; diff --git a/Source/LinqToDB/DataProvider/Sybase/SybaseBulkCopy.cs b/Source/LinqToDB/DataProvider/Sybase/SybaseBulkCopy.cs index 0275495cbb..86420b953e 100644 --- a/Source/LinqToDB/DataProvider/Sybase/SybaseBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/Sybase/SybaseBulkCopy.cs @@ -14,7 +14,15 @@ namespace LinqToDB.DataProvider.Sybase // AseException : Incorrect syntax near ','. class SybaseBulkCopy : BasicBulkCopy { - protected override int MaxSqlLength => 327670; + /// + /// Setting is conservative based on https://maxdb.sap.com/doc/7_6/f6/069940ccd42a54e10000000a1550b0/content.htm + /// Possible to be higher in other versions. + /// + protected override int MaxSqlLength => 65536; + /// + /// Settings based on https://www.jooq.org/doc/3.12/manual/sql-building/dsl-context/custom-settings/settings-inline-threshold/ + /// We subtract 1 based on possibility of provider using parameter for command. + /// protected override int MaxParameters => 1999; private readonly SybaseDataProvider _provider; From 7fab58678c9a0de8026b56ca96d3dc4edb3fd030 Mon Sep 17 00:00:00 2001 From: to11mtm Date: Sat, 29 May 2021 15:25:17 -0400 Subject: [PATCH 07/13] Quick test to confirm Firebird is angry about null + union --- Tests/Linq/Update/BulkCopyTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tests/Linq/Update/BulkCopyTests.cs b/Tests/Linq/Update/BulkCopyTests.cs index 411eb5221b..9e153d1efc 100644 --- a/Tests/Linq/Update/BulkCopyTests.cs +++ b/Tests/Linq/Update/BulkCopyTests.cs @@ -284,8 +284,9 @@ public void UseParametersTest([DataSources(false)] string context) using (db.BeginTransaction()) { var options = new BulkCopyOptions(){ UseParameters = true, MaxBatchSize = 50}; + var start = 111001; var rowsToIns = Enumerable.Range(111001, 149) - .Select(r => new Parent() {ParentID = r}).ToList(); + .Select(r => new Parent() {ParentID = r, Value1 = r-start}).ToList(); db.Parent.BulkCopy(options, rowsToIns); Assert.AreEqual(rowsToIns.Count, db.Parent.Where(r => From 009331e467961b2ca4490563303b4c82ddb9ee1a Mon Sep 17 00:00:00 2001 From: to11mtm Date: Mon, 31 May 2021 15:14:44 -0400 Subject: [PATCH 08/13] Try naieve version of MultipleRowsCopy3 switch where we cast all rows instead of just the first. --- Source/LinqToDB/DataProvider/BasicBulkCopy.cs | 4 ++- .../DataProvider/Firebird/FirebirdBulkCopy.cs | 4 ++- .../DataProvider/MultipleRowsHelper.cs | 26 +++++++++++++++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/Source/LinqToDB/DataProvider/BasicBulkCopy.cs b/Source/LinqToDB/DataProvider/BasicBulkCopy.cs index 2cec9b744c..21740c1719 100644 --- a/Source/LinqToDB/DataProvider/BasicBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/BasicBulkCopy.cs @@ -18,6 +18,8 @@ public class BasicBulkCopy protected virtual int MaxParameters => 999; protected virtual int MaxSqlLength => 100000; + protected virtual bool CastOnUnionAll => false; + public virtual BulkCopyRowsCopied BulkCopy(BulkCopyType bulkCopyType, ITable table, BulkCopyOptions options, IEnumerable source) where T : notnull { @@ -665,7 +667,7 @@ private void MultipleRowsCopy3Add(MultipleRowsHelper helper, object item, string helper.StringBuilder .AppendLine() .Append("\tSELECT "); - helper.BuildColumns(item); + helper.BuildColumns(item,castParameters:CastOnUnionAll); helper.StringBuilder.Append(from); helper.StringBuilder.Append(" UNION ALL"); diff --git a/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs b/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs index 6261d601e4..4103812c59 100644 --- a/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs @@ -19,7 +19,9 @@ class FirebirdBulkCopy : BasicBulkCopy /// Based on https://github.com/FirebirdSQL/firebird/blob/799bca3ca5f9eb604433addc0f2b7cb3b6c07275/src/dsql/DsqlCompilerScratch.cpp#L528 /// Max is 65536/2. We subtract one from that in case ADO provider uses parameter for statemnt. /// - protected override int MaxParameters => 32767; + protected override int MaxParameters => 32767; + + protected override bool CastOnUnionAll => true; protected override BulkCopyRowsCopied MultipleRowsCopy( ITable table, BulkCopyOptions options, IEnumerable source) diff --git a/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs b/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs index 4439ecb4a7..586ca65eb2 100644 --- a/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs +++ b/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs @@ -64,7 +64,9 @@ public void SetHeader() HeaderSize = StringBuilder.Length; } - public virtual void BuildColumns(object item, Func? skipConvert = null) + public virtual void BuildColumns(object item, + Func? skipConvert = null, + bool castParameters = false) { skipConvert ??= (_ => false); @@ -77,7 +79,27 @@ public virtual void BuildColumns(object item, Func? skip { var name = ParameterName == "?" ? ParameterName : ParameterName + ++ParameterIndex; - StringBuilder.Append(name); + if (castParameters) + { + StringBuilder.Append("CAST("); + StringBuilder.Append(name); + StringBuilder.Append(" AS "); + StringBuilder.Append(column.DataType); + if (!string.IsNullOrEmpty(column.DbType)) + StringBuilder.Append($":\"{column.DbType}\""); + + if (column.Length != 0) + StringBuilder.Append('(').Append(column.Length).Append(')'); + else if (column.Precision != 0) + StringBuilder.Append('(').Append(column.Precision).Append(',').Append(column.Scale).Append(')'); + + StringBuilder.Append(')'); + } + else + { + StringBuilder.Append(name); + } + if (value is DataParameter dataParameter) value = dataParameter.Value; From 7d91b5fe721fb2074dda373f7b9ac9c8d0ba267b Mon Sep 17 00:00:00 2001 From: to11mtm Date: Wed, 2 Jun 2021 20:02:18 -0400 Subject: [PATCH 09/13] Use SqlBuilder to build the parameter type info --- .../DataProvider/MultipleRowsHelper.cs | 29 +++++++++---------- .../LinqToDB/SqlProvider/BasicSqlBuilder.cs | 16 ++++++++++ Source/LinqToDB/SqlProvider/ISqlBuilder.cs | 1 + 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs b/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs index 586ca65eb2..47309fb87a 100644 --- a/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs +++ b/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs @@ -64,11 +64,13 @@ public void SetHeader() HeaderSize = StringBuilder.Length; } + private static Func? defaultSkipConvert = (_ => false); + public virtual void BuildColumns(object item, Func? skipConvert = null, bool castParameters = false) { - skipConvert ??= (_ => false); + skipConvert ??= MultipleRowsHelper.defaultSkipConvert; for (var i = 0; i < Columns.Length; i++) { @@ -79,21 +81,9 @@ public virtual void BuildColumns(object item, { var name = ParameterName == "?" ? ParameterName : ParameterName + ++ParameterIndex; - if (castParameters) + if (castParameters && CurrentCount == 0) { - StringBuilder.Append("CAST("); - StringBuilder.Append(name); - StringBuilder.Append(" AS "); - StringBuilder.Append(column.DataType); - if (!string.IsNullOrEmpty(column.DbType)) - StringBuilder.Append($":\"{column.DbType}\""); - - if (column.Length != 0) - StringBuilder.Append('(').Append(column.Length).Append(')'); - else if (column.Precision != 0) - StringBuilder.Append('(').Append(column.Precision).Append(',').Append(column.Scale).Append(')'); - - StringBuilder.Append(')'); + addParameterCasted(name, column); } else { @@ -119,6 +109,15 @@ public virtual void BuildColumns(object item, StringBuilder.Length--; } + private void addParameterCasted(string name, ColumnDescriptor column) + { + StringBuilder.Append("CAST("); + StringBuilder.Append(name); + StringBuilder.Append(" AS "); + SqlBuilder.BuildDataType(StringBuilder, new SqlDataType(column.GetDbDataType(true))); + StringBuilder.Append(')'); + } + public bool Execute() { DataConnection.Execute(StringBuilder.AppendLine().ToString(), Parameters.ToArray()); diff --git a/Source/LinqToDB/SqlProvider/BasicSqlBuilder.cs b/Source/LinqToDB/SqlProvider/BasicSqlBuilder.cs index 50878fa8bc..c1bc4fea4c 100644 --- a/Source/LinqToDB/SqlProvider/BasicSqlBuilder.cs +++ b/Source/LinqToDB/SqlProvider/BasicSqlBuilder.cs @@ -2713,6 +2713,22 @@ void BuildFunction(string name, ISqlExpression[] exprs) #endregion #region BuildDataType + + /// + /// Appends an 's String to a provided + /// + /// + /// + /// The stringbuilder with the type information appended. + public StringBuilder BuildDataType(StringBuilder sb, + SqlDataType dataType) + { + WithStringBuilder(sb, () => + { + BuildDataType(dataType, false); + }); + return sb; + } protected void BuildDataType(SqlDataType type, bool forCreateTable) { if (!string.IsNullOrEmpty(type.Type.DbType)) diff --git a/Source/LinqToDB/SqlProvider/ISqlBuilder.cs b/Source/LinqToDB/SqlProvider/ISqlBuilder.cs index 2c7fe313a5..9d697475c0 100644 --- a/Source/LinqToDB/SqlProvider/ISqlBuilder.cs +++ b/Source/LinqToDB/SqlProvider/ISqlBuilder.cs @@ -15,6 +15,7 @@ public interface ISqlBuilder StringBuilder ConvertTableName (StringBuilder sb, string? server, string? database, string? schema, string table, TableOptions tableOptions); StringBuilder BuildTableName (StringBuilder sb, string? server, string? database, string? schema, string table, TableOptions tableOptions); + StringBuilder BuildDataType (StringBuilder sb, SqlDataType dataType); string? GetTableServerName (SqlTable table); string? GetTableDatabaseName (SqlTable table); string? GetTableSchemaName (SqlTable table); From 12b17480fc735a50e2ca8a0049963ea734152480 Mon Sep 17 00:00:00 2001 From: MaceWindu Date: Thu, 3 Jun 2021 10:40:54 +0200 Subject: [PATCH 10/13] address firebird issue --- Source/LinqToDB/DataProvider/BasicBulkCopy.cs | 9 +++++---- .../DataProvider/Firebird/FirebirdBulkCopy.cs | 3 ++- .../DataProvider/MultipleRowsHelper.cs | 18 ++++++++++-------- .../DataProvider/Oracle/OracleBulkCopy.cs | 4 ++-- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/Source/LinqToDB/DataProvider/BasicBulkCopy.cs b/Source/LinqToDB/DataProvider/BasicBulkCopy.cs index 21740c1719..119acfea2a 100644 --- a/Source/LinqToDB/DataProvider/BasicBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/BasicBulkCopy.cs @@ -18,7 +18,8 @@ public class BasicBulkCopy protected virtual int MaxParameters => 999; protected virtual int MaxSqlLength => 100000; - protected virtual bool CastOnUnionAll => false; + protected virtual bool CastOnUnionAll => false; + protected virtual bool TypeAllUnionParameters => false; public virtual BulkCopyRowsCopied BulkCopy(BulkCopyType bulkCopyType, ITable table, BulkCopyOptions options, IEnumerable source) where T : notnull @@ -548,7 +549,7 @@ private void MultipleRowsCopy1Add(MultipleRowsHelper helper, object item, string helper.StringBuilder .AppendLine() .Append('('); - helper.BuildColumns(item!); + helper.BuildColumns(item); helper.StringBuilder.Append("),"); helper.RowsCopied.RowsCopied++; @@ -612,7 +613,7 @@ private void MultipleRowsCopy2Add(MultipleRowsHelper helper, object item, string helper.StringBuilder .AppendLine() .Append("SELECT "); - helper.BuildColumns(item!); + helper.BuildColumns(item, castParameters: CastOnUnionAll, castAllRows: TypeAllUnionParameters); helper.StringBuilder.Append(from); helper.StringBuilder.Append(" UNION ALL"); @@ -667,7 +668,7 @@ private void MultipleRowsCopy3Add(MultipleRowsHelper helper, object item, string helper.StringBuilder .AppendLine() .Append("\tSELECT "); - helper.BuildColumns(item,castParameters:CastOnUnionAll); + helper.BuildColumns(item, castParameters: CastOnUnionAll, castAllRows : TypeAllUnionParameters); helper.StringBuilder.Append(from); helper.StringBuilder.Append(" UNION ALL"); diff --git a/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs b/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs index 4103812c59..f0dc68374b 100644 --- a/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/Firebird/FirebirdBulkCopy.cs @@ -21,7 +21,8 @@ class FirebirdBulkCopy : BasicBulkCopy /// protected override int MaxParameters => 32767; - protected override bool CastOnUnionAll => true; + protected override bool CastOnUnionAll => true; + protected override bool TypeAllUnionParameters => true; protected override BulkCopyRowsCopied MultipleRowsCopy( ITable table, BulkCopyOptions options, IEnumerable source) diff --git a/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs b/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs index b6c5116187..ec0ba70e20 100644 --- a/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs +++ b/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs @@ -71,13 +71,15 @@ public void SetHeader() HeaderSize = StringBuilder.Length; } - private static Func? defaultSkipConvert = (_ => false); + private static Func defaultSkipConvert = (_ => false); - public virtual void BuildColumns(object item, - Func? skipConvert = null, - bool castParameters = false) + public virtual void BuildColumns( + object item, + Func? skipConvert = null, + bool castParameters = false, + bool castAllRows = false) { - skipConvert ??= MultipleRowsHelper.defaultSkipConvert; + skipConvert ??= defaultSkipConvert; for (var i = 0; i < Columns.Length; i++) { @@ -88,9 +90,9 @@ public virtual void BuildColumns(object item, { var name = ParameterName == "?" ? ParameterName : ParameterName + ++ParameterIndex; - if (castParameters && CurrentCount == 0) + if (castParameters && (CurrentCount == 0 || castAllRows)) { - addParameterCasted(name, column); + AddParameterCasted(name, column); } else { @@ -116,7 +118,7 @@ public virtual void BuildColumns(object item, StringBuilder.Length--; } - private void addParameterCasted(string name, ColumnDescriptor column) + private void AddParameterCasted(string name, ColumnDescriptor column) { StringBuilder.Append("CAST("); StringBuilder.Append(name); diff --git a/Source/LinqToDB/DataProvider/Oracle/OracleBulkCopy.cs b/Source/LinqToDB/DataProvider/Oracle/OracleBulkCopy.cs index f6de18145c..ddb5a94faa 100644 --- a/Source/LinqToDB/DataProvider/Oracle/OracleBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/Oracle/OracleBulkCopy.cs @@ -210,7 +210,7 @@ static void OracleMultipleRowsCopy1Add(MultipleRowsHelper helper, object item, s helper.StringBuilder.Length -= 2; helper.StringBuilder.Append(") VALUES ("); - helper.BuildColumns(item!, _ => _.DataType == DataType.Text || _.DataType == DataType.NText); + helper.BuildColumns(item, _ => _.DataType == DataType.Text || _.DataType == DataType.NText); helper.StringBuilder.AppendLine(")"); helper.RowsCopied.RowsCopied++; @@ -411,7 +411,7 @@ static void OracleMultipleRowsCopy3Add(MultipleRowsHelper helper, object item, s helper.StringBuilder .AppendLine() .Append("\tSELECT "); - helper.BuildColumns(item!, _ => _.DataType == DataType.Text || _.DataType == DataType.NText); + helper.BuildColumns(item, _ => _.DataType == DataType.Text || _.DataType == DataType.NText); helper.StringBuilder.Append(" FROM DUAL "); helper.StringBuilder.Append(" UNION ALL"); From 1a94108a0d5ae7af0df64c923f9733977cfa0a7a Mon Sep 17 00:00:00 2001 From: MaceWindu Date: Thu, 3 Jun 2021 11:53:08 +0200 Subject: [PATCH 11/13] fix warning --- Source/LinqToDB/DataProvider/SqlServer/SqlServerBulkCopy.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/LinqToDB/DataProvider/SqlServer/SqlServerBulkCopy.cs b/Source/LinqToDB/DataProvider/SqlServer/SqlServerBulkCopy.cs index 1209fe1e51..d8fd57d6c1 100644 --- a/Source/LinqToDB/DataProvider/SqlServer/SqlServerBulkCopy.cs +++ b/Source/LinqToDB/DataProvider/SqlServer/SqlServerBulkCopy.cs @@ -18,7 +18,7 @@ class SqlServerBulkCopy : BasicBulkCopy /// protected override int MaxParameters => 2099; /// - /// Based on https://docs.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server?redirectedfrom=MSDN&view=sql-server-ver15 + /// Based on https://docs.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server?redirectedfrom=MSDN&view=sql-server-ver15 /// Default Max is actually (4096*65536) = 256MIB, but we keep a lower number here to avoid the cost of huge statements. /// protected override int MaxSqlLength => 327670; From 6b3a92180bc804c8fab7290ced44e6a20e098e94 Mon Sep 17 00:00:00 2001 From: MaceWindu Date: Thu, 3 Jun 2021 13:02:26 +0200 Subject: [PATCH 12/13] specify copy type in test --- Tests/Linq/Update/BulkCopyTests.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Tests/Linq/Update/BulkCopyTests.cs b/Tests/Linq/Update/BulkCopyTests.cs index 561aebfa44..1db2b9f1b2 100644 --- a/Tests/Linq/Update/BulkCopyTests.cs +++ b/Tests/Linq/Update/BulkCopyTests.cs @@ -283,15 +283,17 @@ public void UseParametersTest([DataSources(false)] string context) using (var db = new TestDataConnection(context)) using (db.BeginTransaction()) { - var options = new BulkCopyOptions(){ UseParameters = true, MaxBatchSize = 50}; - var start = 111001; - var rowsToIns = Enumerable.Range(111001, 149) + var options = new BulkCopyOptions(){ UseParameters = true, MaxBatchSize = 50, BulkCopyType = BulkCopyType.MultipleRows }; + var start = 111001; + + var rowsToInsert = Enumerable.Range(start, 149) .Select(r => new Parent() {ParentID = r, Value1 = r-start}).ToList(); - db.Parent.BulkCopy(options, rowsToIns); - Assert.AreEqual(rowsToIns.Count, - db.Parent.Where(r => - r.ParentID >= rowsToIns.First().ParentID && r.ParentID <= rowsToIns.Last().ParentID).Count()); + db.Parent.BulkCopy(options, rowsToInsert); + + Assert.AreEqual(rowsToInsert.Count, + db.Parent.Where(r => + r.ParentID >= rowsToInsert[0].ParentID && r.ParentID <= rowsToInsert.Last().ParentID).Count()); } } From ed372fc37baf9ea61293c056a47a2643b3fccbf1 Mon Sep 17 00:00:00 2001 From: MaceWindu Date: Thu, 3 Jun 2021 16:08:28 +0200 Subject: [PATCH 13/13] fix column datatype resolve --- Source/LinqToDB/DataProvider/MultipleRowsHelper.cs | 6 +++--- Source/LinqToDB/Mapping/ColumnDescriptor.cs | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs b/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs index ec0ba70e20..1fac5696ae 100644 --- a/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs +++ b/Source/LinqToDB/DataProvider/MultipleRowsHelper.cs @@ -92,7 +92,7 @@ public virtual void BuildColumns( if (castParameters && (CurrentCount == 0 || castAllRows)) { - AddParameterCasted(name, column); + AddParameterCasted(name, ColumnTypes[i]); } else { @@ -118,12 +118,12 @@ public virtual void BuildColumns( StringBuilder.Length--; } - private void AddParameterCasted(string name, ColumnDescriptor column) + private void AddParameterCasted(string name, SqlDataType type) { StringBuilder.Append("CAST("); StringBuilder.Append(name); StringBuilder.Append(" AS "); - SqlBuilder.BuildDataType(StringBuilder, new SqlDataType(column.GetDbDataType(true))); + SqlBuilder.BuildDataType(StringBuilder, type); StringBuilder.Append(')'); } diff --git a/Source/LinqToDB/Mapping/ColumnDescriptor.cs b/Source/LinqToDB/Mapping/ColumnDescriptor.cs index ac22a9549a..1fc4a73051 100644 --- a/Source/LinqToDB/Mapping/ColumnDescriptor.cs +++ b/Source/LinqToDB/Mapping/ColumnDescriptor.cs @@ -490,6 +490,8 @@ public static DataType CalculateDataType(MappingSchema mappingSchema, Type syste if (dataType == DataType.Undefined) dataType = mappingSchema.GetDataType(systemType).Type.DataType; + if (dataType == DataType.Undefined) + dataType = mappingSchema.GetUnderlyingDataType(systemType, out var _).Type.DataType; return dataType; }