Merge pull request #334 from paillave/V2

V2

documentation/src/components/Presentation.tsx

@@ -15,7 +15,7 @@ export default function Presentation() {
         </div> */}
       <div className={styles.paragraphDescription}>
         <h1>Data processing</h1>
-        <p>ETL.NET is a framework for .NET to <b>implement with no effort fast and easy to maintain data processes</b>. All the tooling for <b>normalization</b>, <b>upsert</b>, <b>lookup</b> or <b>join</b> dramatically reduces the effort for any import and transformation purpose. Everything to handle <b>tracing</b>, <b>error tracking</b> is done automatically for the developer.</p>
+        <p>ETL.NET is a framework for .NET to <b>implement with no effort fast, low memory impact and easy to maintain data processes</b>. All the tooling for <b>normalization</b>, <b>upsert</b>, <b>lookup</b> or <b>join</b> dramatically reduces the effort for any import and transformation purpose. Everything to handle <b>tracing</b>, <b>error tracking</b> is done automatically for the developer.</p>
         <div style={{ display: "flex", justifyContent: "space-between" }}>
           <p>Read or write <b>any file type</b> and any data source
             <ul>

src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EntityFrameworkCoreExtension</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.EntityFrameworkCoreExtension/Searcher/SearchDescriptorBase.cs

@@ -17,6 +17,7 @@ public abstract class SearchDescriptorBase<TEntity, TId> : ISearchDescriptor whe
         // \.(?=(?:[^"]*"[^"]*")*(?![^"]*"))
         //private static Regex _regexSplitter = new Regex("\\.(?=(?:[^\"]*\"[^\"]*\")*(?![^\"]*\"))", RegexOptions.Singleline);
         protected DbContext DbContext { get; }
+        private Func<IQueryable<TEntity>, IQueryable<TEntity>> _include = null;
         public SearchDescriptorBase(DbContext dbContext)
         {
             this.DbContext = dbContext;
@@ -44,6 +45,11 @@ public SearchDescriptorBase<TEntity, TId> AddValue<TValue>(string name, bool isO
         }
         public SearchDescriptorBase<TEntity, TId> AddValue(string name, bool isOnKey, Expression<Func<TEntity, string>> getValue)
             => this.AddValue(name, isOnKey, getValue, i => i);
+        public SearchDescriptorBase<TEntity, TId> SetIncludes(Func<IQueryable<TEntity>, IQueryable<TEntity>> include)
+        {
+            _include = include;
+            return this;
+        }
         public SearchDescriptorBase<TEntity, TId> AddNavigation<TTarget, TTargetId>(string name, SearchDescriptorBase<TTarget, TTargetId> levelDescriptor, Expression<Func<TEntity, TTarget>> getTargetExpression) where TTarget : class
         {
             var navigationProperty = new NavigationSelector<TEntity, TTarget, TTargetId>(name, getTargetExpression, levelDescriptor);
@@ -157,8 +163,7 @@ public Dictionary<GroupingValue, List<TEntity>> Search(Dictionary<string, List<s
         private Dictionary<GroupingValue, List<TEntity>> GroupQueryable<TKey>(IQueryable<TEntity> queryable, Expression<Func<TEntity, TKey>> groupingExpression)
         {
             var entityParameterExpression = Expression.Parameter(typeof(TEntity), "entity");
-
-            var keyedRowType = typeof(KeyedRow<>).MakeGenericType(typeof(TEntity), typeof(TId) ,typeof(TKey));
+            var keyedRowType = typeof(KeyedRow<>).MakeGenericType(typeof(TEntity), typeof(TId), typeof(TKey));
             var constructorInfo = keyedRowType.GetConstructor(new Type[] { });
             var callConstructor = Expression.New(constructorInfo);
             var initializedObject = Expression.MemberInit(callConstructor,
@@ -181,9 +186,23 @@ public virtual List<TId> SearchIds(Dictionary<string, List<string>> filters)
         {
             var getId = this.GetIdExpression.Compile();
             var defaultValue = this.DefaultValueProperty.GetValueExpression.Compile() as Func<TEntity, object>;
-            return this.Search(filters, pathToGroupProperty).ToDictionary(i => i.Key, i => i.Value.Select(j => (id: getId(j), label: defaultValue == null ? getId(j).ToString() : defaultValue(j)?.ToString())).ToList());
+            return this.Search(filters, pathToGroupProperty).ToDictionary(i => i.Key, i => i.Value.Select(j =>
+            {
+                var id = getId(j);
+                string label = getId(j).ToString();
+                try { label = defaultValue(j)?.ToString(); }
+                catch { }
+                return (id, label);
+            }).ToList());
+        }
+        protected virtual IQueryable<TEntity> GetQueryable() => Include(this.DbContext.Set<TEntity>().AsQueryable());
+
+        protected IQueryable<TEntity> Include(IQueryable<TEntity> query)
+        {
+            if (_include == null)
+                return query;
+            return _include(query);
         }
-        protected virtual IQueryable<TEntity> GetQueryable() => this.DbContext.Set<TEntity>().AsQueryable();
         private class KeyedRow<TKey>
         {
             public TKey Key { get; set; }

src/Paillave.Etl.Autofac/Paillave.Etl.Autofac.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Autofac</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.Bloomberg/Paillave.Etl.Bloomberg.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Bloomberg</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.Dropbox/Paillave.Etl.Dropbox.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Dropbox</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.EntityFrameworkCore/Paillave.Etl.EntityFrameworkCore.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.EntityFrameworkCore</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.ExcelFile/Paillave.Etl.ExcelFile.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.ExcelFile</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.ExecutionToolkit/Paillave.Etl.ExecutionToolkit.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.ExecutionToolkit</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.FileSystem/Paillave.Etl.FileSystem.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.FileSystem</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.FromConfigurationConnectors/Paillave.Etl.FromConfigurationConnectors.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.FromConfigurationConnectors</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.Ftp/Paillave.Etl.Ftp.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Ftp</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.Mail/Paillave.Etl.Mail.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Mail</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.Pdf/Paillave.Etl.Pdf.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Pdf</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.Sftp/Paillave.Etl.Sftp.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Sftp</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.SqlServer/Paillave.Etl.SqlServer.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.SqlServer</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.TextFile/Paillave.Etl.TextFile.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.TextFile</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.XmlFile/Paillave.Etl.XmlFile.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.XmlFile</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl.Zip/Paillave.Etl.Zip.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Zip</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Etl/Paillave.Etl.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.EtlNet.Core</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Pdf/Paillave.Pdf.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <PackageId>Paillave.Pdf</PackageId>
-    <Version>2.0.23</Version>
+    <Version>2.0.24</Version>
     <Authors>Stéphane Royer</Authors>
     <Company></Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

src/Paillave.Pdf/PdfReader.cs

@@ -47,6 +47,7 @@ internal virtual List<ProcessedBlock> ExtractTextBlocks(Page page)
         {
             var words = page.GetWords(this.WordExtractor);
             var blocks = GetTextGroups(page, words)
+                .Where(i => i.BoundingBox.Rotation < 10 || i.BoundingBox.Rotation > 350)
                 // .OrderByDescending(i => /*i.TextOrientation==Tex*/  i.BoundingBox.Top)
                 .ToList();
             return blocks.Select(i => new ProcessedBlock { KeepTogether = false, TextBlock = i }).ToList();//.SelectMany(i => i.TextLines).ToList();