Fixed minor concat bugs

ConcatenateIndexFiles now removes duplicate files from the directory search. This makes it wasier to omit the directory filter argument which does not have enough expressiveness to select all relevant directories, while not including sub-directories - which would lead to "System.ArgumentException: An entry with the same key already exists." errors. If there are legitimately duplicate dates, I've changed the code so it returns a better, more readable error.
QutEcoacoustics · Mar 25, 2019 · e35260d · e35260d
1 parent 6cbf3da
commit e35260d
Show file tree

Hide file tree

Showing 7 changed files with 43 additions and 16 deletions.
diff --git a/src/Acoustics.Shared/Acoustics.Shared.csproj b/src/Acoustics.Shared/Acoustics.Shared.csproj
@@ -223,6 +223,7 @@
     <Compile Include="Formatters.cs" />
     <Compile Include="GraphicsSegmented.cs" />
     <Compile Include="ImageTooLargeForGraphicsException.cs" />
+    <Compile Include="InvalidDataSetException.cs" />
     <Compile Include="Json.cs" />
     <Compile Include="Logging\Log4NetTextWriter.cs" />
     <Compile Include="Logging\LoggedConsole.cs" />

diff --git a/src/Acoustics.Shared/FileDateHelpers.cs b/src/Acoustics.Shared/FileDateHelpers.cs
@@ -1,4 +1,4 @@
-// --------------------------------------------------------------------------------------------------------------------
+// --------------------------------------------------------------------------------------------------------------------
 // <copyright file="FileDateHelpers.cs" company="QutEcoacoustics">
 // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group).
 // </copyright>
@@ -27,13 +27,13 @@ public class FileDateHelpers
         {
                 "yyyyMMdd[-|T|_]HHmmss (if timezone offset hint provided)",
                 "yyyyMMdd[-|T|_]HHmmssZ",
-            };
+        };
 
         private static readonly string[] AcceptedFormatsTimeZone =
         {
                 "yyyyMMdd[-|T|_]HHmmss[+|-]HH",
                 "yyyyMMdd[-|T|_]HHmmss[+|-]HHmm",
-            };
+        };
 
         internal static readonly DateVariants[] PossibleFormats =
             {
@@ -79,9 +79,15 @@ public static SortedDictionary<DateTimeOffset, FileInfo> FilterFilesForDates(IEn
             var datesAndFiles = new SortedDictionary<DateTimeOffset, FileInfo>();
             foreach (var file in files)
             {
-                DateTimeOffset parsedDate;
-                if (FileNameContainsDateTime(file.Name, out parsedDate, offsetHint))
+                if (FileNameContainsDateTime(file.Name, out var parsedDate, offsetHint))
                 {
+                    if (datesAndFiles.ContainsKey(parsedDate))
+                    {
+                        string message =
+                            $"There was a duplicate date. File {file} with date {parsedDate,'r'} conflicts with existing file {datesAndFiles[parsedDate]}";
+                        throw new InvalidDataSetException(message);
+                    }
+
                     datesAndFiles.Add(parsedDate, file);
                 }
             }

diff --git a/src/Acoustics.Shared/InvalidDataSetException.cs b/src/Acoustics.Shared/InvalidDataSetException.cs
@@ -0,0 +1,16 @@
+// <copyright file="InvalidDataSetException.cs" company="QutEcoacoustics">
+// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group).
+// </copyright>
+
+namespace Acoustics.Shared
+{
+    using System;
+
+    public class InvalidDataSetException : Exception
+    {
+        public InvalidDataSetException(string message)
+            : base(message)
+        {
+        }
+    }
+}
diff --git a/src/AnalysisPrograms/Production/Exceptions.cs b/src/AnalysisPrograms/Production/Exceptions.cs
@@ -88,6 +88,10 @@ private static Dictionary<Type, ExceptionStyle> CreateExceptionMap()
                     typeof(InvalidAudioChannelException),
                     new ExceptionStyle() { ErrorCode = 105, PrintUsage = false }
                 },
+                {
+                    typeof(InvalidDataSetException),
+                    new ExceptionStyle() {ErrorCode = 106, PrintUsage = false }
+                },
                 {
                     typeof(AnalysisOptionDevilException),
                     new ExceptionStyle

diff --git a/src/AudioAnalysisTools/AudioAnalysisTools.csproj b/src/AudioAnalysisTools/AudioAnalysisTools.csproj
@@ -115,6 +115,9 @@
     <Reference Include="Microsoft.Win32.Primitives, Version=4.0.2.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
       <HintPath>..\..\packages\Microsoft.Win32.Primitives.4.3.0\lib\net46\Microsoft.Win32.Primitives.dll</HintPath>
     </Reference>
+    <Reference Include="MoreLinq, Version=3.1.0.0, Culture=neutral, PublicKeyToken=384d532d7e88985d, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\morelinq.3.1.0\lib\net451\MoreLinq.dll</HintPath>
+    </Reference>
     <Reference Include="Newtonsoft.Json, Version=11.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
       <HintPath>..\..\packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
     </Reference>
@@ -205,7 +208,9 @@
       <HintPath>..\..\packages\System.Security.Cryptography.X509Certificates.4.3.0\lib\net461\System.Security.Cryptography.X509Certificates.dll</HintPath>
     </Reference>
     <Reference Include="System.ValueTuple, Version=4.0.2.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\System.ValueTuple.4.4.0-preview2-25405-01\lib\net461\System.ValueTuple.dll</HintPath>
+      <HintPath>..\..\packages\System.ValueTuple.4.4.0\lib\net461\System.ValueTuple.dll</HintPath>
+      <Private>True</Private>
+      <Private>True</Private>
     </Reference>
     <Reference Include="System.Xml.Linq">
       <RequiredTargetFramework>3.5</RequiredTargetFramework>

diff --git a/src/AudioAnalysisTools/Indices/IndexMatrices.cs b/src/AudioAnalysisTools/Indices/IndexMatrices.cs
@@ -21,6 +21,7 @@ namespace AudioAnalysisTools.Indices
     using DSP;
     using StandardSpectrograms;
     using log4net;
+    using MoreLinq.Extensions;
     using TowseyLibrary;
     using Zio;
 
@@ -357,7 +358,7 @@ public static FileInfo[] GetFilesInDirectory(string path, string pattern)
         }
 
         /// <summary>
-        /// Returns a sorted list of file paths, sorted on file name.
+        /// Returns a unique, sorted, list of file paths, sorted on file name.
         /// IMPORTANT: Sorts on alphanumerics, NOT on date or time encoded in the file name.
         /// </summary>
         public static FileInfo[] GetFilesInDirectories(DirectoryInfo[] directories, string pattern)
@@ -378,14 +379,7 @@ public static FileInfo[] GetFilesInDirectories(DirectoryInfo[] directories, stri
                 fileList.AddRange(files);
             }
 
-            //if (fileList.Count == 0)
-            //{
-            //    // No need for this warning. It comes later.
-            //    LoggedConsole.WriteErrorLine($"No file names match pattern <{pattern}>. Returns empty list of files");
-            //}
-
-            FileInfo[] returnList = fileList.ToArray();
-            Array.Sort(returnList, (f1, f2) => f1.Name.CompareTo(f2.Name));
+            FileInfo[] returnList = fileList.DistinctBy(x => x.FullName).OrderBy(x => x.Name).ToArray();
 
             return returnList;
         }

diff --git a/src/AudioAnalysisTools/packages.config b/src/AudioAnalysisTools/packages.config
@@ -19,6 +19,7 @@
   <package id="Microsoft.Net.Http" version="2.2.29" targetFramework="net462" />
   <package id="Microsoft.NETCore.Platforms" version="1.1.0" targetFramework="net462" />
   <package id="Microsoft.Win32.Primitives" version="4.3.0" targetFramework="net462" />
+  <package id="morelinq" version="3.1.0" targetFramework="net462" />
   <package id="NETStandard.Library" version="1.6.1" targetFramework="net462" />
   <package id="Newtonsoft.Json" version="11.0.2" targetFramework="net462" />
   <package id="StyleCop.Analyzers" version="1.1.1-beta.61" targetFramework="net462" developmentDependency="true" />
@@ -64,7 +65,7 @@
   <package id="System.Threading" version="4.3.0" targetFramework="net462" />
   <package id="System.Threading.Tasks" version="4.3.0" targetFramework="net462" />
   <package id="System.Threading.Timer" version="4.3.0" targetFramework="net462" />
-  <package id="System.ValueTuple" version="4.4.0-preview2-25405-01" targetFramework="net462" />
+  <package id="System.ValueTuple" version="4.4.0" targetFramework="net462" />
   <package id="System.Xml.ReaderWriter" version="4.3.0" targetFramework="net462" />
   <package id="System.Xml.XDocument" version="4.3.0" targetFramework="net462" />
   <package id="YamlDotNet" version="5.3.0" targetFramework="net462" />