From e35260d665f071d895aebe5cfc52d06cb1b052d7 Mon Sep 17 00:00:00 2001 From: Anthony Truskinger Date: Mon, 25 Mar 2019 10:29:15 +1000 Subject: [PATCH] Fixed minor concat bugs ConcatenateIndexFiles now removes duplicate files from the directory search. This makes it wasier to omit the directory filter argument which does not have enough expressiveness to select all relevant directories, while not including sub-directories - which would lead to "System.ArgumentException: An entry with the same key already exists." errors. If there are legitimately duplicate dates, I've changed the code so it returns a better, more readable error. --- src/Acoustics.Shared/Acoustics.Shared.csproj | 1 + src/Acoustics.Shared/FileDateHelpers.cs | 16 +++++++++++----- src/Acoustics.Shared/InvalidDataSetException.cs | 16 ++++++++++++++++ src/AnalysisPrograms/Production/Exceptions.cs | 4 ++++ src/AudioAnalysisTools/AudioAnalysisTools.csproj | 7 ++++++- src/AudioAnalysisTools/Indices/IndexMatrices.cs | 12 +++--------- src/AudioAnalysisTools/packages.config | 3 ++- 7 files changed, 43 insertions(+), 16 deletions(-) create mode 100644 src/Acoustics.Shared/InvalidDataSetException.cs diff --git a/src/Acoustics.Shared/Acoustics.Shared.csproj b/src/Acoustics.Shared/Acoustics.Shared.csproj index 725913be9..266e115bb 100644 --- a/src/Acoustics.Shared/Acoustics.Shared.csproj +++ b/src/Acoustics.Shared/Acoustics.Shared.csproj @@ -223,6 +223,7 @@ + diff --git a/src/Acoustics.Shared/FileDateHelpers.cs b/src/Acoustics.Shared/FileDateHelpers.cs index 45fc14e5b..9ea1c7e40 100644 --- a/src/Acoustics.Shared/FileDateHelpers.cs +++ b/src/Acoustics.Shared/FileDateHelpers.cs @@ -1,4 +1,4 @@ -// -------------------------------------------------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). // @@ -27,13 +27,13 @@ public class FileDateHelpers { "yyyyMMdd[-|T|_]HHmmss (if timezone offset hint provided)", "yyyyMMdd[-|T|_]HHmmssZ", - }; + }; private static readonly string[] AcceptedFormatsTimeZone = { "yyyyMMdd[-|T|_]HHmmss[+|-]HH", "yyyyMMdd[-|T|_]HHmmss[+|-]HHmm", - }; + }; internal static readonly DateVariants[] PossibleFormats = { @@ -79,9 +79,15 @@ public static SortedDictionary FilterFilesForDates(IEn var datesAndFiles = new SortedDictionary(); foreach (var file in files) { - DateTimeOffset parsedDate; - if (FileNameContainsDateTime(file.Name, out parsedDate, offsetHint)) + if (FileNameContainsDateTime(file.Name, out var parsedDate, offsetHint)) { + if (datesAndFiles.ContainsKey(parsedDate)) + { + string message = + $"There was a duplicate date. File {file} with date {parsedDate,'r'} conflicts with existing file {datesAndFiles[parsedDate]}"; + throw new InvalidDataSetException(message); + } + datesAndFiles.Add(parsedDate, file); } } diff --git a/src/Acoustics.Shared/InvalidDataSetException.cs b/src/Acoustics.Shared/InvalidDataSetException.cs new file mode 100644 index 000000000..31dd8f2bc --- /dev/null +++ b/src/Acoustics.Shared/InvalidDataSetException.cs @@ -0,0 +1,16 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace Acoustics.Shared +{ + using System; + + public class InvalidDataSetException : Exception + { + public InvalidDataSetException(string message) + : base(message) + { + } + } +} \ No newline at end of file diff --git a/src/AnalysisPrograms/Production/Exceptions.cs b/src/AnalysisPrograms/Production/Exceptions.cs index 3e4e60dc2..5103186ce 100644 --- a/src/AnalysisPrograms/Production/Exceptions.cs +++ b/src/AnalysisPrograms/Production/Exceptions.cs @@ -88,6 +88,10 @@ private static Dictionary CreateExceptionMap() typeof(InvalidAudioChannelException), new ExceptionStyle() { ErrorCode = 105, PrintUsage = false } }, + { + typeof(InvalidDataSetException), + new ExceptionStyle() {ErrorCode = 106, PrintUsage = false } + }, { typeof(AnalysisOptionDevilException), new ExceptionStyle diff --git a/src/AudioAnalysisTools/AudioAnalysisTools.csproj b/src/AudioAnalysisTools/AudioAnalysisTools.csproj index 9b50bd6b0..c9707beed 100644 --- a/src/AudioAnalysisTools/AudioAnalysisTools.csproj +++ b/src/AudioAnalysisTools/AudioAnalysisTools.csproj @@ -115,6 +115,9 @@ ..\..\packages\Microsoft.Win32.Primitives.4.3.0\lib\net46\Microsoft.Win32.Primitives.dll + + ..\..\packages\morelinq.3.1.0\lib\net451\MoreLinq.dll + ..\..\packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll @@ -205,7 +208,9 @@ ..\..\packages\System.Security.Cryptography.X509Certificates.4.3.0\lib\net461\System.Security.Cryptography.X509Certificates.dll - ..\..\packages\System.ValueTuple.4.4.0-preview2-25405-01\lib\net461\System.ValueTuple.dll + ..\..\packages\System.ValueTuple.4.4.0\lib\net461\System.ValueTuple.dll + True + True 3.5 diff --git a/src/AudioAnalysisTools/Indices/IndexMatrices.cs b/src/AudioAnalysisTools/Indices/IndexMatrices.cs index 186540e67..3e9576cdd 100644 --- a/src/AudioAnalysisTools/Indices/IndexMatrices.cs +++ b/src/AudioAnalysisTools/Indices/IndexMatrices.cs @@ -21,6 +21,7 @@ namespace AudioAnalysisTools.Indices using DSP; using StandardSpectrograms; using log4net; + using MoreLinq.Extensions; using TowseyLibrary; using Zio; @@ -357,7 +358,7 @@ public static FileInfo[] GetFilesInDirectory(string path, string pattern) } /// - /// Returns a sorted list of file paths, sorted on file name. + /// Returns a unique, sorted, list of file paths, sorted on file name. /// IMPORTANT: Sorts on alphanumerics, NOT on date or time encoded in the file name. /// public static FileInfo[] GetFilesInDirectories(DirectoryInfo[] directories, string pattern) @@ -378,14 +379,7 @@ public static FileInfo[] GetFilesInDirectories(DirectoryInfo[] directories, stri fileList.AddRange(files); } - //if (fileList.Count == 0) - //{ - // // No need for this warning. It comes later. - // LoggedConsole.WriteErrorLine($"No file names match pattern <{pattern}>. Returns empty list of files"); - //} - - FileInfo[] returnList = fileList.ToArray(); - Array.Sort(returnList, (f1, f2) => f1.Name.CompareTo(f2.Name)); + FileInfo[] returnList = fileList.DistinctBy(x => x.FullName).OrderBy(x => x.Name).ToArray(); return returnList; } diff --git a/src/AudioAnalysisTools/packages.config b/src/AudioAnalysisTools/packages.config index 1a117a3fe..fde30b08e 100644 --- a/src/AudioAnalysisTools/packages.config +++ b/src/AudioAnalysisTools/packages.config @@ -19,6 +19,7 @@ + @@ -64,7 +65,7 @@ - +