Experiments with concatenate

Previous commits have helped fix this problem but not completely. Experimental runs on the HPC revealed a bug in calculation of 98th percentile for CVR and EVN. A hack/fix for this is inserted into IndexDistributions.cs at line 77. Insert option in LDSpectrogramstitching.cs for different search option. I need it for my setup.
QutEcoacoustics · Mar 28, 2019 · d2e3377 · d2e3377
1 parent 9f3455c
commit d2e3377
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 11 deletions.
diff --git a/src/AnalysisConfigFiles/IndexPropertiesConfig.yml b/src/AnalysisConfigFiles/IndexPropertiesConfig.yml
@@ -577,7 +577,7 @@ ACI:
   NormMin: 0.4
   NormMax: 0.7
   CalculateNormMin: true
-  CalculateNormMax: false
+  CalculateNormMax: true
   ProjectID: Acoustic Indices
   Units: ""
 BGN:
@@ -591,7 +591,7 @@ BGN:
   NormMin: -100.0
   NormMax: -30.0
   CalculateNormMin: true
-  CalculateNormMax: false
+  CalculateNormMax: true
   ProjectID: Acoustic Indices
   Units: "dB"
 CVR:
@@ -604,7 +604,7 @@ CVR:
   NormMin: 0.0
   NormMax: 0.7
   CalculateNormMin: true
-  CalculateNormMax: false
+  CalculateNormMax: true
   ProjectID: Acoustic Indices
   Units: ""
 #  ComboWeight: 0.0
@@ -629,7 +629,7 @@ ENT:
   NormMin: 0.0
   NormMax: 0.6
   CalculateNormMin: true
-  CalculateNormMax: false
+  CalculateNormMax: true
   ProjectID: Acoustic Indices
   Units: ""
 EVN:
@@ -675,7 +675,7 @@ PMN:
   DefaultValue: 0.0
   DoDisplay: true
   NormMin: 0.0
-  NormMax: 5.5
+  NormMax: 15.0
   CalculateNormMin: true
   CalculateNormMax: false
   ProjectID: Acoustic Indices

diff --git a/src/AnalysisPrograms/Sandpit.cs b/src/AnalysisPrograms/Sandpit.cs
@@ -65,7 +65,7 @@ public override Task<int> Execute(CommandLineApplication app)
                 Log.WriteLine("# Start Time = " + tStart.ToString(CultureInfo.InvariantCulture));
 
                 //AnalyseFrogDataSet();
-                Audio2CsvOverOneFile();
+                //Audio2CsvOverOneFile();
                 //Audio2CsvOverMultipleFiles();
 
                 // used to get files from availae for Black rail and Least Bittern papers.
@@ -74,7 +74,7 @@ public override Task<int> Execute(CommandLineApplication app)
                 //CodeToPlaceScoreTracksUnderLdfcSpectrograms();
                 //CodeToPlaceScoreTracksUnderSingleImage();
 
-                //ConcatenateIndexFilesAndSpectrograms();
+                ConcatenateIndexFilesAndSpectrograms();
                 //ConcatenateGreyScaleSpectrogramImages();
                 //ConcatenateMarineImages();
                 //ConcatenateImages();
@@ -697,7 +697,7 @@ public static void ConcatenateIndexFilesAndSpectrograms()
 
             // SET DEFAULT COLOUR MAPS
             string colorMap1 = SpectrogramConstants.RGBMap_ACI_ENT_EVN;
-            string colorMap2 = SpectrogramConstants.RGBMap_BGN_PMN_RHZ;
+            string colorMap2 = SpectrogramConstants.RGBMap_BGN_PMN_CVR;
 
             // there are three options for rendering of gaps/missing data: NoGaps, TimedGaps and EchoGaps.
             string gapRendering = "TimedGaps"; // the default
@@ -804,9 +804,9 @@ public static void ConcatenateIndexFilesAndSpectrograms()
                 @"C:\Ecoacoustics\Output\Test\Test24HourRecording\TasmanIslandMez",
             };
 
-            string directoryFilter = "0*"; // this is a directory filter to locate only the required files
+            string directoryFilter = "*"; // this is a directory filter to locate only the required files
             string opFileStem = "TasmanIslandMez";
-            string opPath = @"C:\Ecoacoustics\Output\Test\DebugIssue170";
+            string opPath = @"C:\Ecoacoustics\Output\Test\DebugIssue186";
 
             // there are three options for rendering of gaps/missing data: NoGaps, TimedGaps and EchoGaps.
             gapRendering = "TimedGaps";

diff --git a/src/AudioAnalysisTools/Indices/IndexDistributions.cs b/src/AudioAnalysisTools/Indices/IndexDistributions.cs
@@ -69,6 +69,16 @@ public double GetValueOfNthPercentile(int percentile)
                 this.UpperPercentileBin = percentileBin;
                 double binWidth = (this.Maximum - this.Minimum) / length;
                 double value = this.Minimum + (binWidth * percentileBin);
+
+                // CVR (cover) and EVN (events/sec) have discrete, sparse distributions when calculating for zoomed tiles,
+                // and most values are in the zero bin.
+                // Therefore they return value = 0.0; This is a bug!
+                // To avoid this problem, set value = maximum when percentileBin = 0
+                if (percentileBin == 0)
+                {
+                    value = this.Maximum;
+                }
+
                 return value;
             }
         }

diff --git a/src/AudioAnalysisTools/LongDurationSpectrograms/LDSpectrogramStitching.cs b/src/AudioAnalysisTools/LongDurationSpectrograms/LDSpectrogramStitching.cs
@@ -61,8 +61,11 @@ public static DirectoryInfo[] GetSubDirectoriesForSiteData(IEnumerable<Directory
             //string dateString = String.Format("{0}{1:D2}{2:D2}", dto.Year, dto.Month, dto.Day);
             string searchPattern = "*" + site + "*";
 
+            var searchOption = SearchOption.AllDirectories;
+            //var searchOption = SearchOption.TopDirectoryOnly;
+
             return topLevelDataDirectories
-                .SelectMany(dir => dir.GetDirectories(searchPattern, SearchOption.AllDirectories))
+                .SelectMany(dir => dir.GetDirectories(searchPattern, searchOption))
                 .ToArray();
         }