Final commit for issue #217

Most of the issues have been resolved.
BUT.....
Did NOT generalise the method SummaryIndexvalues.GetDictionaryOfSummaryIndices(List<SummaryIndexValues> summaryIndices)

Anhtony will have to do this. I have increased the number of summary indices displayed.

src/AnalysisConfigFiles/IndexPropertiesConfig.yml

@@ -5,8 +5,8 @@
 # and to the members in the class SummaryIndexValues:SummaryIndexBase
 
 RankOrder:
-  Name: Rank order of audio segment
-  Comment: "Rank of minute segment in temporal order from start of recording."
+  Name: Ordering of audio segments
+  Comment: "Ensures that audio segments are correctly ordered."
   DataType: int
   DefaultValue: 0.0
   DoDisplay: true
@@ -34,7 +34,7 @@ StartOffsetMinute:
   NormMax: 200.0
   ProjectID: Acoustic Indices
   Units: "min"
-StartOffsetSecond:
+ResultStartSeconds:
   Name: Segment start second
   Comment: "Total seconds from start of recording to start of this segment."
   DataType: int
@@ -44,16 +44,16 @@ StartOffsetSecond:
   NormMax: 200.0
   ProjectID: Acoustic Indices
   Units: "sec"
-SegmentDuration:
+SegmentDurationSeconds:
   Name: Time duration of audio segment
-  Comment: "Exact time span (total minutes) of this audio segment - typically 1.0 minutes."
+  Comment: "Time span of audio segment - typically 60 seconds."
   DataType: TimeSpan
   DefaultValue: 0.0
   DoDisplay: false
   NormMin: 0.0
   NormMax: 1.0
   ProjectID: Acoustic Indices
-  Units: "min"
+  Units: "s"
 HighAmplitudeIndex:
   Name: High Amplitude Index
   Comment: "Av number of samples/sec where abs. amplitude is within 10*epislon of the max signal value."
@@ -296,7 +296,7 @@ SptDensity:
   NormMax: 10.0
   ProjectID: Acoustic Indices
   Units: ""
-NDSI:
+Ndsi:
   Name: NDSI
   Comment: "Normalised Difference Soundscape Index) calculated from PSD - originated from Stuart Gage."
   DataType: double

src/AnalysisPrograms/ConcatenateIndexFiles.cs

@@ -157,12 +157,12 @@ public override Task<int> Execute(CommandLineApplication app)
             }
         }
 
+        /// <summary>
+        /// Concatenation is designed only for the output from a "Towsey.Acoustic" analysis.
+        /// </summary>
         public static void Execute(Arguments arguments)
         {
-            // Concatenation is designed only for the output from a "Towsey.Acoustic" analysis.
-
-            // Get the currently available sepctral indices
-            // RHZ, SPT and CVR are correlated with POW and do not add much. CLS not particularly useful. Now using R3D
+            // Get the currently available spectral indices
             string[] keys = LDSpectrogramRGB.GetArrayOfAvailableKeys();
 
             if (arguments == null)
@@ -268,8 +268,9 @@ public static void Execute(Arguments arguments)
 
             string outputFileStem = arguments.FileStemName;
 
-            // SET UP DEFAULT SITE LOCATION INFO    --  DISCUSS IWTH ANTHONY
+            // SET UP DEFAULT SITE LOCATION INFO --  DISCUSS IWTH ANTHONY
             // The following location data is used only to draw the sunrise/sunset tracks on images.
+            // But sun tracks now depracated.
             double? latitude = null;
             double? longitude = null;
             var siteDescription = new SiteDescription
@@ -295,8 +296,7 @@ public static void Execute(Arguments arguments)
 
                 indexPropertiesConfig = arguments.IndexPropertiesConfig.ToFileInfo();
 
-                // prepare the false-colour spgm config file
-                // or set up a default config
+                // prepare the LDFC spgm config file or set up a default config
                 // WARNING: This default config is used when testing. If you alter these defaults, Unit Test results may be affected.
                 var colourSpectrogramConfig = arguments?.FalseColourSpectrogramConfig?.ToFileInfo();
                 ldSpectrogramConfig = (colourSpectrogramConfig?.Exists ?? false)
@@ -341,6 +341,7 @@ public static void Execute(Arguments arguments)
                 var concatenatedSummaryIndices = LdSpectrogramStitching.ConcatenateAllSummaryIndexFiles(summaryIndexFiles, resultsDir, indexGenerationData, outputFileStem);
                 WriteSummaryIndexFile(resultsDir, outputFileStem, AcousticIndices.TowseyAcoustic, concatenatedSummaryIndices);
 
+                // WARNING: call to this method only returns a fixed list of indices.
                 var dictionaryOfSummaryIndices = SummaryIndexValues.ConvertToDictionaryOfSummaryIndices(concatenatedSummaryIndices);
 
                 // REALITY CHECK - check for continuous zero indices or anything else that might indicate defective signal,
@@ -444,6 +445,7 @@ public static void Execute(Arguments arguments)
                 var concatenatedSummaryIndices = LdSpectrogramStitching.ConcatenateAllSummaryIndexFiles(indexFiles, resultsDir, indexGenerationData, outputBaseName);
                 WriteSummaryIndexFile(resultsDir, outputBaseName, AcousticIndices.TowseyAcoustic, concatenatedSummaryIndices);
 
+                // WARNING: call to this method only returns a fixed list of indices.
                 var summaryDict = SummaryIndexValues.ConvertToDictionaryOfSummaryIndices(concatenatedSummaryIndices);
 
                 if (summaryDict == null)

src/AnalysisPrograms/Sandpit.cs

@@ -42,7 +42,7 @@ namespace AnalysisPrograms
     /// audiofilecheck - Writes information about audio files to a csv file.
     /// snr - Calls SnrAnalysis.Execute():  Calculates signal to noise ratio.
     /// audiocutter - Cuts audio into segments of desired length and format
-    /// createfoursonograms
+    /// createfoursonograms.
     /// </summary>
     // TODO: [OPENSOURCE] empty out this file
     public class Sandpit
@@ -62,7 +62,7 @@ public override Task<int> Execute(CommandLineApplication app)
                 Log.WriteLine("# Start Time = " + tStart.ToString(CultureInfo.InvariantCulture));
 
                 //AnalyseFrogDataSet();
-                Audio2CsvOverOneFile();
+                //Audio2CsvOverOneFile();
                 //Audio2CsvOverMultipleFiles();
 
                 // used to get files from availae for Black rail and Least Bittern papers.
@@ -71,7 +71,7 @@ public override Task<int> Execute(CommandLineApplication app)
                 //CodeToPlaceScoreTracksUnderLdfcSpectrograms();
                 //CodeToPlaceScoreTracksUnderSingleImage();
 
-                //ConcatenateIndexFilesAndSpectrograms();
+                ConcatenateIndexFilesAndSpectrograms();
                 //ConcatenateGreyScaleSpectrogramImages();
                 //ConcatenateMarineImages();
                 //ConcatenateImages();
@@ -325,8 +325,8 @@ public static void Audio2CsvOverOneFile()
             //string configPath = @"C:\Work\GitHub\audio-analysis\AudioAnalysis\AnalysisConfigFiles\Towsey.Acoustic.yml";
 
             // Test on STANDARD 24-HOUR RECORDING
-            string recordingPath = @"C:\Ecoacoustics\WavFiles\LizZnidersic\TasmanIsland2015_Unit2_Mez\SM304256_0+1_20151114_071652.wav";
-            string outputPath = @"C:\Ecoacoustics\Output\Test\Test24HourRecording\TasmanIslandMez\08";
+            string recordingPath = @"C:\Ecoacoustics\WavFiles\LizZnidersic\TasmanIsland2015_Unit2_Mez\SM304256_0+1_20151114_131652.wav";
+            string outputPath = @"C:\Ecoacoustics\Output\Test\Test24HourRecording\TasmanIslandMez\14";
             string configPath = @"C:\Work\GitHub\audio-analysis\src\AnalysisConfigFiles\Towsey.Acoustic.yml";
 
             // Ivan Campos recordings
@@ -695,7 +695,7 @@ public static void ConcatenateIndexFilesAndSpectrograms()
 
             // SET DEFAULT COLOUR MAPS
             string colorMap1 = SpectrogramConstants.RGBMap_ACI_ENT_EVN;
-            string colorMap2 = SpectrogramConstants.RGBMap_BGN_PMN_RHZ;
+            string colorMap2 = SpectrogramConstants.RGBMap_BGN_PMN_OSC;
 
             // there are three options for rendering of gaps/missing data: NoGaps, TimedGaps and EchoGaps.
             string gapRendering = "TimedGaps"; // the default
@@ -751,9 +751,9 @@ public static void ConcatenateIndexFilesAndSpectrograms()
             {
                 @"C:\Ecoacoustics\Output\Test\Test24HourRecording\TasmanIslandMez",
             };
-            string directoryFilter = "0*"; // this is a directory filter to locate only the required files
+            string directoryFilter = @"Towsey.Acoustic"; // this is a directory filter to locate only the required files
             string opFileStem = "Testing";
-            string opPath = @"C:\Ecoacoustics\Output\Test\Test24HourRecording\TasmanIslandMez\Concat";
+            string opPath = @"C:\Ecoacoustics\Output\Test\Test24HourRecording\Concat3";
             var falseColourSpgConfig = $"C:\\Work\\GitHub\\audio-analysis\\src\\AnalysisConfigFiles\\SpectrogramFalseColourConfig.yml";
             concatenateEverythingYouCanLayYourHandsOn = true;
 

src/AudioAnalysisTools/Indices/IndexDisplay.cs

@@ -10,6 +10,7 @@ namespace AudioAnalysisTools.Indices
     using System.IO;
     using System.Linq;
     using System.Reflection;
+    using System.Text;
     using AnalysisBase.ResultBases;
     using StandardSpectrograms;
     using log4net;
@@ -85,29 +86,32 @@ public static Bitmap DrawImageOfSummaryIndices(
             TimeSpan indexCalculationDuration,
             DateTimeOffset? recordingStartDate,
             List<GapsAndJoins> errors = null,
-            bool verbose = false)
+            bool verbose = true)
         {
             const int trackHeight = DefaultTrackHeight;
             int scaleLength = 0;
+            var backgroundColour = Color.White;
+
+            // init list of bitmap images to store image tracks
             var bitmapList = new List<Tuple<IndexProperties, Image>>(dictionaryOfSummaryIndices.Keys.Count);
 
+            // set up strings to store info about which indices are used
+            var s1 = new StringBuilder("Indices not found:");
+            var s2 = new StringBuilder("Indices not plotted:");
+
             // accumulate the individual tracks in a List
-            var backgroundColour = Color.White;
             foreach (string key in dictionaryOfSummaryIndices.Keys)
             {
                 if (!listOfIndexProperties.ContainsKey(key))
                 {
-                    if (verbose)
-                    {
-                        Logger.Warn("{0} index configuration could not be found. Index is ignored and not rendered.".Format2(key));
-                    }
-
+                    s1.Append(" {0},".Format2(key));
                     continue;
                 }
 
                 IndexProperties ip = listOfIndexProperties[key];
                 if (!ip.DoDisplay)
                 {
+                    s2.Append(" {0},".Format2(key));
                     continue;
                 }
 
@@ -120,6 +124,12 @@ public static Bitmap DrawImageOfSummaryIndices(
                 bitmapList.Add(Tuple.Create(ip, bitmap));
             }
 
+            if (verbose)
+            {
+                Logger.Warn(s1.ToString());
+                Logger.Warn(s2.ToString());
+            }
+
             var listOfBitmaps = bitmapList
             //    .OrderBy(tuple => tuple.Item1.Order) // don't order because want to preserve alternating gray/white rows.
                 .Select(tuple => tuple.Item2)

src/AudioAnalysisTools/Indices/IndexProperties.cs

@@ -316,7 +316,7 @@ public static FileInfo Find(Config configuration, FileInfo originalConfigFile, b
         }
 
         /// <summary>
-        /// Locate and IndexPropertiesConfig.yml file from the IndexPropertiesConfig key in a config file.
+        /// Locate an IndexPropertiesConfig.yml file from the IndexPropertiesConfig key in a config file.
         /// </summary>
         public static FileEntry Find(IIndexPropertyReferenceConfiguration configuration, string originalConfigpath, bool allowDefault = false)
         {

src/AudioAnalysisTools/Indices/SummaryIndexValues.cs

@@ -108,47 +108,46 @@ public SummaryIndexValues(TimeSpan wavDuration, Dictionary<string, IndexProperti
             }
         }
 
-        public static Dictionary<string, double[]> ConvertToDictionaryOfSummaryIndices(List<SummaryIndexValues> summaryIndices)
-        {
-            // Now add in derived indices i.e. NCDI etc
-            // Decided NOT to do this anymore
-            // dictionaryOfSummaryIndices = IndexMatrices.AddDerivedIndices(dictionaryOfSummaryIndices);
-
-            // Put SUMMARY indices into dictionary. TODO need to generalise the following method
-            // ################# WARNING: THIS METHOD ONLY GETS A "HARD CODED" LIST OF SUMMARY INDICES. See the method.
-            var dictionaryOfSummaryIndices = GetDictionaryOfSummaryIndices(summaryIndices);
-
-            // return the dictionary - it will be used later to produce an index tracks image.
-            return dictionaryOfSummaryIndices;
-        }
-
         /// <summary>
-        /// WARNING: THIS METHOD ONLY GETS FIXED LIST OF INDICES.
-        ///             Also it requires every index to be of type DOUBLE even when htis is not appropriate.
-        /// TODO: This needs to be generalized.
+        /// Put SUMMARY indices into dictionary.
+        /// ################# WARNING: THIS METHOD ONLY GETS A "HARD CODED" LIST OF SUMMARY INDICES. See the method.
+        /// TODO need to generalise the following method.
         /// </summary>
-        public static Dictionary<string, double[]> GetDictionaryOfSummaryIndices(List<SummaryIndexValues> summaryIndices)
+        /// <param name="summaryIndices">a list of summary index values ordered by minute segments and not by name of index.</param>
+        /// <returns>a dictionary whose keys are summary index names and values are arrays of double.</returns>
+        public static Dictionary<string, double[]> ConvertToDictionaryOfSummaryIndices(List<SummaryIndexValues> summaryIndices)
         {
-            var dictionary = new Dictionary<string, double[]>
+            // Put SUMMARY indices into dictionary.
+            var dictionaryOfSummaryIndices = new Dictionary<string, double[]>
             {
                 { GapsAndJoins.KeyZeroSignal, summaryIndices.Select(x => x.ZeroSignal).ToArray() },
                 { "ClippingIndex", summaryIndices.Select(x => x.ClippingIndex).ToArray() },
+                { "HighAmplitudeIndex", summaryIndices.Select(x => x.HighAmplitudeIndex).ToArray() },
+                { "AvgSignalAmplitude", summaryIndices.Select(x => x.AvgSignalAmplitude).ToArray() },
                 { "BackgroundNoise", summaryIndices.Select(x => x.BackgroundNoise).ToArray() },
                 { "Snr", summaryIndices.Select(x => x.Snr).ToArray() },
-                { "EventsPerSecond", summaryIndices.Select(x => x.EventsPerSecond).ToArray() },
+                { "AvgSnrOfActiveFrames", summaryIndices.Select(x => x.AvgSnrOfActiveFrames).ToArray() },
                 { "Activity", summaryIndices.Select(x => x.Activity).ToArray() },
+                { "EventsPerSecond", summaryIndices.Select(x => x.EventsPerSecond).ToArray() },
                 { "HighFreqCover", summaryIndices.Select(x => x.HighFreqCover).ToArray() },
                 { "MidFreqCover", summaryIndices.Select(x => x.MidFreqCover).ToArray() },
                 { "LowFreqCover", summaryIndices.Select(x => x.LowFreqCover).ToArray() },
+                { "AcousticComplexity", summaryIndices.Select(x => x.AcousticComplexity).ToArray() },
                 { "TemporalEntropy", summaryIndices.Select(x => x.TemporalEntropy).ToArray() },
                 { "EntropyOfAverageSpectrum", summaryIndices.Select(x => x.EntropyOfAverageSpectrum).ToArray() },
                 { "EntropyOfPeaksSpectrum", summaryIndices.Select(x => x.EntropyOfPeaksSpectrum).ToArray() },
-                { "AcousticComplexity", summaryIndices.Select(x => x.AcousticComplexity).ToArray() },
                 { "ClusterCount", summaryIndices.Select(x => x.ClusterCount).ToArray() },
                 { "ThreeGramCount", summaryIndices.Select(x => x.ThreeGramCount).ToArray() },
+                { "SptDensity", summaryIndices.Select(x => x.SptDensity).ToArray() },
+                { "Ndsi", summaryIndices.Select(x => x.Ndsi).ToArray() },
             };
 
-            return dictionary;
+            // Now add in derived indices i.e. NCDI etc
+            // Decided NOT to do this anymore
+            // dictionaryOfSummaryIndices = IndexMatrices.AddDerivedIndices(dictionaryOfSummaryIndices);
+
+            // return the dictionary - it will be used later to produce an index tracks image.
+            return dictionaryOfSummaryIndices;
         }
 
         public double ZeroSignal { get; set; }