Access params from the config.yml file

Issue #252 Pretty much the last things to do with Content Description.
Unit tests remain to be done.

src/AnalysisPrograms/ContentDescription.cs

@@ -15,6 +15,8 @@ namespace AnalysisPrograms
     using Acoustics.Shared.Csv;
     using AnalysisBase;
     using AnalysisBase.ResultBases;
+    using AnalysisPrograms.Recognizers;
+    using AnalysisPrograms.Recognizers.Base;
     using AudioAnalysisTools.ContentDescriptionTools;
     using AudioAnalysisTools.Indices;
     using AudioAnalysisTools.LongDurationSpectrograms;
@@ -52,13 +54,13 @@ public override void BeforeAnalyze(AnalysisSettings analysisSettings)
         {
         }
 
-        public AnalyzerConfig ParseConfig(FileInfo file)
+        public override AnalyzerConfig ParseConfig(FileInfo file)
         {
             return ConfigFile.Deserialize<CdConfig>(file);
         }
 
         [Serializable]
-        public class CdConfig : IndexCalculateConfig
+        public class CdConfig : AnalyzerConfig
         {
             public string TemplatesList { get; protected set; }
 
@@ -78,6 +80,7 @@ public override AnalysisResult2 Analyze<T>(AnalysisSettings analysisSettings, Se
             var audioFile = segmentSettings.SegmentAudioFile;
             var recording = new AudioRecording(audioFile.FullName);
 
+            // Calculate six spectral indices.
             var segmentResults = IndexCalculateSixOnly.Analysis(
                 recording,
                 segmentSettings.SegmentStartOffset,
@@ -90,8 +93,9 @@ public override AnalysisResult2 Analyze<T>(AnalysisSettings analysisSettings, Se
                 AnalysisIdentifier = this.Identifier,
                 SpectralIndices = new SpectralIndexBase[1],
             };
-            analysisResults.SpectralIndices[0] = segmentResults.SpectralIndexValues;
 
+            //Transfer the spectral index results to AnalysisResults
+            analysisResults.SpectralIndices[0] = segmentResults.SpectralIndexValues;
             return analysisResults;
         }
 
@@ -105,21 +109,14 @@ public override void WriteSummaryIndicesFile(FileInfo destination, IEnumerable<S
 
         public override List<FileInfo> WriteSpectrumIndicesFiles(DirectoryInfo destination, string fileNameBase, IEnumerable<SpectralIndexBase> results)
         {
+            //get selectors and removed unwanted because these indices were never calculated.
             var selectors = results.First().GetSelectors();
-
-            //remove the following selectors because these indices were never calculated.
-            selectors.Remove("CVR");
-            selectors.Remove("DIF");
-            selectors.Remove("RHZ");
-            selectors.Remove("RVT");
-            selectors.Remove("RPS");
-            selectors.Remove("RNG");
-            selectors.Remove("R3D");
-            selectors.Remove("SPT");
-            selectors.Remove("SUM");
+            foreach (var indexName in ContentSignatures.UnusedIndexNames)
+            {
+                selectors.Remove(indexName);
+            }
 
             var spectralIndexFiles = new List<FileInfo>(selectors.Count);
-
             foreach (var kvp in selectors)
             {
                 // write spectrogram to disk as CSV file
@@ -141,6 +138,9 @@ public override void SummariseResults(
         {
             // below is example of how to access values in ContentDescription config file.
             //sampleRate = analysisSettings.Configuration.GetIntOrNull(AnalysisKeys.ResampleRate) ?? sampleRate;
+            var cdConfiguration = (CdConfig)analysisSettings.Configuration;
+            var templatesFileName = cdConfiguration.TemplatesList;
+            var ldSpectrogramConfig = cdConfiguration.LdSpectrogramConfig;
 
             var cdConfigFile = analysisSettings.ConfigFile;
             var configDirectory = cdConfigFile.DirectoryName;
@@ -170,7 +170,8 @@ public override void SummariseResults(
                 AnalysisStartOffset = inputFileSegment.SegmentStartOffset ?? TimeSpan.Zero,
                 MaximumSegmentDuration = analysisSettings.AnalysisMaxSegmentDuration,
                 BackgroundFilterCoeff = SpectrogramConstants.BACKGROUND_FILTER_COEFF,
-                LongDurationSpectrogramConfig = new LdSpectrogramConfig(),
+                //LongDurationSpectrogramConfig = new LdSpectrogramConfig(),
+                LongDurationSpectrogramConfig = ldSpectrogramConfig,
             };
             var icdPath = FilenameHelpers.AnalysisResultPath(
                 resultsDirectory,
@@ -199,7 +200,7 @@ public override void SummariseResults(
 
             // now get the content description for each minute.
             //TODO TODO TODO TODO GET FILE NAME FROM CONFIG.YML FILE
-            var templatesFile = new FileInfo(Path.Combine(configDirectory, ContentSignatures.TemplatesFileName));
+            var templatesFile = new FileInfo(Path.Combine(configDirectory, templatesFileName));
             var contentDictionary = GetContentDescription(spectralIndices, templatesFile, startTimeOffset);
 
             // Write the results to a csv file
@@ -267,7 +268,6 @@ public static Dictionary<string, double[]> GetContentDescription(
             }
 
             // convert results to dictionary of score arrays
-            // TODO TODO fix the below plotLength and start.
             var dictionaryOfScores = DataProcessing.ConvertResultsToDictionaryOfArrays(results, length, startMinuteId);
             return dictionaryOfScores;
         }
@@ -310,10 +310,11 @@ private static string DrawSpectrogramsFromSpectralIndices(
             string colorMap1 = SpectrogramConstants.RGBMap_ACI_ENT_EVN;
             string colorMap2 = SpectrogramConstants.RGBMap_BGN_PMN_OSC;
 
-            // Set Color Filter: Must lie between +/-1. A good value is -0.25
+            //LdSpectrogramConfig config is accessible through Settings.
             var config = new LdSpectrogramConfig();
             if (config.ColourFilter == null)
             {
+                // Set Color Filter: Must lie between +/-1. A good value is -0.25
                 config.ColourFilter = SpectrogramConstants.BACKGROUND_FILTER_COEFF;
             }
 
@@ -333,8 +334,6 @@ private static string DrawSpectrogramsFromSpectralIndices(
                 }
             }
 
-            // following line is debug purposes only
-            //cs.StartOffset = cs.StartOffset + TimeSpan.FromMinutes(15);
             var indexProperties = IndexCalculateSixOnly.GetIndexProperties();
             cs1.SetSpectralIndexProperties(indexProperties);
 

src/AnalysisPrograms/Sandpit.cs

@@ -368,10 +368,11 @@ public static void Audio2CsvOverOneFile()
             //string configPath = @"C:\Work\GitHub\audio-analysis\src\AnalysisConfigFiles\Towsey.Acoustic.yml";
 
             // Test on Content Description
-            //string recordingPath = @"C:\Ecoacoustics\WavFiles\FlyingFox\20190115_Bellingen_Feeding.wav";
-            string recordingPath = @"C:\Ecoacoustics\WavFiles\LizZnidersic\TasmanIsland2015_Unit2_Mez\SM304256_0+1_20151114_131652+1000.wav";
+            string recordingPath = @"C:\Ecoacoustics\WavFiles\FlyingFox\20190115_Bellingen_Feeding.wav";
+            //string recordingPath = @"C:\Ecoacoustics\WavFiles\LizZnidersic\TasmanIsland2015_Unit2_Mez\SM304256_0+1_20151114_131652+1000.wav";
             //string outputPath = @"C:\Ecoacoustics\Output\Test\Test24HourRecording\TasmanIslandMez\14";
-            string outputPath = @"C:\Ecoacoustics\ContentDescription\Test6IndicesMez13";
+            //string outputPath = @"C:\Ecoacoustics\ContentDescription\Test6IndicesMez13";
+            string outputPath = @"C:\Ecoacoustics\ContentDescription\FlyingFox";
             string configPath = @"C:\Work\GitHub\audio-analysis\src\AnalysisConfigFiles\Towsey.ContentDescription.yml";
 
             // Ivan Campos recordings

src/AudioAnalysisTools/ContentDescriptionTools/ContentAlgorithms.cs

@@ -42,10 +42,11 @@ public static Dictionary<string, double[]> CreateFullBandTemplate1(TemplateManif
         /// <returns>A similarity score.</returns>
         public static double GetFullBandContent1(Dictionary<string, double[]> oneMinuteOfIndices, TemplateManifest template, Dictionary<string, double[]> templateIndices)
         {
-            //TODO TODO Remove next line and change the template manifest to indicate what indices are to be used.
-            oneMinuteOfIndices.Remove("OSC");
+            // copy over the recording indices required by the template.
+            var requiredIndices = DataProcessing.GetRequiredIndices(oneMinuteOfIndices, templateIndices.Keys.ToArray());
 
-            var reducedIndices = DataProcessing.ReduceIndicesByFactor(oneMinuteOfIndices, template.SpectralReductionFactor);
+            //reduce indices and convert to vector.
+            var reducedIndices = DataProcessing.ReduceIndicesByFactor(requiredIndices, template.SpectralReductionFactor);
             var oneMinuteVector = DataProcessing.ConvertDictionaryToVector(reducedIndices);
             var templateVector = DataProcessing.ConvertDictionaryToVector(templateIndices);
 
@@ -96,15 +97,16 @@ public static Dictionary<string, double[]> CreateBroadbandTemplate1(TemplateMani
         /// <returns>A similarity score.</returns>
         public static double GetBroadbandContent1(Dictionary<string, double[]> oneMinuteOfIndices, TemplateManifest template, Dictionary<string, double[]> templateIndices)
         {
-            //TODO TODO Remove next line and change the template manifest to indicate what indices are to be used.
-            oneMinuteOfIndices.Remove("OSC");
+            // copy over the recording indices required by the template.
+            var requiredIndices = DataProcessing.GetRequiredIndices(oneMinuteOfIndices, templateIndices.Keys.ToArray());
 
+            //reduce indices and convert to vector.
             var reductionFactor = template.SpectralReductionFactor;
             int freqBinCount = ContentSignatures.FreqBinCount / reductionFactor;
             int bottomFreq = template.BandMinHz; //Hertz
             int topFreq = template.BandMaxHz; //Hertz
             var freqBinBounds = DataProcessing.GetFreqBinBounds(bottomFreq, topFreq, freqBinCount);
-            var reducedIndices = DataProcessing.ReduceIndicesByFactor(oneMinuteOfIndices, reductionFactor);
+            var reducedIndices = DataProcessing.ReduceIndicesByFactor(requiredIndices, reductionFactor);
 
             // remove top freq bins and bottom freq bins;
             reducedIndices = DataProcessing.ApplyBandPass(reducedIndices, freqBinBounds[0], freqBinBounds[1]);
@@ -161,11 +163,12 @@ public static Dictionary<string, double[]> CreateNarrowBandTemplate1(TemplateMan
         /// <returns>A similarity score.</returns>
         public static double GetNarrowBandContent1(Dictionary<string, double[]> oneMinuteOfIndices, TemplateManifest template, Dictionary<string, double[]> templateIndices)
         {
-            //TODO TODO Remove next line and change the template manifest to indicate what indices are to be used.
-            oneMinuteOfIndices.Remove("OSC");
+            // copy over the recording indices required by the template.
+            var requiredIndices = DataProcessing.GetRequiredIndices(oneMinuteOfIndices, templateIndices.Keys.ToArray());
 
+            //reduce indices and convert to vector.
             var reductionFactor = template.SpectralReductionFactor;
-            var reducedIndices = DataProcessing.ReduceIndicesByFactor(oneMinuteOfIndices, reductionFactor);
+            var reducedIndices = DataProcessing.ReduceIndicesByFactor(requiredIndices, reductionFactor);
 
             // Now pass the template up the full frequency spectrum to get a spectrum of scores.
             var spectralScores = DataProcessing.ScanSpectrumWithTemplate(templateIndices, reducedIndices);

src/AudioAnalysisTools/ContentDescriptionTools/ContentSignatures.cs

@@ -18,12 +18,8 @@ public class ContentSignatures
         public const int FrameSize = 512;
         public const int FreqBinCount = FrameSize / 2;
         public const int IndexCalculationDurationInSeconds = 60; //default seconds value for content description
-
         public const string AnalysisString = "__Towsey.Acoustic.";
 
-        //TODO TODO  GET FILE NAME FROM CONFIG.YML FILE
-        public const string TemplatesFileName = "Towsey.TemplateDefinitions.json";
-
         /// <summary>
         /// The following min and max bounds are same as those defined in the IndexPropertiesConfig.yml file as of August 2019.
         /// </summary>
@@ -39,10 +35,13 @@ public class ContentSignatures
 
         public static string[] IndexNames { get; } = { "ACI", "ENT", "EVN", "BGN", "PMN", "OSC" };
 
+        // Following array is used to remove unwanted selectors.
+        public static string[] UnusedIndexNames { get; } = { "CVR", "DIF", "RHZ", "RVT", "RPS", "RNG", "R3D", "SPT", "SUM" };
+
         /// <summary>
         /// Cycles through a set of acoustic indices in the order listed and calculates one acoustic signature for each minute of recording.
         /// WARNING!!!! It is assumed that the indices are listed in temporal order of the original recordings and that the original recordings were continuous.
-        ///             This means the returned plots contain scores over consecutive minutes.
+        ///             When these conditions satisfied, the returned plots contain scores over consecutive minutes.
         ///             Alternatively could read recording minute from its file name.
         /// </summary>
         /// <param name="listOfIndexFiles">A text file, each line being the path to the acoustic indices derived from one recording.</param>

src/AudioAnalysisTools/ContentDescriptionTools/ContentVisualization.cs

@@ -35,6 +35,9 @@ public static Image DrawLdfcSpectrogramWithContentScoreTracks(Image ldfcSpectrog
             return ImageTools.CombineImagesVertically(imageList);
         }
 
+        /// <summary>
+        /// Can be used for visual checking and debugging purposes.
+        /// </summary>
         public static void DrawNormalisedIndexMatrices(DirectoryInfo dir, string baseName, Dictionary<string, double[,]> dictionary)
         {
             var list = new List<Image>();

src/AudioAnalysisTools/ContentDescriptionTools/DataProcessing.cs

@@ -162,6 +162,21 @@ public static Dictionary<string, double[]> AverageIndicesOverMinutes(Dictionary<
             return opIndices;
         }
 
+        /// <summary>
+        /// Not all templates will use the same indices.
+        /// This method returns a dictionary of the required indices only.
+        /// </summary>
+        public static Dictionary<string, double[]> GetRequiredIndices(Dictionary<string, double[]> oneMinuteOfIndices, string[] keys)
+        {
+            var requiredIndices = new Dictionary<string, double[]>();
+            foreach (var key in keys)
+            {
+                requiredIndices.Add(key, oneMinuteOfIndices[key]);
+            }
+
+            return requiredIndices;
+        }
+
         /// <summary>
         /// Reduces a dictionary of vectors by a factor. It is assumed that the input vectors are a power of 2 in length i.e. FFT spectra.
         /// It is assumed that the factor of reduction will also be a power of 2, typically 8 or 16.
@@ -308,6 +323,9 @@ public static Dictionary<string, Dictionary<string, double[]>> ExtractDictionary
             return opDictionary;
         }
 
+        /// <summary>
+        /// THis method can be used for debugging purposes.
+        /// </summary>
         public static double[,] ConvertDictionaryOfIndicesToMatrix(Dictionary<string, double[]> dictionary)
         {
             var indexCount = ContentSignatures.IndexNames.Length;
@@ -413,6 +431,9 @@ public static Dictionary<string, Plot> ConvertArraysToPlots(Dictionary<string, d
             return plots;
         }
 
+        /// <summary>
+        /// A score normalization option.
+        /// </summary>
         public static List<Plot> SubtractMeanPlusSd(List<Plot> plots)
         {
             var opPlots = new List<Plot>();
@@ -452,8 +473,8 @@ public static List<Plot> SubtractMeanPlusSd(List<Plot> plots)
         }
 
         /// <summary>
-        /// THis method normalizes a score array by subtracting the mode rather than the average of the array.
-        /// THis is because the noise is often not normally distributed but rather skewed.
+        /// This method normalizes a score array by subtracting the mode rather than the average of the array.
+        /// This is because the noise is often not normally distributed but rather skewed.
         /// However, did not work well.
         /// </summary>
         public static List<Plot> SubtractModeAndSd(List<Plot> plots)