Complete refactoring of code to separate manifests from template defi…

…nitions.

Issue #252 Complete refactoring of code to separate manifests from template definitions.
Also add method to ivuslise score distributions.

src/AnalysisPrograms/Sandpit.cs

@@ -136,12 +136,17 @@ public static void ContentDescriptionApplyTemplates()
             var listOfIndexFiles = new FileInfo(@"C:\Ecoacoustics\Output\Test\Test24HourRecording\TasmanIslandMezIndexFiles.txt");
             var contentPlots = ContentDescription.ContentDescriptionOfMultipleRecordingFiles(listOfIndexFiles, templatesFile);
 
+            var images = GraphsAndCharts.DrawPlotDistributions(contentPlots);
+            var plotsImage = ImageTools.CombineImagesVertically(images);
+            var path1 = Path.Combine(@"C:\Ecoacoustics\ContentDescription", "ScoreDistributions.png");
+            plotsImage.Save(path1);
+
             // Attach content description plots to LDFC spectrogram and write to file
             var path = Path.Combine(@"C:\Ecoacoustics\Output\Test\Test24HourRecording", "Testing__2Maps.png");
             var ldfcSpectrogram = Image.FromFile(path);
             var image = ContentVisualization.DrawLdfcSpectrogramWithContentScoreTracks(ldfcSpectrogram, contentPlots);
-            var path1 = Path.Combine(@"C:\Ecoacoustics\ContentDescription", "Testing_2Maps.CONTENTnew05.png");
-            image.Save(path1);
+            var path2 = Path.Combine(@"C:\Ecoacoustics\ContentDescription", "Testing_2Maps.CONTENTnew05.png");
+            image.Save(path2);
             Console.WriteLine("# Finished scanning recording with content description templates");
         }
 

src/AudioAnalysisTools/ContentDescriptionTools/ContentAlgorithms.cs

@@ -26,9 +26,8 @@ public static Dictionary<string, double[]> CreateFullBandTemplate1(TemplateManif
             var startRowId = provenance.StartOffset;
             var endRowId = provenance.EndOffset;
 
-            var reductionFactor = manifest.SpectralReductionFactor;
             var dictionaryOfVector = DataProcessing.AverageIndicesOverMinutes(templateIndices, startRowId, endRowId);
-            var reducedIndices = DataProcessing.ReduceIndicesByFactor(dictionaryOfVector, reductionFactor);
+            var reducedIndices = DataProcessing.ReduceIndicesByFactor(dictionaryOfVector, manifest.SpectralReductionFactor);
 
             return reducedIndices;
         }
@@ -43,8 +42,7 @@ public static Dictionary<string, double[]> CreateFullBandTemplate1(TemplateManif
         /// <returns>A similarity score.</returns>
         public static double GetFullBandContent1(Dictionary<string, double[]> oneMinuteOfIndices, TemplateManifest template, Dictionary<string, double[]> templateIndices)
         {
-            var reductionFactor = template.SpectralReductionFactor;
-            var reducedIndices = DataProcessing.ReduceIndicesByFactor(oneMinuteOfIndices, reductionFactor);
+            var reducedIndices = DataProcessing.ReduceIndicesByFactor(oneMinuteOfIndices, template.SpectralReductionFactor);
             var oneMinuteVector = DataProcessing.ConvertDictionaryToVector(reducedIndices);
             var templateVector = DataProcessing.ConvertDictionaryToVector(templateIndices);
 

src/AudioAnalysisTools/ContentDescriptionTools/ContentDescription.cs

@@ -64,10 +64,10 @@ public static List<Plot> ContentDescriptionOfMultipleRecordingFiles(FileInfo lis
 
             var plotDict = DataProcessing.ConvertResultsToPlots(completeListOfResults, 1440, 0);
             var contentPlots = DataProcessing.ConvertPlotDictionaryToPlotList(plotDict);
-            contentPlots = DataProcessing.SubtractMeanPlusSd(contentPlots);
+            //contentPlots = DataProcessing.SubtractMeanPlusSd(contentPlots);
 
             //the following did not work as well.
-            //contentPlots = SubtractModeAndSd(contentPlots);
+            contentPlots = DataProcessing.SubtractModeAndSd(contentPlots);
             return contentPlots;
         }
 
@@ -95,6 +95,11 @@ public static List<DescriptionResult> AnalyzeMinutes(
                 // now subject the indices to various content searches
                 foreach (var template in templates)
                 {
+                    if (template.UseStatus == false)
+                    {
+                        continue;
+                    }
+
                     var algorithmType = template.FeatureExtractionAlgorithm;
                     var templateIndices = templatesAsDictionary[template.Name];
                     double score;

src/AudioAnalysisTools/ContentDescriptionTools/DataProcessing.cs

@@ -219,31 +219,27 @@ public static Dictionary<string, double[]> ApplyBandPass(Dictionary<string, doub
         }
 
         /// <summary>
-        /// THis method assumes that the passed temp[late contains only one value for each key.
+        /// THis method assumes that the passed template contains only one value for each key.
         /// </summary>
         /// <param name="templateDict"> Each kvp = string, double.</param>
         /// <param name="oneMinuteIndices">the indices.</param>
         /// <returns>A spectrum of similarity-distance scores.</returns>
         public static double[] ScanSpectrumWithTemplate(Dictionary<string, double[]> templateDict, Dictionary<string, double[]> oneMinuteIndices)
         {
-            int templateLength = templateDict.First().Value.Length;
-            if (templateLength != 1)
-            {
-                // Abandon ship!
-            }
-
-            int spectrumLength = oneMinuteIndices.First().Value.Length;
-            var templateVector = ConvertDictionaryToVector(templateDict);
+            // convert the template dictionary to an array of averaged values
+            var dictionaryOfIndexAverages = DataProcessing.AverageIndicesInDictionary(templateDict);
+            var templateVector = ConvertDictionaryToVector(dictionaryOfIndexAverages);
 
             // the score spectrum to be returned
+            int spectrumLength = oneMinuteIndices.First().Value.Length;
             var spectralScores = new double[spectrumLength];
 
             // scan the spectrum of indices
-            // does not appear to make any difference whether use Manhattan or Euclidean distance.
             for (int i = 0; i < spectrumLength; i++)
             {
                 var binVector = GetFreqBinVector(oneMinuteIndices, i);
 
+                // does not appear to make any difference whether use Manhattan or Euclidean distance.
                 //var distance = DataTools.EuclideanDistance(templateVector, binVector);
                 //distance /= Math.Sqrt(templateVector.Length);
                 var distance = DataTools.ManhattanDistance(templateVector, binVector);
@@ -254,6 +250,18 @@ public static double[] ScanSpectrumWithTemplate(Dictionary<string, double[]> tem
             return spectralScores;
         }
 
+        public static Dictionary<string, double> AverageIndicesInDictionary(Dictionary<string, double[]> dictionary)
+        {
+            var dictionaryOfIndexAverages = new Dictionary<string, double>();
+            foreach (var kvp in dictionary)
+            {
+                var array = dictionary[kvp.Key];
+                dictionaryOfIndexAverages.Add(kvp.Key, array.Average());
+            }
+
+            return dictionaryOfIndexAverages;
+        }
+
         public static double[] ConvertDictionaryToVector(Dictionary<string, double[]> dictionary)
         {
             var list = new List<double>();
@@ -270,6 +278,22 @@ public static double[] ConvertDictionaryToVector(Dictionary<string, double[]> di
             return list.ToArray();
         }
 
+        public static double[] ConvertDictionaryToVector(Dictionary<string, double> dictionary)
+        {
+            var list = new List<double>();
+            var keys = dictionary.Keys;
+            foreach (string key in keys)
+            {
+                var success = dictionary.TryGetValue(key, out double index);
+                if (success)
+                {
+                    list.Add(index);
+                }
+            }
+
+            return list.ToArray();
+        }
+
         public static Dictionary<string, Dictionary<string, double[]>> ExtractDictionaryOfTemplateDictionaries(TemplateManifest[] templates)
         {
             var opDictionary = new Dictionary<string, Dictionary<string, double[]>>();

src/AudioAnalysisTools/ContentDescriptionTools/TemplateManifest.cs

@@ -8,22 +8,14 @@ namespace AudioAnalysisTools.ContentDescriptionTools
     using System.Collections.Generic;
     using System.IO;
     using Acoustics.Shared;
-    using Newtonsoft.Json;
 
     public enum EditStatus
     {
-        CalculateTemplate,
         Edit,
         Copy,
         Ignore,
     }
 
-    public enum UseStatus
-    {
-        Use,
-        Ignore,
-    }
-
     public class TemplateManifest
     {
         public static void CreateNewFileOfTemplateDefinitions(FileInfo manifestFile, FileInfo templateDefinitionsFile)
@@ -39,44 +31,50 @@ public static void CreateNewFileOfTemplateDefinitions(FileInfo manifestFile, Fil
             var newTemplateList = new List<TemplateManifest>();
 
             // cycle through all the manifests
-            foreach (var templateManifest in manifests)
+            for (var i = 0; i < manifests.Length; i++)
             {
-                var name = templateManifest.Name;
+                var manifest = manifests[i];
+                var name = manifest.Name;
                 if (!dictionaryOfCurrentTemplates.ContainsKey(name))
                 {
                     // the current manifest is not an existing template - therefore make it.
-                    var newTemplate = CreateNewTemplateFromManifest(templateManifest);
-                    var templateDict = CreateTemplateDeftn(templateManifest);
-                    newTemplate.Template = templateDict;
+                    var newTemplate = CreateNewTemplateFromManifest(manifest);
+                    newTemplate.TemplateId = i;
+                    newTemplate.Template = CreateTemplateDeftn(manifest);
                     newTemplate.MostRecentEdit = DateTime.Now;
                     newTemplateList.Add(newTemplate);
                     continue;
                 }
 
-                if (templateManifest.EditStatus == EditStatus.Copy)
+                if (manifest.EditStatus == EditStatus.Edit)
                 {
-                    // add existing template unchanged.
-                    var existingTemplate = dictionaryOfCurrentTemplates[name];
-                    newTemplateList.Add(existingTemplate);
+                    // edit an existing template but use the manifest.
+                    var newTemplate = CreateNewTemplateFromManifest(manifest);
+                    newTemplate.TemplateId = i;
+                    newTemplate.Template = CreateTemplateDeftn(manifest);
+                    newTemplate.MostRecentEdit = DateTime.Now;
+                    newTemplateList.Add(newTemplate);
                     continue;
                 }
 
-                if (templateManifest.EditStatus == EditStatus.Ignore)
+                if (manifest.EditStatus == EditStatus.Copy)
                 {
-                    // add existing template unchanged except change UseStatus to Ignore.
+                    // add existing template unchanged except for Id.
                     var existingTemplate = dictionaryOfCurrentTemplates[name];
-                    existingTemplate.UseStatus = UseStatus.Ignore;
+                    existingTemplate.TemplateId = i;
+                    existingTemplate.UseStatus = true;
+                    existingTemplate.Provenance = null;
                     newTemplateList.Add(existingTemplate);
                     continue;
                 }
 
-                if (templateManifest.EditStatus == EditStatus.CalculateTemplate)
+                if (manifest.EditStatus == EditStatus.Ignore)
                 {
-                    // add existing template but recalculate the template definition
+                    // add existing template unchanged except change UseStatus to Ignore.
                     var existingTemplate = dictionaryOfCurrentTemplates[name];
-                    existingTemplate.UseStatus = UseStatus.Use;
-                    var templateDict = CreateTemplateDeftn(templateManifest);
-                    existingTemplate.Template = templateDict;
+                    existingTemplate.TemplateId = i;
+                    existingTemplate.Provenance = null;
+                    existingTemplate.UseStatus = false;
                     newTemplateList.Add(existingTemplate);
                 }
             }
@@ -146,14 +144,19 @@ public static TemplateManifest CreateNewTemplateFromManifest(TemplateManifest te
             {
                 Name = templateManifest.Name,
                 TemplateId = templateManifest.TemplateId,
-                EditStatus = templateManifest.EditStatus,
-                UseStatus = templateManifest.UseStatus,
                 FeatureExtractionAlgorithm = templateManifest.FeatureExtractionAlgorithm,
                 SpectralReductionFactor = templateManifest.SpectralReductionFactor,
                 BandMinHz = templateManifest.BandMinHz,
                 BandMaxHz = templateManifest.BandMaxHz,
+                UseStatus = true,
                 Provenance = null,
             };
+
+            if (templateManifest.EditStatus == EditStatus.Ignore)
+            {
+                newTemplate.UseStatus = false;
+            }
+
             return newTemplate;
         }
 
@@ -180,10 +183,10 @@ public static TemplateManifest CreateNewTemplateFromManifest(TemplateManifest te
         public EditStatus EditStatus { get; set; }
 
         /// <summary>
-        /// Gets or sets the template manifest status.
-        /// UseStatus can be "use" or "ignore".
+        /// Gets or sets a value indicating whether to use the template or not.
+        /// UseStatus can be true or false.
         /// </summary>
-        public UseStatus UseStatus { get; set; }
+        public bool UseStatus { get; set; }
 
         public DateTime MostRecentEdit { get; set; }
 

src/TowseyLibrary/GraphsAndCharts.cs

@@ -13,6 +13,29 @@ namespace TowseyLibrary
 
     public static class GraphsAndCharts
     {
+        public static List<Image> DrawPlotDistributions(List<Plot> plots)
+        {
+            int imageWidth = 100;
+            int height = 100;
+            var imageList = new List<Image>();
+
+            foreach (var plot in plots)
+            {
+                DataTools.GetModeAndOneTailedStandardDeviation(plot.data, out int[] histogram, out double min, out double max, out int modalBin, out double mode, out double sd);
+                var statistics = new Dictionary<string, double>
+                {
+                    { "min", min },
+                    { "max", max },
+                    { "mode", mode },
+                    { "sd", sd },
+                };
+                var image = DrawHistogram(plot.title, histogram,  95, statistics, imageWidth,  height);
+                imageList.Add(image);
+            }
+
+            return imageList;
+        }
+
         public static Image DrawHistogram(string label, int[] histogram, int upperPercentileBin, Dictionary<string, double> statistics, int imageWidth, int height)
         {
             int sum = histogram.Sum();
@@ -181,7 +204,7 @@ public static Image DrawWaveform(string label, double[] signal, int imageWidth,
         public static void DrawGraph(double[] rawdata, string label, FileInfo file)
         {
             var normalisedIndex = DataTools.normalise(rawdata);
-            var image2 = GraphsAndCharts.DrawGraph(label, normalisedIndex, 100);
+            var image2 = DrawGraph(label, normalisedIndex, 100);
             image2.Save(file.FullName);
         }