Add another way of calculating distance

Issue #252 Add Manahattan disctance as option but does not appear to differ from the Euclidean.
To some resharper inspired things.

src/AnalysisPrograms/Sandpit.cs

@@ -208,7 +208,7 @@ public static void ContentDescriptionDev()
 
             //Write image + contentPlots to file.
             var image = ContentVisualization.DrawLdfcSpectrogramWithContentScoreTracks(ldfcSpectrogram, contentPlots);
-            var path1 = Path.Combine(@"C:\Ecoacoustics\Output\ContentDescription", "Testing_2Maps.CONTENT9.png");
+            var path1 = Path.Combine(@"C:\Ecoacoustics\Output\ContentDescription", "Testing_2Maps.CONTENT10.png");
             image.Save(path1);
         }
 

src/AudioAnalysisTools/ContentDescriptionTools/ContentDescription.cs

@@ -137,7 +137,7 @@ public static Dictionary<string, double[,]> ReadIndexMatrices(DirectoryInfo dir,
             for (int i = 1; i < IndexNames.Length; i++)
             {
                 key = IndexNames[i];
-                var indexBounds = IndexValueBounds[key];
+                //var indexBounds = IndexValueBounds[key];
 
                 // construct a path to the required matrix
                 path = Path.Combine(dir.FullName, baseName + "__Towsey.Acoustic." + key + ".csv");
@@ -163,8 +163,8 @@ public static Dictionary<string, double[,]> ReadIndexMatrices(DirectoryInfo dir,
 
         public static List<DescriptionResult> AnalyseMinutes(Dictionary<string, double[,]> dictionary, int elapsedMinutes)
         {
-            int rowCount = dictionary[ContentDescription.IndexNames[0]].GetLength(0);
-            int freqBinCount = dictionary[ContentDescription.IndexNames[0]].GetLength(1);
+            int rowCount = dictionary[IndexNames[0]].GetLength(0);
+            //int freqBinCount = dictionary[ContentDescription.IndexNames[0]].GetLength(1);
             var results = new List<DescriptionResult>();
 
             // over all rows assuming one minute per row.
@@ -250,7 +250,7 @@ public static Dictionary<string, double[]> ReduceIndicesByFactor(Dictionary<stri
         }
 
         /// <summary>
-        /// Returns the bin bounds assuming that the full spectrum consists of the defaul value = 256.
+        /// Returns the bin bounds assuming that the full spectrum consists of the default value = 256.
         /// </summary>
         /// <param name="bottomFrequency">Units = Hertz.</param>
         /// <param name="topFrequency">Hertz.</param>
@@ -304,11 +304,15 @@ public static double[] ScanSpectrumWithTemplate(Dictionary<string, double[]> tem
             var spectralScores = new double[spectrumLength];
 
             // scan the spectrum of indices
+            // does not appear to make any difference whether use Manhattan or Euclidean distance.
             for (int i = 0; i < spectrumLength; i++)
             {
                 var binVector = GetFreqBinVector(oneMinuteIndices, i);
-                var distance = DataTools.EuclidianDistance(templateVector, binVector);
-                distance /= Math.Sqrt(templateVector.Length);
+
+                //var distance = DataTools.EuclideanDistance(templateVector, binVector);
+                //distance /= Math.Sqrt(templateVector.Length);
+                var distance = DataTools.ManhattanDistance(templateVector, binVector);
+                distance /= (double)templateVector.Length;
                 spectralScores[i] = 1 - distance;
             }
 
@@ -347,6 +351,13 @@ public static double[] GetFreqBinVector(Dictionary<string, double[]> dictionary,
             return list.ToArray();
         }
 
+        /// <summary>
+        /// Converts individual results to a dictionary of plots.
+        /// It is assumed that the data arrays have been processed in a way that 
+        /// </summary>
+        /// <param name="results">a list of results for each content type in every minute.</param>
+        /// <param name="plotLength">The plot length will the total number of minutes scanned, typically 1440 or one day.</param>
+        /// <param name="plotStart">time start.</param>
         public static Dictionary<string, Plot> ConvertResultsToPlots(List<DescriptionResult> results, int plotLength, int plotStart)
         {
             var plots = new Dictionary<string, Plot>();
@@ -363,7 +374,7 @@ public static Dictionary<string, Plot> ConvertResultsToPlots(List<DescriptionRes
                     if (!plots.ContainsKey(name))
                     {
                         var scores = new double[plotLength];
-                        var plot = new Plot(name, scores, 0.25);
+                        var plot = new Plot(name, scores, 0.25); // NOTE: The threshold can be changed later.
                         plots.Add(name, plot);
                     }
 
@@ -384,7 +395,7 @@ public static List<Plot> SubtractMeanPlusSd(List<Plot> plots)
                 var scores = plot.data;
                 NormalDist.AverageAndSD(scores, out double average, out double sd);
 
-                // normalise the scores to z-scores
+                // normalize the scores to z-scores
                 for (int i = 0; i < scores.Length; i++)
                 {
                     // Convert scores to z-scores
@@ -399,33 +410,36 @@ public static List<Plot> SubtractMeanPlusSd(List<Plot> plots)
                         scores[i] = 4.0;
                     }
 
-                    // normalise full scale to 4 SDs.
+                    // normalize full scale to 4 SDs.
                     scores[i] /= 4.0;
                 }
 
+                // when normalizing the scores this way the range of the plot will be 0 to 4 SD above the mean.
+                // Consequently we set the plot threshold to 0.5, which is two SDs or a p value = 5%.
+                plot.threshold = 0.5;
                 opPlots.Add(plot);
             }
 
             return opPlots;
         }
 
         /// <summary>
-        /// THis method normalises a score array by subtracting the mode rather than the average of the array.
+        /// THis method normalizes a score array by subtracting the mode rather than the average of the array.
         /// THis is because the noise is often not normally distributed but rather skewed.
         /// </summary>
         public static List<Plot> SubtractModeAndSd(List<Plot> plots)
         {
             var opPlots = new List<Plot>();
 
             // subtract average from each plot array
-            foreach (Plot plot in plots)
+            foreach (var plot in plots)
             {
                 var scores = plot.data;
                 var bgn = SNR.CalculateModalBackgroundNoiseInSignal(scores, 1.0);
                 var mode = bgn.NoiseMode;
                 var sd = bgn.NoiseSd;
 
-                // normalise the scores to z-scores
+                // normalize the scores to z-scores
                 for (int i = 0; i < scores.Length; i++)
                 {
                     // Convert scores to z-scores
@@ -440,7 +454,7 @@ public static List<Plot> SubtractModeAndSd(List<Plot> plots)
                         scores[i] = 4.0;
                     }
 
-                    // normalise full scale to 4 SDs.
+                    // normalize full scale to 4 SDs.
                     scores[i] /= 4.0;
                 }
 

src/AudioAnalysisTools/ContentDescriptionTools/ContentTypes/BaseContentType.cs

@@ -18,58 +18,49 @@ public abstract class BaseContentType
         // The TEMPLATE PROVENANCE
         // The source file name from which the indices are extracted.
         private const string BaseName = "BaseNameOfFile";
+        private const string Location = "PlaceName";
+
+        // Template date and Time could also be included where these cannot be inferred from the file name.
 
         //THESE ARE SPECIFIC ROW BOUNDS FOR PREPARING THIS TEMPLATE
-        // The freq bins will be averaged over the time period.
+        // Typically each freq bin will be averaged over the time period.
         private const int StartRowId = 0;
         private const int EndRowId = 59;
 
-        // Full array (256 freq bins) of spectral indices is reduced by the following factor by averaging.
+        // Full array (256 freq bins) of spectral indices is reduced by the following factor by averaging. This is to reduce correlation and computation.
         private const int ReductionFactor = 16;
 
-        // Bandpass filter to be applied
+        // Bandpass filter to be applied where the target content exists only within a narrow band, e.g. 3-4 kHz for Silver-eye band.
         private const int FreqBinCount = 256 / ReductionFactor;
         private const int BottomFreq = 0; //Hertz
         private const int TopFreq = 11000; //Hertz
 
-        // Only want the interval 3-4 kHz for Silver-eye band.
-        // After reducing 256 freq bins to 16, each bin has width 689Hz.
-        // Therefore to get band 3-4 kHz, need to remove the bottom and top bins.
-        // This leaves a template with 2 or 3 freq bins which are then averaged, so that each index has one value.
-        // At the present time this editing is done manually.
-
+        // The all important template that is used to find an acoustic content type.
+        // These are calculate, written to a csv file and then appropriate parts are copied into a dictionary declaration such as this.
+        // The arrays will all be of same length but will vary from length = 1 to 16 or potentially 256.
         private static readonly Dictionary<string, double[]> SilverEyeTemplate = new Dictionary<string, double[]>
         {
-            ["ACI"] = new[] { 0.779 },
-            ["ENT"] = new[] { 0.393 },
-            ["EVN"] = new[] { 0.686 },
-            ["BGN"] = new[] { 0.085 },
-            ["PMN"] = new[] { 0.883 },
+            ["ACI"] = new[] { 0.086, 0.043, 0.041, 0.023, 0.032, 0.027, 0.029, 0.031, 0.032, 0.032, 0.034, 0.069, 0.033, 0.024, 0.018, 0.018 },
+            ["ENT"] = new[] { 0.124, 0.112, 0.146, 0.163, 0.157, 0.157, 0.143, 0.122, 0.113, 0.095, 0.087, 0.121, 0.075, 0.060, 0.054, 0.067 },
+            ["EVN"] = new[] { 0.376, 0.440, 0.590, 0.621, 0.648, 0.621, 0.565, 0.363, 0.273, 0.191, 0.164, 0.221, 0.104, 0.040, 0.017, 0.032 },
+            ["BGN"] = new[] { 0.472, 0.360, 0.273, 0.199, 0.156, 0.121, 0.096, 0.085, 0.075, 0.069, 0.064, 0.061, 0.060, 0.058, 0.054, 0.026 },
+            ["PMN"] = new[] { 0.468, 0.507, 0.687, 0.743, 0.757, 0.751, 0.665, 0.478, 0.391, 0.317, 0.276, 0.367, 0.187, 0.109, 0.071, 0.096 },
         };
 
+        /// <summary>
+        /// THis method changes for each content type.
+        /// </summary>
+        /// <param name="oneMinuteOfIndices">the indices for this minute.</param>
+        /// <returns>A score value for the content in this one minute of recording.</returns>
         public static KeyValuePair<string, double> GetContent(Dictionary<string, double[]> oneMinuteOfIndices)
         {
-            var reducedIndices = ContentDescription.ReduceIndicesByFactor(oneMinuteOfIndices, ReductionFactor);
-
-            //var freqBinBounds = ContentDescription.GetFreqBinBounds(BottomFreq, TopFreq);
-            //reducedIndices = ContentDescription.ApplyBandPass(reducedIndices, freqBinBounds[0], freqBinBounds[1]);
-
-            var oneMinuteVector = ContentDescription.ConvertDictionaryToVector(reducedIndices);
-            var templateVector = ContentDescription.ConvertDictionaryToVector(SilverEyeTemplate);
-
-            //Get Euclidian distance and normalize the distance
-            // Now pass the template up the full frequency spectrum to get a spectrum of scores.
-            var spectralScores = ContentDescription.ScanSpectrumWithTemplate(SilverEyeTemplate, reducedIndices);
-
-            // Now check how much of spectral weight is in the correct freq band ie between 3-4 kHz.
-            var freqBinBounds = ContentDescription.GetFreqBinBounds(BottomFreq, TopFreq, FreqBinCount);
-            double callSum = DataTools.Subarray(spectralScores, freqBinBounds[0], freqBinBounds[1]).Sum();
-            double totalSum = DataTools.Subarray(spectralScores, 1, spectralScores.Length - 3).Sum();
-            double score = callSum / totalSum;
-
+            double score = 0.0;
             return new KeyValuePair<string, double>(Name, score);
         }
 
+        /// <summary>
+        /// THis method is the same for all Content Types but uses constants appropriate the template type.
+        /// </summary>
         public static Dictionary<string, double[]> GetTemplate(DirectoryInfo dir)
         {
             var dictionaryOfIndices = ContentDescription.ReadIndexMatrices(dir, BaseName);
@@ -80,13 +71,17 @@ public static Dictionary<string, double[]> GetTemplate(DirectoryInfo dir)
             return reducedIndices;
         }
 
+        /// <summary>
+        /// THis method is the same for all Content Types.
+        /// </summary>
         public static void WriteTemplateToFile(DirectoryInfo ipDir, DirectoryInfo opDir)
         {
             var template = GetTemplate(ipDir);
             var opPath = Path.Combine(opDir.FullName, Name + "Template.csv");
             FileTools.WriteDictionaryToFile(template, opPath);
         }
 
+        // The following random data was used to try some statistical experiments.
         // get dummy data
         //var rn = new RandomNumber(DateTime.Now.Second + (int)DateTime.Now.Ticks + 333);
         //var distance = rn.GetDouble();

src/AudioAnalysisTools/ContentDescriptionTools/ContentTypes/BirdMorningChorus1.cs

@@ -55,7 +55,7 @@ public static KeyValuePair<string, double> GetContent(Dictionary<string, double[
             var templateVector = ContentDescription.ConvertDictionaryToVector(BirdChorusTemplate);
 
             //Get Euclidian distance and normalize the distance
-            var distance = DataTools.EuclidianDistance(templateVector, oneMinuteVector);
+            var distance = DataTools.EuclideanDistance(templateVector, oneMinuteVector);
             distance /= Math.Sqrt(templateVector.Length);
 
             return new KeyValuePair<string, double>(Name, 1 - distance);

src/AudioAnalysisTools/ContentDescriptionTools/ContentTypes/RainHeavy1.cs

@@ -29,7 +29,7 @@ public static KeyValuePair<string, double> GetRainContent(Dictionary<string, dou
             var templateVector = ContentDescription.ConvertDictionaryToVector(StrongRainTemplate);
 
             //Get Euclidian distance and normalise the distance
-            var distance = DataTools.EuclidianDistance(templateVector, oneMinuteVector);
+            var distance = DataTools.EuclideanDistance(templateVector, oneMinuteVector);
             distance /= Math.Sqrt(templateVector.Length);
 
             // get dummy data

src/AudioAnalysisTools/ContentDescriptionTools/ContentTypes/RainLight1.cs

@@ -43,7 +43,7 @@ public static KeyValuePair<string, double> GetContent(Dictionary<string, double[
             var templateVector = ContentDescription.ConvertDictionaryToVector(LightRainTemplate);
 
             //Get Euclidian distance and normalise the distance
-            var distance = DataTools.EuclidianDistance(templateVector, oneMinuteVector);
+            var distance = DataTools.EuclideanDistance(templateVector, oneMinuteVector);
             distance /= Math.Sqrt(templateVector.Length);
 
             // get dummy data

src/AudioAnalysisTools/ContentDescriptionTools/ContentTypes/WindLight1.cs

@@ -45,7 +45,7 @@ public static KeyValuePair<string, double> GetContent(Dictionary<string, double[
             var oneMinuteVector = ContentDescription.ConvertDictionaryToVector(reducedIndices);
             var templateVector = ContentDescription.ConvertDictionaryToVector(LightWindTemplate);
 
-            var distance = DataTools.EuclidianDistance(templateVector, oneMinuteVector);
+            var distance = DataTools.EuclideanDistance(templateVector, oneMinuteVector);
 
             //normalise the distance
             distance /= Math.Sqrt(templateVector.Length);

src/AudioAnalysisTools/ContentDescriptionTools/ContentTypes/WindStrong1.cs

@@ -42,7 +42,7 @@ public static KeyValuePair<string, double> GetContent(Dictionary<string, double[
             var oneMinuteVector = ContentDescription.ConvertDictionaryToVector(reducedIndices);
             var templateVector = ContentDescription.ConvertDictionaryToVector(StrongWindTemplate);
 
-            var distance = DataTools.EuclidianDistance(templateVector, oneMinuteVector);
+            var distance = DataTools.EuclideanDistance(templateVector, oneMinuteVector);
 
             //normalise the distance
             distance /= Math.Sqrt(templateVector.Length);

src/AudioAnalysisTools/ContentDescriptionTools/ContentVisualization.cs

@@ -18,31 +18,32 @@ public static class ContentVisualization
         public static Image DrawLdfcSpectrogramWithContentScoreTracks(Image ldfcSpectrogram, List<Plot> contentScores)
         {
             int trackHeight = 30;
-            int width = ldfcSpectrogram.Width;
-            var imageList = new List<Image>
-            {
-                ldfcSpectrogram,
-            };
+            //int width = ldfcSpectrogram.Width;
+            //var imageList = new List<Image>
+            //{
+            //    ldfcSpectrogram,
+            //};
 
-            foreach (var plot in contentScores)
+            var image = new Image_MultiTrack(ldfcSpectrogram);
+            if (contentScores != null)
             {
-                var track = new ImageTrack(TrackType.scoreArrayNamed, plot.data)
+                foreach (var plot in contentScores)
                 {
-                    Name = plot.title,
-                    ScoreMin = 0.0, // plot.data.Min(),
-                    ScoreMax = 1.0, // plot.data.Max(),
-                    Height = trackHeight,
-                    topOffset = 0,
-                    ScoreThreshold = plot.threshold,
-                };
-                var bmp = new Bitmap(width, trackHeight);
-                imageList.Add(track.DrawNamedScoreArrayTrack(bmp));
-                //var bmp = ImageTrack.DrawGrayScaleScoreTrack(plot.data, 0.0, 4.0, plot.threshold, plot.title);
-                //imageList.Add(bmp);
+                    var track = new ImageTrack(TrackType.scoreArrayNamed, plot.data)
+                    {
+                        Name = plot.title,
+                        ScoreMin = 0.0, // plot.data.Min(),
+                        ScoreMax = 1.0, // plot.data.Max(),
+                        Height = trackHeight,
+                        topOffset = 0,
+                        ScoreThreshold = plot.threshold,
+                    };
+
+                    image.AddTrack(ImageTrack.GetNamedScoreTrack(plot.data, 0.0, 1.0, plot.threshold, plot.title)); //assumes data normalised in 0,1
+                }
             }
 
-            Image bmp3 = ImageTools.CombineImagesVertically(imageList);
-            return bmp3;
+            return image.GetImage();
         }
 
         public static void DrawNormalisedIndexMatrices(DirectoryInfo dir, string baseName, Dictionary<string, double[,]> dictionary)

src/AudioAnalysisTools/LongDurationSpectrograms/LDSpectrogramDiscreteColour.cs

@@ -1,4 +1,4 @@
-// <copyright file="LDSpectrogramDiscreteColour.cs" company="QutEcoacoustics">
+// <copyright file="LDSpectrogramDiscreteColour.cs" company="QutEcoacoustics">
 // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group).
 // </copyright>
 
@@ -87,7 +87,7 @@ public static void DiscreteColourSpectrograms()
                         colourVector[0] = discreteColourValues[c, 0];
                         colourVector[1] = discreteColourValues[c, 1];
                         colourVector[2] = discreteColourValues[c, 2];
-                        distance[c] = DataTools.EuclidianDistance(imageColorVector, colourVector);
+                        distance[c] = DataTools.EuclideanDistance(imageColorVector, colourVector);
                     }
 
                     int minindex, maxindex;

src/AudioAnalysisTools/LongDurationSpectrograms/LDSpectrogramDistance.cs

@@ -163,7 +163,7 @@ public static void DrawDistanceSpectrogram(
                 cs1,
                 nyquist, herzInterval);
 
-            string outputFileName4 = inputFileName1 + ".EuclidianDistance.png";
+            string outputFileName4 = inputFileName1 + ".EuclideanDistance.png";
             Image deltaSp = DrawDistanceSpectrogram(cs1, cs2);
             Color[] colorArray = LDSpectrogramRGB.ColourChart2Array(GetDifferenceColourChart());
             titleBar = DrawTitleBarOfEuclidianDistanceSpectrogram(
@@ -289,7 +289,7 @@ public static Image DrawDistanceSpectrogram(LDSpectrogramRGB cs1, LDSpectrogramR
                     v2[1] = m2Grn[row, col];
                     v2[2] = m2Blu[row, col];
 
-                    d12Matrix[row, col] = DataTools.EuclidianDistance(v1, v2);
+                    d12Matrix[row, col] = DataTools.EuclideanDistance(v1, v2);
                     d11Matrix[row, col] = (v1[0] + v1[1] + v1[2]) / 3; // get average of the normalised values
                     d22Matrix[row, col] = (v2[0] + v2[1] + v2[2]) / 3;