From 03a304d87c061ede8e4ba9bfc974be5be830e5bc Mon Sep 17 00:00:00 2001 From: towsey Date: Thu, 29 Aug 2019 17:05:23 +1000 Subject: [PATCH] More changes to FF code Issue #238 Changes mostly to get acoustic event information transfered to the FF methods. --- .../Recognizers/PteropusSpecies.cs | 7 ++ src/AnalysisPrograms/Sandpit.cs | 3 +- src/AudioAnalysisTools/AcousticEvent.cs | 113 +++++++---------- src/AudioAnalysisTools/Oscillations2012.cs | 118 +++++++++--------- 4 files changed, 119 insertions(+), 122 deletions(-) diff --git a/src/AnalysisPrograms/Recognizers/PteropusSpecies.cs b/src/AnalysisPrograms/Recognizers/PteropusSpecies.cs index 92eb1bb4e..41ff7aeef 100644 --- a/src/AnalysisPrograms/Recognizers/PteropusSpecies.cs +++ b/src/AnalysisPrograms/Recognizers/PteropusSpecies.cs @@ -213,6 +213,7 @@ private static RecognizerResults TerritorialCall(AudioRecording audioRecording, //iV add additional info to the acoustic events acousticEvents.ForEach(ae => { + ae.FileName = audioRecording.BaseName; ae.SpeciesName = speciesName; ae.Name = abbreviatedSpeciesName + profileName; ae.Profile = profileName; @@ -401,13 +402,19 @@ private static RecognizerResults WingBeats(AudioRecording audioRecording, Config var plots = new List { plot1, plot2 }; // ###################################################################### + + // add additional information about the recording and sonogram properties from which the event is derived. acousticEvents.ForEach(ae => { + ae.FileName = audioRecording.BaseName; ae.SpeciesName = speciesName; ae.Name = abbreviatedSpeciesName + profileName; ae.Profile = profileName; ae.SegmentDurationSeconds = audioRecording.Duration.TotalSeconds; ae.SegmentStartSeconds = segmentStartOffset.TotalSeconds; + var frameOffset = sonogram.FrameStep; + var frameDuration = sonogram.FrameDuration; + ae.SetTimeAndFreqScales(frameOffset, frameDuration, sonogram.FBinWidth); //UNCOMMENT following lines to get spectral profiles of the Wingbeat events. /* double[,] spectrogramData = sonogram.Data; diff --git a/src/AnalysisPrograms/Sandpit.cs b/src/AnalysisPrograms/Sandpit.cs index 8da335cfd..af059067c 100644 --- a/src/AnalysisPrograms/Sandpit.cs +++ b/src/AnalysisPrograms/Sandpit.cs @@ -277,7 +277,8 @@ public static void Audio2CsvOverOneFile() // FLYING FOX RECORDINGS //string recordingPath = @"C:\Ecoacoustics\WavFiles\BradLawData\FlyingFox\20190127_Bellingen_Feeding_SM4.wav"; - string recordingPath = @"C:\Ecoacoustics\WavFiles\FlyingFox\20190115_Bellingen_Feeding.wav"; + //string recordingPath = @"C:\Ecoacoustics\WavFiles\FlyingFox\20190115_Bellingen_Feeding.wav"; + string recordingPath = @"C:\Ecoacoustics\WavFiles\FlyingFox\20190115_Bellingen_Feeding_minute6.wav"; //string recordingPath = @"C:\Ecoacoustics\WavFiles\FlyingFox\20190121_2_Bellingen_Feeding.wav"; //string recordingPath = @"C:\Ecoacoustics\WavFiles\FlyingFox\20190127_Bellingen_Feeding_SM4.wav"; string configPath = @"C:\Work\GitHub\audio-analysis\src\AnalysisConfigFiles\RecognizerConfigFiles\Towsey.PteropusSpecies.yml"; diff --git a/src/AudioAnalysisTools/AcousticEvent.cs b/src/AudioAnalysisTools/AcousticEvent.cs index 02e51195f..df5554a5d 100644 --- a/src/AudioAnalysisTools/AcousticEvent.cs +++ b/src/AudioAnalysisTools/AcousticEvent.cs @@ -11,19 +11,16 @@ namespace AudioAnalysisTools { using System; using System.Collections.Generic; - using System.Collections.ObjectModel; using System.Drawing; using System.IO; using System.Linq; using System.Text; - using System.Text.RegularExpressions; using Acoustics.Shared.Contracts; using Acoustics.Shared.Csv; - using AForge.Imaging.Filters; using AnalysisBase.ResultBases; + using AudioAnalysisTools.DSP; + using AudioAnalysisTools.StandardSpectrograms; using CsvHelper.Configuration; - using DSP; - using StandardSpectrograms; using TowseyLibrary; public class AcousticEvent : EventBase @@ -113,8 +110,8 @@ public void SetEventPositionRelative( } /// - /// Gets or sets units = Hertz - /// Proxied to EventBase.MinHz + /// Gets or sets units = Hertz. + /// Proxied to EventBase.MinHz. /// public new double LowFrequencyHertz { @@ -301,15 +298,12 @@ public void DoMelScale(bool doMelscale, int freqBinCount) public void SetTimeAndFreqScales(int samplingRate, int windowSize, int windowOffset) { - double frameDuration, frameOffset, framesPerSecond; - CalculateTimeScale(samplingRate, windowSize, windowOffset, out frameDuration, out frameOffset, out framesPerSecond); + CalculateTimeScale(samplingRate, windowSize, windowOffset, out var frameDuration, out var frameOffset, out var framesPerSecond); this.FrameDuration = frameDuration; //frame duration in seconds this.FrameOffset = frameOffset; //frame offset in seconds this.FramesPerSecond = framesPerSecond; //inverse of the frame offset - int binCount; - double binWidth; - CalculateFreqScale(samplingRate, windowSize, out binCount, out binWidth); + CalculateFreqScale(samplingRate, windowSize, out var binCount, out var binWidth); this.FreqBinCount = binCount; //required for conversions to & from MEL scale this.FreqBinWidth = binWidth; //required for freq-binID conversions @@ -319,11 +313,21 @@ public void SetTimeAndFreqScales(int samplingRate, int windowSize, int windowOff } } + /// + /// This method assumes that there is no frame overlap i.e. frame duration = frame offset. + /// + /// frames per second assuming no overlap. + /// Number of hertz per freq bin. public void SetTimeAndFreqScales(double framesPerSec, double freqBinWidth) { - //this.FrameDuration = frameDuration; //frame duration in seconds - this.FramesPerSecond = framesPerSec; //inverse of the frame offset - this.FrameOffset = 1 / framesPerSec; //frame offset in seconds + double frameOffset = 1 / framesPerSec; //frame offset in seconds + this.SetTimeAndFreqScales(frameOffset, frameOffset, freqBinWidth); + } + + public void SetTimeAndFreqScales(double frameOffset, double frameDuration, double freqBinWidth) + { + this.FramesPerSecond = 1 / frameOffset; //inverse of the frame offset + this.FrameDuration = frameDuration; //frame duration in seconds //this.FreqBinCount = binCount; //required for conversions to & from MEL scale this.FreqBinWidth = freqBinWidth; //required for freq-binID conversions @@ -343,14 +347,10 @@ public void SetTimeAndFreqScales(double framesPerSec, double freqBinWidth) public static Oblong ConvertEvent2Oblong(AcousticEvent ae) { // Translate time dimension = frames = matrix rows. - int topRow; - int bottomRow; - Time2RowIDs(ae.TimeStart, ae.EventDurationSeconds, ae.FrameOffset, out topRow, out bottomRow); + Time2RowIDs(ae.TimeStart, ae.EventDurationSeconds, ae.FrameOffset, out var topRow, out var bottomRow); //Translate freq dimension = freq bins = matrix columns. - int leftCol; - int rightCol; - Freq2BinIDs(ae.IsMelscale, (int)ae.LowFrequencyHertz, (int)ae.HighFrequencyHertz, ae.FreqBinCount, ae.FreqBinWidth, out leftCol, out rightCol); + Freq2BinIDs(ae.IsMelscale, (int)ae.LowFrequencyHertz, (int)ae.HighFrequencyHertz, ae.FreqBinCount, ae.FreqBinWidth, out var leftCol, out var rightCol); return new Oblong(topRow, leftCol, bottomRow, rightCol); } @@ -934,15 +934,6 @@ public static void CalculateAccuracy(List results, List - /// - /// - /// - /// - /// - /// - /// - /// - /// public static void CalculateAccuracyOnOneRecording(List results, List labels, out int tp, out int fp, out int fn, out double precision, out double recall, out double accuracy, out string resultsText) { @@ -1041,8 +1032,6 @@ public static void CalculateAccuracyOnOneRecording(List results, /// /// duration of event must exceed this to count as an event /// - /// array value must exceed this dB threshold to count as an event - /// name of source file to be added to AcousticEvent class /// a list of acoustic events //public static List ConvertIntensityArray2Events(double[] values, int minHz, int maxHz, // double framesPerSec, double freqBinWidth, @@ -1079,7 +1068,7 @@ public static List ConvertIntensityArray2Events( startFrame = i; } else //check for the end of an event - if (isHit == true && values[i] <= scoreThreshold) //this is end of an event, so initialise it + if (isHit && values[i] <= scoreThreshold) //this is end of an event, so initialise it { isHit = false; double endTime = i * frameOffset; @@ -1146,6 +1135,7 @@ public static List GetEventsAroundMaxima( // convert min an max Hertz durations to freq bins int minBin = (int)Math.Round(minHz / freqBinWidth); int maxBin = (int)Math.Round(maxHz / freqBinWidth); + int binCount = maxBin - minBin + 1; // tried smoothing but not advisable since event onset can be very sudden //values = DataTools.filterMovingAverageOdd(values, 3); @@ -1191,22 +1181,23 @@ public static List GetEventsAroundMaxima( endFrame = i; - int frameDuration = endFrame - startFrame + 1; - if (frameDuration >= minFrames && frameDuration <= maxFrames) + int frameCount = endFrame - startFrame + 1; + if (frameCount >= minFrames && frameCount <= maxFrames) { double startTime = startFrame * frameOffset; // time in seconds - double eventDuration = frameDuration * frameOffset; // time in seconds + double eventDuration = frameCount * frameOffset; // time in seconds AcousticEvent ev = new AcousticEvent(segmentStartOffset, startTime, eventDuration, minHz, maxHz) { Name = "Event", //default name - FrameCount = frameDuration, + FrameCount = frameCount, + FreqBinCount = binCount, Oblong = new Oblong(startFrame, minBin, endFrame, maxBin), }; ev.SetTimeAndFreqScales(framesPerSec, freqBinWidth); //obtain average intensity score. Note-first frame is not actually in the event. - var subArray = DataTools.Subarray(values, startFrame + 1, frameDuration); + var subArray = DataTools.Subarray(values, startFrame + 1, frameCount); ev.Score = subArray.Average(); events.Add(ev); } @@ -1225,18 +1216,15 @@ public static List GetEventsAroundMaxima( /// Some analysis techniques (e.g. OD) have their own methods for extracting events from score arrays. /// /// the array of scores - /// lower freq bound of the acoustic event - /// upper freq bound of the acoustic event - /// the time scale required by AcousticEvent class - /// the freq scale required by AcousticEvent class - /// - /// duration of event must exceed this to count as an event - /// duration of event must be less than this to count as an event - /// - /// score must exceed this threshold to count as an event - /// name of source file to be added to AcousticEvent class - /// name of the event to be added to AcousticEvent class - /// a list of acoustic events + /// lower freq bound of the acoustic event. + /// upper freq bound of the acoustic event. + /// the time scale required by AcousticEvent class. + /// the freq scale required by AcousticEvent class. + /// threshold. + /// duration of event must exceed this to count as an event. + /// duration of event must be less than this to count as an event. + /// offset. + /// a list of acoustic events. public static List ConvertScoreArray2Events( double[] scores, int minHz, @@ -1250,23 +1238,26 @@ public static List ConvertScoreArray2Events( { int count = scores.Length; var events = new List(); - double maxPossibleScore = 5 * scoreThreshold; // used to calcualte a normalised score bewteen 0 - 1.0 + double maxPossibleScore = 5 * scoreThreshold; // used to calculate a normalised score between 0 - 1.0 bool isHit = false; double frameOffset = 1 / framesPerSec; // frame offset in fractions of second double startTime = 0.0; int startFrame = 0; - for (int i = 0; i < count; i++) // pass over all frames + // pass over all frames + for (int i = 0; i < count; i++) { - if (isHit == false && scores[i] >= scoreThreshold) //start of an event + if (isHit == false && scores[i] >= scoreThreshold) { + //start of an event isHit = true; startTime = i * frameOffset; startFrame = i; } else // check for the end of an event - if (isHit == true && scores[i] <= scoreThreshold) // this is end of an event, so initialise it + if (isHit && scores[i] <= scoreThreshold) { + // this is end of an event, so initialise it isHit = false; double endTime = i * frameOffset; double duration = endTime - startTime; @@ -1326,10 +1317,6 @@ public static List ConvertScoreArray2Events( /// The events are required to have the passed name. /// The events are assumed to contain sufficient info about frame rate in order to populate the array. /// - /// - /// - /// - /// public static double[] ExtractScoreArrayFromEvents(List events, int arraySize, string nameOfTargetEvent) { double[] scores = new double[arraySize]; @@ -1341,7 +1328,7 @@ public static double[] ExtractScoreArrayFromEvents(List events, i double windowOffset = events[0].FrameOffset; double frameRate = 1 / windowOffset; //frames per second - int count = events.Count; + //int count = events.Count; foreach ( AcousticEvent ae in events) { if (!ae.Name.Equals(nameOfTargetEvent)) @@ -1359,7 +1346,7 @@ public static double[] ExtractScoreArrayFromEvents(List events, i } return scores; - } //end method + } //############################################################################################################################################## @@ -1367,10 +1354,6 @@ public static double[] ExtractScoreArrayFromEvents(List events, i /// This method is used to do unit test on lists of events. /// First developed for frog recognizers - October 2016. /// - /// - /// - /// - /// public static void TestToCompareEvents(string fileName, DirectoryInfo opDir, string testName, List events) { var testDir = new DirectoryInfo(opDir + $"\\UnitTest_{testName}"); @@ -1385,7 +1368,7 @@ public static void TestToCompareEvents(string fileName, DirectoryInfo opDir, str var eventsFile = new FileInfo(eventsFilePath); Csv.WriteToCsv(eventsFile, events); - LoggedConsole.WriteLine($"# EVENTS TEST: Camparing List of {testName} events with those in benchmark file:"); + LoggedConsole.WriteLine($"# EVENTS TEST: Comparing List of {testName} events with those in benchmark file:"); var benchmarkFile = new FileInfo(benchmarkFilePath); if (!benchmarkFile.Exists) { diff --git a/src/AudioAnalysisTools/Oscillations2012.cs b/src/AudioAnalysisTools/Oscillations2012.cs index def1bc1a2..4f4053984 100644 --- a/src/AudioAnalysisTools/Oscillations2012.cs +++ b/src/AudioAnalysisTools/Oscillations2012.cs @@ -1,4 +1,4 @@ -// +// // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). // @@ -6,17 +6,17 @@ namespace AudioAnalysisTools { using System; using System.Collections.Generic; - using DSP; - using StandardSpectrograms; + using AudioAnalysisTools.DSP; + using AudioAnalysisTools.StandardSpectrograms; using TowseyLibrary; /// /// NOTE: 21st June 2012. /// /// This class contains methods to detect oscillations in a the sonogram of an audio signal. - /// The method Execute() returns all info about oscillaitons in the passed sonogram. + /// The method Execute() returns all info about oscillations in the passed sonogram. /// This method should be called in preference to those in the class OscillationAnalysis. - /// (The latter should be depracated.) + /// (The latter should be deprecated.) /// public static class Oscillations2012 { @@ -118,8 +118,7 @@ public static void Execute( /// threshold - do not accept a DCT coefficient if its value is less than this threshold /// /// - public static double[,] DetectOscillations(SpectrogramStandard sonogram, int minHz, int maxHz, - double dctDuration, int minOscilFreq, int maxOscilFreq, double dctThreshold) + public static double[,] DetectOscillations(SpectrogramStandard sonogram, int minHz, int maxHz, double dctDuration, int minOscilFreq, int maxOscilFreq, double dctThreshold) { int minBin = (int)(minHz / sonogram.FBinWidth); int maxBin = (int)(maxHz / sonogram.FBinWidth); @@ -150,7 +149,8 @@ public static void Execute( //string bmpPath = @"C:\SensorNetworks\Output\cosines.png"; //ImageTools.DrawMatrix(cosines, bmpPath, true); - for (int c = minBin; c <= maxBin; c++) //traverse columns - skip DC column + //traverse columns - skip DC column + for (int c = minBin; c <= maxBin; c++) { var dctArray = new double[dctLength]; @@ -316,7 +316,6 @@ public static double[] DetectOscillations(double[] ipArray, double framesPerSeco } } - //return hits; //dctArray return dctScores; } @@ -324,7 +323,6 @@ public static double[] DetectOscillations(double[] ipArray, double framesPerSeco /// Removes single lines of hits from Oscillation matrix. /// /// the Oscillation matrix - /// public static double[,] RemoveIsolatedOscillations(double[,] matrix) { int rows = matrix.GetLength(0); @@ -351,14 +349,13 @@ public static double[] DetectOscillations(double[] ipArray, double framesPerSeco } //end method RemoveIsolatedOscillations() /// - /// Converts the hits derived from the oscilation detector into a score for each frame. + /// Converts the hits derived from the oscillation detector into a score for each frame. /// NOTE: The oscillation detector skips every second row, so score must be adjusted for this. /// /// sonogram as matrix showing location of oscillation hits /// lower freq bound of the acoustic event /// upper freq bound of the acoustic event /// the freq scale required by AcousticEvent class - /// public static double[] GetOscillationScores(double[,] hits, int minHz, int maxHz, double freqBinWidth) { int rows = hits.GetLength(0); @@ -401,7 +398,9 @@ public static double[] GetOscillationFrequency(double[,] hits, int minHz, int ma { double freq = 0; int count = 0; - for (int c = minBin; c <= maxBin; c++) //traverse columns in required band + + //traverse columns in required band + for (int c = minBin; c <= maxBin; c++) { if (hits[r, c] > 0) { @@ -428,18 +427,18 @@ public static double[] GetOscillationFrequency(double[,] hits, int minHz, int ma /// /// Converts the Oscillation Detector score array to a list of AcousticEvents. /// - /// the array of OD scores - /// - /// lower freq bound of the acoustic event - /// upper freq bound of the acoustic event - /// the time scale required by AcousticEvent class - /// the freq scale required by AcousticEvent class - /// - /// - /// - /// name of source file to be added to AcousticEvent class - /// - /// + /// the array of OD scores. + /// periodicity of oscillation. + /// lower freq bound of the acoustic event. + /// upper freq bound of the acoustic event. + /// the time scale required by AcousticEvent class. + /// the freq scale required by AcousticEvent class. + /// threshold for max score. + /// min duration. + /// max duration. + /// name of source file to be added to AcousticEvent class. + /// absolute start time of the current one minute segment. + /// a list of acoustic events. public static List ConvertOscillationScores2Events( double[] scores, double[] oscFreq, @@ -458,64 +457,75 @@ public static List ConvertOscillationScores2Events( double scoreThreshold = maxScoreThreshold; //set this to the maximum threshold to start with int count = scores.Length; - //int minBin = (int)(minHz / freqBinWidth); - //int maxBin = (int)(maxHz / freqBinWidth); - //int binCount = maxBin - minBin + 1; + int minBin = (int)Math.Floor(minHz / freqBinWidth); + int maxBin = (int)Math.Ceiling(maxHz / freqBinWidth); + int binCount = maxBin - minBin + 1; var events = new List(); bool isHit = false; double frameOffset = 1 / framesPerSec; double startTime = 0.0; int startFrame = 0; - for (int i = 0; i < count; i++) //pass over all frames + //pass over all frames + for (int i = 0; i < count; i++) { - if (isHit == false && scores[i] >= scoreThreshold) //start of an event + //start of an event + if (isHit == false && scores[i] >= scoreThreshold) { isHit = true; startTime = i * frameOffset; startFrame = i; } else //check for the end of an event - if (isHit && (scores[i] < scoreThreshold || i == count - 1)) //this is end of an event, so initialise it + if (isHit && (scores[i] < scoreThreshold || i == count - 1)) { + //this is end of an event, so initialise it isHit = false; //double endTime = i * frameOffset; //double duration = endTime - startTime; - double duration = (i - startFrame + 1) * frameOffset; + int frameCount = i - startFrame + 1; + double duration = frameCount * frameOffset; if (duration < minDurationThreshold) - { - continue; //skip events with duration shorter than threshold - } + { + //skip events with duration shorter than threshold + continue; + } if (duration > maxDurationThreshold) - { - continue; //skip events with duration longer than threshold - } + { + continue; //skip events with duration longer than threshold + } - var ev = new AcousticEvent(segmentStartOffset, startTime, duration, minHz, maxHz); - ev.Name = "Oscillation"; //default name + var ev = new AcousticEvent(segmentStartOffset, startTime, duration, minHz, maxHz) + { + FileName = fileName, + Name = "Oscillation", //default name + FreqBinCount = binCount, + FrameCount = frameCount, + Oblong = new Oblong(startFrame, minBin, startFrame + frameCount - 1, maxBin), + }; - //ev.SetTimeAndFreqScales(framesPerSec, freqBinWidth); - ev.FileName = fileName; + // calling this method assumes there is no frame overlap + ev.SetTimeAndFreqScales(framesPerSec, freqBinWidth); //obtain average score. double av = 0.0; for (int n = startFrame; n <= i; n++) - { - av += scores[n]; - } + { + av += scores[n]; + } - ev.Score = av / (i - startFrame + 1); + ev.Score = av / frameCount; //obtain oscillation freq. av = 0.0; for (int n = startFrame; n <= i; n++) - { - av += oscFreq[n]; - } + { + av += oscFreq[n]; + } - ev.Score2 = av / (i - startFrame + 1); + ev.Score2 = av / frameCount; ev.Intensity = (int)ev.Score2; // store this info for later inclusion in csv file as Event Intensity events.Add(ev); } @@ -533,10 +543,6 @@ public static List ConvertOscillationScores2Events( /// Calculates the optimal frame overlap for the given sample rate, frame width and max oscilation or pulse rate. /// Pulse rate is determined using a DCT and efficient use of the DCT requires that the dominant pulse sit somewhere 3.4 along the array of coefficients. /// - /// - /// - /// - /// public static double CalculateRequiredFrameOverlap(int sr, int frameWidth, double maxOscilation) { double optimumFrameRate = 3 * maxOscilation; //so that max oscillation sits in 3/4 along the array of DCT coefficients @@ -551,5 +557,5 @@ public static double CalculateRequiredFrameOverlap(int sr, int frameWidth, doubl double overlap = (frameWidth - frameOffset) / (double)frameWidth; return overlap; } - }//end class -} //AudioAnalysisTools + } +}