From 04a0808a51b8ecf506e4729c989fbdc4b9e881eb Mon Sep 17 00:00:00 2001 From: towsey Date: Mon, 20 Apr 2020 21:00:47 +1000 Subject: [PATCH] Rename some of the track/event recognizers --- .../Recognizers/Base/OnebinTrackParameters.cs | 194 ++++++++++++++++++ ...rameters.cs => OneframeTrackParameters.cs} | 105 +++++----- .../Recognizers/Base/WhistleParameters.cs | 115 ----------- .../Events/CompositeEvent.cs | 43 ---- 4 files changed, 246 insertions(+), 211 deletions(-) create mode 100644 src/AnalysisPrograms/Recognizers/Base/OnebinTrackParameters.cs rename src/AnalysisPrograms/Recognizers/Base/{ClickParameters.cs => OneframeTrackParameters.cs} (78%) delete mode 100644 src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs delete mode 100644 src/AudioAnalysisTools/Events/CompositeEvent.cs diff --git a/src/AnalysisPrograms/Recognizers/Base/OnebinTrackParameters.cs b/src/AnalysisPrograms/Recognizers/Base/OnebinTrackParameters.cs new file mode 100644 index 000000000..94d01b247 --- /dev/null +++ b/src/AnalysisPrograms/Recognizers/Base/OnebinTrackParameters.cs @@ -0,0 +1,194 @@ +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// + +namespace AnalysisPrograms.Recognizers.Base +{ + using System; + using System.Collections.Generic; + using Acoustics.Shared; + using AudioAnalysisTools; + using AudioAnalysisTools.Events.Tracks; + using AudioAnalysisTools.StandardSpectrograms; + using TowseyLibrary; + + /// + /// Parameters needed from a config file to detect whistle components. + /// A one-bin sounds like a pur-tone whistle. Each track point advances one time step. Points stay in the same frequency bin. + /// + [YamlTypeTag(typeof(OnebinTrackParameters))] + public class OnebinTrackParameters : CommonParameters + { + /// + /// Gets or sets a value indicating whether proximal whistle tracks are to be combined. + /// Proximal means the whistle tracks are in the same frequency band + /// ... and that the gap between their start times is not greater than the specified seconds interval. + /// + public bool CombinePossibleSequence { get; set; } + + /// + /// This method averages dB log values incorrectly but it is faster than doing many log conversions. + /// This method is used to find acoustic events and is accurate enough for the purpose. + /// + public static (List ListOfevents, double[] CombinedIntensityArray) GetOnebinTracks( + SpectrogramStandard sonogram, + int minHz, + int maxHz, + double decibelThreshold, + double minDuration, + double maxDuration, + bool combinePossibleSequence, + TimeSpan segmentStartOffset) + { + var sonogramData = sonogram.Data; + int frameCount = sonogramData.GetLength(0); + int binCount = sonogramData.GetLength(1); + int nyquist = sonogram.NyquistFrequency; + double binWidth = nyquist / (double)binCount; + int minBin = (int)Math.Round(minHz / binWidth); + int maxBin = (int)Math.Round(maxHz / binWidth); + + var converter = new UnitConverters( + segmentStartOffset: segmentStartOffset.TotalSeconds, + sampleRate: sonogram.SampleRate, + frameSize: sonogram.Configuration.WindowSize, + frameOverlap: sonogram.Configuration.WindowOverlap); + + //Find all bin peaks and place in peaks matrix + var peaks = new double[frameCount, binCount]; + for (int tf = 0; tf < frameCount; tf++) + { + for (int bin = minBin + 1; bin < maxBin - 1; bin++) + { + if (sonogramData[tf, bin] < decibelThreshold) + { + continue; + } + + // here we define the amplitude profile of a whistle. The buffer zone around whistle is five bins wide. + var bandIntensity = ((sonogramData[tf, bin - 1] * 0.5) + sonogramData[tf, bin] + (sonogramData[tf, bin + 1] * 0.5)) / 2.0; + var topSidebandIntensity = (sonogramData[tf, bin + 3] + sonogramData[tf, bin + 4] + sonogramData[tf, bin + 5]) / 3.0; + var netAmplitude = 0.0; + if (bin < 4) + { + netAmplitude = bandIntensity - topSidebandIntensity; + } + else + { + var bottomSideBandIntensity = (sonogramData[tf, bin - 3] + sonogramData[tf, bin - 4] + sonogramData[tf, bin - 5]) / 3.0; + netAmplitude = bandIntensity - ((topSidebandIntensity + bottomSideBandIntensity) / 2.0); + } + + if (netAmplitude >= decibelThreshold) + { + peaks[tf, bin] = Math.Max(0.0, netAmplitude); + } + } + } + + var tracks = TrackExtractor.GetOnebinTracks(peaks, minDuration, maxDuration, decibelThreshold, converter); + + /* + // for all frequency bins except top and bottom + for (int bin = minBin + 1; bin < maxBin; bin++) + { + // set up an intensity array for the frequency bin. + double[] intensity = new double[frameCount]; + + // buffer zone around whistle is four bins wide. + if (minBin < 4) + { + // for all time frames in this frequency bin + for (int t = 0; t < frameCount; t++) + { + var bandIntensity = (sonogramData[t, bin - 1] + sonogramData[t, bin] + sonogramData[t, bin + 1]) / 3.0; + var topSideBandIntensity = (sonogramData[t, bin + 3] + sonogramData[t, bin + 4] + sonogramData[t, bin + 5]) / 3.0; + intensity[t] = bandIntensity - topSideBandIntensity; + intensity[t] = Math.Max(0.0, intensity[t]); + } + } + else + { + // for all time frames in this frequency bin + for (int t = 0; t < frameCount; t++) + { + var bandIntensity = (sonogramData[t, bin - 1] + sonogramData[t, bin] + sonogramData[t, bin + 1]) / 3.0; + var topSideBandIntensity = (sonogramData[t, bin + 3] + sonogramData[t, bin + 4] + sonogramData[t, bin + 5]) / 6.0; + var bottomSideBandIntensity = (sonogramData[t, bin - 3] + sonogramData[t, bin - 4] + sonogramData[t, bin - 5]) / 6.0; + intensity[t] = bandIntensity - topSideBandIntensity - bottomSideBandIntensity; + intensity[t] = Math.Max(0.0, intensity[t]); + } + } + + // smooth the decibel array to allow for brief gaps. + intensity = DataTools.filterMovingAverageOdd(intensity, 7); + + //calculate the Hertz bounds of the acoustic events for these freq bins + int bottomHzBound = (int)Math.Floor(sonogram.FBinWidth * (bin - 1)); + int topHzBound = (int)Math.Ceiling(sonogram.FBinWidth * (bin + 2)); + + // list of accumulated acoustic events + var events = new List(); + var combinedIntensityArray = new double[frameCount]; + + //extract the events based on length and threshhold. + // Note: This method does NOT do prior smoothing of the dB array. + var acousticEvents = AcousticEvent.ConvertScoreArray2Events( + intensity, + bottomHzBound, + topHzBound, + sonogram.FramesPerSecond, + sonogram.FBinWidth, + decibelThreshold, + minDuration, + maxDuration, + segmentStartOffset); + + // add to conbined intensity array + for (int t = 0; t < frameCount; t++) + { + //combinedIntensityArray[t] += intensity[t]; + combinedIntensityArray[t] = Math.Max(intensity[t], combinedIntensityArray[t]); + } + + // combine events + events.AddRange(acousticEvents); + } //end for all freq bins + */ + + // initialise tracks as events and get the combined intensity array. + // list of accumulated acoustic events + var events = new List(); + var combinedIntensityArray = new double[frameCount]; + foreach (var track in tracks) + { + var ae = new AcousticEvent(segmentStartOffset, track.StartTimeSeconds, track.TrackDurationSeconds, track.LowFreqHertz, track.HighFreqHertz); + var tr = new List + { + track, + }; + ae.AddTracks(tr); + events.Add(ae); + + // fill the intensity array + var startRow = converter.FrameFromStartTime(track.StartTimeSeconds); + var amplitudeTrack = track.GetAmplitudeOverTimeFrames(); + for (int i = 0; i < amplitudeTrack.Length; i++) + { + combinedIntensityArray[startRow + i] = Math.Max(combinedIntensityArray[startRow + i], amplitudeTrack[i]); + } + } + + // Combine possible related events. + // This will help in some cases. + var startDifference = TimeSpan.FromSeconds(0.5); + var hertzGap = 100; + if (combinePossibleSequence) + { + events = AcousticEvent.CombineSimilarProximalEvents(events, startDifference, hertzGap); + } + + return (events, combinedIntensityArray); + } + } +} \ No newline at end of file diff --git a/src/AnalysisPrograms/Recognizers/Base/ClickParameters.cs b/src/AnalysisPrograms/Recognizers/Base/OneframeTrackParameters.cs similarity index 78% rename from src/AnalysisPrograms/Recognizers/Base/ClickParameters.cs rename to src/AnalysisPrograms/Recognizers/Base/OneframeTrackParameters.cs index eba4eed00..65b568920 100644 --- a/src/AnalysisPrograms/Recognizers/Base/ClickParameters.cs +++ b/src/AnalysisPrograms/Recognizers/Base/OneframeTrackParameters.cs @@ -1,4 +1,4 @@ -// +// // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). // @@ -9,13 +9,14 @@ namespace AnalysisPrograms.Recognizers.Base using System.Linq; using Acoustics.Shared; using AudioAnalysisTools; + using AudioAnalysisTools.Events.Tracks; using AudioAnalysisTools.StandardSpectrograms; /// /// Parameters needed from a config file to detect click components. /// - [YamlTypeTag(typeof(ClickParameters))] - public class ClickParameters : CommonParameters + [YamlTypeTag(typeof(OneframeTrackParameters))] + public class OneframeTrackParameters : CommonParameters { /// /// Gets or sets the minimum bandwidth, units = Hertz. @@ -34,20 +35,15 @@ public class ClickParameters : CommonParameters /// public bool CombineProximalSimilarEvents { get; set; } - public TimeSpan StartDifference { get; set; } - - public int HertzDifference { get; set; } - /// /// A click is a sharp onset broadband sound of brief duration. Geometrically it is similar to a vertical whistle. /// THis method averages dB log values incorrectly but it is faster than doing many log conversions. /// This method is used to find acoustic events and is accurate enough for the purpose. /// - public static (List Events, double[] Intensity) GetClicks( + public static (List Events, double[] Intensity) GetOneFrameTracks( SpectrogramStandard sonogram, int minHz, int maxHz, - int nyquist, double decibelThreshold, int minBandwidthHertz, int maxBandwidthHertz, @@ -57,83 +53,85 @@ public static (List Events, double[] Intensity) GetClicks( var sonogramData = sonogram.Data; int frameCount = sonogramData.GetLength(0); int binCount = sonogramData.GetLength(1); - + var frameStep = sonogram.FrameStep; + int nyquist = sonogram.NyquistFrequency; double binWidth = nyquist / (double)binCount; int minBin = (int)Math.Round(minHz / binWidth); int maxBin = (int)Math.Round(maxHz / binWidth); - int bandwidthBinCount = maxBin - minBin + 1; - // list of accumulated acoustic events - var events = new List(); - var temporalIntensityArray = new double[frameCount]; + var converter = new UnitConverters( + segmentStartOffset: segmentStartOffset.TotalSeconds, + sampleRate: sonogram.SampleRate, + frameSize: sonogram.Configuration.WindowSize, + frameOverlap: sonogram.Configuration.WindowOverlap); + + // Find all frame peaks and place in peaks matrix + // avoid row edge effects. + var peaks = new double[frameCount, binCount]; // for all time frames except 1st and last allowing for edge effects. for (int t = 1; t < frameCount - 1; t++) { - // set up an intensity array for all frequency bins in this frame. - double[] clickIntensity = new double[bandwidthBinCount]; - // buffer zone around click is one frame wide. // for all frequency bins except top and bottom in this time frame for (int bin = minBin; bin < maxBin; bin++) { - // THis is where the profile of a click is defined - // A click requires sudden onset, with maximum amplitude followed by decay. - if (sonogramData[t - 1, bin] > decibelThreshold || sonogramData[t, bin] < sonogramData[t + 1, bin]) + if (sonogramData[t, bin] < decibelThreshold) { continue; } - // THis is where the profile of a vertical ridge is defined - //if (sonogramData[t, bin] < sonogramData[t - 1, bin] || sonogramData[t, bin] < sonogramData[t + 1, bin]) - //{ - // continue; - //} - - clickIntensity[bin - minBin] = sonogramData[t, bin]; - //clickIntensity[bin - minBin] = sonogramData[t, bin] - sonogramData[t - 1, bin]; - clickIntensity[bin - minBin] = Math.Max(0.0, clickIntensity[bin - minBin]); + // THis is where the profile of a click is defined + // A click requires sudden onset, with maximum amplitude followed by decay. + bool isClickPeak = sonogramData[t - 1, bin] < decibelThreshold && sonogramData[t, bin] > sonogramData[t + 1, bin]; + if (isClickPeak) + { + peaks[t, bin] = sonogramData[t, bin]; + } } + } + + //NOTE: the Peaks matrix is same size as the sonogram. + var tracks = TrackExtractor.GetOneFrameTracks(peaks, minBin, maxBin, minBandwidthHertz, maxBandwidthHertz, decibelThreshold, converter); - if (clickIntensity.Max() < decibelThreshold) + // initialise tracks as events and get the combined intensity array. + var events = new List(); + var temporalIntensityArray = new double[frameCount]; + foreach (var track in tracks) + { + var ae = new AcousticEvent(segmentStartOffset, track.StartTimeSeconds, track.TrackDurationSeconds, track.LowFreqHertz, track.HighFreqHertz) { - continue; - } + SegmentDurationSeconds = frameCount * frameStep, + }; - // Extract the events based on bandwidth and threshhold. - var acousticEvents = ConvertSpectralArrayToClickEvents( - clickIntensity, - minHz, - sonogram.FramesPerSecond, - sonogram.FBinWidth, - decibelThreshold, - minBandwidthHertz, - maxBandwidthHertz, - t, - segmentStartOffset); - - // add each event score to combined temporal intensity array - foreach (var ae in acousticEvents) + var tr = new List { - var avClickIntensity = ae.Score; - temporalIntensityArray[t] += avClickIntensity; + track, + }; + ae.AddTracks(tr); + events.Add(ae); + + // fill the intensity array + var startRow = converter.FrameFromStartTime(track.StartTimeSeconds); + var amplitudeTrack = track.GetAmplitudeOverTimeFrames(); + for (int i = 0; i < amplitudeTrack.Length; i++) + { + temporalIntensityArray[startRow + i] += amplitudeTrack[i]; } - - // add new events to list of events - events.AddRange(acousticEvents); } // combine proximal events that occupy similar frequency band - var startDifference = TimeSpan.FromSeconds(1.0); - var hertzDifference = 100; if (combineProximalSimilarEvents) { + TimeSpan startDifference = TimeSpan.FromSeconds(0.5); + int hertzDifference = 500; events = AcousticEvent.CombineSimilarProximalEvents(events, startDifference, hertzDifference); } return (events, temporalIntensityArray); } + /* /// /// A general method to convert an array of score values to a list of AcousticEvents. /// NOTE: The score array is assumed to be a spectrum of dB intensity. @@ -246,5 +244,6 @@ public static List ConvertSpectralArrayToClickEvents( return events; } + */ } } diff --git a/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs b/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs deleted file mode 100644 index c888a60c0..000000000 --- a/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs +++ /dev/null @@ -1,115 +0,0 @@ -// -// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). -// - -namespace AnalysisPrograms.Recognizers.Base -{ - using System; - using System.Collections.Generic; - using Acoustics.Shared; - using AudioAnalysisTools; - using AudioAnalysisTools.StandardSpectrograms; - using TowseyLibrary; - - /// - /// Parameters needed from a config file to detect whistle components. - /// - [YamlTypeTag(typeof(WhistleParameters))] - public class WhistleParameters : CommonParameters - { - /// - /// Calculates the mean intensity in a freq band defined by its min and max freq. - /// THis method averages dB log values incorrectly but it is faster than doing many log conversions. - /// This method is used to find acoustic events and is accurate enough for the purpose. - /// - public static (List, double[]) GetWhistles( - SpectrogramStandard sonogram, - int minHz, - int maxHz, - int nyquist, - double decibelThreshold, - double minDuration, - double maxDuration, - TimeSpan segmentStartOffset) - { - var sonogramData = sonogram.Data; - int frameCount = sonogramData.GetLength(0); - int binCount = sonogramData.GetLength(1); - - double binWidth = nyquist / (double)binCount; - int minBin = (int)Math.Round(minHz / binWidth); - int maxBin = (int)Math.Round(maxHz / binWidth); - - // list of accumulated acoustic events - var events = new List(); - var combinedIntensityArray = new double[frameCount]; - - // for all frequency bins except top and bottom - for (int bin = minBin + 1; bin < maxBin; bin++) - { - // set up an intensity array for the frequency bin. - double[] intensity = new double[frameCount]; - - // buffer zone around whistle is four bins wide. - if (minBin < 4) - { - // for all time frames in this frequency bin - for (int t = 0; t < frameCount; t++) - { - var bandIntensity = (sonogramData[t, bin - 1] + sonogramData[t, bin] + sonogramData[t, bin + 1]) / 3.0; - var topSideBandIntensity = (sonogramData[t, bin + 3] + sonogramData[t, bin + 4] + sonogramData[t, bin + 5]) / 3.0; - intensity[t] = bandIntensity - topSideBandIntensity; - intensity[t] = Math.Max(0.0, intensity[t]); - } - } - else - { - // for all time frames in this frequency bin - for (int t = 0; t < frameCount; t++) - { - var bandIntensity = (sonogramData[t, bin - 1] + sonogramData[t, bin] + sonogramData[t, bin + 1]) / 3.0; - var topSideBandIntensity = (sonogramData[t, bin + 3] + sonogramData[t, bin + 4] + sonogramData[t, bin + 5]) / 6.0; - var bottomSideBandIntensity = (sonogramData[t, bin - 3] + sonogramData[t, bin - 4] + sonogramData[t, bin - 5]) / 6.0; - intensity[t] = bandIntensity - topSideBandIntensity - bottomSideBandIntensity; - intensity[t] = Math.Max(0.0, intensity[t]); - } - } - - // smooth the decibel array to allow for brief gaps. - intensity = DataTools.filterMovingAverageOdd(intensity, 7); - - //calculate the Hertz bounds of the acoustic events for these freq bins - int bottomHzBound = (int)Math.Floor(sonogram.FBinWidth * (bin - 1)); - int topHzBound = (int)Math.Ceiling(sonogram.FBinWidth * (bin + 2)); - - //extract the events based on length and threshhold. - // Note: This method does NOT do prior smoothing of the dB array. - var acousticEvents = AcousticEvent.ConvertScoreArray2Events( - intensity, - bottomHzBound, - topHzBound, - sonogram.FramesPerSecond, - sonogram.FBinWidth, - decibelThreshold, - minDuration, - maxDuration, - segmentStartOffset); - - // add to conbined intensity array - for (int t = 0; t < frameCount; t++) - { - //combinedIntensityArray[t] += intensity[t]; - combinedIntensityArray[t] = Math.Max(intensity[t], combinedIntensityArray[t]); - } - - // combine events - events.AddRange(acousticEvents); - } //end for all freq bins - - // combine adjacent acoustic events - events = AcousticEvent.CombineOverlappingEvents(events, segmentStartOffset); - - return (events, combinedIntensityArray); - } - } -} \ No newline at end of file diff --git a/src/AudioAnalysisTools/Events/CompositeEvent.cs b/src/AudioAnalysisTools/Events/CompositeEvent.cs deleted file mode 100644 index a56dbdfd3..000000000 --- a/src/AudioAnalysisTools/Events/CompositeEvent.cs +++ /dev/null @@ -1,43 +0,0 @@ -// -// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). -// - -namespace AudioAnalysisTools -{ - using System; - using System.Collections.Generic; - using System.Linq; - using AnalysisBase.ResultBases; - using AudioAnalysisTools.Events; - using AudioAnalysisTools.Events.Drawing; - using AudioAnalysisTools.Events.Interfaces; - using SixLabors.ImageSharp.Processing; - - public class CompositeEvent : SpectralEvent - { - public List ComponentEvents { get; set; } = new List(); - - public override double EventStartSeconds => - this.ComponentEvents.Min(x => x.EventStartSeconds); - - public override double EventEndSeconds => - this.ComponentEvents.Max(x => (x as ITemporalEvent)?.EventEndSeconds) ?? double.PositiveInfinity; - - public override double LowFrequencyHertz => - this.ComponentEvents.Min(x => (x as ISpectralEvent)?.LowFrequencyHertz) ?? 0; - - public override double HighFrequencyHertz => - this.ComponentEvents.Max(x => (x as ISpectralEvent)?.HighFrequencyHertz) ?? double.PositiveInfinity; - - public override void Draw(IImageProcessingContext graphics, EventRenderingOptions options) - { - foreach (var @event in this.ComponentEvents) - { - @event.Draw(graphics, options); - } - - // draw a border around all of it - base.Draw(graphics, options); - } - } -} \ No newline at end of file