Skip to content

Commit

Permalink
Start work on the new Event hierarchy
Browse files Browse the repository at this point in the history 
Issue#297 Changes have caused errors in may of the species Recognizers.
WARNING: This commit does not build.
  • Loading branch information
@towsey
towsey committed Apr 22, 2020
1 parent 7d529e6 commit 08a8645
Show file tree
Hide file tree
Showing 30 changed files with 466 additions and 159 deletions.
36 changes: 24 additions & 12 deletions src/AnalysisPrograms/AED.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ namespace AnalysisPrograms
using AnalysisPrograms.Production.Arguments;
using AudioAnalysisTools;
using AudioAnalysisTools.DSP;
using AudioAnalysisTools.Events;
using AudioAnalysisTools.Events.Types;
using AudioAnalysisTools.StandardSpectrograms;
using AudioAnalysisTools.WavTools;
using log4net;
Expand Down Expand Up @@ -117,7 +119,7 @@ public AedConfiguration()
/// </summary>
public override string Identifier => EcosoundsAedIdentifier;

public static Tuple<AcousticEvent[], AudioRecording, BaseSonogram> Detect(
public static Tuple<SpectralEvent[], AudioRecording, BaseSonogram> Detect(
FileInfo audioFile,
AedConfiguration aedConfiguration,
TimeSpan segmentStartOffset)
Expand All @@ -144,12 +146,12 @@ public static Tuple<AcousticEvent[], AudioRecording, BaseSonogram> Detect(
};
var sonogram = (BaseSonogram)new SpectrogramStandard(config, recording.WavReader);

AcousticEvent[] events = CallAed(sonogram, aedConfiguration, segmentStartOffset, segmentDuration);
SpectralEvent[] events = CallAed(sonogram, aedConfiguration, segmentStartOffset, segmentDuration);
Log.Debug("AED # events: " + events.Length);
return Tuple.Create(events, recording, sonogram);
}

public static AcousticEvent[] CallAed(BaseSonogram sonogram, AedConfiguration aedConfiguration, TimeSpan segmentStartOffset, TimeSpan segmentDuration)
public static SpectralEvent[] CallAed(BaseSonogram sonogram, AedConfiguration aedConfiguration, TimeSpan segmentStartOffset, TimeSpan segmentDuration)
{
Log.Info("AED start");

Expand Down Expand Up @@ -180,33 +182,43 @@ public static AcousticEvent[] CallAed(BaseSonogram sonogram, AedConfiguration ae
o.HitElements = null;
}

return new AcousticEvent(
var ae = AcousticEvent.InitializeAcousticEvent(
segmentStartOffset,
o,
sonogram.NyquistFrequency,
sonogram.Configuration.FreqBinCount,
sonogram.FrameDuration,
sonogram.FrameStep,
sonogram.FrameCount)
{
BorderColour = aedConfiguration.AedEventColor,
HitColour = aedConfiguration.AedHitColor,
SegmentDurationSeconds = segmentDuration.TotalSeconds,
};
sonogram.FrameCount);

ae.BorderColour = aedConfiguration.AedEventColor;
ae.HitColour = aedConfiguration.AedHitColor;
ae.SegmentDurationSeconds = segmentDuration.TotalSeconds;

return EventConverters.ConvertAcousticEventToSpectralEvent(ae);
}).ToArray();
return events;
}

public static Image DrawSonogram(BaseSonogram sonogram, IEnumerable<AcousticEvent> events)
public static Image DrawSonogram(BaseSonogram sonogram, IEnumerable<SpectralEvent> events)
{
var image = new Image_MultiTrack(sonogram.GetImage(false, true, doMelScale: false));

image.AddTrack(ImageTrack.GetTimeTrack(sonogram.Duration, sonogram.FramesPerSecond));

////image.AddTrack(ImageTrack.GetWavEnvelopeTrack(sonogram, image.sonogramImage.Width));
image.AddTrack(ImageTrack.GetSegmentationTrack(sonogram));

//############################################################################################ TODO TODO
//convert blob events to acoustic events for drawing purposes
var aeEvents = new List<AcousticEvent>();
foreach (var be in events)
{
aeEvents.Add(EventConverters.ConvertSpectralEventToAcousticEvent(be));
}

image.AddEvents(
events,
aeEvents,
sonogram.NyquistFrequency,
sonogram.Configuration.FreqBinCount,
sonogram.FramesPerSecond);
Expand Down
11 changes: 6 additions & 5 deletions src/AnalysisPrograms/Recognizers/Base/ForwardTrackParameters.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ namespace AnalysisPrograms.Recognizers.Base
using System.Collections.Generic;
using Acoustics.Shared;
using AudioAnalysisTools;
using AudioAnalysisTools.Events.Interfaces;
using AudioAnalysisTools.Events;
using AudioAnalysisTools.Events.Tracks;
using AudioAnalysisTools.StandardSpectrograms;
using TowseyLibrary;
Expand Down Expand Up @@ -44,7 +44,7 @@ public class ForwardTrackParameters : CommonParameters
/// <param name="combinePossibleHarmonics">Combine tracks that are likely to be harmonics/formants.</param>
/// <param name="segmentStartOffset">The start time of the current recording segment under analysis.</param>
/// <returns>A list of acoustic events containing foward tracks.</returns>
public static (List<AcousticEvent> Events, double[] CombinedIntensity) GetForwardTracks(
public static (List<SpectralEvent> Events, double[] CombinedIntensity) GetForwardTracks(
SpectrogramStandard sonogram,
int minHz,
int maxHz,
Expand Down Expand Up @@ -92,11 +92,11 @@ public static (List<AcousticEvent> Events, double[] CombinedIntensity) GetForwar

// initialise tracks as events and get the combined intensity array.
// list of accumulated acoustic events
var events = new List<AcousticEvent>();
var events = new List<SpectralEvent>();
var combinedIntensityArray = new double[frameCount];
foreach (var track in tracks)
{
var ae = new AcousticEvent(segmentStartOffset, track)
var ae = new ChirpEvent(track)
{
SegmentDurationSeconds = frameCount * converter.StepSize,
};
Expand All @@ -119,7 +119,8 @@ public static (List<AcousticEvent> Events, double[] CombinedIntensity) GetForwar
var hertzGap = 200;
if (combinePossibleHarmonics)
{
events = AcousticEvent.CombinePotentialStackedTracks(events, startDifference, hertzGap);
//######################################################################################### TODO TODO
//events = CompositeEvent.CombinePotentialStackedTracks(events, startDifference, hertzGap);
}

return (events, combinedIntensityArray);
Expand Down
19 changes: 12 additions & 7 deletions src/AnalysisPrograms/Recognizers/Base/HarmonicParameters.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ namespace AnalysisPrograms.Recognizers.Base
using System.Collections.Generic;
using Acoustics.Shared;
using AudioAnalysisTools;
using AudioAnalysisTools.Events;
using AudioAnalysisTools.Events.Types;
using AudioAnalysisTools.StandardSpectrograms;
using TowseyLibrary;

Expand All @@ -33,7 +35,7 @@ public class HarmonicParameters : CommonParameters
/// </summary>
public int? MaxFormantGap { get; set; }

public static (List<AcousticEvent>, double[], double[]) GetComponentsWithHarmonics(
public static (List<SpectralEvent> SpectralEvents, double[] AmplitudeArray, double[] HarmonicIntensityScores) GetComponentsWithHarmonics(
SpectrogramStandard spectrogram,
int minHz,
int maxHz,
Expand All @@ -46,7 +48,6 @@ public static (List<AcousticEvent>, double[], double[]) GetComponentsWithHarmoni
int maxFormantGap,
TimeSpan segmentStartOffset)
{

// Event threshold - Determines FP / FN trade-off for events.
//double eventThreshold = 0.2;

Expand Down Expand Up @@ -93,7 +94,7 @@ public static (List<AcousticEvent>, double[], double[]) GetComponentsWithHarmoni

//extract the events based on length and threshhold.
// Note: This method does NOT do prior smoothing of the score array.
var acousticEvents = AcousticEvent.ConvertScoreArray2Events(
var harmonicEvents = AcousticEvent.ConvertScoreArray2Events(
harmonicIntensityScores,
minHz,
maxHz,
Expand All @@ -104,15 +105,19 @@ public static (List<AcousticEvent>, double[], double[]) GetComponentsWithHarmoni
maxDuration,
segmentStartOffset);

var spectralEvents = new List<SpectralEvent>();

// add in temporary names to the events
// These can be altered later.
foreach (var ev in acousticEvents)
foreach (var he in harmonicEvents)
{
ev.SpeciesName = "NoName";
ev.Name = "Harmonics";
var se = EventConverters.ConvertAcousticEventToSpectralEvent(he);
spectralEvents.Add(se);
se.Name = "Harmonics";
//se.ComponentName = "Harmonics";
}

return (acousticEvents, dBArray, harmonicIntensityScores);
return (spectralEvents, dBArray, harmonicIntensityScores);
}
}
}
82 changes: 7 additions & 75 deletions src/AnalysisPrograms/Recognizers/Base/OnebinTrackParameters.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,14 @@ namespace AnalysisPrograms.Recognizers.Base
using System.Collections.Generic;
using Acoustics.Shared;
using AudioAnalysisTools;
using AudioAnalysisTools.Events;
using AudioAnalysisTools.Events.Tracks;
using AudioAnalysisTools.StandardSpectrograms;
using TowseyLibrary;

/// <summary>
/// Parameters needed from a config file to detect whistle components.
/// A one-bin sounds like a pur-tone whistle. Each track point advances one time step. Points stay in the same frequency bin.
/// A one-bin sounds like a pure-tone whistle. Each track point advances one time step. Points stay in the same frequency bin.
/// </summary>
[YamlTypeTag(typeof(OnebinTrackParameters))]
public class OnebinTrackParameters : CommonParameters
Expand All @@ -30,7 +31,7 @@ public class OnebinTrackParameters : CommonParameters
/// This method averages dB log values incorrectly but it is faster than doing many log conversions.
/// This method is used to find acoustic events and is accurate enough for the purpose.
/// </summary>
public static (List<AcousticEvent> ListOfevents, double[] CombinedIntensityArray) GetOnebinTracks(
public static (List<SpectralEvent> ListOfevents, double[] CombinedIntensityArray) GetOnebinTracks(
SpectrogramStandard sonogram,
int minHz,
int maxHz,
Expand Down Expand Up @@ -88,81 +89,12 @@ public static (List<AcousticEvent> ListOfevents, double[] CombinedIntensityArray

var tracks = TrackExtractor.GetOnebinTracks(peaks, minDuration, maxDuration, decibelThreshold, converter);

/*
// for all frequency bins except top and bottom
for (int bin = minBin + 1; bin < maxBin; bin++)
{
// set up an intensity array for the frequency bin.
double[] intensity = new double[frameCount];
// buffer zone around whistle is four bins wide.
if (minBin < 4)
{
// for all time frames in this frequency bin
for (int t = 0; t < frameCount; t++)
{
var bandIntensity = (sonogramData[t, bin - 1] + sonogramData[t, bin] + sonogramData[t, bin + 1]) / 3.0;
var topSideBandIntensity = (sonogramData[t, bin + 3] + sonogramData[t, bin + 4] + sonogramData[t, bin + 5]) / 3.0;
intensity[t] = bandIntensity - topSideBandIntensity;
intensity[t] = Math.Max(0.0, intensity[t]);
}
}
else
{
// for all time frames in this frequency bin
for (int t = 0; t < frameCount; t++)
{
var bandIntensity = (sonogramData[t, bin - 1] + sonogramData[t, bin] + sonogramData[t, bin + 1]) / 3.0;
var topSideBandIntensity = (sonogramData[t, bin + 3] + sonogramData[t, bin + 4] + sonogramData[t, bin + 5]) / 6.0;
var bottomSideBandIntensity = (sonogramData[t, bin - 3] + sonogramData[t, bin - 4] + sonogramData[t, bin - 5]) / 6.0;
intensity[t] = bandIntensity - topSideBandIntensity - bottomSideBandIntensity;
intensity[t] = Math.Max(0.0, intensity[t]);
}
}
// smooth the decibel array to allow for brief gaps.
intensity = DataTools.filterMovingAverageOdd(intensity, 7);
//calculate the Hertz bounds of the acoustic events for these freq bins
int bottomHzBound = (int)Math.Floor(sonogram.FBinWidth * (bin - 1));
int topHzBound = (int)Math.Ceiling(sonogram.FBinWidth * (bin + 2));
// list of accumulated acoustic events
var events = new List<AcousticEvent>();
var combinedIntensityArray = new double[frameCount];
//extract the events based on length and threshhold.
// Note: This method does NOT do prior smoothing of the dB array.
var acousticEvents = AcousticEvent.ConvertScoreArray2Events(
intensity,
bottomHzBound,
topHzBound,
sonogram.FramesPerSecond,
sonogram.FBinWidth,
decibelThreshold,
minDuration,
maxDuration,
segmentStartOffset);
// add to conbined intensity array
for (int t = 0; t < frameCount; t++)
{
//combinedIntensityArray[t] += intensity[t];
combinedIntensityArray[t] = Math.Max(intensity[t], combinedIntensityArray[t]);
}
// combine events
events.AddRange(acousticEvents);
} //end for all freq bins
*/

// initialise tracks as events and get the combined intensity array.
// list of accumulated acoustic events
var events = new List<AcousticEvent>();
var events = new List<SpectralEvent>();
var combinedIntensityArray = new double[frameCount];
foreach (var track in tracks)
{
var ae = new AcousticEvent(segmentStartOffset, track)
var ae = new WhistleEvent(track)
{
SegmentDurationSeconds = frameCount * converter.StepSize,
};
Expand All @@ -174,7 +106,6 @@ public static (List<AcousticEvent> ListOfevents, double[] CombinedIntensityArray
var amplitudeTrack = track.GetAmplitudeOverTimeFrames();
for (int i = 0; i < amplitudeTrack.Length; i++)
{
//combinedIntensityArray[startRow + i] += amplitudeTrack[i];
combinedIntensityArray[startRow + i] = Math.Max(combinedIntensityArray[startRow + i], amplitudeTrack[i]);
}
}
Expand All @@ -185,7 +116,8 @@ public static (List<AcousticEvent> ListOfevents, double[] CombinedIntensityArray
var hertzGap = 100;
if (combinePossibleSequence)
{
events = AcousticEvent.CombineSimilarProximalEvents(events, startDifference, hertzGap);
//################################################################################TODO TODO
//events = AcousticEvent.CombineSimilarProximalEvents(events, startDifference, hertzGap);
}

return (events, combinedIntensityArray);
Expand Down
17 changes: 7 additions & 10 deletions src/AnalysisPrograms/Recognizers/Base/OneframeTrackParameters.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// <copyright file="OneframeTrackParameters.cs" company="QutEcoacoustics">
// <copyright file="OneframeTrackParameters.cs" company="QutEcoacoustics">
// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group).
// </copyright>

Expand All @@ -9,6 +9,7 @@ namespace AnalysisPrograms.Recognizers.Base
using System.Linq;
using Acoustics.Shared;
using AudioAnalysisTools;
using AudioAnalysisTools.Events;
using AudioAnalysisTools.Events.Tracks;
using AudioAnalysisTools.StandardSpectrograms;

Expand Down Expand Up @@ -40,7 +41,7 @@ public class OneframeTrackParameters : CommonParameters
/// THis method averages dB log values incorrectly but it is faster than doing many log conversions.
/// This method is used to find acoustic events and is accurate enough for the purpose.
/// </summary>
public static (List<AcousticEvent> Events, double[] Intensity) GetOneFrameTracks(
public static (List<SpectralEvent> Events, double[] Intensity) GetOneFrameTracks(
SpectrogramStandard sonogram,
int minHz,
int maxHz,
Expand Down Expand Up @@ -95,20 +96,15 @@ public static (List<AcousticEvent> Events, double[] Intensity) GetOneFrameTracks
var tracks = TrackExtractor.GetOneFrameTracks(peaks, minBin, maxBin, minBandwidthHertz, maxBandwidthHertz, decibelThreshold, converter);

// initialise tracks as events and get the combined intensity array.
var events = new List<AcousticEvent>();
var events = new List<SpectralEvent>();
var temporalIntensityArray = new double[frameCount];
foreach (var track in tracks)
{
var ae = new AcousticEvent(segmentStartOffset, track.StartTimeSeconds, track.TrackDurationSeconds, track.LowFreqHertz, track.HighFreqHertz)
var ae = new ClickEvent(track)
{
SegmentDurationSeconds = frameCount * frameStep,
};

var tr = new List<Track>
{
track,
};
ae.AddTracks(tr);
events.Add(ae);

// fill the intensity array
Expand All @@ -125,7 +121,8 @@ public static (List<AcousticEvent> Events, double[] Intensity) GetOneFrameTracks
{
TimeSpan startDifference = TimeSpan.FromSeconds(0.5);
int hertzDifference = 500;
events = AcousticEvent.CombineSimilarProximalEvents(events, startDifference, hertzDifference);
//######################################################################## TODO TODO TODOD
//events = AcousticEvent.CombineSimilarProximalEvents(events, startDifference, hertzDifference);
}

return (events, temporalIntensityArray);
Expand Down
Loading

0 comments on commit 08a8645

Please sign in to comment.