Skip to content

Commit

Permalink
Set up Json reading and writing of template files
Browse files Browse the repository at this point in the history 
Issue #252 Set up reading and writing of Json template files
  • Loading branch information
@towsey
towsey committed Oct 2, 2019
1 parent 6bd20d2 commit d683f11
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 54 deletions.
31 changes: 3 additions & 28 deletions src/AnalysisPrograms/Sandpit.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,49 +123,24 @@ public override Task<int> Execute(CommandLineApplication app)
public static void ContentDescriptionCreateTemplates()
{
var templateManifests = new FileInfo(@"C:\Ecoacoustics\ContentDescription\ContentDescriptionTemplateManifests.yml");
var templateDefinitions = new FileInfo(@"C:\Ecoacoustics\ContentDescription\TemplateDefinitions.yml");
var templateDefinitions = new FileInfo(@"C:\Ecoacoustics\ContentDescription\TemplateDefinitions.json");
TemplateManifest.CreateNewFileOfTemplateDefinitions(templateManifests, templateDefinitions);
Console.WriteLine("# Finished creation of new manifest");

//PREPARE STRONG WIND TEMPLATE
//var ipDir = new DirectoryInfo(@"C:\Ecoacoustics\Output\Test\Test24HourRecording\TasmanIslandMez\Mez02\Towsey.Acoustic");
//var opDir = new DirectoryInfo(@"C:\Ecoacoustics\Output\ContentDescription");
//WindStrong1.WriteTemplateToFile(ipDir, opDir);

//PREPARE LIGHT WIND TEMPLATE
//var ipDir = new DirectoryInfo(@"C:\Ecoacoustics\Output\Test\Test24HourRecording\TasmanIslandMez\Mez03\Towsey.Acoustic");
//var opDir = new DirectoryInfo(@"C:\Ecoacoustics\Output\ContentDescription");
//WindLight1.WriteTemplateToFile(ipDir, opDir);

//PREPARE LIGHT RAIN TEMPLATE
//var ipDir = new DirectoryInfo(@"C:\Ecoacoustics\Output\Test\Test24HourRecording\TasmanIslandMez\Mez08\Towsey.Acoustic");
//var opDir = new DirectoryInfo(@"C:\Ecoacoustics\Output\ContentDescription");
//WindLight1.WriteTemplateToFile(ipDir, opDir);

//PREPARE BIRD MORNING CHORUS1 TEMPLATE
//var ipDir = new DirectoryInfo(@"C:\Ecoacoustics\Output\Test\Test24HourRecording\TasmanIslandMez\Mez05\Towsey.Acoustic");
//var opDir = new DirectoryInfo(@"C:\Ecoacoustics\Output\ContentDescription");
//BirdMorningChorus1.WriteTemplateToFile(ipDir, opDir);

//PREPARE MEZZANINE-TASMAN ISLAND SILVER-EYE TEMPLATE
//var ipDir = new DirectoryInfo(@"C:\Ecoacoustics\Output\Test\Test24HourRecording\TasmanIslandMez\Mez08\Towsey.Acoustic");
//var opDir = new DirectoryInfo(@"C:\Ecoacoustics\Output\ContentDescription");
//SilverEyeMezTasmanIs.WriteTemplateToFile(ipDir, opDir);
}

public static void ContentDescriptionApplyTemplates()
{
Console.WriteLine("# Start scanning with content description templates");

var templatesFile = new FileInfo(@"C:\Ecoacoustics\ContentDescription\TemplateDefinitions.yml");
var templatesFile = new FileInfo(@"C:\Ecoacoustics\ContentDescription\TemplateDefinitions.json");
var listOfIndexFiles = new FileInfo(@"C:\Ecoacoustics\Output\Test\Test24HourRecording\TasmanIslandMezIndexFiles.txt");
var contentPlots = ContentDescription.ContentDescriptionOfMultipleRecordingFiles(listOfIndexFiles, templatesFile);

// Attach content description plots to LDFC spectrogram and write to file
var path = Path.Combine(@"C:\Ecoacoustics\Output\Test\Test24HourRecording", "Testing__2Maps.png");
var ldfcSpectrogram = Image.FromFile(path);
var image = ContentVisualization.DrawLdfcSpectrogramWithContentScoreTracks(ldfcSpectrogram, contentPlots);
var path1 = Path.Combine(@"C:\Ecoacoustics\ContentDescription", "Testing_2Maps.CONTENTnew04.png");
var path1 = Path.Combine(@"C:\Ecoacoustics\ContentDescription", "Testing_2Maps.CONTENTnew05.png");
image.Save(path1);
Console.WriteLine("# Finished scanning recording with content description templates");
}
Expand Down
4 changes: 2 additions & 2 deletions src/AudioAnalysisTools/ContentDescriptionTools/ContentDescription.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ public class ContentDescription
public static List<Plot> ContentDescriptionOfMultipleRecordingFiles(FileInfo listOfIndexFiles, FileInfo templatesFile)
{
// Read in all template manifests
var templates = Yaml.Deserialize<TemplateManifest[]>(templatesFile);
//var templateCollection = ConfigFile.Deserialize<TemplateCollection>(templatesFile);
//var templates = Yaml.Deserialize<TemplateManifest[]>(templatesFile);
var templates = Json.Deserialize<TemplateManifest[]>(templatesFile);
var templatesAsDictionary = DataProcessing.ExtractDictionaryOfTemplateDictionaries(templates);

// Read in list of paths to index files
Expand Down
16 changes: 16 additions & 0 deletions src/AudioAnalysisTools/ContentDescriptionTools/DataProcessing.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,22 @@ namespace AudioAnalysisTools.ContentDescriptionTools

public static class DataProcessing
{
/// <summary>
/// Converts an array of templates to dictionary.
/// </summary>
/// <param name="array">An array of templates.</param>
/// <returns>A dictionary of templates.</returns>
public static Dictionary<string, TemplateManifest> ConvertTemplateArrayToDictionary(TemplateManifest[] array)
{
var dictionary = new Dictionary<string, TemplateManifest>();
foreach (var template in array)
{
dictionary.Add(template.Name, template);
}

return dictionary;
}

/// <summary>
/// Reads in all the index matrices whose keys are in the above array of IndexNames.
/// </summary>
Expand Down
93 changes: 69 additions & 24 deletions src/AudioAnalysisTools/ContentDescriptionTools/TemplateManifest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,48 +8,83 @@ namespace AudioAnalysisTools.ContentDescriptionTools
using System.Collections.Generic;
using System.IO;
using Acoustics.Shared;
using Newtonsoft.Json;

public enum TemplateStatus
public enum EditStatus
{
CalculateTemplate,
Edit,
Locked,
Copy,
Ignore,
}

public enum UseStatus
{
Use,
Ignore,
}

public class TemplateManifest
{
public static void CreateNewFileOfTemplateDefinitions(FileInfo manifestFile, FileInfo templateDefinitions)
public static void CreateNewFileOfTemplateDefinitions(FileInfo manifestFile, FileInfo templateDefinitionsFile)
{
// Read in all template manifests
var manifests = Yaml.Deserialize<TemplateManifest[]>(manifestFile);

// Read current template definitions and convert to dictionary
var arrayOfTemplates = Json.Deserialize<TemplateManifest[]>(templateDefinitionsFile);
var dictionaryOfCurrentTemplates = DataProcessing.ConvertTemplateArrayToDictionary(arrayOfTemplates);

// init a new template list for output.
var newTemplateList = new List<TemplateManifest>();

// cycle through all the manifests
foreach (var templateManifest in manifests)
{
if (templateManifest.Status == TemplateStatus.Locked)
var name = templateManifest.Name;
if (!dictionaryOfCurrentTemplates.ContainsKey(name))
{
newTemplateList.Add(templateManifest);
// the current manifest is not an existing template - therefore make it.
var newTemplate = CreateNewTemplateFromManifest(templateManifest);
var templateDict = CreateTemplateDeftn(templateManifest);
newTemplate.Template = templateDict;
newTemplate.MostRecentEdit = DateTime.Now;
newTemplateList.Add(newTemplate);
continue;
}

var newTemplate = CreateNewTemplateFromManifest(templateManifest);
newTemplate.MostRecentEdit = DateTime.Now;
if (templateManifest.EditStatus == EditStatus.Copy)
{
// add existing template unchanged.
var existingTemplate = dictionaryOfCurrentTemplates[name];
newTemplateList.Add(existingTemplate);
continue;
}

if (templateManifest.Status == TemplateStatus.CalculateTemplate)
if (templateManifest.EditStatus == EditStatus.Ignore)
{
var templateDict = CreateTemplate(templateManifest);
newTemplate.Template = templateDict;
// add existing template unchanged except change UseStatus to Ignore.
var existingTemplate = dictionaryOfCurrentTemplates[name];
existingTemplate.UseStatus = UseStatus.Ignore;
newTemplateList.Add(existingTemplate);
continue;
}

newTemplateList.Add(newTemplate);
if (templateManifest.EditStatus == EditStatus.CalculateTemplate)
{
// add existing template but recalculate the template definition
var existingTemplate = dictionaryOfCurrentTemplates[name];
existingTemplate.UseStatus = UseStatus.Use;
var templateDict = CreateTemplateDeftn(templateManifest);
existingTemplate.Template = templateDict;
newTemplateList.Add(existingTemplate);
}
}

var templatesFilePath = Path.Combine(manifestFile.DirectoryName ?? throw new InvalidOperationException(), "TemplateDefinitions.yml");
var templatesFilePath = Path.Combine(manifestFile.DirectoryName ?? throw new InvalidOperationException(), "TemplateDefinitions.json");

// Save the previous templates file
string backupTemplatesFilePath = Path.Combine(manifestFile.DirectoryName, "TemplateDefinitions.Backup.yml");
string backupTemplatesFilePath = Path.Combine(manifestFile.DirectoryName, "TemplateDefinitions.Backup.json");
if (File.Exists(backupTemplatesFilePath))
{
File.Delete(backupTemplatesFilePath);
Expand All @@ -63,13 +98,15 @@ public static void CreateNewFileOfTemplateDefinitions(FileInfo manifestFile, Fil

// No need to move the backup because serializing over-writes the current templates file.
var templatesFile = new FileInfo(templatesFilePath);
Yaml.Serialize(templatesFile, newTemplateList.ToArray());

//Yaml.Serialize(templatesFile, newTemplateList.ToArray());
Json.Serialise(templatesFile, newTemplateList.ToArray());
}

/// <summary>
/// THis method calculates new template based on passed manifest.
/// </summary>
public static Dictionary<string, double[]> CreateTemplate(TemplateManifest templateManifest)
public static Dictionary<string, double[]> CreateTemplateDeftn(TemplateManifest templateManifest)
{
// Get the template provenance. Assume array contains only one element.
var provenanceArray = templateManifest.Provenance;
Expand All @@ -81,26 +118,26 @@ public static Dictionary<string, double[]> CreateTemplate(TemplateManifest templ
var path = Path.Combine(sourceDirectory, baseName + ContentDescription.AnalysisString);
var dictionaryOfIndices = DataProcessing.ReadIndexMatrices(path);
var algorithmType = templateManifest.FeatureExtractionAlgorithm;
Dictionary<string, double[]> newTemplate;
Dictionary<string, double[]> newTemplateDeftn;

switch (algorithmType)
{
case 1:
newTemplate = ContentAlgorithms.CreateFullBandTemplate1(templateManifest, dictionaryOfIndices);
newTemplateDeftn = ContentAlgorithms.CreateFullBandTemplate1(templateManifest, dictionaryOfIndices);
break;
case 2:
newTemplate = ContentAlgorithms.CreateBroadbandTemplate1(templateManifest, dictionaryOfIndices);
newTemplateDeftn = ContentAlgorithms.CreateBroadbandTemplate1(templateManifest, dictionaryOfIndices);
break;
case 3:
newTemplate = ContentAlgorithms.CreateNarrowBandTemplate1(templateManifest, dictionaryOfIndices);
newTemplateDeftn = ContentAlgorithms.CreateNarrowBandTemplate1(templateManifest, dictionaryOfIndices);
break;
default:
//LoggedConsole.WriteWarnLine("Algorithm " + algorithmType + " does not exist.");
newTemplate = null;
newTemplateDeftn = null;
break;
}

return newTemplate;
return newTemplateDeftn;
}

public static TemplateManifest CreateNewTemplateFromManifest(TemplateManifest templateManifest)
Expand All @@ -109,11 +146,13 @@ public static TemplateManifest CreateNewTemplateFromManifest(TemplateManifest te
{
Name = templateManifest.Name,
TemplateId = templateManifest.TemplateId,
Status = templateManifest.Status,
EditStatus = templateManifest.EditStatus,
UseStatus = templateManifest.UseStatus,
FeatureExtractionAlgorithm = templateManifest.FeatureExtractionAlgorithm,
SpectralReductionFactor = templateManifest.SpectralReductionFactor,
BandMinHz = templateManifest.BandMinHz,
BandMaxHz = templateManifest.BandMaxHz,
Provenance = null,
};
return newTemplate;
}
Expand All @@ -134,11 +173,17 @@ public static TemplateManifest CreateNewTemplateFromManifest(TemplateManifest te
/// </summary>
public string GeneralComment { get; set; }

/// <summary>
/// Gets or sets the template edit status.
/// EditStatus can be "locked", etc.
/// </summary>
public EditStatus EditStatus { get; set; }

/// <summary>
/// Gets or sets the template manifest status.
/// Status can be "locked", etc.
/// UseStatus can be "use" or "ignore".
/// </summary>
public TemplateStatus Status { get; set; }
public UseStatus UseStatus { get; set; }

public DateTime MostRecentEdit { get; set; }

Expand Down

0 comments on commit d683f11

Please sign in to comment.