Skip to content

Commit

Permalink
Attempted to fix unit tests
Browse files Browse the repository at this point in the history 
Didn't actually fix any but I verified their correctness.

These changes are just refactors/enhancements
  • Loading branch information
@atruskie
atruskie committed Dec 3, 2017
1 parent 64ac921 commit b7a07e5
Show file tree
Hide file tree
Showing 6 changed files with 54 additions and 90 deletions.
18 changes: 6 additions & 12 deletions Acoustics/Acoustics.Test/AnalysisPrograms/Concatenation/ConcatenationTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ public void Cleanup()
*/

/// <summary>
/// METHOD TO CHECK Concatenation of spectral and summary index files when ConcatenateEverythingYouCanLayYourHandsOn = true
/// METHOD TO CHECK Concatenation of spectral and summary index files when
/// ConcatenateEverythingYouCanLayYourHandsOn = true
/// </summary>
[TestMethod]
public void ConcatenateEverythingYouCanLayYourHandsOn()
Expand All @@ -107,9 +108,7 @@ public void ConcatenateEverythingYouCanLayYourHandsOn()
ColorMap2 = "BGN-POW-EVN", // POW was depracated post May 2017
ConcatenateEverythingYouCanLayYourHandsOn = true, // join everything found
TimeSpanOffsetHint = TimeSpan.FromHours(8),
SunRiseDataFile = null,
DrawImages = true,
Verbose = true,

// following two lines can be used to add in a recognizer score track
EventDataDirectories = null,
Expand Down Expand Up @@ -154,16 +153,14 @@ public void ConcatenateIndexFilesTest24Hour()
DirectoryFilter = "*.wav",
FileStemName = "Test2_Indonesia",
StartDate = new DateTimeOffset(2016, 07, 26, 0, 0, 0, TimeSpan.Zero),
EndDate = new DateTimeOffset(2016, 07, 26, 0, 0, 0, TimeSpan.Zero),
EndDate = new DateTimeOffset(2016, 07, 27, 0, 0, 0, TimeSpan.Zero),
IndexPropertiesConfig = indexPropertiesConfig,
FalseColourSpectrogramConfig = testConfig,
ColorMap1 = LDSpectrogramRGB.DefaultColorMap1,
ColorMap2 = "BGN-POW-EVN", // POW was depracated post May 2017
ConcatenateEverythingYouCanLayYourHandsOn = false, // 24 hour blocks only
TimeSpanOffsetHint = TimeSpan.FromHours(8),
SunRiseDataFile = null,
DrawImages = true,
Verbose = true,

// following two lines can be used to add in a recognizer score track
EventDataDirectories = null,
Expand All @@ -180,6 +177,7 @@ public void ConcatenateIndexFilesTest24Hour()
Assert.IsTrue(imageFileInfo.Exists);

var actualImage = ImageTools.ReadImage2Bitmap(imageFileInfo.FullName);
// we expect only the second half (past midnight) of the image to be rendered
ImageAssert.IsSize(512, 632, actualImage);
ImageAssert.PixelIsColor(new Point(100, 100), Color.FromArgb(32, 25, 36), actualImage);
ImageAssert.PixelIsColor(new Point(100, 160), Color.FromArgb(0, 80, 132), actualImage);
Expand All @@ -193,7 +191,7 @@ public void ConcatenateIndexFilesTest24Hour()
/// In the case of this dataset, the two partial days of data will be concatenated separately.
/// </summary>
[TestMethod]
public void ConcatenateIndexFilesTest24HourWithDateRange()
public void ConcatenateIndexFilesTest24HourWithoutDateRange()
{
// top level directory
DirectoryInfo[] dataDirs = { this.outputDirectory.Combine("Indonesia_2Reduced") };
Expand All @@ -216,9 +214,7 @@ public void ConcatenateIndexFilesTest24HourWithDateRange()
ColorMap2 = "BGN-POW-EVN", // POW was depracated post May 2017
ConcatenateEverythingYouCanLayYourHandsOn = false, // 24 hour blocks only
TimeSpanOffsetHint = TimeSpan.FromHours(8),
SunRiseDataFile = null,
DrawImages = true,
Verbose = true,

// following two lines can be used to add in a recognizer score track
EventDataDirectories = null,
Expand Down Expand Up @@ -283,14 +279,12 @@ public void ConcatenateIndexFilesTestConfigFileChanges()
DirectoryFilter = "*.wav",
FileStemName = "Test2_Indonesia",
StartDate = new DateTimeOffset(2016, 07, 26, 0, 0, 0, TimeSpan.Zero),
EndDate = new DateTimeOffset(2016, 07, 26, 0, 0, 0, TimeSpan.Zero),
EndDate = new DateTimeOffset(2016, 07, 27, 0, 0, 0, TimeSpan.Zero),
IndexPropertiesConfig = indexPropertiesConfig,
FalseColourSpectrogramConfig = testConfig,
ConcatenateEverythingYouCanLayYourHandsOn = false, // 24 hour blocks only
TimeSpanOffsetHint = TimeSpan.FromHours(8),
SunRiseDataFile = null,
DrawImages = true,
Verbose = true,

// following two lines can be used to add in a recognizer score track
EventDataDirectories = null,
Expand Down
87 changes: 26 additions & 61 deletions AudioAnalysis/AnalysisPrograms/ConcatenateIndexFiles.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,6 @@ public class Arguments
[Production.ArgExistingFile(Extension = ".yml")]
public FileInfo FalseColourSpectrogramConfig { get; set; }

[ArgDescription("User specified file containing times of sunrise & sunset for recording location. Must be correct format!")]
[Production.ArgExistingFile(Extension = ".csv")]
public FileInfo SunRiseDataFile { get; set; }

[ArgDescription("Set true only when concatenating more than 24-hours of data into one image - e.g. PNG/Indonesian data.")]
public bool ConcatenateEverythingYouCanLayYourHandsOn { get; set; }

Expand All @@ -107,16 +103,12 @@ public class Arguments

[ArgDescription("Used only to get Event Recognizer files.")]
public string EventFilePattern { get; set; }

public bool Verbose { get; set; }
}

public static void Execute(Arguments arguments)
{
bool verbose = !arguments.Verbose; // the default value

// Concatenation is designed only for the output from a "Towsey.Acoustic" analysis.
const string analysisType = "Towsey.Acoustic";
const string AnalysisType = Acoustic.TowseyAcoustic;

// Get the currently available sepctral indices
// RHZ, SPT and CVR are correlated with POW and do not add much. CLS not particularly useful. Now using R3D
Expand Down Expand Up @@ -193,30 +185,20 @@ public static void Execute(Arguments arguments)
endDate = arguments.EndDate;
}

if (startDate > endDate)
if (startDate >= endDate)
{
LoggedConsole.WriteErrorLine("# The End Date must be same as, or after, the Start Date when ConcatenateEverythingYouCanLayYourHandsOn = false.");
throw new ArgumentException("FATAL ERROR: End Date must be same as, or after, the Start Date.");
LoggedConsole.WriteErrorLine("# The End Date must be greater than the Start Date when ConcatenateEverythingYouCanLayYourHandsOn = false.");
throw new ArgumentException("FATAL ERROR: End Date must be greater than the Start Date.");
}
}

var startDateTimeOffset = (DateTimeOffset)startDate;

if (verbose)
{
LoggedConsole.WriteLine("\n# Start date = " + startDate.ToString());
LoggedConsole.WriteLine("# End date = " + endDate.ToString());
LoggedConsole.WriteLine("# Time Zone = " + arguments.TimeSpanOffsetHint.ToString());
LoggedConsole.WriteLine("\n# Start date = " + startDate.ToString());
LoggedConsole.WriteLine("# End date = " + endDate.ToString());
LoggedConsole.WriteLine("# Time Zone = " + arguments.TimeSpanOffsetHint.ToString());

if (arguments.SunRiseDataFile != null && arguments.SunRiseDataFile.Exists)
{
LoggedConsole.WriteLine("# Sunrise/sunset data file = " + arguments.TimeSpanOffsetHint.ToString());
}
else
{
LoggedConsole.WriteLine("# WARNING: A sunrise/sunset data file does not exist for time zone >> " + arguments.TimeSpanOffsetHint.ToString());
}
}
LoggedConsole.WriteLine("# WARNING: A sunrise/sunset data file does not exist for time zone >> " + arguments.TimeSpanOffsetHint.ToString());

// create top level output directory if it does not exist.
DirectoryInfo opDir = arguments.OutputDirectory;
Expand Down Expand Up @@ -278,10 +260,7 @@ public static void Execute(Arguments arguments)
if (arguments.ConcatenateEverythingYouCanLayYourHandsOn)
{
var totalTimespan = (DateTimeOffset)endDate - (DateTimeOffset)startDate;
if (verbose)
{
LoggedConsole.WriteLine("# Total duration of available recording = " + totalTimespan.ToString());
}
LoggedConsole.WriteLine("# Total duration of available recording = " + totalTimespan.ToString());

if (totalTimespan > TimeSpan.FromDays(3))
{
Expand Down Expand Up @@ -322,10 +301,7 @@ public static void Execute(Arguments arguments)
tracksImage.Save(imagePath);
}

if (verbose)
{
LoggedConsole.WriteLine("# Finished summary indices. Now start spectral indices.");
}
LoggedConsole.WriteLine("# Finished summary indices. Now start spectral indices.");

// ###### NOW CONCATENATE THE SPECTRAL INDICES, DRAW IMAGES AND SAVE IN RESULTS DIRECTORY
var dictionaryOfSpectralIndices1 = LdSpectrogramStitching.ConcatenateAllSpectralIndexFiles(subDirectories, keys, indexGenerationData);
Expand All @@ -343,26 +319,26 @@ public static void Execute(Arguments arguments)
indexPropertiesConfig,
indexGenerationData,
opFileStem,
analysisType,
AnalysisType,
dictionaryOfSpectralIndices1,
/*summaryIndices = */null,
indexDistributions,
siteDescription,
arguments.SunRiseDataFile,
gapsAndJoins,
ImageChrome.With);
sunriseDataFile: null,
segmentErrors: gapsAndJoins,
imageChrome: ImageChrome.With);
}

WriteSpectralIndexFiles(resultsDir, opFileStem, analysisType, dictionaryOfSpectralIndices1);
WriteSpectralIndexFiles(resultsDir, opFileStem, AnalysisType, dictionaryOfSpectralIndices1);
return;
} // ConcatenateEverythingYouCanLayYourHandsOn
}

// ################################ ConcatenateEverythingYouCanLayYourHandsOn = false
// ################################ That is, CONCATENATE DATA in BLOCKS of 24 hours

var startDateOffset = (DateTimeOffset)startDateTimeOffset.Date;
var endOffset = ((DateTimeOffset)endDate).Date;
int dayCount = (endOffset - startDateOffset).Days + 1;
int dayCount = (int)Math.Ceiling((endOffset - startDateOffset).TotalDays);
LoggedConsole.WriteLine("# Day count = " + dayCount + " (inclusive of start and end days)");
/* Previously used the following line BUT the assumption proved to be a bug, not a feature.
// int dayCount = timespan.Days + 1; // This assumes that the last day has full 24 hours of recording available.
Expand Down Expand Up @@ -423,15 +399,11 @@ public static void Execute(Arguments arguments)
indexPropertiesConfig,
resultsDir,
siteDescription,
arguments.SunRiseDataFile,
indexErrors,
verbose);
sunriseDatafile: null,
erroneousSegments: indexErrors);
}

if (verbose)
{
LoggedConsole.WriteLine("# Finished summary indices. Now start spectral indices.");
}
LoggedConsole.WriteLine("# Finished summary indices. Now start spectral indices.");

// ##############################################################################################################

Expand Down Expand Up @@ -469,14 +441,14 @@ public static void Execute(Arguments arguments)
indexPropertiesConfig,
indexGenerationData,
opFileStem1,
analysisType,
AnalysisType,
dictionaryOfSpectralIndices2,
/*summaryIndices = */null,
indexDistributions,
siteDescription,
arguments.SunRiseDataFile,
indexErrors,
ImageChrome.With);
sunriseDataFile: null,
segmentErrors: indexErrors,
imageChrome: ImageChrome.With);

if (arguments.EventDataDirectories != null)
{
Expand Down Expand Up @@ -513,7 +485,7 @@ public static void Execute(Arguments arguments)
}
}

WriteSpectralIndexFiles(resultsDir, opFileStem1, analysisType, dictionaryOfSpectralIndices2);
WriteSpectralIndexFiles(resultsDir, opFileStem1, AnalysisType, dictionaryOfSpectralIndices2);
LoggedConsole.WriteLine(" Completed Spectral Indices");
} // over days
} // Execute()
Expand Down Expand Up @@ -782,6 +754,7 @@ public static double[] ConvertEventsToSummaryIndices(List<string> events)
return eventsPerUnitTime;
}

// TODO: [OPENSOURCE] remove all of the following
// ######################################## CONCATENATE INDEX FILES TEST METHODS BELOW HERE ######################################################

/*
Expand Down Expand Up @@ -866,9 +839,7 @@ public static void TESTMETHOD_ConcatenateIndexFilesTest1()
ColorMap2 = "BGN-POW-SPT", // This color map dates pre-May 2017.
ConcatenateEverythingYouCanLayYourHandsOn = true,
TimeSpanOffsetHint = TimeSpan.FromHours(8),
SunRiseDataFile = null,
DrawImages = true,
Verbose = true,

// following used to add in a recognizer score track
EventDataDirectories = null,
Expand Down Expand Up @@ -918,9 +889,7 @@ public static void TESTMETHOD_ConcatenateIndexFilesTest2()
ColorMap2 = "BGN-POW-SPT", // This color map dates pre-May 2017.
ConcatenateEverythingYouCanLayYourHandsOn = false, // 24 hour blocks only
TimeSpanOffsetHint = TimeSpan.FromHours(8),
SunRiseDataFile = null,
DrawImages = true,
Verbose = true,

// following used to add in a recognizer score track
EventDataDirectories = null,
Expand Down Expand Up @@ -965,9 +934,7 @@ public static void TESTMETHOD_ConcatenateIndexFilesTest3()
ColorMap2 = "BGN-POW-SPT", // This color map dates pre-May 2017.
ConcatenateEverythingYouCanLayYourHandsOn = false, // 24 hour blocks only
TimeSpanOffsetHint = TimeSpan.FromHours(8),
SunRiseDataFile = null,
DrawImages = true,
Verbose = true,

// following used to add in a recognizer score track
EventDataDirectories = null,
Expand Down Expand Up @@ -1024,9 +991,7 @@ public static void TESTMETHOD_ConcatenateIndexFilesTest4()
ColorMap2 = "BGN-POW-SPT", // This color map dates pre-May 2017.
ConcatenateEverythingYouCanLayYourHandsOn = true,
TimeSpanOffsetHint = TimeSpan.FromHours(8),
SunRiseDataFile = null,
DrawImages = true,
Verbose = true,

// following used to add in a recognizer score track
EventDataDirectories = null,
Expand Down
5 changes: 1 addition & 4 deletions AudioAnalysis/AnalysisPrograms/Sandpit.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ namespace AnalysisPrograms
/// audiocutter - Cuts audio into segments of desired length and format
/// createfoursonograms
/// </summary>
// TODO: [OPENSOURCE] empty out this file
public class Sandpit
{
//public const string imageViewer = @"C:\Windows\system32\mspaint.exe";
Expand Down Expand Up @@ -152,9 +153,7 @@ public static void Audio2CsvOverMultipleFiles()
ConcatenateEverythingYouCanLayYourHandsOn = false,
GapRendering = (ConcatMode)Enum.Parse(typeof(ConcatMode), gapRendering),
TimeSpanOffsetHint = TimeSpan.FromHours(-5), // default = Brisbane time,
SunRiseDataFile = null,
DrawImages = true,
Verbose = true,

// following used to add in a recognizer score track
// Used only to get Event Recognizer files - set eventDirs=null if not used
Expand Down Expand Up @@ -950,9 +949,7 @@ public static void ConcatenateIndexFilesAndSpectrograms()
ConcatenateEverythingYouCanLayYourHandsOn = concatenateEverythingYouCanLayYourHandsOn,
GapRendering = (ConcatMode)Enum.Parse(typeof(ConcatMode), gapRendering),
TimeSpanOffsetHint = timeSpanOffsetHint,
SunRiseDataFile = sunriseDatafile,
DrawImages = drawImages,
Verbose = true,

// following used to add in a recognizer score track
EventDataDirectories = eventDirs,
Expand Down
32 changes: 19 additions & 13 deletions AudioAnalysis/AudioAnalysisTools/Indices/GapsAndJoins.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,28 @@ namespace AudioAnalysisTools.Indices

/// <summary>
/// Choices in how recording gaps are visualised.
/// NoGaps: Recording gaps will be ignored. Segments joined without space. Continuity of the time scale will be broken.
/// This will be best option when viewing 12-hour night-time recordings.
/// TimedGaps: Recording gaps will be filled with a grey "gap" segment of same duration as gap. Time scale remains linear and complete.
/// this is, continuity of the time scale is preserved. This is the default mode for visualisation
/// EchoGaps: Recording gaps are filled with a repeat of the last three-index spectrum prior to the gap.
/// Continuity of the time scale is preserved. Use this when recordings are one minute in 10, for example.
/// </summary>
public enum ConcatMode
{
NoGaps,
TimedGaps,
EchoGaps,
/// <summary>
/// TimedGaps (default): Recording gaps will be filled with a grey "gap" segment of same duration as gap. Time
/// scale remains linear and complete. This is, continuity of the time scale is preserved.
/// This is the default mode for visualisation.
/// </summary>
TimedGaps = 0,

/// <summary>
/// NoGaps: Recording gaps will be ignored. Segments joined without space. Continuity of the time scale will
/// be broken. This will be best option when you want to show source data as an uninterrupted visual stream.
/// </summary>
NoGaps = 1,

/// <summary>
/// EchoGaps: Recording gaps are filled with a repeat of the last three-index spectrum prior to the gap.
/// Continuity of the time scale is preserved. Use there are many small, short, non-contigious blocks of
/// source data (e.g. Sampling one minute every 10).
/// </summary>
EchoGaps = 2,
}

public class GapsAndJoins
Expand Down Expand Up @@ -52,10 +62,6 @@ public class GapsAndJoins
/// </summary>
public ConcatMode GapRendering { get; set; }

// #####################################################################################################################
// STATIC METHODS BELOW
// #####################################################################################################################

/// <summary>
/// Does several data integrity checks.
/// </summary>
Expand Down
Loading

2 comments on commit b7a07e5

@towsey
Copy link
Contributor

@towsey towsey commented on b7a07e5 Dec 3, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am do not see why you needed to change the end date so that it is now "exclusive", i.e. outside the required interval. But having done it line 342 should be changed: LoggedConsole.WriteLine("# Day count = " + dayCount + " (inclusive of start and end days)");

@atruskie
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @towsey - good pick.

As for exclusive ended ranges by default:

  • For the same reason we start from zero, we use exclusive ended ranges (see Dijkstra 1982)
  • This program is primarily used by other programs (most of which also index to zero), therefore our general practice is to use exclusive ended ranges by default on all external APIs
  • It should be noted that Dates are never just dates but are always DateTimes. This leaves an open question of what the implicit time component of a date is. Using an closed/inclusive range means that the implicit time component for the start and end dates is 00:00:00.000000 and 23:59:59.999999 respectively. Whereas if we adhere to our standard there are no special rules for end dates - both time components are 00:00:00.000000.

It should be noted that the unit test failed despite that change. It seems the ConcatenateIndexFilesTest24Hour test ignores whatever dates are provided to it and just produces the whole image.

Please sign in to comment.