More work on zooming spectrograms

For #120

- Refactored several methods to accept Zio FileSystemEntry types
- IndexGenerationData now includes the original file's basename
- IndexGenerationData changed member name BGNoiseNeighbourhood [breaking]
- Fixed a bug in the tiler for rendering very short recordings
- Made the log verbosity function public so we can dial logs down in integration tests
-

Acoustics/Acoustics.Shared/ConfigFile/ConfigFile.cs

@@ -19,6 +19,8 @@ namespace Acoustics.Shared.ConfigFile
     using YamlDotNet.Dynamic;
     using YamlDotNet.RepresentationModel;
 
+    using Zio;
+
     public static class ConfigFile
     {
         public const string ProfilesKey = "Profiles";
@@ -46,10 +48,7 @@ public static string ConfigFolder
                     $"Cannot find currently set ConfigFiles directory. {defaultConfigFolder} does not exist!");
             }
 
-            set
-            {
-                defaultConfigFolder = value;
-            }
+            set => defaultConfigFolder = value;
         }
 
         public static dynamic GetProfile(dynamic configuration, string profileName)
@@ -143,12 +142,11 @@ public static FileInfo ResolveConfigFile(FileInfo file, params DirectoryInfo[] s
 
         public static FileInfo ResolveConfigFile(string file, params DirectoryInfo[] searchPaths)
         {
-            FileInfo configFile;
-            var success = TryResolveConfigFile(file, searchPaths, out configFile);
+            var success = TryResolveConfigFile(file, searchPaths.Select(x => x.ToDirectoryEntry()), out FileEntry configFile);
 
             if (success)
             {
-                return configFile;
+                return configFile.ToFileInfo();
             }
 
             var searchedIn =
@@ -159,17 +157,21 @@ public static FileInfo ResolveConfigFile(string file, params DirectoryInfo[] sea
             throw new ConfigFileException(message, file);
         }
 
-        public static bool TryResolveConfigFile(string file, DirectoryInfo[] searchPaths, out FileInfo configFile)
+        public static bool TryResolveConfigFile(string file, IEnumerable<DirectoryEntry> searchPaths, out FileEntry configFile)
         {
             configFile = null;
             if (string.IsNullOrWhiteSpace(file))
             {
                 return false;
             }
 
-            if (File.Exists(file))
+            // this is a holdover from concrete file systems. The concept of a working directory has no real
+            // equivalent in a virtual file system but this is implemented for compatibility
+            var workingDirectory = Directory.GetCurrentDirectory().ToDirectoryEntry();
+            var localConfig = workingDirectory.CombineFile(file);
+            if (localConfig.Exists)
             {
-                configFile = new FileInfo(file);
+                configFile = localConfig;
                 return true;
             }
 
@@ -180,23 +182,24 @@ public static bool TryResolveConfigFile(string file, DirectoryInfo[] searchPaths
                 return false;
             }
 
-            if (searchPaths != null && searchPaths.Length > 0)
+            if (searchPaths != null)
             {
-                foreach (var directoryInfo in searchPaths)
+                foreach (var directory in searchPaths)
                 {
-                    var searchPath = Path.GetFullPath(Path.Combine(directoryInfo.FullName, file));
-                    if (File.Exists(searchPath))
+                    var searchPath = directory.CombineFile(file);
+                    if (searchPath.Exists)
                     {
-                        configFile = new FileInfo(searchPath);
+                        configFile = searchPath;
                         return true;
                     }
                 }
             }
 
+            // config files are always packaged with the app so use a physical file system
             var defaultConfigFile = Path.GetFullPath(Path.Combine(ConfigFolder, file));
             if (File.Exists(defaultConfigFile))
             {
-                configFile = new FileInfo(defaultConfigFile);
+                configFile = defaultConfigFile.ToFileEntry();
                 return true;
             }
 
@@ -206,7 +209,7 @@ public static bool TryResolveConfigFile(string file, DirectoryInfo[] searchPaths
                 var nestedDefaultConfigFile = Path.Combine(directory, file);
                 if (File.Exists(nestedDefaultConfigFile))
                 {
-                    configFile = new FileInfo(nestedDefaultConfigFile);
+                    configFile = nestedDefaultConfigFile.ToFileEntry();
                     return true;
                 }
             }

Acoustics/Acoustics.Shared/Extensions/ZioExtensions.cs

@@ -26,6 +26,11 @@ public static UPath ToUPath(this FileSystemInfo file)
             return FileSystem.ConvertPathFromInternal(file.FullName);
         }
 
+        public static string ToOsPath(this UPath path)
+        {
+            return FileSystem.ConvertPathToInternal(path);
+        }
+
         public static DirectoryEntry ToDirectoryEntry(this DirectoryInfo directory)
         {
             return new DirectoryEntry(FileSystem, directory.ToUPath());
@@ -46,6 +51,16 @@ public static FileEntry ToFileEntry(this string file)
             return new FileEntry(FileSystem, FileSystem.ConvertPathFromInternal(file));
         }
 
+        public static FileInfo ToFileInfo(this FileEntry file)
+        {
+            Contract.Requires(file != null);
+            Contract.Requires(
+                file.FileSystem is PhysicalFileSystem,
+                $"To convert the path {file} back to a physical filesystem, it must be from a physical file system");
+
+            return new FileInfo(file.Path.ToOsPath());
+        }
+
         public static DirectoryEntry Combine(this DirectoryEntry directoryInfo, params string[] str)
         {
             Contract.Requires(directoryInfo != null);

Acoustics/Acoustics.Shared/FileNameHelpers.cs

@@ -11,6 +11,8 @@ namespace Acoustics.Shared
     using System.Text;
     using System.Text.RegularExpressions;
 
+    using Zio;
+
     /// <summary>
     /// A set of helper methods used to create/read consistently encoded filenames
     /// </summary>
@@ -96,6 +98,15 @@ public static void ParseAnalysisFileName(
             ParseAnalysisFileName(file.Name, out originalBaseName, out analysisTag, out otherSegments);
         }
 
+        public static void ParseAnalysisFileName(
+            FileEntry file,
+            out string originalBaseName,
+            out string analysisTag,
+            out string[] otherSegments)
+        {
+            ParseAnalysisFileName(file.Name, out originalBaseName, out analysisTag, out otherSegments);
+        }
+
         public static void ParseAnalysisFileName(string fileName, out string originalBaseName, out string analysisTag, out string[] otherSegments)
         {
             if (!TryParseAnalysisFileName(fileName, out originalBaseName, out analysisTag, out otherSegments))

Acoustics/Acoustics.Shared/Json.cs

@@ -15,6 +15,8 @@ namespace Acoustics.Shared
     using Newtonsoft.Json;
     using Newtonsoft.Json.Linq;
 
+    using Zio;
+
     public static class Json
     {
         public static void Serialise<T>(FileInfo file, T obj)
@@ -46,6 +48,11 @@ public static string SerialiseToString<T>(T obj, bool prettyPrint = true)
         }
 
         public static T Deserialise<T>(FileInfo file)
+        {
+            return Deserialise<T>(file.ToFileEntry());
+        }
+
+        public static T Deserialise<T>(FileEntry file)
         {
             var serializer = new JsonSerializer();
 

Acoustics/Acoustics.Shared/Yaml.cs

@@ -17,6 +17,8 @@ namespace Acoustics.Shared
     using YamlDotNet.RepresentationModel;
     using YamlDotNet.Serialization;
 
+    using Zio;
+
     public class Yaml
     {
         public static DynamicYaml Deserialise(FileInfo file)
@@ -56,6 +58,11 @@ public static void SerialiseDynamic(FileInfo file, dynamic obj)
         }
 
         public static T Deserialise<T>(FileInfo file)
+        {
+            return Deserialise<T>(file.ToFileEntry());
+        }
+
+        public static T Deserialise<T>(FileEntry file)
         {
             using (var stream = file.OpenText())
             {

Acoustics/Acoustics.Test/AnalysisPrograms/AnalyzeLongRecordings/TestAnalyzeLongRecording.cs

@@ -17,6 +17,8 @@ namespace Acoustics.Test.AnalysisPrograms.AnalyzeLongRecordings
     using TestHelpers;
     using TowseyLibrary;
 
+    using Zio;
+
     /// <summary>
     /// Test methods for the various standard Sonograms or Spectrograms
     /// Notes on TESTS: (from Anthony in email @ 05/04/2017)
@@ -138,6 +140,9 @@ public void TestAnalyzeSr22050Recording()
             // draw array just to check peaks are in correct places - just for debugging purposes
             var ldsBgnSpectrumFile = this.outputDirectory.CombineFile("Spectrum1.png");
             GraphsAndCharts.DrawGraph(array, "LD BGN SPECTRUM Linear", ldsBgnSpectrumFile);
+
+            var generationData = Json.Deserialise<IndexGenerationData>(IndexGenerationData.FindFile(resultsDirectory.ToDirectoryEntry()));
+            Assert.AreEqual("TemporaryRecording1", generationData.RecordingBasename);
         }
 
         /// <summary>

Acoustics/Acoustics.Test/AnalysisPrograms/Draw/Zooming/DrawZoomingTests.cs

@@ -8,8 +8,12 @@ namespace Acoustics.Test.AnalysisPrograms.Draw.Zooming
     using System.IO;
     using System.Linq;
     using Acoustics.Shared;
+
+    using global::AnalysisPrograms;
     using global::AnalysisPrograms.AnalyseLongRecordings;
     using global::AnalysisPrograms.Draw.Zooming;
+    using global::AnalysisPrograms.Production;
+
     using global::AudioAnalysisTools.DSP;
     using global::AudioAnalysisTools.Indices;
     using global::AudioAnalysisTools.LongDurationSpectrograms;
@@ -20,10 +24,10 @@ namespace Acoustics.Test.AnalysisPrograms.Draw.Zooming
     [TestClass]
     public class DrawZoomingTests : OutputDirectoryTest
     {
-        private DirectoryInfo resultsDirectory;
+        public static DirectoryInfo ResultsDirectory { get; set; }
 
         [ClassInitialize]
-        public void ClassInitialize()
+        public static void ClassInitialize(TestContext context)
         {
             // calculate indices
             var recordingPath = PathHelper.ResolveAsset("Recordings", "OxleyCreek_site_1_1060_244333_20140529T081358+1000_120_0.wav");
@@ -32,24 +36,27 @@ public void ClassInitialize()
             {
                 Source = recordingPath,
                 Config = configPath,
-                Output = this.outputDirectory,
-                TempDir = this.outputDirectory.Combine("Temp"),
+                Output = SharedDirectory,
+                TempDir = SharedDirectory.Combine("Temp"),
             };
 
+            context.WriteLine($"{DateTime.Now} generating indices fixture data");
+            MainEntry.SetLogVerbosity(LogVerbosity.Warn, true);
             AnalyseLongRecording.Execute(arguments);
+            MainEntry.SetLogVerbosity(LogVerbosity.Debug, true);
+            context.WriteLine($"{DateTime.Now} finished generting fixture");
 
-            this.resultsDirectory = this.outputDirectory.Combine("Towsey.Acoustic");
+            ResultsDirectory = SharedDirectory.Combine("Towsey.Acoustic");
 
             // do some basic checks that the indices were generated
-            var listOfFiles = this.resultsDirectory.EnumerateFiles().ToArray();
-            Assert.AreEqual(33, listOfFiles.Length);
+            var listOfFiles = ResultsDirectory.EnumerateFiles().ToArray();
+            Assert.AreEqual(20, listOfFiles.Length);
             var csvCount = listOfFiles.Count(f => f.Name.EndsWith(".csv"));
             Assert.AreEqual(16, csvCount);
             var jsonCount = listOfFiles.Count(f => f.Name.EndsWith(".json"));
             Assert.AreEqual(2, jsonCount);
             var pngCount = listOfFiles.Count(f => f.Name.EndsWith(".png"));
-            Assert.AreEqual(0, pngCount);
-
+            Assert.AreEqual(2, pngCount);
         }
 
         /// <summary>
@@ -65,7 +72,7 @@ public void TestGenerateTiles()
                 new DrawZoomingSpectrograms.Arguments()
                 {
                     Output = zoomOutput.FullName,
-                    SourceDirectory = resultsDirectory.FullName,
+                    SourceDirectory = ResultsDirectory.FullName,
                     SpectrogramZoomingConfig = PathHelper.ResolveConfigFile("SpectrogramZoomingConfig.yml"),
                     ZoomAction = DrawZoomingSpectrograms.Arguments.ZoomActionType.Tile,
                 });
@@ -86,7 +93,7 @@ public void TestGenerateTilesSqlite()
                 new DrawZoomingSpectrograms.Arguments()
                 {
                     Output = zoomOutput.FullName,
-                    SourceDirectory = resultsDirectory.FullName,
+                    SourceDirectory = ResultsDirectory.FullName,
                     SpectrogramZoomingConfig = PathHelper.ResolveConfigFile("SpectrogramZoomingConfig.yml"),
                     ZoomAction = DrawZoomingSpectrograms.Arguments.ZoomActionType.Tile,
                 });

Acoustics/Acoustics.Test/AudioAnalysisTools/TileImage/TilerTests.cs

@@ -547,6 +547,46 @@ public void EnsureSameThreeTilesWrittenForTwoOddlySizedSuperTiles()
 
         }
 
+        [TestMethod]
+        public void EnsureReallyShortRecordingsWork()
+        {
+            var testBitmap = new Bitmap(2, 300);
+            using (var graphics = Graphics.FromImage(testBitmap))
+            {
+                var graphicsUnit = GraphicsUnit.Pixel;
+                graphics.FillRectangle(Brushes.Red, testBitmap.GetBounds(ref graphicsUnit));
+            }
+
+            var superTile = new TimeOffsetSingleLayerSuperTile()
+                {
+                    Image = testBitmap,
+                    Scale = TimeSpan.FromSeconds(60.0),
+                    TimeOffset = TimeSpan.Zero,
+                    SpectrogramType = SpectrogramType.Index,
+                };
+            this.tiler.Tile(superTile);
+
+            var producedFiles = this.outputDirectory.GetFiles();
+
+            Assert.AreEqual(1, producedFiles.Length);
+
+            // produced image should have 180px of transparency, 2px of color, and then 118px of transparency
+            var expected = new Bitmap(300, 300);
+            using (var graphics = Graphics.FromImage(expected))
+            {
+                graphics.FillRectangle(Brushes.Red, new Rectangle(180, 0, 2, 300));
+            }
+
+            var expectedImages = new[] { expected };
+
+            for (int i = 0; i < expectedImages.Length; i++)
+            {
+                var producedImage = Image.FromFile(producedFiles[i].FullName);
+                var areEqual = BitmapEquals((Bitmap)expectedImages[i], (Bitmap)producedImage);
+                Assert.IsTrue(areEqual, "Bitmaps were not equal {0}, {1}", expectedImages[i], producedFiles[i].Name);
+            }
+        }
+
         /// <summary>
         /// 2B.abcd = Rect.FromLTRB(100, 100, 200, 200)
         ///

Acoustics/Acoustics.Test/Shared/ConfigFileTests.cs

@@ -19,6 +19,8 @@ namespace Acoustics.Test.Shared
 
     using MSTestExtensions;
 
+    using Zio;
+
     [TestClass]
     public class ConfigFileTests : BaseTest
     {
@@ -99,8 +101,7 @@ public void IfAbsolutePathReturnsSameFileEvenFailing()
             // the below file will not exist
             var config = TempFileHelper.NewTempFile();
 
-            FileInfo foundConfig;
-            var actual = ConfigFile.TryResolveConfigFile(config.FullName, null, out foundConfig);
+            var actual = ConfigFile.TryResolveConfigFile(config.FullName, null, out _);
 
             Assert.IsFalse(actual);
         }
@@ -140,8 +141,7 @@ public void TheTryMethodDoesNotThrow()
         {
             string config = "doesNotExist.yml";
 
-            FileInfo foundConfig;
-            Assert.IsFalse(ConfigFile.TryResolveConfigFile(config, null, out foundConfig));
+            Assert.IsFalse(ConfigFile.TryResolveConfigFile(config, null, out _));
         }
 
         [TestCleanup]