WIP progress refactoring zooming spectrograms

Work for #120

Started dissemenating a virtual file system into the zooming code and have been generally refactoring.

Acoustics/Acoustics.Shared/Acoustics.Shared.csproj

@@ -220,8 +220,12 @@
     <Reference Include="YamlDotNet.Dynamic">
       <HintPath>..\..\Extra Assemblies\YamlDotNet\YamlDotNet.Dynamic.dll</HintPath>
     </Reference>
+    <Reference Include="Zio, Version=0.3.1.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\..\AudioAnalysis\packages\Zio.0.3.1\lib\net45\Zio.dll</HintPath>
+    </Reference>
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="AnalysisIo.cs" />
     <Compile Include="AppConfigHelper.cs" />
     <Compile Include="Base58.cs" />
     <Compile Include="Binary.cs" />
@@ -257,6 +261,7 @@
     <Compile Include="Extensions\StringCaseExtensions.cs" />
     <Compile Include="Extensions\TaskExtensions.cs" />
     <Compile Include="Extensions\TupleExtensions.cs" />
+    <Compile Include="Extensions\ZioExtensions.cs" />
     <Compile Include="FileDateHelpers.cs" />
     <Compile Include="FileNameHelpers.cs" />
     <Compile Include="Formatters.cs" />
@@ -275,7 +280,6 @@
     <Compile Include="PropertyComparer.cs" />
     <Compile Include="Range.cs" />
     <Compile Include="ResultProperty.cs" />
-    <Compile Include="SqlServerTypes\Loader.cs" />
     <Compile Include="StatDescriptive.cs" />
     <Compile Include="StringKeyValueStore.cs" />
     <Compile Include="TempFileHelper.cs" />

Acoustics/Acoustics.Shared/AnalysisIo.cs

@@ -0,0 +1,29 @@
+// <copyright file="AnalysisIo.cs" company="QutEcoacoustics">
+// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group).
+// </copyright>
+
+namespace Acoustics.Shared
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+    using System.Text;
+    using System.Threading.Tasks;
+    using Zio;
+
+    public class AnalysisIo
+    {
+        public AnalysisIo(IFileSystem input, IFileSystem output, IFileSystem temp)
+        {
+            this.Input = input;
+            this.Output = output;
+            this.Temp = temp ?? input;
+        }
+
+        public IFileSystem Input { get; }
+
+        public IFileSystem Output { get; }
+
+        public IFileSystem Temp { get; }
+    }
+}

Acoustics/Acoustics.Shared/Extensions/ExtensionsString.cs

@@ -396,19 +396,19 @@ public static Color ParseAsColor(this string str)
                     throw new FormatException("The color format was not recognized");
             }
 
-
             return color;
         }
 
         public static string ToCommaSeparatedList<T>(this IEnumerable<T> items)
         {
-            return items == null ? string.Empty : items.Aggregate(string.Empty, (seed, b) => seed  + b + ",");
+            return items == null ? string.Empty : items.Aggregate(string.Empty, (seed, b) => seed + b.ToString() + ",");
         }
 
         public static string Format(this string format, params object[] args)
         {
             return string.Format(format, args);
         }
+
         public static string Format2(this string format, params object[] args)
         {
             return string.Format(format, args);
@@ -430,5 +430,4 @@ public static string NormalizeToCrLf(this string str)
             return normalized;
         }
     }
-}
-
+}

Acoustics/Acoustics.Shared/Extensions/ZioExtensions.cs

@@ -0,0 +1,24 @@
+// <copyright file="ZioExtensions.cs" company="QutEcoacoustics">
+// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group).
+// </copyright>
+
+namespace Zio
+{
+    using System;
+    using System.Collections.Generic;
+    using System.IO;
+    using System.Linq;
+    using System.Text;
+    using System.Threading.Tasks;
+    using FileSystems;
+
+    public static class ZioExtensions
+    {
+        private static readonly PhysicalFileSystem FileSystem = new PhysicalFileSystem();
+
+        public static UPath ToUPath(this FileSystemInfo file)
+        {
+            return FileSystem.ConvertPathFromInternal(file.FullName);
+        }
+    }
+}

Acoustics/Acoustics.Shared/packages.config

@@ -12,4 +12,5 @@
   <package id="StyleCop.Analyzers" version="1.1.0-beta004" targetFramework="net462" developmentDependency="true" />
   <package id="System.ValueTuple" version="4.4.0-preview2-25405-01" targetFramework="net462" />
   <package id="YamlDotNet" version="3.9.0" targetFramework="net462" />
+  <package id="Zio" version="0.3.1" targetFramework="net462" />
 </packages>

Acoustics/Acoustics.Test/Acoustics.Test.csproj

@@ -307,6 +307,8 @@
     <Compile Include="AnalysisPrograms\AcousticWorkbench.Orchestration\EventMetadataResolverTests.cs" />
     <Compile Include="AnalysisPrograms\AnalyzeLongRecordings\TestAnalyzeLongRecording.cs" />
     <Compile Include="AnalysisPrograms\Concatenation\ConcatenationTests.cs" />
+    <Compile Include="AnalysisPrograms\Draw\Zooming\DrawZoomingTests.cs" />
+    <Compile Include="AnalysisPrograms\Production\FileSystemProviderTests.cs" />
     <Compile Include="AnalysisPrograms\SourcePreparers\RemoteSourcePreparerTests.cs" />
     <Compile Include="AnalysisPrograms\SourcePreparers\LocalSourcePreparerTests.cs" />
     <Compile Include="ArrayExtensionsTests.cs" />
@@ -388,6 +390,10 @@
       <Project>{86855CBC-7243-492A-A500-2DE36CC7C825}</Project>
       <Name>AudioAnalysisTools</Name>
     </ProjectReference>
+    <ProjectReference Include="..\..\AudioAnalysis\SqLiteFileSystem\SqliteFileSystem.csproj">
+      <Project>{364B812E-4E3D-415E-BE10-2928028EE7B6}</Project>
+      <Name>SqliteFileSystem</Name>
+    </ProjectReference>
     <ProjectReference Include="..\..\AudioAnalysis\TowseyLibrary\TowseyLibrary.csproj">
       <Project>{7C6831FD-F60C-4F7B-8E6A-35E850174411}</Project>
       <Name>TowseyLibrary</Name>

Acoustics/Acoustics.Test/AnalysisPrograms/Draw/Zooming/DrawZoomingTests.cs

@@ -0,0 +1,100 @@
+// <copyright file="TestAnalyzeLongRecording.cs" company="QutEcoacoustics">
+// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group).
+// </copyright>
+
+namespace Acoustics.Test.AnalysisPrograms.Draw.Zooming
+{
+    using System;
+    using System.IO;
+    using System.Linq;
+    using Acoustics.Shared;
+    using global::AnalysisPrograms.AnalyseLongRecordings;
+    using global::AnalysisPrograms.Draw.Zooming;
+    using global::AudioAnalysisTools.DSP;
+    using global::AudioAnalysisTools.Indices;
+    using global::AudioAnalysisTools.LongDurationSpectrograms;
+    using global::TowseyLibrary;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+    using TestHelpers;
+
+    [TestClass]
+    public class DrawZoomingTests : OutputDirectoryTest
+    {
+        private DirectoryInfo resultsDirectory;
+
+        [ClassInitialize]
+        public void ClassInitialize()
+        {
+            // calculate indices
+            var recordingPath = PathHelper.ResolveAsset("Recordings", "OxleyCreek_site_1_1060_244333_20140529T081358+1000_120_0.wav");
+            var configPath = PathHelper.ResolveConfigFile("Towsey.Acoustic.HiRes.yml");
+            var arguments = new AnalyseLongRecording.Arguments
+            {
+                Source = recordingPath,
+                Config = configPath,
+                Output = this.outputDirectory,
+                TempDir = this.outputDirectory.Combine("Temp"),
+            };
+
+            AnalyseLongRecording.Execute(arguments);
+
+            this.resultsDirectory = this.outputDirectory.Combine("Towsey.Acoustic");
+
+            // do some basic checks that the indices were generated
+            var listOfFiles = this.resultsDirectory.EnumerateFiles().ToArray();
+            Assert.AreEqual(33, listOfFiles.Length);
+            var csvCount = listOfFiles.Count(f => f.Name.EndsWith(".csv"));
+            Assert.AreEqual(16, csvCount);
+            var jsonCount = listOfFiles.Count(f => f.Name.EndsWith(".json"));
+            Assert.AreEqual(2, jsonCount);
+            var pngCount = listOfFiles.Count(f => f.Name.EndsWith(".png"));
+            Assert.AreEqual(0, pngCount);
+
+        }
+
+        /// <summary>
+        /// Tests the rendering of zooming spectrograms for a minute of audio indices
+        /// </summary>
+        [TestMethod]
+        [Timeout(45_000)]
+        public void TestGenerateTiles()
+        {
+            // generate the zooming spectrograms
+            var zoomOutput = this.outputDirectory.Combine("Zooming");
+            DrawZoomingSpectrograms.Execute(
+                new DrawZoomingSpectrograms.Arguments()
+                {
+                    Output = zoomOutput.FullName,
+                    SourceDirectory = resultsDirectory.FullName,
+                    SpectrogramZoomingConfig = PathHelper.ResolveConfigFile("SpectrogramZoomingConfig.yml"),
+                    ZoomAction = DrawZoomingSpectrograms.Arguments.ZoomActionType.Tile,
+                });
+
+            Assert.Fail();
+        }
+
+        /// <summary>
+        /// Tests the rendering of zooming spectrograms for a minute of audio indices
+        /// </summary>
+        [TestMethod]
+        [Timeout(45_000)]
+        public void TestGenerateTilesSqlite()
+        {
+            // generate the zooming spectrograms
+            var zoomOutput = this.outputDirectory.CombineFile("Zooming/tiles.sqlite3");
+            DrawZoomingSpectrograms.Execute(
+                new DrawZoomingSpectrograms.Arguments()
+                {
+                    Output = zoomOutput.FullName,
+                    SourceDirectory = resultsDirectory.FullName,
+                    SpectrogramZoomingConfig = PathHelper.ResolveConfigFile("SpectrogramZoomingConfig.yml"),
+                    ZoomAction = DrawZoomingSpectrograms.Arguments.ZoomActionType.Tile,
+                });
+
+            Assert.IsTrue(File.Exists(zoomOutput.FullName));
+            Assert.Fail();
+        }
+
+
+    }
+}

Acoustics/Acoustics.Test/AnalysisPrograms/Production/FileSystemProviderTests.cs

@@ -0,0 +1,58 @@
+// <copyright file="FileSystemProviderTests.cs" company="QutEcoacoustics">
+// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group).
+// </copyright>
+
+namespace Acoustics.Test.AnalysisPrograms.Production
+{
+    using System;
+    using System.IO;
+    using global::AnalysisPrograms.Production;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+    using TestHelpers;
+    using Zio;
+    using Zio.FileSystems;
+    using Zio.FileSystems.Community.SqliteFileSystem;
+
+    [TestClass]
+    public class FileSystemProviderTests : OutputDirectoryTest
+    {
+        [DataTestMethod]
+        [DataRow("")]
+        [DataRow(null)]
+        public void TestFullFileSystem(string path)
+        {
+            var fs = FileSystemProvider.DetermineFileSystem(path);
+
+            Assert.IsInstanceOfType(fs, typeof(PhysicalFileSystem));
+        }
+
+        [TestMethod]
+        public void TestSubFileSystem()
+        {
+            var fs = FileSystemProvider.DetermineFileSystem(this.outputDirectory.FullName);
+
+            Assert.IsInstanceOfType(fs, typeof(SubFileSystem));
+
+            Assert.AreEqual(this.outputDirectory.ToUPath(), ((SubFileSystem)fs).SubPath);
+        }
+
+        [TestMethod]
+        public void TestSqliteFileSystem()
+        {
+            var path = this.outputDirectory.FullName + "\\test.sqlite3";
+            var fs = FileSystemProvider.DetermineFileSystem(path);
+
+            Assert.IsInstanceOfType(fs, typeof(SqliteFileSystem));
+
+            StringAssert.Contains(((SqliteFileSystem)fs).ConnectionString, path);
+            Assert.IsTrue(File.Exists(path));
+        }
+
+        [TestMethod]
+        public void TestInvalidFileSystem()
+        {
+            Assert.ThrowsException<NotSupportedException>(
+            () => FileSystemProvider.DetermineFileSystem(this.outputDirectory.FullName + "\\test.zip"));
+        }
+    }
+}

Acoustics/Acoustics.Test/Shared/CsvTests.cs

@@ -299,7 +299,7 @@ public void TestCsvClassMapsAreAutomaticallyRegistered()
             var partialExpected = new[]
             {
                 typeof(AcousticEvent.AcousticEventClassMap),
-                typeof(EventStatisticsClassMap),
+                typeof(EventStatistics.EventStatisticsClassMap),
                 typeof(ImportedEvent.ImportedEventNameClassMap),
             };