More work on Zooming Spectrogram

Analysis IO now has a subclass that ensure the input is a directory. Future work will add file equivalent.

Changed the naming format of the tiles to be closer to ISO8601.

Allowed filename helper to accept an empty basename for when we intentionally want to not prepend basename to a result file. Useful for results that are stored in a package (that is named with the basename).

Acoustics/Acoustics.Shared/AnalysisIo.cs

@@ -9,6 +9,9 @@ namespace Acoustics.Shared
     using System.Linq;
     using System.Text;
     using System.Threading.Tasks;
+
+    using Acoustics.Shared.Contracts;
+
     using Zio;
 
     public class AnalysisIo
@@ -25,16 +28,39 @@ public AnalysisIo((IFileSystem FileSystem, FileSystemEntry Base) input, (IFileSy
             this.TempBase = (DirectoryEntry)(temp ?? input).Base;
         }
 
-        public FileSystemEntry InputBase { get; set; }
+        public FileSystemEntry InputBase { get; }
 
-        public DirectoryEntry OutputBase { get; set; }
+        public DirectoryEntry OutputBase { get; }
 
-        public DirectoryEntry TempBase { get; set; }
+        public DirectoryEntry TempBase { get; }
 
         public IFileSystem Input { get; }
 
         public IFileSystem Output { get; }
 
         public IFileSystem Temp { get; }
+
+        public AnalysisIoInputDirectory EnsureInputIsDirectory()
+        {
+            Contract.Requires(this.InputBase != null, $"{nameof(this.InputBase)} must not be null");
+
+            if (this.Input.DirectoryExists(this.InputBase.Path))
+            {
+                return new AnalysisIoInputDirectory((this.Input, this.InputBase), (this.Output, this.OutputBase), (this.Temp, this.TempBase));
+            }
+
+            throw new ArgumentException($"Expected `{this.InputBase}` to be a directory, but it is not");
+        }
+    }
+
+    public class AnalysisIoInputDirectory
+        : AnalysisIo
+    {
+        internal AnalysisIoInputDirectory((IFileSystem FileSystem, FileSystemEntry Base) input, (IFileSystem FileSystem, DirectoryEntry Base) output, (IFileSystem FileSystem, DirectoryEntry Base)? temp)
+            : base(input, output, temp)
+        {
+        }
+
+        public new DirectoryEntry InputBase => (DirectoryEntry)base.InputBase;
     }
 }

Acoustics/Acoustics.Shared/AppConfigHelper.cs

@@ -30,7 +30,10 @@ public static class AppConfigHelper
         /// </summary>
         public const string Iso8601FileCompatibleDateFormat = "yyyyMMddTHHmmsszzz";
 
+        public const string Iso8601FileCompatibleDateFormatUtcWithFractionalSeconds = "yyyyMMddTHHmmss.FFF\\Z";
+
         public const string StandardDateFormatUtc = "yyyyMMdd-HHmmssZ";
+
         public const string StandardDateFormatUtcWithFractionalSeconds = "yyyyMMdd-HHmmss.FFFZ";
 
         public static string FileDateFormatUtc

Acoustics/Acoustics.Shared/Extensions/ZioExtensions.cs

@@ -12,6 +12,9 @@ namespace Zio
     using System.Linq;
     using System.Text;
     using System.Threading.Tasks;
+
+    using Acoustics.Shared.Contracts;
+
     using FileSystems;
 
     public static class ZioExtensions
@@ -28,9 +31,47 @@ public static DirectoryEntry ToDirectoryEntry(this DirectoryInfo directory)
             return new DirectoryEntry(FileSystem, directory.ToUPath());
         }
 
-        public static DirectoryEntry ToFileEntry(this FileInfo file)
+        public static FileEntry ToFileEntry(this FileInfo file)
+        {
+            return new FileEntry(FileSystem, file.ToUPath());
+        }
+
+        public static DirectoryEntry ToDirectoryEntry(this string directory)
+        {
+            return new DirectoryEntry(FileSystem, FileSystem.ConvertPathFromInternal(directory));
+        }
+
+        public static FileEntry ToFileEntry(this string file)
         {
-            return new DirectoryEntry(FileSystem, file.ToUPath());
+            return new FileEntry(FileSystem, FileSystem.ConvertPathFromInternal(file));
+        }
+
+        public static DirectoryEntry Combine(this DirectoryEntry directoryInfo, params string[] str)
+        {
+            Contract.Requires(directoryInfo != null);
+            Contract.Requires(str != null && str.Length > 0);
+
+            UPath merged = directoryInfo.Path;
+            foreach (var fragment in str)
+            {
+                merged = merged / fragment;
+            }
+
+            return new DirectoryEntry(directoryInfo.FileSystem, merged);
+        }
+
+        public static FileEntry CombineFile(this DirectoryEntry directoryInfo, params string[] str)
+        {
+            Contract.Requires(directoryInfo != null);
+            Contract.Requires(str != null && str.Length > 0);
+
+            UPath merged = directoryInfo.Path;
+            foreach (var fragment in str)
+            {
+                merged = merged / fragment;
+            }
+
+            return new FileEntry(directoryInfo.FileSystem, merged);
         }
 
         /// <summary>
@@ -71,5 +112,20 @@ public static void Save(this System.Drawing.Bitmap bitmap, IFileSystem fileSyste
                 bitmap.Save(fileStream, format);
             }
         }
+
+        public static StreamReader OpenText(this IFileSystem fileSystem, UPath path)
+        {
+            using (var stream = fileSystem.OpenFile(path, FileMode.Open, FileAccess.Read, FileShare.Read))
+            {
+                return new StreamReader(stream, Encoding.UTF8, true);
+            }
+        }
+
+        public static StreamReader OpenText(this FileEntry file)
+        {
+            var stream = file.FileSystem.OpenFile(file.Path, FileMode.Open, FileAccess.Read, FileShare.Read);
+
+            return new StreamReader(stream, Encoding.UTF8, true);
+        }
     }
 }

Acoustics/Acoustics.Shared/FileNameHelpers.cs

@@ -55,7 +55,7 @@ public static string AnalysisResultPath(
         /// </summary>
         public static string AnalysisResultName(string baseName, string analysisTag, string newExtension, params string[] otherSegments)
         {
-            if (string.IsNullOrWhiteSpace(baseName))
+            if (baseName == null)
             {
                 throw new ArgumentException("Invalid file stem / base name supplied");
             }
@@ -70,7 +70,7 @@ public static string AnalysisResultName(string baseName, string analysisTag, str
                 baseName = baseName.Replace(BasenameSeparator, SegmentSeparator);
             }
 
-            var filename = baseName + BasenameSeparator + analysisTag;
+            var filename = baseName + (string.IsNullOrEmpty(baseName) ? string.Empty : BasenameSeparator) + analysisTag;
 
             if (otherSegments.Length > 0)
             {

Acoustics/Acoustics.Test/AnalysisPrograms/AcousticWorkbench.Orchestration/RemoteSegmentTests.cs

@@ -2,7 +2,7 @@
 // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group).
 // </copyright>
 
-namespace AnalysisPrograms.AcousticWorkbench.Orchestration.Tests
+namespace Acoustics.Test.AnalysisPrograms.AcousticWorkbench.Orchestration.Tests
 {
     using System;
     using System.Collections.Generic;
@@ -12,6 +12,9 @@ namespace AnalysisPrograms.AcousticWorkbench.Orchestration.Tests
     using Acoustics.Shared;
     using Acoustics.Test.TestHelpers.Factories;
     using global::AcousticWorkbench.Models;
+
+    using global::AnalysisPrograms.AcousticWorkbench.Orchestration;
+
     using Microsoft.VisualStudio.TestTools.UnitTesting;
     using Orchestration;
 

Acoustics/Acoustics.Test/AnalysisPrograms/AnalyzeLongRecordings/TestAnalyzeLongRecording.cs

@@ -259,7 +259,7 @@ public void TestAnalyzeSr64000Recording()
             // finally read in the dictionary of spectra
             string analysisType = "Towsey.Acoustic";
             var keys = LDSpectrogramRGB.GetArrayOfAvailableKeys();
-            var dictionaryOfSpectra = IndexMatrices.ReadCsvFiles(resultsDirectory, recordingName + "__" + analysisType, keys);
+            var dictionaryOfSpectra = IndexMatrices.ReadSpectralIndices(resultsDirectory, recordingName, analysisType, keys);
 
             LDSpectrogramRGB.DrawSpectrogramsFromSpectralIndices(
                     inputDirectory: resultsDirectory,

Acoustics/Acoustics.Test/AudioAnalysisTools/TileImage/AbsoluteDateTimeTilerTests.cs

@@ -17,6 +17,7 @@ public class AbsoluteDateTimeTilerTests
         private Tiler tiler;
         private DirectoryInfo outputDirectory;
         private AbsoluteDateTilingProfile tilingProfileNotRoundStart;
+        private readonly DateTimeOffset dateTimeOffset = new DateTimeOffset(2015, 04, 10, 3, 30, 15, 123, TimeSpan.FromHours(10));
 
         [TestInitialize]
         public void Setup()
@@ -30,7 +31,7 @@ public void Setup()
             this.tilingProfileNotRoundStart = new AbsoluteDateTilingProfile(
                 "Filename",
                 "Tile",
-                new DateTimeOffset(2015, 04, 10, 3, 30, 0, TimeSpan.FromHours(10)),
+                this.dateTimeOffset,
                 256,
                 60);
 
@@ -53,6 +54,18 @@ public void Cleanup()
             PathHelper.DeleteTempDir(this.outputDirectory);
         }
 
+        [TestMethod]
+        public void TestNamingPattern()
+        {
+            var profile = new AbsoluteDateTilingProfile("Basename", "Tag", this.dateTimeOffset, 256, 300);
+
+            Assert.AreEqual("Basename__Tag_20150409T173015.123Z_60", profile.GetFileBaseName(this.tiler.CalculatedLayers, this.tiler.CalculatedLayers.First(), new Point(0, 0)));
+
+            var profile2 = new AbsoluteDateTilingProfile("", "Tag", this.dateTimeOffset, 256, 300);
+
+            Assert.AreEqual("Tag_20150409T173015.123Z_60", profile2.GetFileBaseName(this.tiler.CalculatedLayers, this.tiler.CalculatedLayers.First(), new Point(0, 0)));
+        }
+
         [TestMethod]
         public void TestItShouldCutAndPadRightWithTransparency()
         {

AudioAnalysis/AnalysisPrograms/Draw/Zooming/DrawZoomingSpectrograms.cs

@@ -64,11 +64,15 @@ public static void Execute(Arguments arguments)
             // get the indexDistributions and the indexGenerationData AND the common.OriginalBasename
             common.CheckForNeededFiles(arguments.SourceDirectory.ToDirectoryInfo());
 
+            common.OmitBasename = !string.IsNullOrEmpty(arguments.OutputFormat);
+
             LoggedConsole.WriteLine("# File name of recording      : " + common.OriginalBasename);
 
+            // create file systems for reading input and writing output
             var io = FileSystemProvider.GetInputOutputFileSystems(
                 arguments.SourceDirectory,
-                Path.Combine(arguments.Output));
+                FileSystemProvider.MakePath(arguments.Output, common.OriginalBasename, arguments.OutputFormat))
+                .EnsureInputIsDirectory();
 
             switch (arguments.ZoomAction)
             {
@@ -90,7 +94,8 @@ public static void Execute(Arguments arguments)
                         arguments.Output.ToDirectoryInfo(),
                         common,
                         focalTime,
-                        ImageWidth);
+                        ImageWidth,
+                        Acoustic.TowseyAcoustic);
                     break;
                 case Arguments.ZoomActionType.Tile:
                     // Create the super tiles for a full set of recordings

AudioAnalysis/AnalysisPrograms/DrawLongDurationSpectrograms.cs

@@ -310,7 +310,7 @@ public static Image DrawGrayScaleSpectrograms(Arguments arguments, string fileSt
 
             //double backgroundFilter = 0.0; // 0.0 means small values are removed.
             double backgroundFilter = 0.75;  // 0.75 means small values are accentuated.
-            string analysisType = "Towsey.Acoustic";
+            string analysisType = Acoustic.TowseyAcoustic;
             string[] keys = LDSpectrogramRGB.GetArrayOfAvailableKeys();
 
             //LoggedConsole.WriteLine("# Spectrogram Config      file: " + arguments.SpectrogramConfigPath);
@@ -320,12 +320,8 @@ public static Image DrawGrayScaleSpectrograms(Arguments arguments, string fileSt
 
             if (spectra == null)
             {
-                var sw = Stopwatch.StartNew();
-
                 //C:\SensorNetworks\Output\BIRD50\Training\ID0001\Towsey.Acoustic\ID0001__Towsey.Acoustic.ACI
-                spectra = IndexMatrices.ReadCsvFiles(inputDirectory, fileStem + "__" + analysisType, keys);
-                sw.Stop();
-                LoggedConsole.WriteLine("Time to read spectral index files = " + sw.Elapsed.TotalSeconds + " seconds");
+                spectra = IndexMatrices.ReadSpectralIndices(inputDirectory, fileStem, analysisType, keys);
             }
 
             // note: the spectra are oriented as per visual orientation, i.e. xAxis = time frames
@@ -461,7 +457,7 @@ public static int DrawRidgeSpectrograms(Arguments arguments, string fileStem, Di
 
         public static Image DrawRidgeSpectrograms(DirectoryInfo inputDirectory, FileInfo ipConfig, string fileStem, double scale, Dictionary<string, double[,]> spectra = null)
         {
-            string analysisType = "Towsey.Acoustic";
+            string analysisType = Acoustic.TowseyAcoustic;
 
             //double backgroundFilter = 0.0; // 0.0 means small values are removed.
             double backgroundFilter = 0.75;  // 0.75 means small values are accentuated.
@@ -473,12 +469,8 @@ public static Image DrawRidgeSpectrograms(DirectoryInfo inputDirectory, FileInfo
             // read the csv files of the indices in keys array
             if (spectra == null)
             {
-                var sw = Stopwatch.StartNew();
-
                 //C:\SensorNetworks\Output\BIRD50\Training\ID0001\Towsey.Acoustic\ID0001__Towsey.Acoustic.ACI
-                spectra = IndexMatrices.ReadCsvFiles(inputDirectory, fileStem + "__" + analysisType, keys);
-                sw.Stop();
-                LoggedConsole.WriteLine("Time to read spectral index files = " + sw.Elapsed.TotalSeconds + " seconds");
+                spectra = IndexMatrices.ReadSpectralIndices(inputDirectory, fileStem, analysisType, keys);
             }
 
             var cs1 = new LDSpectrogramRGB(minuteOffset: TimeSpan.Zero, xScale: dataScale, sampleRate: 22050, frameWidth: 512, colourMap: null)

AudioAnalysis/AnalysisPrograms/HerveGlotinCollaboration.cs

@@ -180,11 +180,13 @@ public static void HiRes1()
                         Directory.CreateDirectory(zoomingArguments.Output);
                     }
 
-                    ZoomFocusedSpectrograms.DrawStackOfZoomedSpectrograms(zoomingArguments.SourceDirectory.ToDirectoryInfo(),
-                                                                            zoomingArguments.Output.ToDirectoryInfo(),
-                                                                            common,
-                                                                            TimeSpan.FromMinutes(focalMinute),
-                                                                            imageWidth);
+                    ZoomFocusedSpectrograms.DrawStackOfZoomedSpectrograms(
+                        zoomingArguments.SourceDirectory.ToDirectoryInfo(),
+                        zoomingArguments.Output.ToDirectoryInfo(),
+                        common,
+                        TimeSpan.FromMinutes(focalMinute),
+                        imageWidth,
+                        Acoustic.TowseyAcoustic);
 
                     // DRAW THE VARIOUS IMAGES
                     string fileStem = fileName;

AudioAnalysis/AnalysisPrograms/HighResolutionAcousticIndices.cs

@@ -251,11 +251,13 @@ public static void Execute(Arguments arguments)
                         Directory.CreateDirectory(zoomingArguments.Output);
                     }
 
-                    ZoomFocusedSpectrograms.DrawStackOfZoomedSpectrograms(zoomingArguments.SourceDirectory.ToDirectoryInfo(),
-                                                                            zoomingArguments.Output.ToDirectoryInfo(),
-                                                                            common,
-                                                                            TimeSpan.FromMinutes(focalMinute),
-                                                                            imageWidth);
+                    ZoomFocusedSpectrograms.DrawStackOfZoomedSpectrograms(
+                        zoomingArguments.SourceDirectory.ToDirectoryInfo(),
+                        zoomingArguments.Output.ToDirectoryInfo(),
+                        common,
+                        TimeSpan.FromMinutes(focalMinute),
+                        imageWidth,
+                        Acoustic.TowseyAcoustic);
 
                     // DRAW THE VARIOUS IMAGES
                     // i.e. greyscale images, ridge spectrogram and two-maps spectrograms.

AudioAnalysis/AnalysisPrograms/Production/FileSystemProvider.cs

@@ -30,6 +30,8 @@ public static class FileSystemProvider
 
         public const string SqlitePattern = "sqlite3";
 
+        public static readonly string[] AllFormats = { SqlitePattern };
+
         private static readonly ILog Log = LogManager.GetLogger(nameof(FileSystemProvider));
 
         public enum Options
@@ -101,6 +103,18 @@ public static AnalysisIo GetInputOutputFileSystems(string inputPath, string outp
             var output = DetermineFileSystem(outputPath);
             return new AnalysisIo(input, output, null);
         }
+
+        public static string MakePath(string directory, string baseName, string format)
+        {
+            if (string.IsNullOrEmpty(format))
+            {
+                return directory;
+            }
+
+            Contract.Requires(AllFormats.Contains(format));
+
+            return Path.Combine(directory, baseName + "." + format);
+        }
     }
 
 }

AudioAnalysis/AudioAnalysisTools/LongDurationSpectrograms/Zooming/ZoomArguments.cs

@@ -23,6 +23,8 @@ public class ZoomArguments
 
         public FileInfo IndexDistributionsFile { get; set; }
 
+        public bool OmitBasename { get; set; }
+
         /// <summary>
         /// Read in required files.
         /// We expect a valid indices output directory (the input directory in this action)