Furhter development of SqliteFileSystem

Now implements File.Open via a database backed custom stream
implementation.

Also partially implements more of the IFileSystem interface.

Work for #120

Acoustics/Acoustics.Test/Acoustics.Test.csproj

@@ -286,6 +286,7 @@
     <Compile Include="AudioAnalysisTools\TileImage\TilerTests.cs" />
     <Compile Include="Shared\RangeTests.cs" />
     <Compile Include="SqliteFileSystem\AdapterTests.cs" />
+    <Compile Include="SqliteFileSystem\DatabaseBackedMemoryStreamTests.cs" />
     <Compile Include="SqliteFileSystem\SqliteFileSystemTests.cs" />
     <Compile Include="SqliteFileSystem\SqliteFileSystemConstructionTests.cs" />
     <Compile Include="TestHelpers\Factories\AudioRecordingFactory.cs" />

Acoustics/Acoustics.Test/SqliteFileSystem/AdapterTests.cs

@@ -8,12 +8,13 @@ namespace Acoustics.Test.SqliteFileSystem
 {
     using System.Diagnostics;
     using System.IO;
-    using global::SqliteFileSystem;
     using Microsoft.Data.Sqlite;
     using Microsoft.VisualStudio.TestTools.UnitTesting;
     using TestHelpers;
+    using Zio;
     using Zio.FileSystems;
-    using Zio.FileSystems.Additional;
+    using Zio.FileSystems.Community.SqliteFileSystem;
+    using static Zio.FileSystems.Community.SqliteFileSystem.Date;
 
     [TestClass]
     public class AdapterTests
@@ -22,17 +23,15 @@ public class AdapterTests
         protected DirectoryInfo outputDirectory;
         private FileInfo testFile;
         private System.Random random;
-        private byte[] sampleBlob;
+
 
         [TestInitialize]
         public void Setup()
         {
             this.testFile = PathHelper.GetTempFile("sqlite3");
             this.outputDirectory = this.testFile.Directory;
             this.random = Random.GetRandom();
-            this.sampleBlob = new byte[1024];
-
-            this.random.NextBytes(this.sampleBlob);
+
         }
 
         [TestCleanup]
@@ -43,9 +42,162 @@ public void Cleanup()
             PathHelper.DeleteTempDir(this.outputDirectory);
         }
 
+        /// <summary>
+        ///  manually add a blob so we can test other parts of the code
+        /// </summary>
+        /// <param name="random"></param>
+        /// <returns></returns>
+        public static (string ConnectionString, UPath blobPath, byte[] blobData) PrepareDatabaseAndBlob(
+            string testFile,
+            System.Random random)
+        {
+            var testData = SqliteFileSystemTests.GenerateTestData(random, "/test.blob");
+
+            // create a new empty database - mainly doing this to get a schema
+            using (var fs = new SqliteFileSystem(testFile, SqliteOpenMode.ReadWriteCreate))
+            {
+            }
+
+            var connectionString = $"Data source='{testFile}';Mode={SqliteOpenMode.ReadWrite}";
+            using (var connection = new SqliteConnection(connectionString))
+            {
+                connection.Open();
+
+                InsertBlobManually(connection, testData);
+            }
+
+            return (connectionString, testData.Path, testData.Data);
+        }
+
+        public static void InsertBlobManually(SqliteConnection connection, (UPath Path, byte[] Data) testData)
+        {
+// add a blob we can read
+            using (var command = new SqliteCommand(
+                "INSERT INTO files VALUES ('" + testData.Path.FullName + "', @blob, 0, 0, 0)",
+                connection))
+            {
+                var parameter = new SqliteParameter("blob", SqliteType.Blob) { Value = testData.Data };
+                command.Parameters.Add(parameter);
+
+                command.ExecuteNonQuery();
+            }
+        }
+
+        private static void AssertBlobMetadata(
+            SqliteConnection connection,
+            int expectedLength,
+            long expectedAccessed,
+            long expectedCreated,
+            long expectedModified,
+            string path = "/test.blob")
+        {
+            using (var command = new SqliteCommand(
+                $"SELECT length(blob), accessed, created, modified FROM files WHERE path = '{path}'",
+                connection))
+            {
+                var reader = command.ExecuteReader();
+
+                Assert.IsTrue(reader.Read());
+                Assert.AreEqual(expectedLength, reader.GetInt32(0));
+                Assert.That.AreClose(expectedAccessed, reader.GetInt64(1), 15_000_0);
+                Assert.That.AreClose(expectedCreated, reader.GetInt64(2),  15_000_0);
+                Assert.That.AreClose(expectedModified, reader.GetInt64(3), 15_000_0);
+            }
+        }
+
+        [TestMethod]
+        public void GetBlobTest()
+        {
+            var prepared = PrepareDatabaseAndBlob(this.testFile.FullName, this.random);
+
+            using (var connection = new SqliteConnection(prepared.ConnectionString))
+            {
+                connection.Open();
+
+                // before test, everything should be set up according to mock data
+                AssertBlobMetadata(connection, 1024, 0, 0, 0);
+
+                var now = Now;
+                var blob = Adapter.GetBlob(connection, UPath.Root / "test.blob");
+
+                CollectionAssert.AreEqual(prepared.blobData, blob);
+
+                // now the accessed date should have been updated
+                AssertBlobMetadata(connection, 1024, now, 0, 0);
+            }
+        }
+
         [TestMethod]
-        public void AdapterCanStreamBlobs()
+        public void SetBlobTestInsert()
         {
+            var prepared = PrepareDatabaseAndBlob(this.testFile.FullName, this.random);
+
+            using (var connection = new SqliteConnection(prepared.ConnectionString))
+            {
+                connection.Open();
+
+                // before test, everything should be set up according to mock data
+                AssertBlobMetadata(connection, 1024, 0, 0, 0);
+
+                // insert an entirely new blob
+                var newBlob = SqliteFileSystemTests.GenerateTestData(this.random);
+
+                var now = Now;
+                Adapter.SetBlob(connection, UPath.Root / newBlob.Path, newBlob.Data);
+
+                // now all dates should have been updated on the new blob
+                AssertBlobMetadata(connection, 1024, 0, 0, 0);
+                AssertBlobMetadata(connection, 1024, now, now, now, newBlob.Path.FullName);
+
+                var now2 = Now;
+                var blob = Adapter.GetBlob(connection, newBlob.Path.FullName);
+
+                CollectionAssert.AreEqual(newBlob.Data, blob);
+
+                // now the accessed date should have been updated
+                AssertBlobMetadata(connection, 1024, 0, 0, 0);
+                AssertBlobMetadata(connection, 1024, now2, now, now, newBlob.Path.FullName);
+            }
+        }
+
+        [TestMethod]
+        public void SetBlobTestUpdate()
+        {
+            var prepared = PrepareDatabaseAndBlob(this.testFile.FullName, this.random);
+
+            using (var connection = new SqliteConnection(prepared.ConnectionString))
+            {
+                connection.Open();
+
+                // before test, everything should be set up according to mock data
+                AssertBlobMetadata(connection, 1024, 0, 0, 0);
+
+                // overwrite the blob
+                var newBlob = SqliteFileSystemTests.GenerateTestData(this.random, "/test.blob");
+
+                var now = Now;
+                Adapter.SetBlob(connection, UPath.Root / "test.blob", newBlob.Data);
+
+                // now the accessed date and modified date should have been updated
+                AssertBlobMetadata(connection, 1024, now, 0, now);
+
+                var now2 = Now;
+                var blob = Adapter.GetBlob(connection, UPath.Root / "test.blob");
+
+                CollectionAssert.AreEqual(newBlob.Data, blob);
+
+                // now the accessed date should have been updated
+                AssertBlobMetadata(connection, 1024, now2, 0, now);
+            }
+        }
+
+        [TestMethod]
+        [Ignore]
+        public void AdapterCanStreamReadBlobs()
+        {
+            // https://github.com/aspnet/Microsoft.Data.Sqlite/issues/18
+            throw new NotImplementedException("Sqlite adapter does not yet support blob streaming");
+            /*
             // create a new empty database - mainly doing this to get a schema
             using (var fs = new SqliteFileSystem(this.testFile.FullName, SqliteOpenMode.ReadWriteCreate))
             {
@@ -57,8 +209,7 @@ public void AdapterCanStreamBlobs()
                 connection.Open();
 
                 // add a blob we can read
-
-                using (var command = new SqliteCommand("INSERT INTO files VALUES ('/test.blob', @blob)"))
+                using (var command = new SqliteCommand("INSERT INTO files VALUES ('/test.blob', @blob, 0, 0, 0)", connection))
                 {
                     var parameter = new SqliteParameter("blob", SqliteType.Blob) { Value = this.sampleBlob };
                     command.Parameters.Add(parameter);
@@ -69,7 +220,7 @@ public void AdapterCanStreamBlobs()
 
                 // verify we can stream the response back
                 byte[] result;
-                using (var readStream = Adapter.ExecuteNonQueryStream(connection, "SELECT blob FROM files LIMIT 1"))
+                using (var readStream = Adapter.GetBlob(connection, "SELECT blob FROM files LIMIT 1"))
                 {
                     // make sure we can read, get length, can't write
                     Assert.AreEqual(1024, readStream.Length);
@@ -91,7 +242,15 @@ public void AdapterCanStreamBlobs()
                // finally make sure the blob was read back accurately
                CollectionAssert.AreEqual(this.sampleBlob, result);
             }
+            */
         }
 
+        [TestMethod]
+        [Ignore]
+        public void AdapterCanStreamWriteBlobs()
+        {
+            // https://github.com/aspnet/Microsoft.Data.Sqlite/issues/18
+            throw new NotImplementedException("Sqlite adapter does not yet support blob streaming");
+        }
     }
 }

Acoustics/Acoustics.Test/SqliteFileSystem/DatabaseBackedMemoryStreamTests.cs

@@ -0,0 +1,129 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Acoustics.Test.SqliteFileSystem
+{
+    using System;
+    using System.Diagnostics;
+    using System.IO;
+    using Acoustics.Tools.Wav;
+    using Microsoft.Data.Sqlite;
+    using TestHelpers;
+    using Zio;
+    using Zio.FileSystems.Community.SqliteFileSystem;
+    using Random = TestHelpers.Random;
+
+    [TestClass]
+    public class DatabaseBackedMemoryStreamTests
+    {
+        protected DirectoryInfo outputDirectory;
+        private FileInfo testFile;
+        private System.Random random;
+        private byte[] sampleBlob;
+
+        [TestInitialize]
+        public void Setup()
+        {
+            this.testFile = PathHelper.GetTempFile("sqlite3");
+            this.outputDirectory = this.testFile.Directory;
+            this.random = Random.GetRandom();
+            this.sampleBlob = new byte[1024];
+
+            this.random.NextBytes(this.sampleBlob);
+        }
+
+        [TestCleanup]
+        public void Cleanup()
+        {
+            Debug.WriteLine("Deleting output directory:" + this.outputDirectory.FullName);
+
+            PathHelper.DeleteTempDir(this.outputDirectory);
+        }
+
+        [TestMethod]
+        public void TestDatabaseBackedMemoryStream()
+        {
+            var prepared = AdapterTests.PrepareDatabaseAndBlob(this.testFile.FullName, this.random);
+            using (var connection = new SqliteConnection(prepared.ConnectionString))
+            {
+                connection.Open();
+                using (var stream = new DatabaseBackedMemoryStream(connection, UPath.Root / "test.blob", true, true))
+                {
+
+                    Assert.IsTrue(stream.CanRead);
+                    Assert.IsTrue(stream.CanSeek);
+                    Assert.IsTrue(stream.CanWrite);
+                    Assert.AreEqual(0, stream.Position);
+                    Assert.AreEqual(1024, stream.Length);
+
+                    var actualBlob = new byte[stream.Length];
+                    var read = stream.Read(actualBlob, 0, 1024);
+
+                    Assert.AreEqual(1024, read);
+                    CollectionAssert.AreEqual(prepared.blobData, actualBlob);
+                }
+            }
+        }
+
+
+        [TestMethod]
+        public void TestDatabaseBackedMemoryStreamWriting()
+        {
+            var prepared = AdapterTests.PrepareDatabaseAndBlob(this.testFile.FullName, this.random);
+
+            var additionalBytes = new byte[512];
+            this.random.NextBytes(additionalBytes);
+
+            using (var connection = new SqliteConnection(prepared.ConnectionString))
+            {
+                connection.Open();
+                using (var stream = new DatabaseBackedMemoryStream(connection, UPath.Root / "test.blob", true, true))
+                {
+                    // we're going to append extra data onto the stream
+                    stream.Seek(0, SeekOrigin.End);
+                    Assert.AreEqual(1024, stream.Position);
+                    Assert.AreEqual(1024, stream.Length);
+
+                    stream.Write(additionalBytes, 0, 256);
+
+                    Assert.AreEqual(1024 + 256, stream.Position);
+                    Assert.AreEqual(1024 + 256, stream.Length);
+
+                    // no changes should have happened to the database
+                    AssertDatabaseBlobLength(connection, 1024);
+
+                    // when changes are flushed we expect an update
+                    stream.Flush();
+
+                    Assert.AreEqual(1024 + 256, stream.Position);
+                    Assert.AreEqual(1024 + 256, stream.Length);
+
+                    AssertDatabaseBlobLength(connection, 1024 + 256);
+
+                    // write another set of bytes
+                    stream.Write(additionalBytes, 256, 256);
+
+                    Assert.AreEqual(1024 + 512, stream.Position);
+                    Assert.AreEqual(1024 + 512, stream.Length);
+
+                    // no flush, so no db update
+                    AssertDatabaseBlobLength(connection, 1024 + 256);
+                }
+
+                // dispose should flush
+                AssertDatabaseBlobLength(connection, 1024 + 512);
+            }
+        }
+
+        private static void AssertDatabaseBlobLength(SqliteConnection connection, int expectedLength)
+        {
+            using (var command = new SqliteCommand(
+                "SELECT length(blob) FROM files WHERE path = '/test.blob'",
+                connection))
+            {
+                long dbBlobLength = (long)command.ExecuteScalar();
+                Assert.AreEqual(expectedLength, dbBlobLength);
+            }
+        }
+    }
+}
+

Acoustics/Acoustics.Test/SqliteFileSystem/SqliteFileSystemConstructionTests.cs

@@ -8,14 +8,13 @@ namespace Acoustics.Test.SqliteFileSystem
 {
     using System.Diagnostics;
     using System.IO;
-    using global::SqliteFileSystem;
     using Microsoft.Data.Sqlite;
     using Microsoft.VisualStudio.TestTools.UnitTesting;
-    using SqLiteFileSystem;
     using TestHelpers;
     using Zio;
     using Zio.FileSystems;
-    using Zio.FileSystems.Additional;
+    using Zio.FileSystems.Community;
+    using Zio.FileSystems.Community.SqliteFileSystem;
 
     [TestClass]
     public class SqliteFileSystemConstructionTests