extended AddArius with AriusConfiguration, added StateDbRepositoryFac…

…tory, boilerplate ArchiveCommand

src/Arius.Core.Domain/AriusConfiguration.cs

@@ -0,0 +1,10 @@
+namespace Arius.Core.Domain;
+
+public record AriusConfiguration
+{
+    public required DirectoryInfo LocalConfigRoot { get; set; }
+
+
+    public DirectoryInfo StateDbRoot                              => LocalConfigRoot.CreateSubdirectory("StateDbs");
+    public DirectoryInfo GetStateDbForRepositoryName(string name) => StateDbRoot.CreateSubdirectory(name);
+}

src/Arius.Core.Domain/Repositories/IStateDbRepositoryFactory.cs

@@ -0,0 +1,13 @@
+using Arius.Core.Domain.Storage;
+
+namespace Arius.Core.Domain.Repositories;
+
+public interface IStateDbRepositoryFactory
+{
+    public Task<IStateDbRepository> CreateAsync(RepositoryOptions repository);
+}
+
+public interface IStateDbRepository
+{
+
+}

src/Arius.Core.Infrastructure/Arius.Core.Infrastructure.csproj

@@ -8,6 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Azure.Storage.Blobs" Version="12.21.2" />
+    <PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
     <PackageReference Include="Nito.AsyncEx.Coordination" Version="5.1.2" />
     <PackageReference Include="System.Linq.Async" Version="6.0.1" />
   </ItemGroup>

src/Arius.Core.Infrastructure/Repositories/StateDbRepositoryFactory.cs

@@ -0,0 +1,25 @@
+using Arius.Core.Domain;
+using Arius.Core.Domain.Repositories;
+using Arius.Core.Domain.Storage;
+using Microsoft.Extensions.Options;
+
+namespace Arius.Core.Infrastructure.Repositories;
+
+public class SqliteStateDbRepositoryFactory : IStateDbRepositoryFactory
+{
+    private readonly AriusConfiguration config;
+
+    public SqliteStateDbRepositoryFactory(IOptions<AriusConfiguration> config)
+    {
+        this.config = config.Value;
+    }
+    public Task<IStateDbRepository> CreateAsync(RepositoryOptions repository)
+    {
+        throw new NotImplementedException();
+    }
+}
+
+public class SqliteStateDbRepository : IStateDbRepository
+{
+
+}

src/Arius.Core.New.UnitTests/Fixtures/AriusFixture.cs

@@ -50,7 +50,7 @@ void InitializeMockedServices()
             mockedTestRepositoryOptions = new Lazy<TestRepositoryOptions>(configuration.GetSection("RepositoryOptions").Get<TestRepositoryOptions>()!);
 
             // Register the actual services
-            services.AddArius();
+            services.AddArius(c => c.LocalConfigRoot = UnitTestRoot);
             services.AddLogging();
 
             // Add additional services
@@ -73,7 +73,7 @@ void InitializeRealServices()
             realTestRepositoryOptions = new Lazy<TestRepositoryOptions>(configuration.GetSection("RepositoryOptions").Get<TestRepositoryOptions>()!);
 
             // Register the actual services
-            services.AddArius();
+            services.AddArius(c => c.LocalConfigRoot = UnitTestRoot);
             services.AddLogging();
 
             // Add additional services

src/Arius.Core.New/Commands/Archive/ArchiveCommand.cs

@@ -0,0 +1,78 @@
+using Arius.Core.Domain.Storage;
+using Arius.Core.New.Commands.DownloadStateDb;
+using FluentValidation;
+using MediatR;
+
+namespace Arius.Core.New.Commands.Archive;
+
+public record ArchiveCommand : IRequest
+{
+    public required RepositoryOptions Repository  { get; init; }
+    public required bool              FastHash    { get; init; }
+    public required bool              RemoveLocal { get; init; }
+    public required StorageTier       Tier        { get; init; }
+    public required DirectoryInfo     LocalRoot   { get; init; }
+    public required RepositoryVersion VersionName { get; init; }
+
+
+    internal int IndexBlock_Parallelism => Environment.ProcessorCount * 8; //index AND hash options. A low count doesnt achieve a high throughput when there are a lot of small files
+
+    internal int BinariesToUpload_BufferSize => 100; //apply backpressure if we cannot upload fast enough
+
+    internal int UploadBinaryFileBlock_BinaryFileParallelism => Environment.ProcessorCount * 2;
+    internal int TransferChunked_ChunkBufferSize             => 1024; //put lower on systems with low memory -- if unconstrained, it will load all the BinaryFiles in memory
+    internal int TransferChunked_ParallelChunkTransfers      => 128; // 128 * 2; -- NOTE sep22 this was working before but now getting ResourceUnavailable errors --> throttling?
+
+    internal int PointersToCreate_BufferSize => 1000;
+
+    internal int CreatePointerFileIfNotExistsBlock_Parallelism => 1;
+
+    internal int PointerFileEntriesToCreate_BufferSize => 1000;
+
+    internal int CreatePointerFileEntryIfNotExistsBlock_Parallelism => 1;
+
+    internal int BinariesToDelete_BufferSize => 1000;
+
+    internal int DeleteBinaryFilesBlock_Parallelism => 1;
+
+    internal int CreateDeletedPointerFileEntryForDeletedPointerFilesBlock_Parallelism => 1;
+
+    internal int UpdateTierBlock_Parallelism => 10;
+}
+
+internal class ArchiveCommandValidator : AbstractValidator<ArchiveCommand>
+{
+    public ArchiveCommandValidator()
+    {
+        RuleFor(command => command.Repository).SetValidator(new RepositoryOptionsValidator());
+        RuleFor(command => command.LocalRoot)
+            .NotEmpty()
+            .Must(localRoot => localRoot.Exists).WithMessage("The local root does not exist.");
+    }
+}
+
+internal class ArchiveCommandHandler : IRequestHandler<ArchiveCommand>
+{
+    private readonly IMediator                      mediator;
+    private readonly ILogger<ArchiveCommandHandler> logger;
+
+    public ArchiveCommandHandler(IMediator mediator, ILogger<ArchiveCommandHandler> logger)
+    {
+        this.mediator = mediator;
+        this.logger   = logger;
+    }
+
+    public async Task Handle(ArchiveCommand request, CancellationToken cancellationToken)
+    {
+        await new ArchiveCommandValidator().ValidateAndThrowAsync(request, cancellationToken);
+
+        // Download latest state database
+        var downloadStateDbCommand = new DownloadStateDbCommand
+        {
+            Repository = request.Repository,
+            LocalPath  = request.LocalRoot.FullName
+        };
+
+        await mediator.Send(downloadStateDbCommand);
+    }
+}

src/Arius.Core.New/Commands/DownloadStateDb/DownloadStateDbCommand.cs

@@ -1,8 +1,11 @@
-using Arius.Core.Domain.Storage;
+using Arius.Core.Domain;
+using Arius.Core.Domain.Repositories;
+using Arius.Core.Domain.Storage;
 using Arius.Core.New.Services;
 using Azure;
 using FluentValidation;
 using MediatR;
+using Microsoft.Extensions.Options;
 
 namespace Arius.Core.New.Commands.DownloadStateDb;
 
@@ -43,7 +46,7 @@ internal class DownloadStateDbCommandHandler : IRequestHandler<DownloadStateDbCo
     private readonly ICryptoService                         cryptoService;
     private readonly ILogger<DownloadStateDbCommandHandler> logger;
 
-    public DownloadStateDbCommandHandler(IStorageAccountFactory storageAccountFactory, ICryptoService cryptoService, ILogger<DownloadStateDbCommandHandler> logger)
+    public DownloadStateDbCommandHandler(IStateDbRepositoryFactory c, IStorageAccountFactory storageAccountFactory, ICryptoService cryptoService, ILogger<DownloadStateDbCommandHandler> logger)
     {
         this.storageAccountFactory = storageAccountFactory;
         this.cryptoService         = cryptoService;

src/Arius.Core.New/IServiceCollectionExtensions.cs

@@ -4,7 +4,9 @@
 using FluentValidation;
 using Microsoft.Extensions.DependencyInjection;
 using System.Reflection;
-using System.Runtime.CompilerServices;
+using Arius.Core.Domain;
+using Arius.Core.Domain.Repositories;
+using Arius.Core.Infrastructure.Repositories;
 
 ///*
 // * This is required to test the internals of the Arius.Core assembly
@@ -16,8 +18,10 @@ namespace Arius.Core.New;
 
 public static class IServiceCollectionExtensions
 {
-    public static IServiceCollection AddArius(this IServiceCollection services)
+    public static IServiceCollection AddArius(this IServiceCollection services, Action<AriusConfiguration> configureOptions)
     {
+        services.Configure(configureOptions); // TODO add validation
+
         // Add FluentValidation validators
         services.AddValidatorsFromAssembly(Assembly.GetExecutingAssembly());
 
@@ -33,6 +37,7 @@ public static IServiceCollection AddArius(this IServiceCollection services)
 
         services.AddSingleton<IStorageAccountFactory, AzureStorageAccountFactory>();
         services.AddSingleton<ICryptoService, CryptoService>();
+        services.AddSingleton<IStateDbRepositoryFactory, SqliteStateDbRepositoryFactory>();
 
         return services;
     }