Add options for elasticsearch synchronization (#487)

* Add options for elasticsearch synchronization * Change structure for elasticsearch settings * Change settings after solve conflicts after rebase * fix productivity Co-authored-by: Sergii Novytskyi <[email protected]>
ita-social-projects · Jan 14, 2022 · 66dd061 · 66dd061
1 parent 0d383dc
commit 66dd061
Show file tree

Hide file tree

Showing 8 changed files with 66 additions and 14 deletions.
diff --git a/OutOfSchool/OutOfSchool.WebApi/Config/ElasticsearchSynchronizationSchedulerConfig.cs b/OutOfSchool/OutOfSchool.WebApi/Config/ElasticsearchSynchronizationSchedulerConfig.cs
@@ -0,0 +1,13 @@
+using Microsoft.Extensions.Configuration;
+
+namespace OutOfSchool.WebApi.Config
+{
+    public class ElasticsearchSynchronizationSchedulerConfig
+    {
+        public static string SectionName { get; } = ElasticConfig.Name + ConfigurationPath.KeyDelimiter + "SynchronizationScheduler";
+
+        public int OperationsPerTask { get; set; }
+
+        public int DelayBetweenTasksInMilliseconds { get; set; }
+    }
+}
diff --git a/OutOfSchool/OutOfSchool.WebApi/Extensions/ElasticsearchSynchronizationExtension.cs b/OutOfSchool/OutOfSchool.WebApi/Extensions/ElasticsearchSynchronizationExtension.cs
@@ -0,0 +1,27 @@
+using System;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
+using OutOfSchool.WebApi.Config;
+using OutOfSchool.WebApi.Services;
+
+namespace OutOfSchool.WebApi.Extensions
+{
+    public static class ElasticsearchSynchronizationExtension
+    {
+        public static IServiceCollection AddElasticsearchSynchronization(
+            this IServiceCollection services,
+            Action<OptionsBuilder<ElasticsearchSynchronizationSchedulerConfig>> elasticsearchSynchronizationSchedulerConfig)
+        {
+            services.AddHostedService<ElasticsearchSynchronizationHostedService>();
+            services.AddTransient<IElasticsearchSynchronizationService, ElasticsearchSynchronizationService>();
+            if (elasticsearchSynchronizationSchedulerConfig == null)
+            {
+                throw new ArgumentNullException(nameof(elasticsearchSynchronizationSchedulerConfig));
+            }
+
+            var elasticsearchSynchronizationSchedulerConfigBuilder = services.AddOptions<ElasticsearchSynchronizationSchedulerConfig>();
+            elasticsearchSynchronizationSchedulerConfig(elasticsearchSynchronizationSchedulerConfigBuilder);
+            return services;
+        }
+    }
+}
diff --git a/OutOfSchool/OutOfSchool.WebApi/Services/Elasticsearch/ElasticsearchSynchronizationService.cs b/OutOfSchool/OutOfSchool.WebApi/Services/Elasticsearch/ElasticsearchSynchronizationService.cs
@@ -6,12 +6,14 @@
 using AutoMapper;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using Nest;
 using OutOfSchool.ElasticsearchData;
 using OutOfSchool.ElasticsearchData.Models;
 using OutOfSchool.Services.Enums;
 using OutOfSchool.Services.Models;
 using OutOfSchool.Services.Repository;
+using OutOfSchool.WebApi.Config;
 
 namespace OutOfSchool.WebApi.Services
 {
@@ -20,27 +22,27 @@ namespace OutOfSchool.WebApi.Services
     /// </summary>
     public class ElasticsearchSynchronizationService : IElasticsearchSynchronizationService
     {
-        // TODO: move to config
-        private const int NumberOfOperation = 10;
-
         private readonly IWorkshopService databaseService;
         private readonly IElasticsearchSyncRecordRepository elasticsearchSyncRecordRepository;
         private readonly IElasticsearchProvider<WorkshopES, WorkshopFilterES> esProvider;
         private readonly ILogger<ElasticsearchSynchronizationService> logger;
         private readonly IMapper mapper;
+        private readonly IOptions<ElasticsearchSynchronizationSchedulerConfig> options;
 
         public ElasticsearchSynchronizationService(
             IWorkshopService workshopService,
             IElasticsearchSyncRecordRepository elasticsearchSyncRecordRepository,
             IElasticsearchProvider<WorkshopES, WorkshopFilterES> esProvider,
             ILogger<ElasticsearchSynchronizationService> logger,
-            IMapper mapper)
+            IMapper mapper,
+            IOptions<ElasticsearchSynchronizationSchedulerConfig> options)
         {
             this.databaseService = workshopService;
             this.elasticsearchSyncRecordRepository = elasticsearchSyncRecordRepository;
             this.esProvider = esProvider;
             this.logger = logger;
             this.mapper = mapper;
+            this.options = options;
         }
 
         public async Task<bool> Synchronize()
@@ -49,7 +51,7 @@ public async Task<bool> Synchronize()
 
             var elasticsearchSyncRecords = await elasticsearchSyncRecordRepository.GetByEntity(
                 ElasticsearchSyncEntity.Workshop,
-                NumberOfOperation).ConfigureAwait(false);
+                options.Value.OperationsPerTask).ConfigureAwait(false);
 
             var resultCreate = await SynchronizeAndDeleteRecords(elasticsearchSyncRecords, ElasticsearchSyncOperation.Create).ConfigureAwait(false);
             if (!resultCreate)
@@ -117,13 +119,13 @@ public async Task Synchronize(CancellationToken cancellationToken)
 
                 logger.LogInformation("Elasticsearch synchronization finished");
 
-                await Task.Delay(10000, cancellationToken).ConfigureAwait(false);
+                await Task.Delay(options.Value.DelayBetweenTasksInMilliseconds, cancellationToken).ConfigureAwait(false);
             }
         }
 
         private async Task<bool> Synchronize(IEnumerable<ElasticsearchSyncRecord> elasticsearchSyncRecords, ElasticsearchSyncOperation elasticsearchSyncOperation)
         {
-            var ids = elasticsearchSyncRecords.Where(es => es.Operation == elasticsearchSyncOperation).Select(es => es.RecordId);
+            var ids = elasticsearchSyncRecords.Where(es => es.Operation == elasticsearchSyncOperation).Select(es => es.RecordId).ToList();
 
             if (!ids.Any())
             {

diff --git a/OutOfSchool/OutOfSchool.WebApi/Startup.cs b/OutOfSchool/OutOfSchool.WebApi/Startup.cs
@@ -169,8 +169,8 @@ public void ConfigureServices(IServiceCollection services)
             services.AddTransient<IElasticsearchProvider<WorkshopES, WorkshopFilterES>, ESWorkshopProvider>();
             services.AddTransient<IElasticsearchService<WorkshopES, WorkshopFilterES>, ESWorkshopService>();
 
-            services.AddHostedService<ElasticsearchSynchronizationHostedService>();
-            services.AddTransient<IElasticsearchSynchronizationService, ElasticsearchSynchronizationService>();
+            services.AddElasticsearchSynchronization(
+                builder => builder.Bind(Configuration.GetSection(ElasticsearchSynchronizationSchedulerConfig.SectionName)));
 
             // entities services
             services.AddTransient<IAddressService, AddressService>();

diff --git a/OutOfSchool/OutOfSchool.WebApi/appsettings.Compose.json b/OutOfSchool/OutOfSchool.WebApi/appsettings.Compose.json
@@ -26,6 +26,10 @@
       "http://elasticsearch:9200/"
     ],
     "User": "admin",
-    "Password": "admin"
+    "Password": "admin",
+    "SynchronizationScheduler": {
+      "OperationsPerTask": 10,
+      "DelayBetweenTasksInMilliseconds": 60000
+    }
   }
 }
diff --git a/OutOfSchool/OutOfSchool.WebApi/appsettings.Development.json b/OutOfSchool/OutOfSchool.WebApi/appsettings.Development.json
@@ -39,6 +39,10 @@
       "http://localhost:9200/"
     ],
     "User": "admin",
-    "Password": "admin"
+    "Password": "admin",
+    "SynchronizationScheduler": {
+      "OperationsPerTask": 10,
+      "DelayBetweenTasksInMilliseconds": 60000
+    }
   }
 }
diff --git a/OutOfSchool/OutOfSchool.WebApi/appsettings.Google.json b/OutOfSchool/OutOfSchool.WebApi/appsettings.Google.json
@@ -35,6 +35,10 @@
     ],
     "User": "admin",
     "Password": "admin",
-    "PingIntervalSeconds": 30
+    "PingIntervalSeconds": 30,
+    "SynchronizationScheduler": {
+      "OperationsPerTask": 10,
+      "DelayBetweenTasksInMilliseconds": 60000
+    }
   }
 }
diff --git a/OutOfSchool/OutOfSchool.WebApi/appsettings.json b/OutOfSchool/OutOfSchool.WebApi/appsettings.json
@@ -76,5 +76,3 @@
 
   "MySQLServerVersion": "8.0.27"
 }
-
-