Merge pull request #4757 from freelawproject/add-independent-sweep-in…

…dexer-wait-setting Added a separate setting for waiting between sweep indexer chunks
freelawproject · Nov 29, 2024 · 61e5660 · 61e5660
2 parents 8c15b92 + ceb049e
commit 61e5660
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 3 deletions.
diff --git a/cl/corpus_importer/signals.py b/cl/corpus_importer/signals.py
@@ -76,6 +76,10 @@ def update_latest_case_id_and_schedule_iquery_sweep(docket: Docket) -> None:
                 countdown=task_scheduled_countdown,
                 queue=settings.CELERY_IQUERY_QUEUE,
             )
+            logger.info(
+                f"Enqueued iquery docket case ID: {iquery_pacer_case_id_current} "
+                f"for court {court_id} with countdown {task_scheduled_countdown}"
+            )
 
         # Update the iquery_pacer_case_id_current in Redis
         r.hset(

diff --git a/cl/search/management/commands/sweep_indexer.py b/cl/search/management/commands/sweep_indexer.py
@@ -406,13 +406,13 @@ def process_queryset(
 
                 accumulated_chunk += len(chunk)
                 if not testing_mode:
-                    # Wait for 1/ELASTICSEARCH_SWEEP_INDEXER_POLL_INTERVAL
+                    # Wait for 1/ELASTICSEARCH_SWEEP_INDEXER_WAIT_BETWEEN_CHUNKS
                     # before processing the next chunk.
                     # e.g: With a poll interval of 10 and a chunk size of 10,
                     # it will wait for 0.1 seconds for every 10 documents processed,
                     # maintaining an index rate of 100 documents per second.
                     time.sleep(
-                        1 / settings.ELASTICSEARCH_SWEEP_INDEXER_POLL_INTERVAL  # type: ignore
+                        1 / settings.ELASTICSEARCH_SWEEP_INDEXER_WAIT_BETWEEN_CHUNKS  # type: ignore
                     )
                 self.stdout.write(
                     "\rProcessed {}/{}, ({:.0%}), last {} ID indexed: {},".format(

diff --git a/cl/settings/third_party/elasticsearch.py b/cl/settings/third_party/elasticsearch.py
@@ -265,7 +265,10 @@
     "ELASTICSEARCH_SWEEP_INDEXER_HEADS_RATE", default=60
 )
 ELASTICSEARCH_SWEEP_INDEXER_POLL_INTERVAL = env(
-    "ELASTICSEARCH_SWEEP_INDEXER_POLL_INTERVAL", default=5
+    "ELASTICSEARCH_SWEEP_INDEXER_POLL_INTERVAL", default=10
+)
+ELASTICSEARCH_SWEEP_INDEXER_WAIT_BETWEEN_CHUNKS = env(
+    "ELASTICSEARCH_SWEEP_INDEXER_WAIT_BETWEEN_CHUNKS", default=3
 )
 ELASTICSEARCH_SWEEP_INDEXER_MODELS = env(
     "ELASTICSEARCH_SWEEP_INDEXER_MODELS",