add referential integrity checking to API operations

Makefile

@@ -101,7 +101,7 @@ mongorestore-nmdc-db:
 	mkdir -p /tmp/remote-mongodump/nmdc
 	# SSH into the remote server, stream the dump directory as a gzipped tar archive, and extract it locally.
 	ssh -i ~/.ssh/nersc ${NERSC_USERNAME}@dtn01.nersc.gov \
-		'tar -czf - -C /global/cfs/projectdirs/m3408/nmdc-mongodumps/dump_nmdc-prod_2024-07-29_20-12-07/nmdc .' \
+		'tar -czf - -C /global/cfs/projectdirs/m3408/nmdc-mongodumps/dump_nmdc-prod_2024-08-19_20-12-02/nmdc .' \
 		| tar -xzv -C /tmp/remote-mongodump/nmdc
 	mongorestore -v -h localhost:27018 -u admin -p root --authenticationDatabase=admin \
 		--drop --nsInclude='nmdc.*' --dir /tmp/remote-mongodump

nmdc_runtime/api/main.py

@@ -1,5 +1,5 @@
+import logging
 import os
-import re
 from contextlib import asynccontextmanager
 from importlib import import_module
 from importlib.metadata import version
@@ -8,7 +8,6 @@
 import fastapi
 import requests
 import uvicorn
-from bs4 import BeautifulSoup
 from fastapi import APIRouter, FastAPI, Cookie
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.openapi.docs import get_swagger_ui_html
@@ -49,6 +48,7 @@
 from nmdc_runtime.api.v1.router import router_v1
 from nmdc_runtime.minter.bootstrap import bootstrap as minter_bootstrap
 from nmdc_runtime.minter.entrypoints.fastapi_app import router as minter_router
+from nmdc_runtime.site.repository import repo as runtime_dagster_code_location
 
 api_router = APIRouter()
 api_router.include_router(users.router, tags=["users"])
@@ -388,11 +388,24 @@ def ensure_default_api_perms():
         db["_runtime.api.allow"].create_index("action")
 
 
+def ensure_alldocs_exists():
+    db = get_mongo_db()
+    alldocs_coll = db["alldocs"]
+    if alldocs_coll.estimated_document_count() > 0:
+        return
+
+    logging.info(
+        "ensure_alldocs_exists: executing `ensure_alldocs` dagster job in process"
+    )
+    runtime_dagster_code_location.get_job("ensure_alldocs").execute_in_process()
+
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     ensure_initial_resources_on_boot()
     ensure_attribute_indexes()
     ensure_default_api_perms()
+    ensure_alldocs_exists()
     yield
 
 

nmdc_runtime/site/dagster.yaml

@@ -5,6 +5,13 @@ scheduler:
 run_coordinator:
   module: dagster.core.run_coordinator
   class: QueuedRunCoordinator
+  config:
+    tag_concurrency_limits:
+      - key: "do-not-overlap-runs"
+        value:
+          applyLimitPerUniqueValue: true
+        limit: 1
+
 
 run_launcher:
   module: dagster.core.launcher

nmdc_runtime/site/repository.py

@@ -451,7 +451,9 @@ def repo():
         ensure_jobs.to_job(**preset_normal),
         apply_metadata_in.to_job(**preset_normal),
         export_study_biosamples_metadata.to_job(**preset_normal),
-        ensure_alldocs.to_job(**preset_normal),
+        ensure_alldocs.to_job(
+            tags={"do-not-overlap-runs": "ensure_alldocs"}, **preset_normal
+        ),
     ]
     schedules = [housekeeping_weekly]
     sensors = [