Customize and use fix_encumbrances_quesnelia script

[jgreben]

To be merged after prod upgrade to Quesnelia.

--- fix_encumbrances.py	2025-02-19 14:45:58
+++ fix_encumbrances_quesnelia.py	2025-02-19 14:54:25
@@ -20,17 +20,17 @@
 ITEM_MAX = 2147483647
 MAX_BY_CHUNK = 1000
 
-okapi_url = Variable.get("OKAPI_URL")
+okapi_url = f"{Variable.get('OKAPI_URL')}/"
 headers = {}
 client = httpx.AsyncClient()
 dryrun = False
 
 # request timeout in seconds
-ASYNC_CLIENT_TIMEOUT = 60
+ASYNC_CLIENT_TIMEOUT = 30
 
 # limit the number of parallel threads.
 # Try different values. Bigger values - for increasing performance, but could produce "Connection timeout exception"
-MAX_ACTIVE_THREADS = 7
+MAX_ACTIVE_THREADS = Variable.get("FIX_ENC_THREADS", 1)
 
 
 # ---------------------------------------------------
@@ -45,7 +45,7 @@
     login_headers = {'x-okapi-tenant': tenant, 'Content-Type': 'application/json'}
     data = {'username': username, 'password': password}
     try:
-        r = requests.post(okapi_url + '/authn/login', headers=login_headers, json=data)
+        r = requests.post(okapi_url + 'authn/login', headers=login_headers, json=data)
         if r.status_code != 201:
             raise_exception_for_reply(r)
         logger.info('Logged in successfully.')
@@ -89,7 +89,9 @@
             return resp.json()
         else:
             print(f'Error getting records by {url} ?query= "{query}": \n{resp.text} ')
-            logger.error(f'Error getting records by {url} ?query= "{query}": \n{resp.text} ')
+            logger.error(
+                f'Error getting records by {url} ?query= "{query}": \n{resp.text} '
+            )
             raise Exception("Exiting Fix Encumbrances script.")
     except Exception as err:
         print(f'Error getting records by {url}?query={query}: {err=}')
@@ -97,39 +99,44 @@
         raise Exception("Exiting Fix Encumbrances script.")
 
 
-async def put_request(url: str, data):
+async def post_request(url: str, data):
     if dryrun:
         return
     try:
-        resp = await client.put(
+        resp = await client.post(
             url, headers=headers, data=json.dumps(data), timeout=ASYNC_CLIENT_TIMEOUT
         )
-        if resp.status_code == HTTPStatus.NO_CONTENT:
+        if (
+            resp.status_code == HTTPStatus.CREATED
+            or resp.status_code == HTTPStatus.NO_CONTENT
+        ):
             return
-        print(f'Error updating record {url} "{data}": {resp.text}')
-        logger.error(f'Error updating record {url} "{data}": {resp.text}')
+        print(f'Error in POST request {url} "{data}": {resp.text}')
+        logger.error(f'Error in POST request {url} "{data}": {resp.text}')
         raise Exception("Exiting Fix Encumbrances script.")
 
     except Exception as err:
-        print(f'Error updating record {url} "{data}": {err=}')
-        logger.error(f'Error updating record {url} "{data}": {err=}')
+        print(f'Error in POST request {url} "{data}": {err=}')
+        logger.error(f'Error in POST request {url} "{data}": {err=}')
         raise Exception("Exiting Fix Encumbrances script.")
 
 
-async def delete_request(url: str):
+async def put_request(url: str, data):
     if dryrun:
         return
     try:
-        resp = await client.delete(url, headers=headers, timeout=ASYNC_CLIENT_TIMEOUT)
+        resp = await client.put(
+            url, headers=headers, data=json.dumps(data), timeout=ASYNC_CLIENT_TIMEOUT
+        )
         if resp.status_code == HTTPStatus.NO_CONTENT:
             return
-        print(f'Error deleting record {url}: {resp.text}')
-        logger.error(f'Error deleting record {url}: {resp.text}')
+        print(f'Error updating record {url} "{data}": {resp.text}')
+        logger.error(f'Error updating record {url} "{data}": {resp.text}')
         raise Exception("Exiting Fix Encumbrances script.")
 
     except Exception as err:
-        print(f'Error deleting record {url}: {err=}')
-        logger.error(f'Error deleting record {url}: {err=}')
+        print(f'Error updating record {url} "{data}": {err=}')
+        logger.error(f'Error updating record {url} "{data}": {err=}')
         raise Exception("Exiting Fix Encumbrances script.")
 
 
@@ -137,7 +144,7 @@
     params = {'query': query, 'offset': '0', 'limit': ITEM_MAX}
     try:
         r = requests.get(
-            okapi_url + '/finance-storage/fiscal-years', headers=headers, params=params
+            okapi_url + 'finance-storage/fiscal-years', headers=headers, params=params
         )
         if r.status_code != 200:
             raise_exception_for_reply(r)
@@ -177,7 +184,7 @@
 def get_order_ids_by_query(query) -> list:
     try:
         orders = get_by_chunks(
-            okapi_url + '/orders-storage/purchase-orders', query, 'purchaseOrders'
+            okapi_url + 'orders-storage/purchase-orders', query, 'purchaseOrders'
         )
         ids = []
         for order in orders:
@@ -190,7 +197,7 @@
 
 
 async def get_encumbrances_by_query(query) -> list:
-    url = okapi_url + '/finance-storage/transactions'
+    url = okapi_url + 'finance-storage/transactions'
     response = await get_request(url, query)
     return response['transactions']
 
@@ -202,7 +209,7 @@
             query = query + f"id=={enc_id} OR "
         else:
             query = query + f"id=={enc_id}"
-    resp = await get_request(okapi_url + '/finance-storage/transactions', query)
+    resp = await get_request(okapi_url + 'finance-storage/transactions', query)
 
     return resp['transactions']
 
@@ -228,10 +235,7 @@
     print('Retrieving closed order ids...')
     query = 'workflowStatus=="Closed"'
     closed_orders_ids = get_order_ids_by_query(query)
-    if len(closed_orders_ids):
-        print('  Closed orders:', len(closed_orders_ids))
-    else:
-        print('No closed orders')
+    print('  Closed orders:', len(closed_orders_ids))
     return closed_orders_ids
 
 
@@ -239,21 +243,24 @@
     print('Retrieving open order ids...')
     query = 'workflowStatus=="Open"'
     open_orders_ids = get_order_ids_by_query(query)
-    if len(open_orders_ids):
-        print('  Open orders:', len(open_orders_ids))
-    else:
-        print('No open orders')
+    print('  Open orders:', len(open_orders_ids))
     return open_orders_ids
 
 
-async def transaction_summary(order_id, num_transactions):
-    data = {'id': order_id, 'numTransactions': num_transactions}
-    url = f'{okapi_url}/finance-storage/order-transaction-summaries/{order_id}'
-    await put_request(url, data)
+async def batch_update(transactions: list):
+    batch = {'transactionsToUpdate': transactions}
+    url = f'{okapi_url}finance-storage/transactions/batch-all-or-nothing'
+    await post_request(url, batch)
 
 
+async def batch_delete(transaction_ids: list):
+    batch = {'idsOfTransactionsToDelete': transaction_ids}
+    url = f'{okapi_url}finance-storage/transactions/batch-all-or-nothing'
+    await post_request(url, batch)
+
+
 async def get_budgets_by_query(query) -> list:
-    budget_collection = await get_request(okapi_url + '/finance-storage/budgets', query)
+    budget_collection = await get_request(okapi_url + 'finance-storage/budgets', query)
     return budget_collection['budgets']
 
 
@@ -277,7 +284,7 @@
 
 
 async def get_order_encumbrances(order_id, fiscal_year_id, sem=None) -> list:
-    url = okapi_url + '/finance-storage/transactions'
+    url = okapi_url + 'finance-storage/transactions'
     query = f'encumbrance.sourcePurchaseOrderId=={order_id} AND fiscalYearId=={fiscal_year_id}'
     response = await get_request(url, query)
     if sem is not None:
@@ -346,7 +353,7 @@
 async def update_poline_encumbrance(encumbrance_to_remove, replace_by, poline=None):
     url = (
         okapi_url
-        + f"/orders-storage/po-lines/{encumbrance_to_remove['encumbrance']['sourcePoLineId']}"
+        + f"orders-storage/po-lines/{encumbrance_to_remove['encumbrance']['sourcePoLineId']}"
     )
     if poline is None:
         poline = await get_request_without_query(url)
@@ -359,6 +366,7 @@
 
 async def remove_encumbrances_and_update_polines(encumbrance_changes):
     futures = []
+    ids_to_delete = []
     for change in encumbrance_changes:
         encumbrance_to_remove = change['remove']
         replace_by = change['replace_by']
@@ -367,9 +375,9 @@
                 update_poline_encumbrance(encumbrance_to_remove, replace_by)
             )
         )
-        url = f"{okapi_url}/finance-storage/transactions/{encumbrance_to_remove['id']}"
-        futures.append(asyncio.ensure_future(delete_request(url)))
+        ids_to_delete.append(encumbrance_to_remove['id'])
     await asyncio.gather(*futures)
+    await batch_delete(ids_to_delete)
 
 
 async def remove_duplicate_encumbrances_in_order(order_id, fiscal_year_id, sem) -> int:
@@ -413,7 +421,7 @@
 
 async def get_polines_by_order_id(order_id) -> list:
     query = f'purchaseOrderId=={order_id}'
-    resp = await get_request(okapi_url + '/orders-storage/po-lines', query)
+    resp = await get_request(okapi_url + 'orders-storage/po-lines', query)
     po_lines = resp['poLines']
     return po_lines
 
@@ -421,13 +429,10 @@
 async def update_encumbrance_fund_id(encumbrance, new_fund_id, poline):
     encumbrance['fromFundId'] = new_fund_id
     encumbrance_id = encumbrance['id']
-    order_id = poline['purchaseOrderId']
     print(
         f"  Fixing fromFundId for po line {poline['id']} ({poline['poLineNumber']}) encumbrance {encumbrance_id}"
     )
-    await transaction_summary(order_id, 1)
-    url = f"{okapi_url}/finance-storage/transactions/{encumbrance_id}"
-    await put_request(url, encumbrance)
+    await batch_update([encumbrance])
 
 
 # Remove a duplicate encumbrance if it has a wrong fromFundId, and update the poline fd if needed
@@ -452,14 +457,15 @@
         )
         return
     replace_by = encumbrances_with_right_fund[0]
+    ids_to_delete = []
     for encumbrance_to_remove in encumbrances_with_bad_fund:
         print(
             f"  Removing encumbrance {encumbrance_to_remove['id']} for po line {poline['id']} "
             f"({poline['poLineNumber']})"
         )
         await update_poline_encumbrance(encumbrance_to_remove, replace_by, poline)
-        url = f"{okapi_url}/finance-storage/transactions/{encumbrance_to_remove['id']}"
-        await delete_request(url)
+        ids_to_delete.append(encumbrance_to_remove['id'])
+    await batch_delete(ids_to_delete)
 
 
 # Fix encumbrance fromFundId if it doesn't match the po line fund distribution (see MODFISTO-384, MODFISTO-385)
@@ -488,7 +494,11 @@
     poline_id = poline['id']
     fd_fund_id = fd['fundId']
     for enc in order_encumbrances:
-        # if enc['encumbrance']['sourcePoLineId'] == poline_id and float(enc['amount']) != 0.0 and \
+        # if (
+        #     enc['encumbrance']['sourcePoLineId'] == poline_id
+        #     and float(enc['amount']) != 0.0
+        #     and enc['fromFundId'] == fd_fund_id
+        # ):
         if (
             enc['encumbrance']['sourcePoLineId'] == poline_id
             and enc['fromFundId'] == fd_fund_id
@@ -496,8 +506,10 @@
             fd_encumbrance_id = fd['encumbrance']
             if enc['id'] == fd_encumbrance_id:
                 return False
-            # print(f"  Updating poline {poline_id} ({poline['poLineNumber']}) encumbrance {fd_encumbrance_id} "
-            #       f"with new value {enc['id']}")
+            # print(
+            #     f"  Updating poline {poline_id} ({poline['poLineNumber']}) encumbrance {fd_encumbrance_id} "
+            #     f"with new value {enc['id']}"
+            # )
             fd['encumbrance'] = enc['id']
             return True
     return False
@@ -518,7 +530,7 @@
 
     # update poline if one or more fund distributions modified
     if poline_needs_updates:
-        url = f"{okapi_url}/orders-storage/po-lines/{poline['id']}"
+        url = f"{okapi_url}orders-storage/po-lines/{poline['id']}"
         await put_request(url, poline)
 
 
@@ -582,18 +594,15 @@
         f'encumbrance.orderStatus<>"Closed" AND encumbrance.sourcePurchaseOrderId=={order_id} AND '
         f'fiscalYearId=={fiscal_year_id}'
     )
-    url = okapi_url + '/finance-storage/transactions'
+    url = okapi_url + 'finance-storage/transactions'
 
     return await get_request(url, query)
 
 
-async def unrelease_order_encumbrances(order_id, encumbrances) -> list:
-    await transaction_summary(order_id, len(encumbrances))
-
+async def unrelease_order_encumbrances(encumbrances) -> list:
     for encumbrance in encumbrances:
         encumbrance['encumbrance']['status'] = 'Unreleased'
-        url = f"{okapi_url}/finance-storage/transactions/{encumbrance['id']}"
-        await put_request(url, encumbrance)
+    await batch_update(encumbrances)
 
     # the encumbrance amounts get modified (and the version in MG), so we need to get a fresh version
     modified_encumbrance_futures = []
@@ -630,13 +639,10 @@
 
 async def fix_order_status_and_release_encumbrances(order_id, encumbrances):
     try:
-        await transaction_summary(order_id, len(encumbrances))
-
         for encumbrance in encumbrances:
             encumbrance['encumbrance']['status'] = 'Released'
             encumbrance['encumbrance']['orderStatus'] = 'Closed'
-            url = f"{okapi_url}/finance-storage/transactions/{encumbrance['id']}"
-            await put_request(url, encumbrance)
+        await batch_update(encumbrances)
 
     except Exception as err:
         print(
@@ -658,9 +664,7 @@
         # print(f'\n  Fixing the following encumbrance(s) for order {order_id} :')
         for encumbrance in encumbrances:
             print(f"    {encumbrance['id']}")
-        modified_encumbrances = await unrelease_order_encumbrances(
-            order_id, encumbrances
-        )
+        modified_encumbrances = await unrelease_order_encumbrances(encumbrances)
         if len(modified_encumbrances) != 0:
             await fix_order_status_and_release_encumbrances(
                 order_id, modified_encumbrances
@@ -718,13 +722,8 @@
     # print(f'\n  Unreleasing the following encumbrance(s) for order {order_id} :')
     for encumbrance in encumbrances:
         print(f"    {encumbrance['id']}")
-
-    await transaction_summary(order_id, len(encumbrances))
-
-    for encumbrance in encumbrances:
         encumbrance['encumbrance']['status'] = 'Unreleased'
-        url = f"{okapi_url}/finance-storage/transactions/{encumbrance['id']}"
-        await put_request(url, encumbrance)
+    await batch_update(encumbrances)
 
 
 async def unrelease_encumbrances_with_non_zero_amounts(
@@ -735,7 +734,7 @@
         f'fiscalYearId=={fiscal_year_id}'
     )
     transactions_response = await get_request(
-        okapi_url + '/finance-storage/transactions', query
+        okapi_url + 'finance-storage/transactions', query
     )
 
     order_encumbrances = transactions_response['transactions']
@@ -779,16 +778,11 @@
 
 
 async def release_encumbrances(order_id, encumbrances):
-    # print\(f'\n  Releasing the following encumbrances for order {order_id} :')
+    # print(f'\n  Releasing the following encumbrances for order {order_id} :')
     for encumbrance in encumbrances:
         print(f"    {encumbrance['id']}")
-
-    await transaction_summary(order_id, len(encumbrances))
-
-    for encumbrance in encumbrances:
         encumbrance['encumbrance']['status'] = 'Released'
-        url = f"{okapi_url}/finance-storage/transactions/{encumbrance['id']}"
-        await put_request(url, encumbrance)
+    await batch_update(encumbrances)
 
 
 async def release_encumbrances_with_negative_amounts(
@@ -800,7 +794,7 @@
         f'encumbrance.sourcePurchaseOrderId=={order_id} AND fiscalYearId=={fiscal_year_id}'
     )
     transactions_response = await get_request(
-        okapi_url + '/finance-storage/transactions', query
+        okapi_url + 'finance-storage/transactions', query
     )
 
     order_encumbrances = transactions_response['transactions']
@@ -907,10 +901,12 @@
 
     # Cast into decimal values, so 0 == 0.0 == 0.00 will return true
     if Decimal(str(budget['encumbered'])) != Decimal(encumbered):
-        # print(f"    Budget \"{budget['name']}\": changing encumbered from {budget['encumbered']} to {encumbered}")
+        # print(
+        #     f"    Budget \"{budget['name']}\": changing encumbered from {budget['encumbered']} to {encumbered}"
+        # )
         budget['encumbered'] = encumbered
 
-        url = f"{okapi_url}/finance-storage/budgets/{budget['id']}"
+        url = f"{okapi_url}finance-storage/budgets/{budget['id']}"
         await put_request(url, budget)
         nb_modified = 1
     sem.release()
@@ -1022,9 +1018,10 @@
     await fix_poline_encumbrances_relations(
         open_orders_ids, fiscal_year_id, fy_is_current
     )
-    await fix_encumbrance_order_status_for_closed_orders(
-        closed_orders_ids, fiscal_year_id
-    )
+    if fy_is_current:
+        await fix_encumbrance_order_status_for_closed_orders(
+            closed_orders_ids, fiscal_year_id
+        )
     await unrelease_open_orders_encumbrances_with_nonzero_amounts(
         fiscal_year_id, open_orders_ids
     )
@@ -1098,10 +1095,15 @@
             open_orders_ids, fiscal_year_id, fy_is_current
         )
     elif choice == 4:
-        closed_orders_ids = get_closed_orders_ids()
-        await fix_encumbrance_order_status_for_closed_orders(
-            closed_orders_ids, fiscal_year_id
-        )
+        if not fy_is_current:
+            print(
+                'Fiscal year is not current - fixing encumbrance order status is not needed.'
+            )
+        else:
+            closed_orders_ids = get_closed_orders_ids()
+            await fix_encumbrance_order_status_for_closed_orders(
+                closed_orders_ids, fiscal_year_id
+            )
     elif choice == 5:
         open_orders_ids = get_open_orders_ids()
         await unrelease_open_orders_encumbrances_with_nonzero_amounts(