From de6065a3530bc7badea836a4224687147cf9b0b1 Mon Sep 17 00:00:00 2001
From: joseph-sentry <joseph.sawaya@sentry.io>
Date: Fri, 13 Sep 2024 14:16:41 -0400
Subject: [PATCH] perf: use lpop with count and don't fetch upload in Upload
 task

we don't need to fetch the report upload because all we're doing is
setting the upload_pk to upload_id in the normalized args dict

we also can batch our redis lpops into groups of 50 so we don't have
to make as many calls to redis
---
 tasks/tests/unit/test_upload_task.py | 18 ++++++++++++++----
 tasks/upload.py                      | 11 +++++------
 2 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/tasks/tests/unit/test_upload_task.py b/tasks/tests/unit/test_upload_task.py
index 3fdc7a0c8..40163648a 100644
--- a/tasks/tests/unit/test_upload_task.py
+++ b/tasks/tests/unit/test_upload_task.py
@@ -85,14 +85,24 @@ def get(self, key):
             return res.encode()
         return res
 
-    def lpop(self, key):
+    def lpop(self, key, count=None):
         list = self.lists.get(key)
         if not list:
             return None
 
-        res = list.pop(0)
-        if list == []:
-            del self.lists[key]
+        res = None
+        if count:
+            res = []
+            for _ in range(count):
+                res.append(list.pop(0))
+                if list == []:
+                    del self.lists[key]
+                    break
+        else:
+            res = list.pop(0)
+            if list == []:
+                del self.lists[key]
+
         return res
 
     def delete(self, key):
diff --git a/tasks/upload.py b/tasks/upload.py
index 725f973f7..48eadf2c1 100644
--- a/tasks/upload.py
+++ b/tasks/upload.py
@@ -175,8 +175,9 @@ def arguments_list(self):
         """
         uploads_list_key = self.upload_location
         log.debug("Fetching arguments from redis %s", uploads_list_key)
-        while arguments := self.redis_connection.lpop(uploads_list_key):
-            yield loads(arguments)
+        while arguments := self.redis_connection.lpop(uploads_list_key, count=50):
+            for arg in arguments:
+                yield loads(arg)
 
     def normalize_arguments(self, commit: Commit, arguments: dict[str, Any]):
         """
@@ -502,15 +503,13 @@ def run_impl_within_lock(
         for arguments in upload_context.arguments_list():
             normalized_arguments = upload_context.normalize_arguments(commit, arguments)
             if "upload_id" in normalized_arguments:
-                upload = report_service.fetch_report_upload(
-                    commit_report, normalized_arguments["upload_id"]
-                )
+                normalized_arguments["upload_pk"] = normalized_arguments["upload_id"]
             else:
                 upload = report_service.create_report_upload(
                     normalized_arguments, commit_report
                 )
 
-            normalized_arguments["upload_pk"] = upload.id_
+                normalized_arguments["upload_pk"] = upload.id_
             argument_list.append(normalized_arguments)
 
         if argument_list: