improved performance of loading list of flows

server/kraken/migrations/versions/ab9326d5fc02_added_index_to_flows.py

@@ -0,0 +1,48 @@
+"""added index to flows
+
+Revision ID: ab9326d5fc02
+Revises: 9d8d8713c1ad
+Create Date: 2021-05-28 07:44:40.012276
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'ab9326d5fc02'
+down_revision = '9d8d8713c1ad'
+branch_labels = None
+depends_on = None
+
+
+INDEXES = [
+    # create INDEX ix_flows_branch_id_kind on flows(branch_id, kind);
+    ['ix_flows_branch_id_kind', 'flows', ['branch_id', 'kind']],
+    # create INDEX ix_flows_created on flows(created);
+    ['ix_flows_created', 'flows', ['created']],
+    # create INDEX ix_runs_flow_id on runs(flow_id);
+    ['ix_runs_flow_id', 'runs', ['flow_id']],
+    # create INDEX ix_artifacts_run_id on artifacts(run_id);
+    ['ix_artifacts_run_id', 'artifacts', ['run_id']],
+    # create INDEX ix_artifacts_flow_id on artifacts(flow_id);
+    ['ix_artifacts_flow_id', 'artifacts', ['flow_id']],
+]
+
+
+def upgrade():
+    for name, table, columns in INDEXES:
+        try:
+            print('creating index %s' % name)
+            op.create_index(name, table, columns)
+        except Exception:
+            pass
+
+
+def downgrade():
+    for name, table, _ in INDEXES:
+        try:
+            print('dropping index %s' % name)
+            op.drop_index(name, table_name=table)
+        except Exception:
+            pass

server/kraken/server/execution.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import os
+import time
 import logging
 from urllib.parse import urljoin, urlparse
 
@@ -46,39 +47,46 @@ def create_flow(branch_id, kind, body):
 
 
 def get_flows(branch_id, kind, start=0, limit=10, middle=None):
+    t0 = time.time()
+
     flows = []
     if kind == 'dev':
         kind = 1
     else:
         kind = 0
     q = Flow.query.filter_by(branch_id=branch_id, kind=kind)
     q = q.options(joinedload('branch'),
-                  joinedload('branch.project'),
+                  # joinedload('branch.project'),
                   joinedload('branch.stages'),
                   joinedload('artifacts_files'),
-                  joinedload('runs'),
-                  joinedload('runs.artifacts_files'))
+                  joinedload('runs'))
+                  # joinedload('runs.artifacts_files'))
     if middle is None:
+        # log.info('A' * 120)
         total = q.count()
         q = q.order_by(desc(Flow.created))
         q = q.offset(start).limit(limit)
         for flow in q.all():
-            flows.append(flow.get_json())
+            flows.append(flow.get_json(with_project=False, with_branch=False))
+
+        # log.info('B' * 120)
     else:
         q1 = q.filter(Flow.id >= middle)
         q1 = q1.order_by(asc(Flow.created))
         q1 = q1.offset(0).limit(limit)
         for flow in reversed(q1.all()):
-            flows.append(flow.get_json())
+            flows.append(flow.get_json(with_project=False, with_branch=False))
 
         q2 = q.filter(Flow.id < middle)
         q2 = q2.order_by(desc(Flow.created))
         q2 = q2.offset(0).limit(limit)
         for flow in q2.all():
-            flows.append(flow.get_json())
+            flows.append(flow.get_json(with_project=False, with_branch=False))
 
         total = 0
 
+    log.info('get_flows time %s', time.time() - t0)
+
     return {'items': flows, 'total': total}, 200
 
 

server/kraken/server/models.py

@@ -219,15 +219,13 @@ class Stage(db.Model, DatesMixin):
     sequences = relationship("BranchSequence", back_populates="stage")
     # services
 
-    def get_json(self):
-        return dict(id=self.id,
+    def get_json(self, with_schema=True):
+        data = dict(id=self.id,
                     created=self.created.strftime("%Y-%m-%dT%H:%M:%SZ") if self.created else None,
                     deleted=self.deleted.strftime("%Y-%m-%dT%H:%M:%SZ") if self.deleted else None,
                     name=self.name,
                     description=self.description,
                     enabled=self.enabled,
-                    schema=self.schema,
-                    schema_code=self.schema_code,
                     schema_from_repo_enabled=self.schema_from_repo_enabled,
                     repo_url=self.repo_url,
                     repo_branch=self.repo_branch,
@@ -238,6 +236,12 @@ def get_json(self):
                     repo_version=self.repo_version,
                     schema_file=self.schema_file)
 
+        if with_schema:
+            data['schema'] = self.schema
+            data['schema_code'] = self.schema_code
+
+        return data
+
     def __repr__(self):
         return "<Stage %s, '%s'>" % (self.id, self.name)
 
@@ -298,7 +302,7 @@ class Flow(db.Model, DatesMixin):
     def get_label(self):
         return self.label if self.label else ("%d." % self.id)
 
-    def get_json(self):
+    def get_json(self, with_project=True, with_branch=True):
         if self.state == consts.FLOW_STATE_COMPLETED:
             duration = self.finished - self.created
         else:
@@ -308,25 +312,31 @@ def get_json(self):
         if self.trigger_data:
             trigger = self.trigger_data.data[0]
 
-        return dict(id=self.id,
+        data = dict(id=self.id,
                     label=self.get_label(),
                     created=self.created.strftime("%Y-%m-%dT%H:%M:%SZ") if self.created else None,
                     finished=self.finished.strftime("%Y-%m-%dT%H:%M:%SZ") if self.finished else None,
                     deleted=self.deleted.strftime("%Y-%m-%dT%H:%M:%SZ") if self.deleted else None,
                     state=consts.FLOW_STATES_NAME[self.state],
                     kind='ci' if self.kind == 0 else 'dev',
                     duration=duration_to_txt(duration),
-                    branch_id=self.branch_id,
-                    base_branch_name=self.branch.name,
                     branch_name=self.branch_name,
-                    project_id=self.branch.project_id,
-                    project_name=self.branch.project.name,
                     args=self.args,
-                    stages=[s.get_json() for s in self.branch.stages if s.deleted is None],
-                    runs=[r.get_json() for r in self.runs],
+                    stages=[s.get_json(with_schema=False) for s in self.branch.stages if s.deleted is None],
+                    runs=[r.get_json(with_project=False, with_branch=False, with_artifacts=False) for r in self.runs],
                     trigger=trigger,
                     artifacts=self.artifacts)
 
+        if with_project:
+            data['project_id'] = self.branch.project_id
+            data['project_name'] = self.branch.project.name
+
+        if with_branch:
+            data['branch_id'] = self.branch_id
+            data['base_branch_name'] = self.branch.name
+
+        return data
+
 
 class Run(db.Model, DatesMixin):
     __tablename__ = "runs"
@@ -364,7 +374,7 @@ class Run(db.Model, DatesMixin):
     repo_data = relationship('RepoChanges')
     reason = Column(JSONB, nullable=False)
 
-    def get_json(self):
+    def get_json(self, with_project=True, with_branch=True, with_artifacts=True):
         jobs_processing = 0
         jobs_executing = 0
         jobs_waiting = 0
@@ -416,10 +426,6 @@ def get_json(self):
                     stage_id=self.stage_id,
                     flow_id=self.flow_id,
                     flow_kind='ci' if self.flow.kind == 0 else 'dev',
-                    branch_id=self.flow.branch_id,
-                    branch_name=self.flow.branch.name,
-                    project_id=self.flow.branch.project_id,
-                    project_name=self.flow.branch.project.name,
                     args=self.args,
                     jobs_total=jobs_total,
                     jobs_waiting=jobs_waiting,
@@ -431,7 +437,6 @@ def get_json(self):
                     tests_not_run=self.tests_not_run,
                     issues_total=self.issues_total,
                     issues_new=self.issues_new,
-                    artifacts_total=len(self.artifacts_files),
                     new_cnt=self.new_cnt,
                     no_change_cnt=self.no_change_cnt,
                     regr_cnt=self.regr_cnt,
@@ -440,6 +445,17 @@ def get_json(self):
                     reason=self.reason['reason'],
                     note=self.note)
 
+        if with_project:
+            data['project_id'] = self.flow.branch.project_id
+            data['project_name'] = self.flow.branch.project.name
+
+        if with_branch:
+            data['branch_id'] = self.flow.branch_id
+            data['branch_name'] = self.flow.branch.name
+
+        if with_artifacts:
+            data['artifacts_total'] = len(self.artifacts_files)
+
         return data