diff --git a/server/kraken/migrations/versions/ab9326d5fc02_added_index_to_flows.py b/server/kraken/migrations/versions/ab9326d5fc02_added_index_to_flows.py new file mode 100644 index 00000000..10f0ba30 --- /dev/null +++ b/server/kraken/migrations/versions/ab9326d5fc02_added_index_to_flows.py @@ -0,0 +1,48 @@ +"""added index to flows + +Revision ID: ab9326d5fc02 +Revises: 9d8d8713c1ad +Create Date: 2021-05-28 07:44:40.012276 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'ab9326d5fc02' +down_revision = '9d8d8713c1ad' +branch_labels = None +depends_on = None + + +INDEXES = [ + # create INDEX ix_flows_branch_id_kind on flows(branch_id, kind); + ['ix_flows_branch_id_kind', 'flows', ['branch_id', 'kind']], + # create INDEX ix_flows_created on flows(created); + ['ix_flows_created', 'flows', ['created']], + # create INDEX ix_runs_flow_id on runs(flow_id); + ['ix_runs_flow_id', 'runs', ['flow_id']], + # create INDEX ix_artifacts_run_id on artifacts(run_id); + ['ix_artifacts_run_id', 'artifacts', ['run_id']], + # create INDEX ix_artifacts_flow_id on artifacts(flow_id); + ['ix_artifacts_flow_id', 'artifacts', ['flow_id']], +] + + +def upgrade(): + for name, table, columns in INDEXES: + try: + print('creating index %s' % name) + op.create_index(name, table, columns) + except Exception: + pass + + +def downgrade(): + for name, table, _ in INDEXES: + try: + print('dropping index %s' % name) + op.drop_index(name, table_name=table) + except Exception: + pass diff --git a/server/kraken/server/execution.py b/server/kraken/server/execution.py index eca8327f..6d302439 100644 --- a/server/kraken/server/execution.py +++ b/server/kraken/server/execution.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import time import logging from urllib.parse import urljoin, urlparse @@ -46,6 +47,8 @@ def create_flow(branch_id, kind, body): def get_flows(branch_id, kind, start=0, limit=10, middle=None): + t0 = time.time() + flows = [] if kind == 'dev': kind = 1 @@ -53,32 +56,37 @@ def get_flows(branch_id, kind, start=0, limit=10, middle=None): kind = 0 q = Flow.query.filter_by(branch_id=branch_id, kind=kind) q = q.options(joinedload('branch'), - joinedload('branch.project'), + # joinedload('branch.project'), joinedload('branch.stages'), joinedload('artifacts_files'), - joinedload('runs'), - joinedload('runs.artifacts_files')) + joinedload('runs')) + # joinedload('runs.artifacts_files')) if middle is None: + # log.info('A' * 120) total = q.count() q = q.order_by(desc(Flow.created)) q = q.offset(start).limit(limit) for flow in q.all(): - flows.append(flow.get_json()) + flows.append(flow.get_json(with_project=False, with_branch=False)) + + # log.info('B' * 120) else: q1 = q.filter(Flow.id >= middle) q1 = q1.order_by(asc(Flow.created)) q1 = q1.offset(0).limit(limit) for flow in reversed(q1.all()): - flows.append(flow.get_json()) + flows.append(flow.get_json(with_project=False, with_branch=False)) q2 = q.filter(Flow.id < middle) q2 = q2.order_by(desc(Flow.created)) q2 = q2.offset(0).limit(limit) for flow in q2.all(): - flows.append(flow.get_json()) + flows.append(flow.get_json(with_project=False, with_branch=False)) total = 0 + log.info('get_flows time %s', time.time() - t0) + return {'items': flows, 'total': total}, 200 diff --git a/server/kraken/server/models.py b/server/kraken/server/models.py index 7f14892a..98c5fdc7 100644 --- a/server/kraken/server/models.py +++ b/server/kraken/server/models.py @@ -219,15 +219,13 @@ class Stage(db.Model, DatesMixin): sequences = relationship("BranchSequence", back_populates="stage") # services - def get_json(self): - return dict(id=self.id, + def get_json(self, with_schema=True): + data = dict(id=self.id, created=self.created.strftime("%Y-%m-%dT%H:%M:%SZ") if self.created else None, deleted=self.deleted.strftime("%Y-%m-%dT%H:%M:%SZ") if self.deleted else None, name=self.name, description=self.description, enabled=self.enabled, - schema=self.schema, - schema_code=self.schema_code, schema_from_repo_enabled=self.schema_from_repo_enabled, repo_url=self.repo_url, repo_branch=self.repo_branch, @@ -238,6 +236,12 @@ def get_json(self): repo_version=self.repo_version, schema_file=self.schema_file) + if with_schema: + data['schema'] = self.schema + data['schema_code'] = self.schema_code + + return data + def __repr__(self): return "" % (self.id, self.name) @@ -298,7 +302,7 @@ class Flow(db.Model, DatesMixin): def get_label(self): return self.label if self.label else ("%d." % self.id) - def get_json(self): + def get_json(self, with_project=True, with_branch=True): if self.state == consts.FLOW_STATE_COMPLETED: duration = self.finished - self.created else: @@ -308,7 +312,7 @@ def get_json(self): if self.trigger_data: trigger = self.trigger_data.data[0] - return dict(id=self.id, + data = dict(id=self.id, label=self.get_label(), created=self.created.strftime("%Y-%m-%dT%H:%M:%SZ") if self.created else None, finished=self.finished.strftime("%Y-%m-%dT%H:%M:%SZ") if self.finished else None, @@ -316,17 +320,23 @@ def get_json(self): state=consts.FLOW_STATES_NAME[self.state], kind='ci' if self.kind == 0 else 'dev', duration=duration_to_txt(duration), - branch_id=self.branch_id, - base_branch_name=self.branch.name, branch_name=self.branch_name, - project_id=self.branch.project_id, - project_name=self.branch.project.name, args=self.args, - stages=[s.get_json() for s in self.branch.stages if s.deleted is None], - runs=[r.get_json() for r in self.runs], + stages=[s.get_json(with_schema=False) for s in self.branch.stages if s.deleted is None], + runs=[r.get_json(with_project=False, with_branch=False, with_artifacts=False) for r in self.runs], trigger=trigger, artifacts=self.artifacts) + if with_project: + data['project_id'] = self.branch.project_id + data['project_name'] = self.branch.project.name + + if with_branch: + data['branch_id'] = self.branch_id + data['base_branch_name'] = self.branch.name + + return data + class Run(db.Model, DatesMixin): __tablename__ = "runs" @@ -364,7 +374,7 @@ class Run(db.Model, DatesMixin): repo_data = relationship('RepoChanges') reason = Column(JSONB, nullable=False) - def get_json(self): + def get_json(self, with_project=True, with_branch=True, with_artifacts=True): jobs_processing = 0 jobs_executing = 0 jobs_waiting = 0 @@ -416,10 +426,6 @@ def get_json(self): stage_id=self.stage_id, flow_id=self.flow_id, flow_kind='ci' if self.flow.kind == 0 else 'dev', - branch_id=self.flow.branch_id, - branch_name=self.flow.branch.name, - project_id=self.flow.branch.project_id, - project_name=self.flow.branch.project.name, args=self.args, jobs_total=jobs_total, jobs_waiting=jobs_waiting, @@ -431,7 +437,6 @@ def get_json(self): tests_not_run=self.tests_not_run, issues_total=self.issues_total, issues_new=self.issues_new, - artifacts_total=len(self.artifacts_files), new_cnt=self.new_cnt, no_change_cnt=self.no_change_cnt, regr_cnt=self.regr_cnt, @@ -440,6 +445,17 @@ def get_json(self): reason=self.reason['reason'], note=self.note) + if with_project: + data['project_id'] = self.flow.branch.project_id + data['project_name'] = self.flow.branch.project.name + + if with_branch: + data['branch_id'] = self.flow.branch_id + data['branch_name'] = self.flow.branch.name + + if with_artifacts: + data['artifacts_total'] = len(self.artifacts_files) + return data