From 9894fc743beaa38eba9dc6f2aa6ce24f30f9486d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tibor=20Cserv=C3=A1k?= Date: Fri, 12 Jul 2024 14:01:46 +0200 Subject: [PATCH] [Fix] Report sorting in unique mode Fixing report sorting on unique mode. After modifying the DB schema in #4089 PR, the unique mode query of getRunResults endpoint has been changed, therefore, the report sorting is not working properly. Now, the unique mode query is redesigned and it use row_number() function to filter unique reports correctly. Where clause is also modified. It is getting rid of report annotation filter. Filtering annotation remains in having clause. --- .../codechecker_server/api/report_server.py | 139 ++++++++---------- 1 file changed, 65 insertions(+), 74 deletions(-) diff --git a/web/server/codechecker_server/api/report_server.py b/web/server/codechecker_server/api/report_server.py index 6d7b167f69..eed6ff3202 100644 --- a/web/server/codechecker_server/api/report_server.py +++ b/web/server/codechecker_server/api/report_server.py @@ -1962,45 +1962,29 @@ def getRunResults(self, run_ids, limit, offset, sort_types, ReportAnnotations.value)])).label(f"annotation_{col}") if report_filter.isUnique: + # A report annotation filter cannot set in WHERE clause if we + # use annotation parameters in aggregate functions to create + # pivot table. Instead filtering report annotations in WHERE + # clause, we should use HAVING clause only for filtering + # aggregate functions. + # TODO: Fixing report annotation filter in every report server + # enpoint function. + report_filter_annotations = report_filter.annotations + report_filter.annotations = None filter_expression, join_tables = process_report_filter( - session, run_ids, report_filter, cmp_data, - keep_all_annotations=False) + session, run_ids, report_filter, cmp_data) sort_types, sort_type_map, order_type_map = \ get_sort_map(sort_types, True) - selects = [func.max(Report.id).label('id')] - for sort in sort_types: - sorttypes = sort_type_map.get(sort.type) - for sorttype in sorttypes: - if sorttype[0] != 'bug_path_length': - selects.append(func.max(sorttype[0]) - .label(sorttype[1])) - - unique_reports = session.query(*selects) - unique_reports = apply_report_filter(unique_reports, - filter_expression, - join_tables) - if report_filter.annotations is not None: - unique_reports = unique_reports.outerjoin( - ReportAnnotations, - Report.id == ReportAnnotations.report_id) - unique_reports = unique_reports \ - .group_by(Report.bug_id) \ - .subquery() - - # Sort the results. - sorted_reports = session.query(unique_reports.c.id) - sorted_reports = sort_results_query(sorted_reports, - sort_types, - sort_type_map, - order_type_map, - True) - sorted_reports = sorted_reports \ - .limit(limit).offset(offset).subquery() - + # TODO: Create a helper function for common section of unique + # and non unique modes. q = session.query(Report, File.filename, + func.row_number().over( + partition_by=Report.bug_id, + order_by=desc(Report.id) + ).label("row_num"), *annotation_cols.values()) \ .join(Checker, Report.checker_id == Checker.id) \ @@ -2010,14 +1994,18 @@ def getRunResults(self, run_ids, limit, offset, sort_types, Report.file_id == File.id) \ .outerjoin( ReportAnnotations, - Report.id == ReportAnnotations.report_id) \ - .outerjoin(sorted_reports, - sorted_reports.c.id == Report.id) \ - .filter(sorted_reports.c.id.isnot(None)) + Report.id == ReportAnnotations.report_id) - if report_filter.annotations is not None: + q = apply_report_filter(q, + filter_expression, + join_tables, + [File, Checker]) + + q = q.group_by(Report.id, File.id, Checker.id) + + if report_filter_annotations: annotations = defaultdict(list) - for annotation in report_filter.annotations: + for annotation in report_filter_annotations: annotations[annotation.first].append(annotation.second) OR = [] @@ -2025,55 +2013,58 @@ def getRunResults(self, run_ids, limit, offset, sort_types, OR.append(annotation_cols[key].in_(values)) q = q.having(or_(*OR)) - # We have to sort the results again because an ORDER BY in a - # subtable is broken by the JOIN. q = sort_results_query(q, sort_types, sort_type_map, order_type_map) - q = q.group_by(Report.id, File.id, Checker.id) + + q = q.limit(limit).offset(offset) query_result = q.all() # Get report details if it is required. report_details = {} if get_details: - report_ids = [r[0].id for r in query_result] + report_ids = [r[0].id for r in query_result if r[2] == 1] report_details = get_report_details(session, report_ids) for row in query_result: - report, filename = row[0], row[1] - annotations = { - k: v for k, v in zip(annotation_keys, row[2:]) - if v is not None} - - review_data = create_review_data( - report.review_status, - report.review_status_message, - report.review_status_author, - report.review_status_date, - report.review_status_is_in_source) - - results.append( - ReportData(runId=report.run_id, - bugHash=report.bug_id, - checkedFile=filename, - checkerMsg=report.checker_message, - reportId=report.id, - fileId=report.file_id, - line=report.line, - column=report.column, - analyzerName=report.checker.analyzer_name, - checkerId=report.checker.checker_name, - severity=report.checker.severity, - reviewData=review_data, - detectionStatus=detection_status_enum( - report.detection_status), - detectedAt=str(report.detected_at), - fixedAt=str(report.fixed_at), - bugPathLength=report.path_length, - details=report_details.get(report.id), - annotations=annotations)) + # Filtering row_num = 1 to get unique reports. + row_num = row[2] + if row_num == 1: + report, filename = row[0], row[1] + annotations = { + k: v for k, v in zip(annotation_keys, row[3:]) + if v is not None} + + review_data = create_review_data( + report.review_status, + report.review_status_message, + report.review_status_author, + report.review_status_date, + report.review_status_is_in_source) + + results.append( + ReportData(runId=report.run_id, + bugHash=report.bug_id, + checkedFile=filename, + checkerMsg=report.checker_message, + reportId=report.id, + fileId=report.file_id, + line=report.line, + column=report.column, + analyzerName=report + .checker.analyzer_name, + checkerId=report.checker.checker_name, + severity=report.checker.severity, + reviewData=review_data, + detectionStatus=detection_status_enum( + report.detection_status), + detectedAt=str(report.detected_at), + fixedAt=str(report.fixed_at), + bugPathLength=report.path_length, + details=report_details.get(report.id), + annotations=annotations)) else: # not is_unique filter_expression, join_tables = process_report_filter( session, run_ids, report_filter, cmp_data,