Skip to content

Commit

Permalink
Improve performances of the charts view (25x faster) #569
Browse files Browse the repository at this point in the history 
Signed-off-by: Thomas Druez <[email protected]>
  • Loading branch information
@tdruez
tdruez committed May 17, 2023
1 parent ab8df20 commit 7b54a4d
Show file tree
Hide file tree
Showing 4 changed files with 110 additions and 104 deletions.
1 change: 1 addition & 0 deletions scanpipe/templates/scanpipe/base.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
#project-extra-data pre {background-color: initial; color: initial; padding: initial; white-space: pre-wrap; word-break: break-all;}
#inputs-panel .panel-block.dropdown:hover {background-color: #f5f5f5;}
#inputs-panel .dropdown-menu {width: 85%;}
a.panel-block {word-break: break-all;}
.is-wider .dropdown-menu {min-width: 18rem;}
.is-tooltip .dropdown-content {background-color: #363636;}
.is-tooltip .dropdown-item {font-weight: normal; color: #fff;}
Expand Down
69 changes: 33 additions & 36 deletions scanpipe/templates/scanpipe/project_charts.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

{% if project.package_count %}
<hr>
<h3 class="title is-4 has-text-centered">
<h3 id="package-charts" class="title is-4 has-text-centered">
Discovered Packages
<span class="tag is-link is-light is-rounded ml-1">{{ project.package_count|intcomma }}</span>
</h3>
Expand All @@ -18,13 +18,13 @@ <h3 class="title is-4 has-text-centered">

{% if project.dependency_count %}
<hr>
<h3 class="title is-4 has-text-centered">
<h3 id="dependency-charts" class="title is-4 has-text-centered">
Discovered Dependencies
<span class="tag is-link is-light is-rounded ml-1">{{ project.dependency_count|intcomma }}</span>
</h3>
<div class="columns is-gapless">
<div class="column">
<div id="dependency_package_type_chart" data-url="{% url 'project_dependencies' project.uuid %}" data-lookup_field="package_type"></div>
<div id="dependency_type_chart" data-url="{% url 'project_dependencies' project.uuid %}" data-lookup_field="type"></div>
</div>
<div class="column">
<div id="dependency_is_runtime_chart" data-url="{% url 'project_dependencies' project.uuid %}" data-lookup_field="is_runtime"></div>
Expand Down Expand Up @@ -71,24 +71,18 @@ <h3 class="title is-4 has-text-centered mb-3">
{% endif %}

{% block scripts %}
{% if project.file_count %}
{{ programming_languages|json_script:"programming_languages" }}
{{ mime_types|json_script:"mime_types" }}
{{ holders|json_script:"holders" }}
{{ copyrights|json_script:"copyrights" }}
{{ file_license_expressions|json_script:"file_license_expressions" }}
{{ file_compliance_alert|json_script:"file_compliance_alert" }}
{% endif %}
{% if project.package_count %}
{{ package_licenses|json_script:"package_licenses" }}
{{ package_types|json_script:"package_types" }}
{% endif %}
{% if project.dependency_count %}
{{ dependency_package_type|json_script:"dependency_package_type" }}
{{ dependency_is_runtime|json_script:"dependency_is_runtime" }}
{{ dependency_is_optional|json_script:"dependency_is_optional" }}
{{ dependency_is_resolved|json_script:"dependency_is_resolved" }}
{% endif %}
{{ file_programming_language|json_script:"file_programming_language" }}
{{ file_mime_type|json_script:"file_mime_type" }}
{{ file_holders|json_script:"file_holders" }}
{{ file_copyrights|json_script:"file_copyrights" }}
{{ file_detected_license_expression|json_script:"file_detected_license_expression" }}
{{ file_compliance_alert|json_script:"file_compliance_alert" }}
{{ package_declared_license_expression|json_script:"package_declared_license_expression" }}
{{ package_type|json_script:"package_type" }}
{{ dependency_type|json_script:"dependency_type" }}
{{ dependency_is_runtime|json_script:"dependency_is_runtime" }}
{{ dependency_is_optional|json_script:"dependency_is_optional" }}
{{ dependency_is_resolved|json_script:"dependency_is_resolved" }}
<script>
let makeChart = function(data_source_id, element_id, title) {
let data_source = document.getElementById(data_source_id);
Expand Down Expand Up @@ -157,20 +151,23 @@ <h3 class="title is-4 has-text-centered mb-3">
});
};

// Packages
makeChart("package_types", "#package_type_chart", "Package\nType");
makeChart("package_licenses", "#package_license_chart", "Package\nLicense\nExpression");
// Dependencies
makeChart("dependency_package_type", "#dependency_package_type_chart", "Package\nType");
makeChart("dependency_is_runtime", "#dependency_is_runtime_chart", "Runtime\nDependencies");
makeChart("dependency_is_optional", "#dependency_is_optional_chart", "Optional\nDependencies");
makeChart("dependency_is_resolved", "#dependency_is_resolved_chart", "Resolved\nDependencies");
// Resources
makeChart("programming_languages", "#programming_language_chart", "Programming\nLanguage");
makeChart("mime_types", "#mime_type_chart", "Mime\nType");
makeChart("holders", "#holders_chart", "Holder");
makeChart("copyrights", "#copyrights_chart", "Copyright");
makeChart("file_license_expressions", "#detected_license_expression_chart", "Detected\nLicense\nExpression");
makeChart("file_compliance_alert", "#compliance_alert_chart", "Compliance\nAlert");
{% if project.package_count %}
makeChart("package_type", "#package_type_chart", "Package\nType");
makeChart("package_declared_license_expression", "#package_license_chart", "Package\nLicense\nExpression");
{% endif %}
{% if project.dependency_count %}
makeChart("dependency_type", "#dependency_type_chart", "Package\nType");
makeChart("dependency_is_runtime", "#dependency_is_runtime_chart", "Runtime\nDependencies");
makeChart("dependency_is_optional", "#dependency_is_optional_chart", "Optional\nDependencies");
makeChart("dependency_is_resolved", "#dependency_is_resolved_chart", "Resolved\nDependencies");
{% endif %}
{% if project.resource_count %}
makeChart("file_programming_language", "#programming_language_chart", "Programming\nLanguage");
makeChart("file_mime_type", "#mime_type_chart", "Mime\nType");
makeChart("file_holders", "#holders_chart", "Holder");
makeChart("file_copyrights", "#copyrights_chart", "Copyright");
makeChart("file_detected_license_expression", "#detected_license_expression_chart", "Detected\nLicense\nExpression");
makeChart("file_compliance_alert", "#compliance_alert_chart", "Compliance\nAlert");
{% endif %}
</script>
{% endblock %}
31 changes: 26 additions & 5 deletions scanpipe/tests/test_views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
from scanpipe.models import Project
from scanpipe.pipes import make_relation
from scanpipe.pipes.input import copy_inputs
from scanpipe.tests import license_policies_index
from scanpipe.tests import make_resource_file
from scanpipe.views import ProjectDetailView

Expand Down Expand Up @@ -212,25 +211,47 @@ def test_scanpipe_views_project_details_add_pipelines(self):
self.assertEqual("docker", run.pipeline_name)
self.assertIsNone(run.task_start_date)

def test_scanpipe_views_project_details_charts_view(self):
url = reverse("project_charts", args=[self.project1.uuid])

with self.assertNumQueries(9):
response = self.client.get(url)

self.assertNotContains(response, 'id="package-charts"')
self.assertNotContains(response, 'id="dependency-charts"')
self.assertNotContains(response, 'id="resource-charts-charts"')

CodebaseResource.objects.create(
project=self.project1,
programming_language="Python",
type=CodebaseResource.Type.FILE,
)

with self.assertNumQueries(12):
response = self.client.get(url)
self.assertContains(response, '{"Python": 1}')

def test_scanpipe_views_project_details_charts_compliance_alert(self):
url = reverse("project_charts", args=[self.project1.uuid])
expected = 'id="compliance_alert_chart"'

scanpipe_app.license_policies_index = None
response = self.client.get(url)
self.assertNotContains(response, expected)

scanpipe_app.license_policies_index = license_policies_index
response = self.client.get(url)
self.assertNotContains(response, expected)

CodebaseResource.objects.create(
resource = CodebaseResource.objects.create(
project=self.project1,
compliance_alert="error",
type=CodebaseResource.Type.FILE,
)
CodebaseResource.objects.filter(id=resource.id).update(
compliance_alert=CodebaseResource.Compliance.ERROR
)

response = self.client.get(url)
self.assertContains(response, expected)
self.assertContains(response, '{"error": 1}')

def test_scanpipe_views_project_details_scan_summary_panels(self):
url = self.project1.get_absolute_url()
Expand Down
113 changes: 50 additions & 63 deletions scanpipe/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -610,9 +610,15 @@ class ProjectChartsView(ConditionalLoginRequired, ProjectViewMixin, generic.Deta

@staticmethod
def get_summary(values_list, limit=settings.SCANCODEIO_MOST_COMMON_LIMIT):
most_common = dict(Counter(values_list).most_common(limit))
counter = Counter(values_list)

other = len(values_list) - sum(most_common.values())
has_only_empty_string = list(counter.keys()) == [""]
if has_only_empty_string:
return {}

most_common = dict(counter.most_common(limit))

other = counter.total() - sum(most_common.values())
if other > 0:
most_common["Other"] = other

Expand All @@ -626,72 +632,53 @@ def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
project = self.object

files_qs = project.codebaseresources.files()
file_filter = self.request.GET.get("file-filter")
context["file_filter"] = file_filter

file_filter = self.request.GET.get("file-filter", "all")
files = project.codebaseresources.files()
if file_filter == "in-a-package":
files_qs = files_qs.in_package()
files = files.in_package()
elif file_filter == "not-in-a-package":
files_qs = files_qs.not_in_package()

files = files_qs.only(
"programming_language",
"mime_type",
"holders",
"copyrights",
"detected_license_expression",
)

packages = project.discoveredpackages.all().only(
"type",
"declared_license_expression",
)

dependencies = project.discovereddependencies.all().only(
"is_runtime",
"is_optional",
"is_resolved",
)

file_languages = files.values_list("programming_language", flat=True)
file_mime_types = files.values_list("mime_type", flat=True)
file_holders = files.values_from_json_field("holders", "holder")
file_copyrights = files.values_from_json_field("copyrights", "copyright")
file_license_expressions = files.values_list(
"detected_license_expression", flat=True
)

file_compliance_alert = []
if scanpipe_app.policies_enabled:
file_compliance_alert = files.values_list("compliance_alert", flat=True)

package_licenses = packages.values_list(
"declared_license_expression", flat=True
)
package_types = packages.values_list("type", flat=True)
files = files.not_in_package()

charts = {
"file": {
"queryset": files,
"fields": [
"programming_language",
"mime_type",
"holders",
"copyrights",
"detected_license_expression",
"compliance_alert",
],
},
"package": {
"queryset": project.discoveredpackages,
"fields": ["type", "declared_license_expression"],
},
"dependency": {
"queryset": project.discovereddependencies,
"fields": ["type", "is_runtime", "is_optional", "is_resolved"],
},
}

dependency_package_type = dependencies.values_list("type", flat=True)
dependency_is_runtime = dependencies.values_list("is_runtime", flat=True)
dependency_is_optional = dependencies.values_list("is_optional", flat=True)
dependency_is_resolved = dependencies.values_list("is_resolved", flat=True)
for group_name, spec in charts.items():
fields = spec["fields"]
# Clear the un-needed ordering to get faster queries
qs_values = spec["queryset"].values(*fields).order_by()

for field_name in fields:
if field_name in ["holders", "copyrights"]:
field_values = (
data.get(field_name[:-1])
for entry in qs_values
for data in entry.get(field_name, [])
)
else:
field_values = (entry[field_name] for entry in qs_values)

context.update(
{
"programming_languages": self.get_summary(file_languages),
"mime_types": self.get_summary(file_mime_types),
"holders": self.get_summary(file_holders),
"copyrights": self.get_summary(file_copyrights),
"file_license_expressions": self.get_summary(file_license_expressions),
"file_compliance_alert": self.get_summary(file_compliance_alert),
"package_licenses": self.get_summary(package_licenses),
"package_types": self.get_summary(package_types),
"dependency_package_type": self.get_summary(dependency_package_type),
"dependency_is_runtime": self.get_summary(dependency_is_runtime),
"dependency_is_optional": self.get_summary(dependency_is_optional),
"dependency_is_resolved": self.get_summary(dependency_is_resolved),
"file_filter": file_filter,
}
)
context[f"{group_name}_{field_name}"] = self.get_summary(field_values)

return context

Expand Down

0 comments on commit 7b54a4d

Please sign in to comment.