Catch inspect ELF errors and create messages #1121

Signed-off-by: tdruez <[email protected]>

docs/built-in-pipelines.rst

@@ -45,7 +45,7 @@ Analyse Docker Windows Image
 .. _pipeline_collect_symbols:
 
 Collect Codebase Symbols (addon)
----------------------------------
+--------------------------------
 .. autoclass:: scanpipe.pipelines.collect_symbols.CollectSymbols()
     :members:
     :member-order: bysource

docs/scanpipe-pipes.rst

@@ -33,28 +33,53 @@ Docker
 .. automodule:: scanpipe.pipes.docker
     :members:
 
+ELF
+---
+.. automodule:: scanpipe.pipes.elf
+    :members:
+
 Fetch
 -----
 .. automodule:: scanpipe.pipes.fetch
     :members:
     :exclude-members: Download
 
+Flag
+----
+.. automodule:: scanpipe.pipes.flag
+    :members:
+
 Input
 -----
 .. automodule:: scanpipe.pipes.input
     :members:
 
+JS
+--
+.. automodule:: scanpipe.pipes.js
+    :members:
+
 JVM
 ---
 .. automodule:: scanpipe.pipes.jvm
     :members:
 
+MatchCode
+---------
+.. automodule:: scanpipe.pipes.matchcode
+    :members:
+
 Output
 ------
 .. automodule:: scanpipe.pipes.output
     :members:
     :exclude-members: JSONResultsGenerator
 
+PathMap
+-------
+.. automodule:: scanpipe.pipes.pathmap
+    :members:
+
 PurlDB
 ------
 .. automodule:: scanpipe.pipes.purldb
@@ -80,9 +105,9 @@ SPDX
 .. automodule:: scanpipe.pipes.spdx
     :members:
 
-Flag
-----
-.. automodule:: scanpipe.pipes.flag
+Symbols
+-------
+.. automodule:: scanpipe.pipes.symbols
     :members:
 
 VulnerableCode

scanpipe/models.py

@@ -2540,7 +2540,6 @@ def create_and_add_package(self, package_data):
                 model=DiscoveredPackage,
                 details={
                     "codebase_resource_path": self.path,
-                    "codebase_resource_pk": self.pk,
                     **package_data,
                 },
                 exception=exception,

scanpipe/pipelines/__init__.py

@@ -230,9 +230,15 @@ def download_missing_inputs(self):
         if errors:
             raise InputFileError(errors)
 
-    def add_error(self, exception):
+    def add_error(self, exception, resource=None):
         """Create a ``ProjectMessage`` ERROR record on the current `project`."""
-        self.project.add_error(model=self.pipeline_name, exception=exception)
+        details = {}
+        if resource:
+            details["codebase_resource_path"] = resource.path
+
+        self.project.add_error(
+            model=self.pipeline_name, details=details, exception=exception
+        )
 
     @contextmanager
     def save_errors(self, *exceptions):

scanpipe/pipelines/inspect_elf_binaries.py

@@ -20,11 +20,11 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
-from pathlib import Path
-
-from elf_inspector.dwarf import get_dwarf_paths
+from elftools.common.exceptions import DWARFError
+from elftools.common.exceptions import ELFError
 
 from scanpipe.pipelines import Pipeline
+from scanpipe.pipes.elf import collect_dwarf_source_path_references
 
 
 class InspectELFBinaries(Pipeline):
@@ -38,10 +38,9 @@ def steps(cls):
         return (cls.collect_dwarf_source_path_references,)
 
     def collect_dwarf_source_path_references(self):
-        """
-        Update ``extra_data`` of ELF files with
-        dwarf data extracted from ELF files.
-        """
-        for elf in self.project.codebaseresources.elfs():
-            dwarf_paths = get_dwarf_paths(Path(self.project.codebase_path / elf.path))
-            elf.update_extra_data(dwarf_paths)
+        """Collect DWARF paths from ELF files and set values on the extra_data field."""
+        for elf_resource in self.project.codebaseresources.elfs():
+            try:
+                collect_dwarf_source_path_references(elf_resource)
+            except (ELFError, DWARFError) as error:
+                self.add_error(error, resource=elf_resource)

scanpipe/pipes/elf.py

@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/nexB/scancode.io for support and download.
+
+from elf_inspector.dwarf import get_dwarf_paths
+
+
+def collect_dwarf_source_path_references(resource):
+    """Collect and store the DWARF debug paths of the provided ELF ``resource``."""
+    dwarf_paths = get_dwarf_paths(resource.location_path)
+    resource.update_extra_data(dwarf_paths)
+    return dwarf_paths

scanpipe/templates/scanpipe/message_list.html

@@ -38,10 +38,10 @@
             </td>
             <td class="break-all" style="max-width: 450px;">
               <div style="max-height: 200px; overflow-y: scroll;">
-                {% if message.details.codebase_resource_pk and message.details.codebase_resource_path %}
+                {% if message.details.codebase_resource_path %}
                   <div>
-                    <strong>Codebase resource</strong>:
-                    <a href="{% url 'resource_detail' project.pk message.details.codebase_resource_pk %}" target="_blank">
+                    <strong>Resource</strong>:
+                    <a href="{% url 'resource_detail' project.slug message.details.codebase_resource_path %}" target="_blank">
                       {{ message.details.codebase_resource_path }}
                     </a>
                   </div>

scanpipe/tests/test_models.py

@@ -2475,6 +2475,5 @@ def test_scanpipe_codebase_resource_create_and_add_package_warnings(self):
         expected_message = "value too long for type character varying(100)"
         self.assertEqual(expected_message, message.description)
         self.assertEqual(bad_data["version"], message.details["version"])
-        self.assertTrue(message.details["codebase_resource_pk"])
         self.assertEqual(resource.path, message.details["codebase_resource_path"])
         self.assertIn("in save", message.traceback)