feat(upload): Implement PDB and PE upload

src/sentry/api/endpoints/chunk.py

@@ -28,6 +28,7 @@
 CHUNK_UPLOAD_ACCEPT = (
     'debug_files',  # DIF assemble
     'release_files',  # Artifacts assemble
+    'pdbs',  # PDB upload and debug id override
 )
 
 

src/sentry/api/endpoints/debug_files.py

@@ -251,6 +251,7 @@ def post(self, request, project):
                     "required": ["name", "chunks"],
                     "properties": {
                         "name": {"type": "string"},
+                        "debug_id": {"type": "string"},
                         "chunks": {
                             "type": "array",
                             "items": {
@@ -259,7 +260,7 @@ def post(self, request, project):
                             }
                         }
                     },
-                    "additionalProperties": False
+                    "additionalProperties": True
                 }
             },
             "additionalProperties": False
@@ -279,6 +280,7 @@ def post(self, request, project):
 
         for checksum, file_to_assemble in six.iteritems(files):
             name = file_to_assemble.get('name', None)
+            debug_id = file_to_assemble.get('debug_id', None)
             chunks = file_to_assemble.get('chunks', [])
 
             # First, check the cached assemble status. During assembling, a
@@ -348,6 +350,7 @@ def post(self, request, project):
                 kwargs={
                     'project_id': project.id,
                     'name': name,
+                    'debug_id': debug_id,
                     'checksum': checksum,
                     'chunks': chunks,
                 }

src/sentry/constants.py

@@ -229,6 +229,8 @@ def get_all_languages():
     'text/x-breakpad': 'breakpad',
     'application/x-mach-binary': 'macho',
     'application/x-elf-binary': 'elf',
+    'application/x-dosexec': 'pe',
+    'application/x-ms-pdb': 'pdb',
     'text/x-proguard+plain': 'proguard',
 }
 

src/sentry/models/debugfile.py

@@ -21,7 +21,7 @@
 
 from django.db import models
 
-from symbolic import Archive, SymbolicError, ObjectErrorUnsupportedObject
+from symbolic import Archive, SymbolicError, ObjectErrorUnsupportedObject, normalize_debug_id
 
 from sentry import options
 from sentry.constants import KNOWN_DIF_FORMATS
@@ -139,16 +139,25 @@ def file_format(self):
         ct = self.file.headers.get('Content-Type', 'unknown').lower()
         return KNOWN_DIF_FORMATS.get(ct, 'unknown')
 
+    @property
+    def file_type(self):
+        if self.data:
+            return self.data.get('type')
+
     @property
     def file_extension(self):
         if self.file_format == 'breakpad':
             return '.sym'
         if self.file_format == 'macho':
-            return '.dSYM'
+            return '' if self.file_type == 'exe' else '.dSYM'
         if self.file_format == 'proguard':
             return '.txt'
         if self.file_format == 'elf':
-            return '.debug'
+            return '' if self.file_type == 'exe' else '.debug'
+        if self.file_format == 'pe':
+            return '.exe' if self.file_type == 'exe' else '.dll'
+        if self.file_format == 'pdb':
+            return '.pdb'
 
         return ''
 
@@ -189,7 +198,7 @@ def create_dif_from_id(project, meta, fileobj=None, file=None):
     """
     if meta.file_format == 'proguard':
         object_name = 'proguard-mapping'
-    elif meta.file_format in ('macho', 'elf'):
+    elif meta.file_format in ('macho', 'elf', 'pdb', 'pe'):
         object_name = meta.name
     elif meta.file_format == 'breakpad':
         object_name = meta.name[:-4] if meta.name.endswith('.sym') else meta.name
@@ -284,11 +293,24 @@ def __init__(self, file_format, arch, debug_id, path, code_id=None, name=None, d
             self.name = os.path.basename(path)
 
     @classmethod
-    def from_object(cls, obj, path, name=None):
+    def from_object(cls, obj, path, name=None, debug_id=None):
+        if debug_id is not None:
+            try:
+                debug_id = normalize_debug_id(debug_id)
+            except SymbolicError:
+                debug_id = None
+
+        # Only allow overrides in the debug_id's age if the rest of the debug id
+        # matches with what we determine from the object file. We generally
+        # trust the server more than the client.
+        obj_id = obj.debug_id
+        if obj_id and debug_id and obj_id[:36] == debug_id[:36]:
+            obj_id = debug_id
+
         return cls(
             file_format=obj.file_format,
             arch=obj.arch,
-            debug_id=obj.debug_id,
+            debug_id=obj_id,
             code_id=obj.code_id,
             path=path,
             # TODO: Extract the object name from the object
@@ -304,7 +326,7 @@ def basename(self):
         return os.path.basename(self.path)
 
 
-def detect_dif_from_path(path, name=None):
+def detect_dif_from_path(path, name=None, debug_id=None):
     """This detects which kind of dif(Debug Information File) the path
     provided is. It returns an array since an Archive can contain more than
     one Object.
@@ -335,7 +357,7 @@ def detect_dif_from_path(path, name=None):
     else:
         objs = []
         for obj in archive.iter_objects():
-            objs.append(DifMeta.from_object(obj, path, name=name))
+            objs.append(DifMeta.from_object(obj, path, name=name, debug_id=debug_id))
         return objs
 
 

src/sentry/tasks/assemble.py

@@ -79,7 +79,7 @@ def set_assemble_status(task, scope, checksum, state, detail=None):
 
 
 @instrumented_task(name='sentry.tasks.assemble.assemble_dif', queue='assemble')
-def assemble_dif(project_id, name, checksum, chunks, **kwargs):
+def assemble_dif(project_id, name, checksum, chunks, debug_id=None, **kwargs):
     """
     Assembles uploaded chunks into a ``ProjectDebugFile``.
     """
@@ -109,7 +109,11 @@ def assemble_dif(project_id, name, checksum, chunks, **kwargs):
             # We only permit split difs to hit this endpoint.  The
             # client is required to split them up first or we error.
             try:
-                result = debugfile.detect_dif_from_path(temp_file.name, name=name)
+                result = debugfile.detect_dif_from_path(
+                    temp_file.name,
+                    name=name,
+                    debug_id=debug_id,
+                )
             except BadDif as e:
                 set_assemble_status(AssembleTask.DIF, project.id, checksum,
                                     ChunkFileState.ERROR, detail=e.args[0])

tests/sentry/api/endpoints/test_dif_assemble.py

@@ -252,6 +252,7 @@ def test_assemble(self, mock_assemble_dif):
                 'name': 'test',
                 'chunks': chunks,
                 'checksum': total_checksum,
+                'debug_id': None,
             }
         )
 

tests/sentry/tasks/test_assemble.py

@@ -151,6 +151,30 @@ def test_assemble_duplicate_blobs(self):
         assert f.checksum == file_checksum.hexdigest()
         assert f.type == 'dummy.type'
 
+    def test_assemble_debug_id_override(self):
+        sym_file = self.load_fixture('crash.sym')
+        blob1 = FileBlob.from_file(ContentFile(sym_file))
+        total_checksum = sha1(sym_file).hexdigest()
+
+        assemble_dif(
+            project_id=self.project.id,
+            name='crash.sym',
+            checksum=total_checksum,
+            chunks=[blob1.checksum],
+            debug_id='67e9247c-814e-392b-a027-dbde6748fcbf-beef'
+        )
+
+        status, _ = get_assemble_status(AssembleTask.DIF, self.project.id, total_checksum)
+        assert status == ChunkFileState.OK
+
+        dif = ProjectDebugFile.objects.filter(
+            project=self.project,
+            file__checksum=total_checksum,
+        ).get()
+
+        assert dif.file.headers == {'Content-Type': 'text/x-breakpad'}
+        assert dif.debug_id == '67e9247c-814e-392b-a027-dbde6748fcbf-beef'
+
 
 class AssembleArtifactsTest(BaseAssembleTest):
     def setUp(self):