ARXIVNG-1108 implemented s3-backed storage for compilation products

.coveragerc

@@ -0,0 +1,7 @@
+[run]
+omit =
+    */app.py
+    */wsgi.py
+    */config.py
+    docs/*
+    */tests*

Pipfile

@@ -8,10 +8,15 @@ arxiv-base = "*"
 flask = "*"
 requests = "*"
 arxiv-auth = "==0.1.0rc6"
+"boto3" = "*"
+moto = "==1.3.4"
 
 [dev-packages]
 jsonschema = "*"
 openapi-spec-validator = "*"
+"nose2" = "*"
+coveralls = "*"
+coverage = "*"
 
 [requires]
 python_version = "3.6"

compiler/config.py

@@ -34,3 +34,14 @@
     f'/{FILE_MANAGER_PATH}'
 )
 FILE_MANAGER_VERIFY = bool(int(os.environ.get('FILE_MANAGER_VERIFY', '1')))
+
+# Configuration for object store.
+S3_ENDPOINT = os.environ.get('S3_ENDPOINT', None)
+S3_VERIFY = bool(int(os.environ.get('S3_VERIFY', 1)))
+S3_BUCKETS = [
+    ('arxiv', 'arxiv-compiler'),
+    ('submission', 'arxiv-compiler-submission')
+]
+AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID', None)
+AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY', None)
+AWS_REGION = os.environ.get('AWS_REGION', 'us-east-1')

compiler/domain.py

@@ -1,20 +1,88 @@
 """Domain class for the compiler service."""
 
-from typing import NamedTuple
-
+from typing import NamedTuple, Optional
+import io
+from datetime import datetime
 from .util import ResponseStream
 
 
+class CompilationStatus(NamedTuple):
+    """Represents the state of a compilation product in the store."""
+
+    # These are intended as fixed class attributes, not slots.
+    PDF = "pdf"   # type: ignore
+    DVI = "dvi"   # type: ignore
+    PS = "ps"   # type: ignore
+
+    CURRENT = "current"   # type: ignore
+    IN_PROGRESS = "in_progress"   # type: ignore
+    FAILED = "failed"   # type: ignore
+
+    # Here are the actual slots/fields.
+    source_id: str
+
+    format: str
+    """
+    The target format of the compilation.
+
+    One of :attr:`PDF`, :attr:`DVI`, or :attr:`PS`.
+    """
+
+    source_checksum: str
+    """Checksum of the source tarball from the file management service."""
+
+    task_id: str
+    """If a task exists for this compilation, the unique task ID."""
+
+    status: str
+    """
+    The status of the compilation.
+
+    One of :attr:`CURRENT`, :attr:`IN_PROGRESS`, or :attr:`FAILED`.
+
+    If :attr:`CURRENT`, the current file corresponding to the format of this
+    compilation status is the product of this compilation.
+    """
+
+    @property
+    def ext(self) -> str:
+        """Filename extension for the compilation product."""
+        return self.format
+
+    def to_dict(self) -> dict:
+        """Generate a dict representation of this object."""
+        return {
+            'source_id': self.source_id,
+            'format': self.format,
+            'source_checksum': self.source_checksum,
+            'task_id': self.task_id,
+            'status': self.status
+        }
+
+
+class CompilationProduct(NamedTuple):
+    """Content of a compilation product itself."""
+
+    stream: io.BytesIO
+    """Readable buffer with the product content."""
+
+    checksum: Optional[str] = None
+    """The B64-encoded MD5 hash of the compilation product."""
+
+    status: Optional[CompilationStatus] = None
+    """Status information about the product."""
+
+
 class SourcePackage(NamedTuple):
     """Source package content, retrieved from file management service."""
 
-    upload_id: str
+    source_id: str
     stream: ResponseStream
     etag: str
 
 
 class SourcePackageInfo(NamedTuple):
     """Current state of the source package in the file managment service."""
 
-    upload_id: str
+    source_id: str
     etag: str

compiler/services/filemanager.py

@@ -194,7 +194,7 @@ def get_upload_content(self, upload_id: str) -> SourcePackage:
                                       status.HTTP_200_OK)
         logger.debug('Got response with status %s', response.status_code)
         return SourcePackage(
-            upload_id=upload_id,
+            source_id=upload_id,
             stream=ResponseStream(response.iter_content(chunk_size=None)),
             etag=response.headers['ETag']
         )
@@ -215,7 +215,7 @@ def get_upload_info(self, upload_id: str) -> SourcePackageInfo:
         logger.debug('Get upload info for: %s', upload_id)
         response, headers = self.request('head', f'/{upload_id}/content')
         logger.debug('Got response with etag %s', headers['ETag'])
-        return SourcePackageInfo(upload_id=upload_id, etag=headers['ETag'])
+        return SourcePackageInfo(source_id=upload_id, etag=headers['ETag'])
 
 
 def init_app(app: object = None) -> None: