WIP: packager

.github/workflows/deploy-lambdas.yaml

@@ -2,8 +2,8 @@ name: Deploy lambdas to S3 and ECR
 
 on:
   push:
-    branches:
-      - master
+    # branches:
+    #   - master
     paths:
       - '.github/workflows/deploy-lambdas.yaml'
       - 'lambdas/**'
@@ -13,15 +13,15 @@ jobs:
     strategy:
       matrix:
         path:
-          - access_counts
-          - indexer
-          - pkgevents
+          # - access_counts
+          # - indexer
+          # - pkgevents
           - pkgpush
-          - preview
-          - s3hash
-          - status_reports
-          - tabular_preview
-          - transcode
+          # - preview
+          # - s3hash
+          # - status_reports
+          # - tabular_preview
+          # - transcode
     runs-on: ubuntu-latest
     # These permissions are needed to interact with GitHub's OIDC Token endpoint.
     permissions:
@@ -60,32 +60,32 @@ jobs:
           s3_key="${{ matrix.path }}/${{ github.sha }}.zip"
           ./lambdas/scripts/upload_zip.sh ./out.zip "$AWS_REGION" "$s3_key"
 
-  deploy-lambda-ecr:
-    strategy:
-      matrix:
-        path:
-          - thumbnail
-    runs-on: ubuntu-latest
-    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
-    permissions:
-      id-token: write
-      contents: read
-    steps:
-      - uses: actions/checkout@v4
-      - name: Build Docker image
-        working-directory: ./lambdas/${{ matrix.path }}
-        run: docker buildx build -t "quiltdata/lambdas/${{ matrix.path }}:${{ github.sha }}" -f Dockerfile .
-      - name: Configure AWS credentials from Prod account
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: arn:aws:iam::730278974607:role/github/GitHub-Quilt
-          aws-region: us-east-1
-      - name: Push Docker image to Prod ECR
-        run: ./lambdas/scripts/upload_ecr.sh 730278974607 "quiltdata/lambdas/${{ matrix.path }}:${{ github.sha }}"
-      - name: Configure AWS credentials from GovCloud account
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: arn:aws-us-gov:iam::313325871032:role/github/GitHub-Quilt
-          aws-region: us-gov-east-1
-      - name: Push Docker image to GovCloud ECR
-        run: ./lambdas/scripts/upload_ecr.sh 313325871032 "quiltdata/lambdas/${{ matrix.path }}:${{ github.sha }}"
+  # deploy-lambda-ecr:
+  #   strategy:
+  #     matrix:
+  #       path:
+  #         - thumbnail
+  #   runs-on: ubuntu-latest
+  #   # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+  #   permissions:
+  #     id-token: write
+  #     contents: read
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - name: Build Docker image
+  #       working-directory: ./lambdas/${{ matrix.path }}
+  #       run: docker buildx build -t "quiltdata/lambdas/${{ matrix.path }}:${{ github.sha }}" -f Dockerfile .
+  #     - name: Configure AWS credentials from Prod account
+  #       uses: aws-actions/configure-aws-credentials@v4
+  #       with:
+  #         role-to-assume: arn:aws:iam::730278974607:role/github/GitHub-Quilt
+  #         aws-region: us-east-1
+  #     - name: Push Docker image to Prod ECR
+  #       run: ./lambdas/scripts/upload_ecr.sh 730278974607 "quiltdata/lambdas/${{ matrix.path }}:${{ github.sha }}"
+  #     - name: Configure AWS credentials from GovCloud account
+  #       uses: aws-actions/configure-aws-credentials@v4
+  #       with:
+  #         role-to-assume: arn:aws-us-gov:iam::313325871032:role/github/GitHub-Quilt
+  #         aws-region: us-gov-east-1
+  #     - name: Push Docker image to GovCloud ECR
+  #       run: ./lambdas/scripts/upload_ecr.sh 313325871032 "quiltdata/lambdas/${{ matrix.path }}:${{ github.sha }}"

lambdas/pkgpush/src/t4_lambda_pkgpush/__init__.py

@@ -39,6 +39,7 @@
     PackageConstructEntry,
     PackageConstructParams,
     PackagePromoteParams,
+    PackagePushParams,
     PackagePushResult,
     S3CopyLambdaParams,
     S3HashLambdaParams,
@@ -527,3 +528,21 @@ def create_package(req_file: T.IO[bytes]) -> PackagePushResult:
 
     # XXX: return mtime?
     return PackagePushResult(top_hash=TopHash(top_hash))
+
+
+class PackagePrefixParams(PackagePushParams):
+    prefix: str  # XXX: validate with regex?
+
+
+def package_prefix_sqs(event, context):
+    with setup_user_boto_session(boto3.Session()):
+        for record in event["Records"]:
+            params = PackagePrefixParams.parse_raw(record["body"])
+            pkg = quilt3.Package()
+            pkg.set_meta(params.user_meta or {})
+            pkg.set_dir(".", params.prefix)
+            pkg._build(
+                name=params.name,
+                registry=f"s3://{params.bucket}",
+                message=params.message,
+            )