Merge branch 'no-wheelhouse' into develop

.github/workflows/build.yaml

@@ -0,0 +1,55 @@
+name: Build and push images
+
+on:
+  push:
+    branches:
+      - '*'
+    tags:
+      - 'image-v*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # NOTE: Only one entry per major version.
+        nodejs_version: ['18.15.0', '12.22.12']
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: set up buildx
+        uses: docker/setup-buildx-action@v2
+      - name: extract nodejs major version
+        id: nodemajver
+        run: |
+          echo ${{ matrix.nodejs_version }} | cut -d. -f1 | sed 's/^/nodemajver=/' >> "$GITHUB_OUTPUT"
+      - name: extract image version from tag
+        id: imagever
+        if: ${{ github.ref_type == 'tag' }}
+        run: |
+          echo ${{ github.ref_name }} | sed 's/^image-/imagever=/' >> "$GITHUB_OUTPUT"
+      - name: construct image metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ghcr.io/${{ github.repository }}
+          tags: |
+            type=semver,pattern=node${{ steps.nodemajver.outputs.nodemajver }}-{{raw}},value=${{ steps.imagever.outputs.imagever }}
+            type=sha,prefix=node${{ steps.nodemajver.outputs.nodemajver }}-{{branch}}-,enable=${{ github.ref_type != 'tag' }}
+      - name: login to ghcr
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: build and push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          build-args: |
+            NODEJS_MAJOR=${{ steps.nodemajver.outputs.nodemajver }}
+            NODEJS_VERSION=${{ matrix.nodejs_version }}

.github/workflows/ci.yaml

@@ -6,32 +6,30 @@ on:
 jobs:
   tests:
     name: tests
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['2.x']
-        nodejs-version: ['4', '6', '8']
-        include:
-          - python-version: 'pypy2'
-            nodejs-version: '8'
+        nodejs-version: ['12', '18']
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: setup python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: pypy-2.7
       - name: setup redis
         uses: shogo82148/actions-setup-redis@v1
         with:
           redis-version: '6.x'
       - name: setup nodejs
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.nodejs-version }}
       - name: install deps
         run: |
           pip install wheel
+          # We need the backport of the typing module to build Twisted.
+          pip install typing==3.10.0.0
           pip install -e .
       - name: run tests
         env:

Dockerfile

@@ -1,11 +1,36 @@
-FROM praekeltfoundation/vumi
+FROM ghcr.io/praekeltfoundation/pypy-base-nw:2-buster AS builder
+
+RUN apt-get update
+RUN apt-get -yy install build-essential libssl-dev libffi-dev
+
+COPY . ./
+
+RUN pip install --upgrade pip
+# We need the backport of the typing module to build Twisted.
+RUN pip install typing==3.10.0.0
+
+RUN pip wheel -w /wheels -r /requirements.txt
+
+
+FROM ghcr.io/praekeltfoundation/vumi-base:0.1.1
 MAINTAINER Praekelt Foundation <[email protected]>
 
-# Install nodejs 0.10 from the Debian repo & make executable available as 'node'
-RUN apt-get-install.sh nodejs && \
-    update-alternatives --install /usr/bin/node node /usr/bin/nodejs 50
+# We need both of these outside the build, so we may as well pass them both in
+# and save some work extracting the major version.
+ARG NODEJS_MAJOR="18"
+ARG NODEJS_VERSION="18.15.0"
+
+# Install nodejs from upstream apt repo.
+RUN apt-get-install.sh apt-transport-https curl gnupg2 && \
+    curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key \
+        | apt-key add - && \
+    echo "deb https://deb.nodesource.com/node_${NODEJS_MAJOR}.x buster main" \
+        > /etc/apt/sources.list.d/nodesource.list && \
+    apt-get-purge.sh curl gnupg2
+
+RUN apt-get-install.sh nodejs=${NODEJS_VERSION}*
 
-ENV VXSANDBOX_VERSION "0.6.1"
-RUN pip install vxsandbox==$VXSANDBOX_VERSION
+COPY --from=builder /wheels /wheels
+RUN pip install -f /wheels vxsandbox
 
 ENV WORKER_CLASS "vxsandbox.worker.StandaloneJsFileSandbox"

setup.py

@@ -12,6 +12,7 @@
     packages=find_packages(),
     include_package_data=True,
     install_requires=[
+        'automat<22.10.0',  # Twisted dep, newer versions don't work with ancient Python.
         'Twisted>=20.3.0,<21.0.0',
         'vumi>=0.6.19',
     ],

vxsandbox/tests/test_worker.py

@@ -386,9 +386,14 @@ def test_sandbox_command_does_not_parse_timestamps(self):
 
 class JsSandboxTestMixin(object):
 
+    # We don't know what the actual limits are, and since we now control CPU
+    # and memory externally having limits here is less important.
     BIGGER_RLIMITS = {
-        "RLIMIT_STACK": [2 * 1024 * 1024] * 2,
-        "RLIMIT_AS": [786 * 1024 * 1024] * 2,
+        "RLIMIT_DATA": [-1, -1],
+        "RLIMIT_STACK": [-1, -1],
+        "RLIMIT_AS": [-1, -1],
+        "RLIMIT_CPU": [-1, -1],
+        "RLIMIT_NOFILE": [-1, -1],
     }
 
     @inlineCallbacks