misc: add concourse ci

* misc: add concourse ci
* fix: update drawio with functional install link

PR ref: algoo#279
Co-authored-by: raphj <[email protected]>

README.rst

@@ -93,7 +93,8 @@ To install all previews builders dependencies:
 
   pip install preview-generator[all]
   sudo apt-get install libreoffice inkscape ufraw-batch ffmpeg xvfb
-  DRAWIO_VERSION="12.6.5" && curl -LO https://github.com/jgraph/drawio-desktop/releases/download/v${DRAWIO_VERSION}/draw.io-amd64-${DRAWIO_VERSION}.deb && sudo dpkg -i draw.io-amd64-${DRAWIO_VERSION}.deb
+  DRAWIO_VERSION="15.7.3" && curl -LO https://github.com/jgraph/drawio-desktop/releases/download/v${DRAWIO_VERSION}/drawio-x86_64-${DRAWIO_VERSION}.AppImage && mv drawio-x86_64-${DRAWIO_VERSION}.AppImage /usr/local/bin/drawio
+
 
 
 To check dependencies, you can run:
@@ -186,7 +187,7 @@ on debian:
 .. code:: console
 
   apt install xvfb
-  DRAWIO_VERSION="12.6.5" && curl -LO https://github.com/jgraph/drawio-desktop/releases/download/v${DRAWIO_VERSION}/draw.io-amd64-${DRAWIO_VERSION}.deb && sudo dpkg -i draw.io-amd64-${DRAWIO_VERSION}.deb
+  DRAWIO_VERSION="15.7.3" && curl -LO https://github.com/jgraph/drawio-desktop/releases/download/v${DRAWIO_VERSION}/drawio-x86_64-${DRAWIO_VERSION}.AppImage && mv drawio-x86_64-${DRAWIO_VERSION}.AppImage /usr/local/bin/drawio
   pip install preview-generator[drawio]
 
 

concourse/pipelines/pull-request.yml

@@ -0,0 +1,138 @@
+## YAML definitions re-used in plans
+definitions:
+  main_python_version: &main_python_version "3.9.6"
+  python_versions: &python_versions ["3.6.14", "3.7.11", "3.8.11", "3.9.6"]
+
+# plan step to put the current job name as pending in github's PR
+pending_status_notification: &pending_status_notification
+  put: preview_generator-status-update
+  resource: pull-request
+  inputs: [pull-request]
+  params: {path: pull-request, status: pending, context: $BUILD_JOB_NAME}
+  get_params: {skip_download: true}
+
+# sync job statuses as success/failure/error in github's PR.
+status_notifications: &status_notifications
+  on_success:
+    put: pull-request
+    inputs: [pull-request]
+    params: {path: pull-request, status: success, context: $BUILD_JOB_NAME}
+    get_params: {skip_download: true}
+  on_failure:
+    put: pull-request
+    inputs: [pull-request]
+    params: {path: pull-request, status: failure, context: $BUILD_JOB_NAME}
+    get_params: {skip_download: true}
+  on_abort:
+    put: pull-request
+    inputs: [pull-request]
+    params: {path: pull-request, status: error, context: $BUILD_JOB_NAME}
+    get_params: {skip_download: true}
+## END YAML definitions
+
+
+resource_types:
+- name: pull-request
+  type: registry-image
+  source:
+    repository: aoldershaw/github-pr-resource
+- name: abort-running-builds
+  type: registry-image
+  source:
+    repository: algooci/abort-running-builds-resource
+
+
+resources:
+# the github's pull-request
+- name: pull-request
+  type: pull-request
+  check_every: 30s
+  # TODO - S.G. - 2021/08/02
+  # The token is not used as one webhook would have to be created PER PR in github.
+  # This could be automated, but is not so easy.
+  webhook_token: ((github-webhook-token))
+  source:
+    repository: algoo/preview-generator
+    access_token: ((github-access-token))
+    number: ((number))
+- name: test-image
+  type: registry-image
+  source:
+    repository: algooci/preview-generator
+# resource used to abort the running builds before starting our test jobs
+- name: abort-running-builds
+  type: abort-running-builds
+  source:
+    username: ((concourse-bot-user))
+    password: ((concourse-bot-password))
+
+
+# Pipeline jobs:
+# 1. update this pipeline if the PR's file has changed
+# 2. abort any running build matching the current PR
+# 3. run backend (lint -> quick tests -> full tests) checks
+#    run frontend (lint + unit tests) checks
+#    run end-to-end (cypress) checks
+#    All of these in parallel.
+# Each check has its own status in the github's PR.
+jobs:
+- name: reconfigure-self
+  plan:
+  - get: pull-request
+    trigger: true
+    params:
+        list_changed_files: true
+
+  - put: abort-running-builds
+
+  - set_pipeline: self
+    file: pull-request/concourse/pipelines/pull-request.yml
+    vars:
+      github-webhook-token: ((github-webhook-token))
+      github-access-token: ((github-access-token))
+      concourse-bot-user: ((concourse-bot-user))
+      concourse-bot-password: ((concourse-bot-password))
+      docker-registry: ((docker-registry))
+    instance_vars:
+      number: ((number))
+
+- name: tests
+  <<: *status_notifications
+  plan:
+
+  - in_parallel:
+    - get: pull-request
+      passed: [reconfigure-self]
+      trigger: true
+      params:
+        list_changed_files: true
+    - get: test-image
+
+  - *pending_status_notification
+
+  - task: lint
+    image: test-image
+    config:
+      platform: linux
+      inputs:
+        - name: pull-request
+      run:
+        dir: pull-request
+        path: concourse/scripts/lint
+
+  - across:
+    - var: python_version
+      values: *python_versions
+      max_in_flight: all
+    task: tests_((.:python_version))
+    image: test-image
+    config:
+      platform: linux
+      inputs:
+        - name: pull-request
+      run:
+        dir: pull-request
+        path: concourse/scripts/tests
+        args:
+          - ((.:python_version))
+          - ((docker-registry))

concourse/scripts/install_os_packages

@@ -0,0 +1,10 @@
+# !/bin/bash
+set -e
+packages_dir="$(realpath $(dirname $0))/../../system_packages/debian"
+
+apt-get update
+apt-get install -qy $(cat "$packages_dir/os_packages.list")
+apt-get install -qy $(cat "$packages_dir/optional_preview_packages.list")
+# echo "install drawio"
+# Custom install of drawio: Reintroduce it when possible (fuse related issue)
+# DRAWIO_VERSION="15.7.3" && curl -LO https://github.com/jgraph/drawio-desktop/releases/download/v${DRAWIO_VERSION}/drawio-x86_64-${DRAWIO_VERSION}.AppImage && mv drawio-x86_64-${DRAWIO_VERSION}.AppImage /usr/local/bin/drawio

concourse/scripts/install_pyenv

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -e
+
+apt-get update
+apt-get install -qy git
+
+git clone https://github.com/pyenv/pyenv.git "${HOME}/.pyenv"
+
+cat > "${HOME}/.bashrc" <<'EOF'
+export PATH="${HOME}/.pyenv/bin:$PATH"
+eval "$(pyenv init --path)"
+eval "$(pyenv init -)"
+EOF
+
+# dependencies needed to build python
+apt-get install -qy \
+        make build-essential libssl-dev zlib1g-dev \
+        libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
+        libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
+
+
+echo "Path is now: $PATH, shell is $SHELL"
+exit 0

concourse/scripts/install_python_packages

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+source "$HOME/.bashrc"
+command -v pyenv || (echo "need pyenv installed" && exit 1)
+for version in $@; do
+  pyenv install "$version"
+  pyenv shell "$version"
+  pip install ".[all, testing]"
+done

concourse/scripts/lint

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e
+
+script_dir="$(realpath $(dirname $0))"
+source "$script_dir/util-lib.sh"
+
+source "$HOME/.bashrc"
+python_version=${1:-3.6.14}
+pyenv shell "$python_version"
+
+
+pip install black isort flake8 mypy==0.770 types-filelock
+# black
+black --version
+black -l 100 --exclude '/(\..*)/' --diff --check preview_generator tests setup.py
+# isort
+echo -n "isort " && isort --version-number
+isort --df -c **/*.py
+# flake8
+flake8 --version
+flake8
+# mypy
+mypy --version
+mypy --ignore-missing-imports --disallow-untyped-defs .

concourse/scripts/tests

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -e
+
+python_version=${1:-3.6.14}
+docker_registry=${2-""}
+
+script_dir="$(realpath $(dirname $0))"
+source "$script_dir/util-lib.sh"
+skip_if_no_changed_file_match "^(preview_generator|tests|concourse|setup.py|setup.cfg|pytest.ini)"
+"$script_dir/install_os_packages"
+
+source "$HOME/.bashrc"
+pyenv shell "$python_version"
+
+pip install ".[all, testing]"
+# Configuration to get most builder working
+GLTF_EXPERIMENTAL_SUPPORT_ENABLED=1
+rm -f /etc/ImageMagick-6/policy.xml
+export DISPLAY=:99.0
+which Xvfb
+Xvfb :99 -screen 0 1x1x16 > /dev/null 2>&1 &
+
+pytest tests

concourse/scripts/util-lib.sh

@@ -0,0 +1,11 @@
+
+# Exits the calling script with a 0 return code if no changed file matches the given arguments
+# Arguments are given to grep so a regex/complex expression is possible.
+# Changed files are read in concourse's resource change_files.
+skip_if_no_changed_file_match() {
+    if [ -f .git/resource/changed_files ] && ! $(egrep -q "$@" .git/resource/changed_files); then
+        echo "Changed files do not match '$@', skipping the task."
+        exit 0
+    fi
+    echo "Found files matching '$@', running the task"
+}

containers/concourse/Dockerfile

@@ -0,0 +1,20 @@
+# This image should be built from the root of tracim repository, e.g.:
+# docker build -f containers/concourse/Dockerfile -t algooci/preview-generator:latest .
+FROM debian:bullseye AS base_install
+
+# HOME, used by pyenv scripts
+ENV HOME=/root
+# Tracim needs UTF-8 to properly work
+ENV LANG C.UTF-8
+ENV LANGUAGE C.UTF-8
+ENV LC_ALL C.UTF-8
+
+COPY . preview_generator
+WORKDIR /preview_generator
+RUN ./concourse/scripts/install_os_packages && \
+    # needed to install different python versions \
+    ./concourse/scripts/install_pyenv
+
+FROM base_install AS main
+    # tested python versions
+RUN ./concourse/scripts/install_python_packages 3.6.14 3.7.11 3.8.11 3.9.6

preview_generator/preview/builder/document__drawio.py

@@ -31,7 +31,7 @@ def check_dependencies(cls) -> None:
         if not executable_is_available("xvfb-run"):
             raise BuilderDependencyNotFound("this builder requires xvfb-run to be available")
 
-        if not executable_is_available("/usr/bin/drawio"):
+        if not executable_is_available("drawio"):
             raise BuilderDependencyNotFound("this builder requires drawio to be available")
 
     @classmethod
@@ -67,14 +67,17 @@ def build_jpeg_preview(
             with Xvfb():
                 build_jpg_result_code = check_call(
                     [
-                        "/usr/bin/drawio",
-                        "--no-sandbox",
+                        "drawio",
                         "-x",
                         "-f",
                         "jpg",
                         "-o",
                         tmp_jpg.name,
                         file_path,
+                        # INFO - G.M - 12/11/2021 - Add no-sandbox at the end as putting it before
+                        # doesn't work, see:
+                        # https://github.com/jgraph/drawio-desktop/issues/249#issuecomment-695179747
+                        "--no-sandbox",
                     ],
                     stdout=DEVNULL,
                     stderr=STDOUT,

preview_generator/preview/builder/document__scribus.py

@@ -10,6 +10,8 @@
 from subprocess import check_output
 import typing
 
+import pytest
+
 from preview_generator.exception import BuilderDependencyNotFound
 from preview_generator.extension import mimetypes_storage
 from preview_generator.preview.builder.document_generic import DocumentPreviewBuilder
@@ -30,6 +32,7 @@
 SCRIPT_PATH = os.path.join(parent_dir, SCRIPT_FOLDER_NAME, SCRIPT_NAME)
 
 
+@pytest.mark.xfail(reason="Broken builder ?")
 class DocumentPreviewBuilderScribus(DocumentPreviewBuilder):
 
     weight = 110

system_packages/debian/optional_preview_packages.list

@@ -0,0 +1,4 @@
+ffmpeg
+xvfb
+inkscape
+libreoffice

system_packages/debian/os_packages.list

@@ -0,0 +1,18 @@
+build-essential
+libjpeg-dev
+zlib1g-dev
+python3-dev
+python3-venv
+curl
+ghostscript
+git
+imagemagick
+iproute2
+libfile-mimeinfo-perl
+libimage-exiftool-perl
+locales
+poppler-utils
+python3
+python3-pip
+qpdf
+libmagic1