diff --git a/README.rst b/README.rst index 699c5156..fdfc1428 100644 --- a/README.rst +++ b/README.rst @@ -93,7 +93,8 @@ To install all previews builders dependencies: pip install preview-generator[all] sudo apt-get install libreoffice inkscape ufraw-batch ffmpeg xvfb - DRAWIO_VERSION="12.6.5" && curl -LO https://github.com/jgraph/drawio-desktop/releases/download/v${DRAWIO_VERSION}/draw.io-amd64-${DRAWIO_VERSION}.deb && sudo dpkg -i draw.io-amd64-${DRAWIO_VERSION}.deb + DRAWIO_VERSION="15.7.3" && curl -LO https://github.com/jgraph/drawio-desktop/releases/download/v${DRAWIO_VERSION}/drawio-x86_64-${DRAWIO_VERSION}.AppImage && mv drawio-x86_64-${DRAWIO_VERSION}.AppImage /usr/local/bin/drawio + To check dependencies, you can run: @@ -186,7 +187,7 @@ on debian: .. code:: console apt install xvfb - DRAWIO_VERSION="12.6.5" && curl -LO https://github.com/jgraph/drawio-desktop/releases/download/v${DRAWIO_VERSION}/draw.io-amd64-${DRAWIO_VERSION}.deb && sudo dpkg -i draw.io-amd64-${DRAWIO_VERSION}.deb + DRAWIO_VERSION="15.7.3" && curl -LO https://github.com/jgraph/drawio-desktop/releases/download/v${DRAWIO_VERSION}/drawio-x86_64-${DRAWIO_VERSION}.AppImage && mv drawio-x86_64-${DRAWIO_VERSION}.AppImage /usr/local/bin/drawio pip install preview-generator[drawio] diff --git a/concourse/pipelines/pull-request.yml b/concourse/pipelines/pull-request.yml new file mode 100644 index 00000000..5d3b0f9e --- /dev/null +++ b/concourse/pipelines/pull-request.yml @@ -0,0 +1,138 @@ +## YAML definitions re-used in plans +definitions: + main_python_version: &main_python_version "3.9.6" + python_versions: &python_versions ["3.6.14", "3.7.11", "3.8.11", "3.9.6"] + +# plan step to put the current job name as pending in github's PR +pending_status_notification: &pending_status_notification + put: preview_generator-status-update + resource: pull-request + inputs: [pull-request] + params: {path: pull-request, status: pending, context: $BUILD_JOB_NAME} + get_params: {skip_download: true} + +# sync job statuses as success/failure/error in github's PR. +status_notifications: &status_notifications + on_success: + put: pull-request + inputs: [pull-request] + params: {path: pull-request, status: success, context: $BUILD_JOB_NAME} + get_params: {skip_download: true} + on_failure: + put: pull-request + inputs: [pull-request] + params: {path: pull-request, status: failure, context: $BUILD_JOB_NAME} + get_params: {skip_download: true} + on_abort: + put: pull-request + inputs: [pull-request] + params: {path: pull-request, status: error, context: $BUILD_JOB_NAME} + get_params: {skip_download: true} +## END YAML definitions + + +resource_types: +- name: pull-request + type: registry-image + source: + repository: aoldershaw/github-pr-resource +- name: abort-running-builds + type: registry-image + source: + repository: algooci/abort-running-builds-resource + + +resources: +# the github's pull-request +- name: pull-request + type: pull-request + check_every: 30s + # TODO - S.G. - 2021/08/02 + # The token is not used as one webhook would have to be created PER PR in github. + # This could be automated, but is not so easy. + webhook_token: ((github-webhook-token)) + source: + repository: algoo/preview-generator + access_token: ((github-access-token)) + number: ((number)) +- name: test-image + type: registry-image + source: + repository: algooci/preview-generator +# resource used to abort the running builds before starting our test jobs +- name: abort-running-builds + type: abort-running-builds + source: + username: ((concourse-bot-user)) + password: ((concourse-bot-password)) + + +# Pipeline jobs: +# 1. update this pipeline if the PR's file has changed +# 2. abort any running build matching the current PR +# 3. run backend (lint -> quick tests -> full tests) checks +# run frontend (lint + unit tests) checks +# run end-to-end (cypress) checks +# All of these in parallel. +# Each check has its own status in the github's PR. +jobs: +- name: reconfigure-self + plan: + - get: pull-request + trigger: true + params: + list_changed_files: true + + - put: abort-running-builds + + - set_pipeline: self + file: pull-request/concourse/pipelines/pull-request.yml + vars: + github-webhook-token: ((github-webhook-token)) + github-access-token: ((github-access-token)) + concourse-bot-user: ((concourse-bot-user)) + concourse-bot-password: ((concourse-bot-password)) + docker-registry: ((docker-registry)) + instance_vars: + number: ((number)) + +- name: tests + <<: *status_notifications + plan: + + - in_parallel: + - get: pull-request + passed: [reconfigure-self] + trigger: true + params: + list_changed_files: true + - get: test-image + + - *pending_status_notification + + - task: lint + image: test-image + config: + platform: linux + inputs: + - name: pull-request + run: + dir: pull-request + path: concourse/scripts/lint + + - across: + - var: python_version + values: *python_versions + max_in_flight: all + task: tests_((.:python_version)) + image: test-image + config: + platform: linux + inputs: + - name: pull-request + run: + dir: pull-request + path: concourse/scripts/tests + args: + - ((.:python_version)) + - ((docker-registry)) diff --git a/concourse/scripts/install_os_packages b/concourse/scripts/install_os_packages new file mode 100755 index 00000000..eb688721 --- /dev/null +++ b/concourse/scripts/install_os_packages @@ -0,0 +1,10 @@ +# !/bin/bash +set -e +packages_dir="$(realpath $(dirname $0))/../../system_packages/debian" + +apt-get update +apt-get install -qy $(cat "$packages_dir/os_packages.list") +apt-get install -qy $(cat "$packages_dir/optional_preview_packages.list") +# echo "install drawio" +# Custom install of drawio: Reintroduce it when possible (fuse related issue) +# DRAWIO_VERSION="15.7.3" && curl -LO https://github.com/jgraph/drawio-desktop/releases/download/v${DRAWIO_VERSION}/drawio-x86_64-${DRAWIO_VERSION}.AppImage && mv drawio-x86_64-${DRAWIO_VERSION}.AppImage /usr/local/bin/drawio diff --git a/concourse/scripts/install_pyenv b/concourse/scripts/install_pyenv new file mode 100755 index 00000000..b775e7d0 --- /dev/null +++ b/concourse/scripts/install_pyenv @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +apt-get update +apt-get install -qy git + +git clone https://github.com/pyenv/pyenv.git "${HOME}/.pyenv" + +cat > "${HOME}/.bashrc" <<'EOF' +export PATH="${HOME}/.pyenv/bin:$PATH" +eval "$(pyenv init --path)" +eval "$(pyenv init -)" +EOF + +# dependencies needed to build python +apt-get install -qy \ + make build-essential libssl-dev zlib1g-dev \ + libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ + libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev + + +echo "Path is now: $PATH, shell is $SHELL" +exit 0 diff --git a/concourse/scripts/install_python_packages b/concourse/scripts/install_python_packages new file mode 100755 index 00000000..8fb2f484 --- /dev/null +++ b/concourse/scripts/install_python_packages @@ -0,0 +1,9 @@ +#!/bin/bash +set -e +source "$HOME/.bashrc" +command -v pyenv || (echo "need pyenv installed" && exit 1) +for version in $@; do + pyenv install "$version" + pyenv shell "$version" + pip install ".[all, testing]" +done diff --git a/concourse/scripts/lint b/concourse/scripts/lint new file mode 100755 index 00000000..aec525b0 --- /dev/null +++ b/concourse/scripts/lint @@ -0,0 +1,24 @@ +#!/bin/bash +set -e + +script_dir="$(realpath $(dirname $0))" +source "$script_dir/util-lib.sh" + +source "$HOME/.bashrc" +python_version=${1:-3.6.14} +pyenv shell "$python_version" + + +pip install black isort flake8 mypy==0.770 types-filelock +# black +black --version +black -l 100 --exclude '/(\..*)/' --diff --check preview_generator tests setup.py +# isort +echo -n "isort " && isort --version-number +isort --df -c **/*.py +# flake8 +flake8 --version +flake8 +# mypy +mypy --version +mypy --ignore-missing-imports --disallow-untyped-defs . diff --git a/concourse/scripts/tests b/concourse/scripts/tests new file mode 100755 index 00000000..d68a1e1e --- /dev/null +++ b/concourse/scripts/tests @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +python_version=${1:-3.6.14} +docker_registry=${2-""} + +script_dir="$(realpath $(dirname $0))" +source "$script_dir/util-lib.sh" +skip_if_no_changed_file_match "^(preview_generator|tests|concourse|setup.py|setup.cfg|pytest.ini)" +"$script_dir/install_os_packages" + +source "$HOME/.bashrc" +pyenv shell "$python_version" + +pip install ".[all, testing]" +# Configuration to get most builder working +GLTF_EXPERIMENTAL_SUPPORT_ENABLED=1 +rm -f /etc/ImageMagick-6/policy.xml +export DISPLAY=:99.0 +which Xvfb +Xvfb :99 -screen 0 1x1x16 > /dev/null 2>&1 & + +pytest tests diff --git a/concourse/scripts/util-lib.sh b/concourse/scripts/util-lib.sh new file mode 100644 index 00000000..c978dd4c --- /dev/null +++ b/concourse/scripts/util-lib.sh @@ -0,0 +1,11 @@ + +# Exits the calling script with a 0 return code if no changed file matches the given arguments +# Arguments are given to grep so a regex/complex expression is possible. +# Changed files are read in concourse's resource change_files. +skip_if_no_changed_file_match() { + if [ -f .git/resource/changed_files ] && ! $(egrep -q "$@" .git/resource/changed_files); then + echo "Changed files do not match '$@', skipping the task." + exit 0 + fi + echo "Found files matching '$@', running the task" +} diff --git a/containers/concourse/Dockerfile b/containers/concourse/Dockerfile new file mode 100644 index 00000000..1a9e2c97 --- /dev/null +++ b/containers/concourse/Dockerfile @@ -0,0 +1,20 @@ +# This image should be built from the root of tracim repository, e.g.: +# docker build -f containers/concourse/Dockerfile -t algooci/preview-generator:latest . +FROM debian:bullseye AS base_install + +# HOME, used by pyenv scripts +ENV HOME=/root +# Tracim needs UTF-8 to properly work +ENV LANG C.UTF-8 +ENV LANGUAGE C.UTF-8 +ENV LC_ALL C.UTF-8 + +COPY . preview_generator +WORKDIR /preview_generator +RUN ./concourse/scripts/install_os_packages && \ + # needed to install different python versions \ + ./concourse/scripts/install_pyenv + +FROM base_install AS main + # tested python versions +RUN ./concourse/scripts/install_python_packages 3.6.14 3.7.11 3.8.11 3.9.6 diff --git a/preview_generator/preview/builder/document__drawio.py b/preview_generator/preview/builder/document__drawio.py index e1523edd..eedd4d58 100644 --- a/preview_generator/preview/builder/document__drawio.py +++ b/preview_generator/preview/builder/document__drawio.py @@ -31,7 +31,7 @@ def check_dependencies(cls) -> None: if not executable_is_available("xvfb-run"): raise BuilderDependencyNotFound("this builder requires xvfb-run to be available") - if not executable_is_available("/usr/bin/drawio"): + if not executable_is_available("drawio"): raise BuilderDependencyNotFound("this builder requires drawio to be available") @classmethod @@ -67,14 +67,17 @@ def build_jpeg_preview( with Xvfb(): build_jpg_result_code = check_call( [ - "/usr/bin/drawio", - "--no-sandbox", + "drawio", "-x", "-f", "jpg", "-o", tmp_jpg.name, file_path, + # INFO - G.M - 12/11/2021 - Add no-sandbox at the end as putting it before + # doesn't work, see: + # https://github.com/jgraph/drawio-desktop/issues/249#issuecomment-695179747 + "--no-sandbox", ], stdout=DEVNULL, stderr=STDOUT, diff --git a/preview_generator/preview/builder/document__scribus.py b/preview_generator/preview/builder/document__scribus.py index 2aaf6feb..9c3644ce 100644 --- a/preview_generator/preview/builder/document__scribus.py +++ b/preview_generator/preview/builder/document__scribus.py @@ -10,6 +10,8 @@ from subprocess import check_output import typing +import pytest + from preview_generator.exception import BuilderDependencyNotFound from preview_generator.extension import mimetypes_storage from preview_generator.preview.builder.document_generic import DocumentPreviewBuilder @@ -30,6 +32,7 @@ SCRIPT_PATH = os.path.join(parent_dir, SCRIPT_FOLDER_NAME, SCRIPT_NAME) +@pytest.mark.xfail(reason="Broken builder ?") class DocumentPreviewBuilderScribus(DocumentPreviewBuilder): weight = 110 diff --git a/system_packages/debian/optional_preview_packages.list b/system_packages/debian/optional_preview_packages.list new file mode 100644 index 00000000..928d3455 --- /dev/null +++ b/system_packages/debian/optional_preview_packages.list @@ -0,0 +1,4 @@ +ffmpeg +xvfb +inkscape +libreoffice diff --git a/system_packages/debian/os_packages.list b/system_packages/debian/os_packages.list new file mode 100644 index 00000000..82a830ec --- /dev/null +++ b/system_packages/debian/os_packages.list @@ -0,0 +1,18 @@ +build-essential +libjpeg-dev +zlib1g-dev +python3-dev +python3-venv +curl +ghostscript +git +imagemagick +iproute2 +libfile-mimeinfo-perl +libimage-exiftool-perl +locales +poppler-utils +python3 +python3-pip +qpdf +libmagic1 diff --git a/tests/input/binary/test_binary.py b/tests/input/binary/test_binary.py index f42a47b9..7c8d6f54 100644 --- a/tests/input/binary/test_binary.py +++ b/tests/input/binary/test_binary.py @@ -22,63 +22,63 @@ def setup_function(function: typing.Callable) -> None: @pytest.mark.parametrize("file_path", [BINARY_FILE_PATH, BINARY_FILE_PATH_WITHOUT_EXT]) -def test_to_jpeg(file_path) -> None: +def test_to_jpeg(file_path: str) -> None: manager = PreviewManager(cache_folder_path=CACHE_DIR, create_folder=True) with pytest.raises(UnsupportedMimeType): manager.get_jpeg_preview(file_path=file_path, height=256, width=512) @pytest.mark.parametrize("file_path", [BINARY_FILE_PATH, BINARY_FILE_PATH_WITHOUT_EXT]) -def test_get_nb_page(file_path) -> None: +def test_get_nb_page(file_path: str) -> None: manager = PreviewManager(cache_folder_path=CACHE_DIR, create_folder=True) with pytest.raises(UnsupportedMimeType): manager.get_page_nb(file_path=file_path, file_ext=".bin") @pytest.mark.parametrize("file_path", [BINARY_FILE_PATH, BINARY_FILE_PATH_WITHOUT_EXT]) -def test_to_jpeg__default_size(file_path) -> None: +def test_to_jpeg__default_size(file_path: str) -> None: manager = PreviewManager(cache_folder_path=CACHE_DIR, create_folder=True) with pytest.raises(UnsupportedMimeType): manager.get_jpeg_preview(file_path=file_path, file_ext=".bin") @pytest.mark.parametrize("file_path", [BINARY_FILE_PATH, BINARY_FILE_PATH_WITHOUT_EXT]) -def test_to_json(file_path) -> None: +def test_to_json(file_path: str) -> None: manager = PreviewManager(cache_folder_path=CACHE_DIR, create_folder=True) with pytest.raises(UnsupportedMimeType): manager.get_json_preview(file_path=file_path, force=True) @pytest.mark.parametrize("file_path", [BINARY_FILE_PATH, BINARY_FILE_PATH_WITHOUT_EXT]) -def test_to_pdf(file_path) -> None: +def test_to_pdf(file_path: str) -> None: manager = PreviewManager(cache_folder_path=CACHE_DIR, create_folder=True) with pytest.raises(UnsupportedMimeType): manager.get_pdf_preview(file_path=file_path, force=True) @pytest.mark.parametrize("file_path", [BINARY_FILE_PATH, BINARY_FILE_PATH_WITHOUT_EXT]) -def test_has_pdf_preview(file_path) -> None: +def test_has_pdf_preview(file_path: str) -> None: manager = PreviewManager(cache_folder_path=CACHE_DIR, create_folder=True) with pytest.raises(UnsupportedMimeType): manager.has_pdf_preview(file_path=file_path, file_ext=".bin") @pytest.mark.parametrize("file_path", [BINARY_FILE_PATH, BINARY_FILE_PATH_WITHOUT_EXT]) -def test_has_jpeg_preview(file_path) -> None: +def test_has_jpeg_preview(file_path: str) -> None: manager = PreviewManager(cache_folder_path=CACHE_DIR, create_folder=True) with pytest.raises(UnsupportedMimeType): manager.has_jpeg_preview(file_path=file_path, file_ext=".bin") @pytest.mark.parametrize("file_path", [BINARY_FILE_PATH, BINARY_FILE_PATH_WITHOUT_EXT]) -def test_has_json_preview(file_path) -> None: +def test_has_json_preview(file_path: str) -> None: manager = PreviewManager(cache_folder_path=CACHE_DIR, create_folder=True) with pytest.raises(UnsupportedMimeType): manager.has_json_preview(file_path=file_path, file_ext=".bin") @pytest.mark.parametrize("file_path", [BINARY_FILE_PATH, BINARY_FILE_PATH_WITHOUT_EXT]) -def test_has_html_preview(file_path) -> None: +def test_has_html_preview(file_path: str) -> None: manager = PreviewManager(cache_folder_path=CACHE_DIR, create_folder=True) with pytest.raises(UnsupportedMimeType): manager.has_html_preview(file_path=file_path, file_ext=".bin") diff --git a/tests/input/drawio/test_drawio.py b/tests/input/drawio/test_drawio.py index 284b1bb4..998ab4df 100644 --- a/tests/input/drawio/test_drawio.py +++ b/tests/input/drawio/test_drawio.py @@ -6,10 +6,15 @@ import typing from PIL import Image +import pytest from preview_generator.manager import PreviewManager +from preview_generator.utils import executable_is_available from tests import test_utils +if not executable_is_available("drawio"): + pytest.skip("drawio is not available.", allow_module_level=True) + CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) CACHE_DIR = "/tmp/preview-generator-tests/cache" IMAGE_FILE_PATH = os.path.join(CURRENT_DIR, "test_drawio.drawio") diff --git a/tests/input/svg/test_svg_inkscape_install.py b/tests/input/svg/test_svg_inkscape_install.py index 78ce2ef9..77dd05b2 100644 --- a/tests/input/svg/test_svg_inkscape_install.py +++ b/tests/input/svg/test_svg_inkscape_install.py @@ -4,23 +4,29 @@ import pytest -import preview_generator.preview.builder.image__inkscape as inkscape_builder_module +from preview_generator.preview.builder import image__inkscape as inkscape_builder_module @pytest.mark.parametrize( "side_effect, inkscape_version, options", [ (lambda _: b"Inkscape 1.0", "1.0", inkscape_builder_module.INKSCAPE_100_SVG_TO_PNG_OPTIONS), - (lambda _: b"Inkscape 0.92", "0.92", inkscape_builder_module.INKSCAPE_0x_SVG_TO_PNG_OPTIONS), - (FileNotFoundError(), "not_installed", inkscape_builder_module.INKSCAPE_100_SVG_TO_PNG_OPTIONS), + ( + lambda _: b"Inkscape 0.92", + "0.92", + inkscape_builder_module.INKSCAPE_0x_SVG_TO_PNG_OPTIONS, + ), + ( + FileNotFoundError(), + "not_installed", + inkscape_builder_module.INKSCAPE_100_SVG_TO_PNG_OPTIONS, + ), ], ) def test_inkscape_installation( side_effect: typing.Callable, inkscape_version: str, options: typing.Tuple[str, ...] ) -> None: - with unittest.mock.patch( - "subprocess.check_output" - ) as check_output_mock: + with unittest.mock.patch("subprocess.check_output") as check_output_mock: check_output_mock.side_effect = side_effect builder_module = importlib.reload(inkscape_builder_module) builder_class = builder_module.ImagePreviewBuilderInkscape # type: ignore