From 609ac5b5aa5c0b08dbd04c5911584cfb723c3c53 Mon Sep 17 00:00:00 2001 From: Daniel Hollas Date: Mon, 14 Oct 2024 19:14:57 +0100 Subject: [PATCH] WIP: Reenable notebook tests --- .github/workflows/ci.yml | 73 ++++++++-- tests_notebooks/conftest.py | 127 ++++++++++++++++++ tests_notebooks/docker-compose.yml | 18 +++ {tests => tests_notebooks}/test_manage_app.py | 12 +- {tests => tests_notebooks}/test_start.py | 9 +- {tests => tests_notebooks}/test_terminal.py | 3 +- 6 files changed, 216 insertions(+), 26 deletions(-) create mode 100644 tests_notebooks/conftest.py create mode 100644 tests_notebooks/docker-compose.yml rename {tests => tests_notebooks}/test_manage_app.py (50%) rename {tests => tests_notebooks}/test_start.py (85%) rename {tests => tests_notebooks}/test_terminal.py (86%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bad3e2..3eab0ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,25 +3,80 @@ name: continuous-integration -on: [push, pull_request] +on: + push: + branches: + - master + pull_request: + +env: + FORCE_COLOR: "1" + +# https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - pre-commit: + test-notebooks: + + strategy: + matrix: + browser: [Chrome, Firefox] + fail-fast: false runs-on: ubuntu-latest + timeout-minutes: 30 steps: - - uses: actions/checkout@v4 - - name: Setup Python + - name: Check out app + uses: actions/checkout@v4 + + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: '3.11' + + - name: Setup uv + uses: astral-sh/setup-uv@v3 + with: + version: '0.4.20' - - name: Install dependencies + - name: Install package test dependencies + # Notebook tests happen in the container, here we only need to install + # the pytest-docker dependency. Unfortunately, uv/pip does not allow to + # only install [dev] dependencies so we end up installing all the rest as well. + run: uv pip install --system .[dev] + + - name: Set jupyter token env + run: echo "JUPYTER_TOKEN=$(openssl rand -hex 32)" >> $GITHUB_ENV + + # The Firefox and its engine geckodrive need do be installed manually to run + # selenium tests. + - name: Install Firefox + uses: browser-actions/setup-firefox@latest + with: + firefox-version: '96.0' + if: matrix.browser == 'Firefox' + + - name: Install geckodriver run: | - pip install .[dev] + wget -c https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz + tar xf geckodriver-v0.30.0-linux64.tar.gz -C /usr/local/bin + if: matrix.browser == 'Firefox' + + - name: Run pytest + run: pytest -v --driver ${{ matrix.browser }} tests_notebooks + env: + TAG: edge - - name: Run pre-commit - run: pre-commit run --all-files || ( git status --short ; git diff ; exit 1 ) + - name: Upload screenshots as artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: Screenshots-${{ matrix.browser }} + path: screenshots/ + if-no-files-found: error diff --git a/tests_notebooks/conftest.py b/tests_notebooks/conftest.py new file mode 100644 index 0000000..cefa72e --- /dev/null +++ b/tests_notebooks/conftest.py @@ -0,0 +1,127 @@ +import os +from pathlib import Path +from urllib.parse import urljoin + +import pytest +import requests +import selenium.webdriver.support.expected_conditions as ec +from requests.exceptions import ConnectionError +from selenium.webdriver.common.by import By +from selenium.webdriver.support.wait import WebDriverWait + + +def is_responsive(url): + try: + response = requests.get(url) + if response.status_code == 200: + return True + except ConnectionError: + return False + + +@pytest.fixture(scope="session") +def screenshot_dir(): + sdir = Path.joinpath(Path.cwd(), "screenshots") + try: + os.mkdir(sdir) + except FileExistsError: + pass + return sdir + + +@pytest.fixture(scope="session") +def docker_compose_file(pytestconfig): + return str(Path(pytestconfig.rootdir) / "tests_notebooks" / "docker-compose.yml") + + +@pytest.fixture(scope="session") +def docker_compose(docker_services): + return docker_services._docker_compose + + +@pytest.fixture(scope="session") +def aiidalab_exec(docker_compose): + def execute(command, user=None, **kwargs): + workdir = "/home/jovyan/apps/home" + if user: + command = f"exec --workdir {workdir} -T --user={user} aiidalab {command}" + else: + command = f"exec --workdir {workdir} -T aiidalab {command}" + + return docker_compose.execute(command, **kwargs) + + return execute + + +@pytest.fixture(scope="session", autouse=True) +def notebook_service(docker_ip, docker_services, aiidalab_exec): + """Ensure that HTTP service is up and responsive.""" + # Directory ~/apps/home/ is mounted by docker, + # make it writeable for jovyan user, needed for `pip install` + aiidalab_exec("chmod -R a+rw /home/jovyan/apps/home", user="root") + + aiidalab_exec("pip install --no-cache-dir .") + + # `port_for` takes a container port and returns the corresponding host port + port = docker_services.port_for("aiidalab", 8888) + url = f"http://{docker_ip}:{port}" + token = os.environ["JUPYTER_TOKEN"] + docker_services.wait_until_responsive( + timeout=30.0, pause=0.1, check=lambda: is_responsive(url) + ) + return url, token + + +@pytest.fixture(scope="function") +def selenium_driver(selenium, notebook_service): + def _selenium_driver(nb_path): + url, token = notebook_service + url_with_token = urljoin( + url, f"apps/apps/home/{nb_path}?token={token}" + ) + selenium.get(f"{url_with_token}") + # By default, let's allow selenium functions to retry for 60s + # till a given element is loaded, see: + # https://selenium-python.readthedocs.io/waits.html#implicit-waits + selenium.implicitly_wait(60) + window_width = 800 + window_height = 600 + selenium.set_window_size(window_width, window_height) + + selenium.find_element(By.ID, "ipython-main-app") + selenium.find_element(By.ID, "notebook-container") + selenium.find_element(By.ID, "appmode-busy") + # We wait until the appmode spinner disappears. However, + # this does not seem to be fully robust, as the spinner might flash + # while the page is still loading. So we add explicit sleep here as well. + WebDriverWait(selenium, 240).until( + ec.invisibility_of_element((By.ID, "appmode-busy")) + ) + + return selenium + + return _selenium_driver + + +@pytest.fixture +def final_screenshot(request, screenshot_dir, selenium): + """Take screenshot at the end of the test. + Screenshot name is generated from the test function name + by stripping the 'test_' prefix + """ + screenshot_name = f"{request.function.__name__[5:]}.png" + screenshot_path = Path.joinpath(screenshot_dir, screenshot_name) + yield + selenium.get_screenshot_as_file(screenshot_path) + + +@pytest.fixture +def firefox_options(firefox_options): + firefox_options.add_argument("--headless") + return firefox_options + + +@pytest.fixture +def chrome_options(chrome_options): + chrome_options.add_argument("--headless") + return chrome_options diff --git a/tests_notebooks/docker-compose.yml b/tests_notebooks/docker-compose.yml new file mode 100644 index 0000000..3c348e1 --- /dev/null +++ b/tests_notebooks/docker-compose.yml @@ -0,0 +1,18 @@ +--- +version: '3.4' + +services: + + aiidalab: + image: ghcr.io/full-stack:${TAG:-latest} + environment: + RMQHOST: messaging + TZ: Europe/Zurich + DOCKER_STACKS_JUPYTER_CMD: notebook + SETUP_DEFAULT_AIIDA_PROFILE: 'true' + AIIDALAB_DEFAULT_APPS: '' + JUPYTER_TOKEN: ${JUPYTER_TOKEN} + volumes: + - ..:/home/jovyan/apps/home + ports: + - 8998:8888 diff --git a/tests/test_manage_app.py b/tests_notebooks/test_manage_app.py similarity index 50% rename from tests/test_manage_app.py rename to tests_notebooks/test_manage_app.py index cdc6db7..54efafe 100755 --- a/tests/test_manage_app.py +++ b/tests_notebooks/test_manage_app.py @@ -2,14 +2,8 @@ from selenium.webdriver.common.by import By -def test_uninstall_install_widgets_base(selenium, url): +def test_single_app(selenium, url, final_screenshot): selenium.get(url("apps/apps/home/single_app.ipynb?app=aiidalab-widgets-base")) selenium.set_window_size(1440, 828) - selenium.find_element(By.XPATH, "//button[contains(.,'Uninstall')]").click() - selenium.get_screenshot_as_file( - "screenshots/manage-app-aiidalab-widgets-base-uninstalled.png" - ) - selenium.find_element(By.XPATH, "//button[contains(.,'Install')]").click() - selenium.get_screenshot_as_file( - "screenshots/manage-app-aiidalab-widgets-base-installed.png" - ) + selenium.find_element(By.XPATH, "//button[contains(.,'Uninstall')]") + selenium.find_element(By.XPATH, "//button[contains(.,'Install')]") diff --git a/tests/test_start.py b/tests_notebooks/test_start.py similarity index 85% rename from tests/test_start.py rename to tests_notebooks/test_start.py index 44e23a2..5afbec7 100755 --- a/tests/test_start.py +++ b/tests_notebooks/test_start.py @@ -14,7 +14,7 @@ def get_new_windows(selenium, timeout=2): handles.update(set(selenium.window_handles).difference(wh_before)) -def test_click_appstore(selenium, url): +def test_click_appstore(selenium, url, final_screenshot): selenium.get(url("apps/apps/home/start.ipynb")) with get_new_windows(selenium) as handles: selenium.find_element(By.CSS_SELECTOR, ".fa-puzzle-piece").click() @@ -29,10 +29,9 @@ def test_click_appstore(selenium, url): selenium.find_element(By.CSS_SELECTOR, ".widget-button:nth-child(1)").click() selenium.find_element(By.CSS_SELECTOR, ".widget-html-content > h1").click() time.sleep(5) - selenium.get_screenshot_as_file("screenshots/app-store.png") -def test_click_help(selenium, url): +def test_click_help(selenium, url, final_screenshot): selenium.get(url("apps/apps/home/start.ipynb")) selenium.set_window_size(1200, 941) with get_new_windows(selenium) as handles: @@ -40,10 +39,9 @@ def test_click_help(selenium, url): assert len(handles) == 1 selenium.switch_to.window(handles.pop()) selenium.find_element(By.CSS_SELECTOR, ".mr-md-2").click() - selenium.get_screenshot_as_file("screenshots/help.png") -def test_click_filemanager(selenium, url): +def test_click_filemanager(selenium, url, final_screenshot): selenium.get(url("apps/apps/home/start.ipynb")) selenium.set_window_size(1200, 941) with get_new_windows(selenium) as handles: @@ -53,4 +51,3 @@ def test_click_filemanager(selenium, url): selenium.find_element(By.LINK_TEXT, "Running").click() selenium.find_element(By.LINK_TEXT, "Clusters").click() selenium.find_element(By.LINK_TEXT, "Files").click() - selenium.get_screenshot_as_file("screenshots/file-manager.png") diff --git a/tests/test_terminal.py b/tests_notebooks/test_terminal.py similarity index 86% rename from tests/test_terminal.py rename to tests_notebooks/test_terminal.py index ef19180..3dda850 100755 --- a/tests/test_terminal.py +++ b/tests_notebooks/test_terminal.py @@ -5,7 +5,7 @@ from selenium.webdriver.common.keys import Keys -def test_terminal(selenium, url): +def test_terminal(selenium, url, final_screenshot): selenium.get(url("apps/apps/home/start.ipynb")) selenium.set_window_size(1575, 907) selenium.find_element(By.CSS_SELECTOR, ".fa-terminal").click() @@ -20,4 +20,3 @@ def test_terminal(selenium, url): Keys.ENTER ) sleep(1) - selenium.get_screenshot_as_file("screenshots/aiidalab-terminal.png")