WIP: Reenable notebook tests

.github/workflows/ci.yml

@@ -3,25 +3,80 @@
 
 name: continuous-integration
 
-on: [push, pull_request]
+on:
+    push:
+        branches:
+            - master
+    pull_request:
+
+env:
+    FORCE_COLOR: "1"
+
+# https://docs.github.com/en/actions/using-jobs/using-concurrency
+concurrency:
+    # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
 
 jobs:
 
-    pre-commit:
+    test-notebooks:
+
+        strategy:
+            matrix:
+                browser: [Chrome, Firefox]
+            fail-fast: false
 
         runs-on: ubuntu-latest
+        timeout-minutes: 30
 
         steps:
-            - uses: actions/checkout@v4
 
-            - name: Setup Python
+            - name: Check out app
+              uses: actions/checkout@v4
+
+            - name: Set up Python
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.11
+                  python-version: '3.11'
+
+            - name: Setup uv
+              uses: astral-sh/setup-uv@v3
+              with:
+                  version: '0.4.20'
 
-            - name: Install dependencies
+            - name: Install package test dependencies
+              # Notebook tests happen in the container, here we only need to install
+              # the pytest-docker dependency. Unfortunately, uv/pip does not allow to
+              # only install [dev] dependencies so we end up installing all the rest as well.
+              run: uv pip install --system .[dev]
+
+            - name: Set jupyter token env
+              run: echo "JUPYTER_TOKEN=$(openssl rand -hex 32)" >> $GITHUB_ENV
+
+            # The Firefox and its engine geckodrive need do be installed manually to run
+            # selenium tests.
+            - name: Install Firefox
+              uses: browser-actions/setup-firefox@latest
+              with:
+                  firefox-version: '96.0'
+              if: matrix.browser == 'Firefox'
+
+            - name: Install geckodriver
               run: |
-                  pip install .[dev]
+                  wget -c https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz
+                  tar xf geckodriver-v0.30.0-linux64.tar.gz -C /usr/local/bin
+              if: matrix.browser == 'Firefox'
+
+            - name: Run pytest
+              run: pytest -v --driver ${{ matrix.browser }} tests_notebooks
+              env:
+                  TAG: edge
 
-            - name: Run pre-commit
-              run: pre-commit run --all-files || ( git status --short ; git diff ; exit 1 )
+            - name: Upload screenshots as artifacts
+              if: always()
+              uses: actions/upload-artifact@v4
+              with:
+                  name: Screenshots-${{ matrix.browser }}
+                  path: screenshots/
+                  if-no-files-found: error

tests_notebooks/conftest.py

@@ -0,0 +1,127 @@
+import os
+from pathlib import Path
+from urllib.parse import urljoin
+
+import pytest
+import requests
+import selenium.webdriver.support.expected_conditions as ec
+from requests.exceptions import ConnectionError
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.wait import WebDriverWait
+
+
+def is_responsive(url):
+    try:
+        response = requests.get(url)
+        if response.status_code == 200:
+            return True
+    except ConnectionError:
+        return False
+
+
+@pytest.fixture(scope="session")
+def screenshot_dir():
+    sdir = Path.joinpath(Path.cwd(), "screenshots")
+    try:
+        os.mkdir(sdir)
+    except FileExistsError:
+        pass
+    return sdir
+
+
+@pytest.fixture(scope="session")
+def docker_compose_file(pytestconfig):
+    return str(Path(pytestconfig.rootdir) / "tests_notebooks" / "docker-compose.yml")
+
+
+@pytest.fixture(scope="session")
+def docker_compose(docker_services):
+    return docker_services._docker_compose
+
+
+@pytest.fixture(scope="session")
+def aiidalab_exec(docker_compose):
+    def execute(command, user=None, **kwargs):
+        workdir = "/home/jovyan/apps/home"
+        if user:
+            command = f"exec --workdir {workdir} -T --user={user} aiidalab {command}"
+        else:
+            command = f"exec --workdir {workdir} -T aiidalab {command}"
+
+        return docker_compose.execute(command, **kwargs)
+
+    return execute
+
+
+@pytest.fixture(scope="session", autouse=True)
+def notebook_service(docker_ip, docker_services, aiidalab_exec):
+    """Ensure that HTTP service is up and responsive."""
+    # Directory ~/apps/home/ is mounted by docker,
+    # make it writeable for jovyan user, needed for `pip install`
+    aiidalab_exec("chmod -R a+rw /home/jovyan/apps/home", user="root")
+
+    aiidalab_exec("pip install --no-cache-dir .")
+
+    # `port_for` takes a container port and returns the corresponding host port
+    port = docker_services.port_for("aiidalab", 8888)
+    url = f"http://{docker_ip}:{port}"
+    token = os.environ["JUPYTER_TOKEN"]
+    docker_services.wait_until_responsive(
+        timeout=30.0, pause=0.1, check=lambda: is_responsive(url)
+    )
+    return url, token
+
+
+@pytest.fixture(scope="function")
+def selenium_driver(selenium, notebook_service):
+    def _selenium_driver(nb_path):
+        url, token = notebook_service
+        url_with_token = urljoin(
+            url, f"apps/apps/home/{nb_path}?token={token}"
+        )
+        selenium.get(f"{url_with_token}")
+        # By default, let's allow selenium functions to retry for 60s
+        # till a given element is loaded, see:
+        # https://selenium-python.readthedocs.io/waits.html#implicit-waits
+        selenium.implicitly_wait(60)
+        window_width = 800
+        window_height = 600
+        selenium.set_window_size(window_width, window_height)
+
+        selenium.find_element(By.ID, "ipython-main-app")
+        selenium.find_element(By.ID, "notebook-container")
+        selenium.find_element(By.ID, "appmode-busy")
+        # We wait until the appmode spinner disappears. However,
+        # this does not seem to be fully robust, as the spinner might flash
+        # while the page is still loading. So we add explicit sleep here as well.
+        WebDriverWait(selenium, 240).until(
+            ec.invisibility_of_element((By.ID, "appmode-busy"))
+        )
+
+        return selenium
+
+    return _selenium_driver
+
+
+@pytest.fixture
+def final_screenshot(request, screenshot_dir, selenium):
+    """Take screenshot at the end of the test.
+    Screenshot name is generated from the test function name
+    by stripping the 'test_' prefix
+    """
+    screenshot_name = f"{request.function.__name__[5:]}.png"
+    screenshot_path = Path.joinpath(screenshot_dir, screenshot_name)
+    yield
+    selenium.get_screenshot_as_file(screenshot_path)
+
+
+@pytest.fixture
+def firefox_options(firefox_options):
+    firefox_options.add_argument("--headless")
+    return firefox_options
+
+
+@pytest.fixture
+def chrome_options(chrome_options):
+    chrome_options.add_argument("--headless")
+    return chrome_options

tests_notebooks/docker-compose.yml

@@ -0,0 +1,18 @@
+---
+version: '3.4'
+
+services:
+
+    aiidalab:
+        image: ghcr.io/full-stack:${TAG:-latest}
+        environment:
+            RMQHOST: messaging
+            TZ: Europe/Zurich
+            DOCKER_STACKS_JUPYTER_CMD: notebook
+            SETUP_DEFAULT_AIIDA_PROFILE: 'true'
+            AIIDALAB_DEFAULT_APPS: ''
+            JUPYTER_TOKEN: ${JUPYTER_TOKEN}
+        volumes:
+            - ..:/home/jovyan/apps/home
+        ports:
+            - 8998:8888

tests/test_manage_app.py → tests_notebooks/test_manage_app.py

@@ -2,14 +2,8 @@
 from selenium.webdriver.common.by import By
 
 
-def test_uninstall_install_widgets_base(selenium, url):
+def test_single_app(selenium, url, final_screenshot):
     selenium.get(url("apps/apps/home/single_app.ipynb?app=aiidalab-widgets-base"))
     selenium.set_window_size(1440, 828)
-    selenium.find_element(By.XPATH, "//button[contains(.,'Uninstall')]").click()
-    selenium.get_screenshot_as_file(
-        "screenshots/manage-app-aiidalab-widgets-base-uninstalled.png"
-    )
-    selenium.find_element(By.XPATH, "//button[contains(.,'Install')]").click()
-    selenium.get_screenshot_as_file(
-        "screenshots/manage-app-aiidalab-widgets-base-installed.png"
-    )
+    selenium.find_element(By.XPATH, "//button[contains(.,'Uninstall')]")
+    selenium.find_element(By.XPATH, "//button[contains(.,'Install')]")

tests/test_start.py → tests_notebooks/test_start.py

@@ -14,7 +14,7 @@ def get_new_windows(selenium, timeout=2):
     handles.update(set(selenium.window_handles).difference(wh_before))
 
 
-def test_click_appstore(selenium, url):
+def test_click_appstore(selenium, url, final_screenshot):
     selenium.get(url("apps/apps/home/start.ipynb"))
     with get_new_windows(selenium) as handles:
         selenium.find_element(By.CSS_SELECTOR, ".fa-puzzle-piece").click()
@@ -29,21 +29,19 @@ def test_click_appstore(selenium, url):
     selenium.find_element(By.CSS_SELECTOR, ".widget-button:nth-child(1)").click()
     selenium.find_element(By.CSS_SELECTOR, ".widget-html-content > h1").click()
     time.sleep(5)
-    selenium.get_screenshot_as_file("screenshots/app-store.png")
 
 
-def test_click_help(selenium, url):
+def test_click_help(selenium, url, final_screenshot):
     selenium.get(url("apps/apps/home/start.ipynb"))
     selenium.set_window_size(1200, 941)
     with get_new_windows(selenium) as handles:
         selenium.find_element(By.CSS_SELECTOR, ".fa-question").click()
     assert len(handles) == 1
     selenium.switch_to.window(handles.pop())
     selenium.find_element(By.CSS_SELECTOR, ".mr-md-2").click()
-    selenium.get_screenshot_as_file("screenshots/help.png")
 
 
-def test_click_filemanager(selenium, url):
+def test_click_filemanager(selenium, url, final_screenshot):
     selenium.get(url("apps/apps/home/start.ipynb"))
     selenium.set_window_size(1200, 941)
     with get_new_windows(selenium) as handles:
@@ -53,4 +51,3 @@ def test_click_filemanager(selenium, url):
     selenium.find_element(By.LINK_TEXT, "Running").click()
     selenium.find_element(By.LINK_TEXT, "Clusters").click()
     selenium.find_element(By.LINK_TEXT, "Files").click()
-    selenium.get_screenshot_as_file("screenshots/file-manager.png")

tests/test_terminal.py → tests_notebooks/test_terminal.py

@@ -5,7 +5,7 @@
 from selenium.webdriver.common.keys import Keys
 
 
-def test_terminal(selenium, url):
+def test_terminal(selenium, url, final_screenshot):
     selenium.get(url("apps/apps/home/start.ipynb"))
     selenium.set_window_size(1575, 907)
     selenium.find_element(By.CSS_SELECTOR, ".fa-terminal").click()
@@ -20,4 +20,3 @@ def test_terminal(selenium, url):
         Keys.ENTER
     )
     sleep(1)
-    selenium.get_screenshot_as_file("screenshots/aiidalab-terminal.png")