Add Playwright tests for OIDC-aware & OIDC-native (#12252)

* Resolve race condition between opening settings & well-known check in OIDC mode

Signed-off-by: Michael Telatynski <[email protected]>

* Add OIDC-aware and OIDC-native tests using MAS

Signed-off-by: Michael Telatynski <[email protected]>

---------

Signed-off-by: Michael Telatynski <[email protected]>

docs/playwright.md

@@ -85,7 +85,7 @@ to be left with some stray containers if, for example, you terminate a test such
 that the `after()` does not run and also exit Playwright uncleanly. All the containers
 it starts are prefixed, so they are easy to recognise. They can be removed safely.
 
-After each test run, logs from the Synapse instances are saved in `playwright/synapselogs`
+After each test run, logs from the Synapse instances are saved in `playwright/logs/synapse`
 with each instance in a separate directory named after its ID. These logs are removed
 at the start of each test run.
 

playwright/.gitignore

@@ -1,6 +1,6 @@
 /test-results/
 /html-report/
-/synapselogs/
+/logs/
 # Only commit snapshots from Linux
 /snapshots/**/*.png
 !/snapshots/**/*-linux.png

playwright/e2e/oidc/index.ts

@@ -0,0 +1,104 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { API, Messages } from "mailhog";
+import { Page } from "@playwright/test";
+
+import { test as base, expect } from "../../element-web-test";
+import { MatrixAuthenticationService } from "../../plugins/matrix-authentication-service";
+import { StartHomeserverOpts } from "../../plugins/homeserver";
+
+export const test = base.extend<{
+    masPrepare: MatrixAuthenticationService;
+    mas: MatrixAuthenticationService;
+}>({
+    // There's a bit of a chicken and egg problem between MAS & Synapse where they each need to know how to reach each other
+    // so spinning up a MAS is split into the prepare & start stage: prepare mas -> homeserver -> start mas to disentangle this.
+    masPrepare: async ({ context }, use) => {
+        const mas = new MatrixAuthenticationService(context);
+        await mas.prepare();
+        await use(mas);
+    },
+    mas: [
+        async ({ masPrepare: mas, homeserver, mailhog }, use, testInfo) => {
+            await mas.start(homeserver, mailhog.instance);
+            await use(mas);
+            await mas.stop(testInfo);
+        },
+        { auto: true },
+    ],
+    startHomeserverOpts: async ({ masPrepare }, use) => {
+        await use({
+            template: "mas-oidc",
+            variables: {
+                MAS_PORT: masPrepare.port,
+            },
+        });
+    },
+    config: async ({ homeserver, startHomeserverOpts, context }, use) => {
+        const issuer = `http://localhost:${(startHomeserverOpts as StartHomeserverOpts).variables["MAS_PORT"]}/`;
+        const wellKnown = {
+            "m.homeserver": {
+                base_url: homeserver.config.baseUrl,
+            },
+            "org.matrix.msc2965.authentication": {
+                issuer,
+                account: `${issuer}account`,
+            },
+        };
+
+        // Ensure org.matrix.msc2965.authentication is in well-known
+        await context.route("https://localhost/.well-known/matrix/client", async (route) => {
+            await route.fulfill({ json: wellKnown });
+        });
+
+        await use({
+            default_server_config: wellKnown,
+        });
+    },
+});
+
+export { expect };
+
+export async function registerAccountMas(
+    page: Page,
+    mailhog: API,
+    username: string,
+    email: string,
+    password: string,
+): Promise<void> {
+    await expect(page.getByText("Please sign in to continue:")).toBeVisible();
+
+    await page.getByRole("link", { name: "Create Account" }).click();
+    await page.getByRole("textbox", { name: "Username" }).fill(username);
+    await page.getByRole("textbox", { name: "Email address" }).fill(email);
+    await page.getByRole("textbox", { name: "Password", exact: true }).fill(password);
+    await page.getByRole("textbox", { name: "Confirm Password" }).fill(password);
+    await page.getByRole("button", { name: "Continue" }).click();
+
+    let messages: Messages;
+    await expect(async () => {
+        messages = await mailhog.messages();
+        expect(messages.items).toHaveLength(1);
+    }).toPass();
+    expect(messages.items[0].to).toEqual(`${username} <${email}>`);
+    const [code] = messages.items[0].text.match(/(\d{6})/);
+
+    await page.getByRole("textbox", { name: "6-digit code" }).fill(code);
+    await page.getByRole("button", { name: "Continue" }).click();
+    await expect(page.getByText("Allow access to your account?")).toBeVisible();
+    await page.getByRole("button", { name: "Continue" }).click();
+}

playwright/e2e/oidc/oidc-aware.spec.ts

@@ -0,0 +1,42 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { test, expect, registerAccountMas } from ".";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
+
+test.describe("OIDC Aware", () => {
+    test.skip(isDendrite, "does not yet support MAS");
+    test.slow(); // trace recording takes a while here
+
+    test("can register an account and manage it", async ({ context, page, homeserver, mailhog, app }) => {
+        await page.goto("/#/login");
+        await page.getByRole("button", { name: "Continue" }).click();
+        await registerAccountMas(page, mailhog.api, "alice", "[email protected]", "Pa$sW0rD!");
+
+        // Eventually, we should end up at the home screen.
+        await expect(page).toHaveURL(/\/#\/home$/, { timeout: 10000 });
+        await expect(page.getByRole("heading", { name: "Welcome alice", exact: true })).toBeVisible();
+
+        // Open settings and navigate to account management
+        await app.settings.openUserSettings("General");
+        const newPagePromise = context.waitForEvent("page");
+        await page.getByRole("button", { name: "Manage account" }).click();
+
+        // Assert new tab opened
+        const newPage = await newPagePromise;
+        await expect(newPage.getByText("Primary email")).toBeVisible();
+    });
+});

playwright/e2e/oidc/oidc-native.spec.ts

@@ -0,0 +1,74 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { test, expect, registerAccountMas } from ".";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
+
+test.describe("OIDC Native", () => {
+    test.skip(isDendrite, "does not yet support MAS");
+    test.slow(); // trace recording takes a while here
+
+    test.use({
+        labsFlags: ["feature_oidc_native_flow"],
+    });
+
+    test("can register the oauth2 client and an account", async ({ context, page, homeserver, mailhog, app, mas }) => {
+        const tokenUri = `http://localhost:${mas.port}/oauth2/token`;
+        const tokenApiPromise = page.waitForRequest(
+            (request) => request.url() === tokenUri && request.postDataJSON()["grant_type"] === "authorization_code",
+        );
+
+        await page.goto("/#/login");
+        await page.getByRole("button", { name: "Continue" }).click();
+        await registerAccountMas(page, mailhog.api, "alice", "[email protected]", "Pa$sW0rD!");
+
+        // Eventually, we should end up at the home screen.
+        await expect(page).toHaveURL(/\/#\/home$/, { timeout: 10000 });
+        await expect(page.getByRole("heading", { name: "Welcome alice", exact: true })).toBeVisible();
+
+        const tokenApiRequest = await tokenApiPromise;
+        expect(tokenApiRequest.postDataJSON()["grant_type"]).toBe("authorization_code");
+
+        const deviceId = await page.evaluate<string>(() => window.localStorage.mx_device_id);
+
+        await app.settings.openUserSettings("General");
+        const newPagePromise = context.waitForEvent("page");
+        await page.getByRole("button", { name: "Manage account" }).click();
+        await app.settings.closeDialog();
+
+        // Assert MAS sees the session as OIDC Native
+        const newPage = await newPagePromise;
+        await newPage.getByText("Sessions").click();
+        await newPage.getByText(deviceId).click();
+        await expect(newPage.getByText("Element")).toBeVisible();
+        await expect(newPage.getByText("oauth2_session:")).toBeVisible();
+        await expect(newPage.getByText("http://localhost:8080/")).toBeVisible();
+        await newPage.close();
+
+        // Assert logging out revokes both tokens
+        const revokeUri = `http://localhost:${mas.port}/oauth2/revoke`;
+        const revokeAccessTokenPromise = page.waitForRequest(
+            (request) => request.url() === revokeUri && request.postDataJSON()["token_type_hint"] === "access_token",
+        );
+        const revokeRefreshTokenPromise = page.waitForRequest(
+            (request) => request.url() === revokeUri && request.postDataJSON()["token_type_hint"] === "refresh_token",
+        );
+        const locator = await app.settings.openUserMenu();
+        await locator.getByRole("menuitem", { name: "Sign out", exact: true }).click();
+        await revokeAccessTokenPromise;
+        await revokeRefreshTokenPromise;
+    });
+});

playwright/plugins/homeserver/synapse/index.ts

@@ -134,7 +134,7 @@ export class Synapse implements Homeserver, HomeserverInstance {
     public async stop(): Promise<string[]> {
         if (!this.config) throw new Error("Missing existing synapse instance, did you call stop() before start()?");
         const id = this.config.serverId;
-        const synapseLogsPath = path.join("playwright", "synapselogs", id);
+        const synapseLogsPath = path.join("playwright", "logs", "synapse", id);
         await fse.ensureDir(synapseLogsPath);
         await this.docker.persistLogsToFile({
             stdoutFile: path.join(synapseLogsPath, "stdout.log"),

playwright/plugins/homeserver/synapse/templates/mas-oidc/README.md

@@ -0,0 +1 @@
+A synapse configured with auth delegated to via matrix authentication service