feat: add demo for bridge mode (#27)

* fix: update fixture of playwright demo * feat: add demo for bridge mode * feat: add demo for bridge mode * feat: add 'current tab' demo * feat: add sample code file * chore: use latest version in bridge moe
web-infra-dev · Jan 7, 2025 · 2906d3b · 2906d3b
1 parent 3600fa7
commit 2906d3b
Show file tree

Hide file tree

Showing 13 changed files with 161 additions and 436 deletions.
diff --git a/bridge-mode-demo/.gitignore b/bridge-mode-demo/.gitignore
@@ -0,0 +1,8 @@
+package-lock.json
+.env
+
+# Midscene.js dump files
+midscene_run/midscene-report
+midscene_run/dump-logger
+
+midscene_run/cache
diff --git a/bridge-mode-demo/README.md b/bridge-mode-demo/README.md
@@ -0,0 +1,38 @@
+# Bridge mode demo
+
+This is a demo to show how to use bridge mode to control the page on your desktop Chrome.
+
+## Steps
+
+Install Midscene extension from chrome web store: [Midscene](https://chromewebstore.google.com/detail/midscene/gbldofcpkknbggpkmbdaefngejllnief)
+
+create `.env` file
+
+```shell
+# replace by your own
+OPENAI_API_KEY="YOUR_TOKEN"
+```
+
+install deps
+
+```bash
+npm install 
+```
+
+Run demo to create new tabs. 
+Remember to click the "Allow connection" button from Chrome extension while running.
+
+```bash
+npm run demo
+```
+
+Run demo to connect to the active tab.
+
+```bash
+npm run demo-current-tab
+```
+
+# Reference 
+
+https://midscenejs.com/bridge-mode-by-chrome-extension
+https://midscenejs.com/api
diff --git a/bridge-mode-demo/demo-current-tab.ts b/bridge-mode-demo/demo-current-tab.ts
@@ -0,0 +1,20 @@
+import { AgentOverChromeBridge } from "@midscene/web/bridge-mode";
+import "dotenv/config";
+
+Promise.resolve(
+  (async () => {
+    const agent = new AgentOverChromeBridge();
+
+    // This will connect to **the current active tab** on your desktop Chrome
+    // remember to start your chrome extension, click 'allow connection' button.
+    await agent.connectCurrentTab();
+    // After connected, you can see this log. Otherwise you will get an timeout error.
+    console.log("connected to the active tab !");
+
+    const content = await agent.aiQuery(
+      "what is the title of the page? answer in {title: string}"
+    );
+    console.log(content);
+    await agent.destroy();
+  })()
+);
diff --git a/bridge-mode-demo/demo-new-tab.ts b/bridge-mode-demo/demo-new-tab.ts
@@ -0,0 +1,22 @@
+import { AgentOverChromeBridge } from "@midscene/web/bridge-mode";
+import "dotenv/config";
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+Promise.resolve(
+  (async () => {
+    const agent = new AgentOverChromeBridge();
+
+    // This will connect to a new tab on your desktop Chrome
+    // remember to start your chrome extension, click 'allow connection' button.
+    await agent.connectNewTabWithUrl("https://www.bing.com");
+    // After connected, you can see this log. Otherwise you will get an timeout error.
+    console.log("connected to a new tab !");
+
+    // these are the same as normal Midscene agent
+    await agent.aiAction('type "AI 101" and hit Enter');
+    await sleep(3000);
+
+    await agent.aiAssert("there are some search results");
+    await agent.destroy();
+  })()
+);
diff --git a/bridge-mode-demo/package.json b/bridge-mode-demo/package.json
@@ -0,0 +1,19 @@
+{
+    "name": "bridge-mode-demo",
+    "private": true,
+    "version": "1.0.0",
+    "description": "> quick start",
+    "main": "index.js",
+    "type": "module",
+    "scripts": {
+        "demo": "npx tsx demo-new-tab.ts",
+        "demo-current-tab": "npx tsx demo-current-tab.ts"
+    },
+    "author": "",
+    "license": "MIT",
+    "devDependencies": {
+        "@midscene/web": "latest",
+        "dotenv": "^16.4.5",
+        "tsx": "^4.19.1"
+    }
+}
diff --git a/connectivity-test/tests/connectivity.test.ts b/connectivity-test/tests/connectivity.test.ts
@@ -100,7 +100,7 @@ describe("Use Midscene wrapped OpenAI SDK", () => {
 });
 
 // remove the ".skip" if you want to test Azure OpenAI Service
-describe.skip("Azure OpenAI Service", () => {
+describe.skip("Azure OpenAI Service by ADT Credential", () => {
   it("basic call", async () => {
     // sample code: https://github.com/Azure/azure-sdk-for-js/blob/main/sdk/openai/openai/samples/cookbook/simpleCompletionsPage/app.js
     const scope = process.env.MIDSCENE_AZURE_OPENAI_SCOPE;

diff --git a/playwright-demo/e2e/fixture.ts b/playwright-demo/e2e/fixture.ts
@@ -1,5 +1,5 @@
-import { test as base } from '@playwright/test';
-import type { PlayWrightAiFixtureType } from '@midscene/web';
-import { PlaywrightAiFixture } from '@midscene/web';
+import { test as base } from "@playwright/test";
+import type { PlayWrightAiFixtureType } from "@midscene/web/playwright";
+import { PlaywrightAiFixture } from "@midscene/web/playwright";
 
 export const test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture());
diff --git a/playwright-demo/midscene_run/cache/ebay-search-en.spec.ts(search headphone on ebay).json b/playwright-demo/midscene_run/cache/ebay-search-en.spec.ts(search headphone on ebay).json
@@ -21,21 +21,21 @@
             "actions": [
               {
                 "locate": {
-                  "id": "834f7a74",
-                  "prompt": "the search bar with placeholder text '搜索任何物品'"
+                  "id": "c528580a",
+                  "prompt": "the search bar"
                 },
                 "param": {
                   "value": "Headphones"
                 },
-                "thought": "Input 'Headphones' into the search bar to search for the item.",
+                "thought": "Input 'Headphones' into the search bar to search for headphones.",
                 "type": "Input"
               },
               {
                 "locate": null,
                 "param": {
                   "value": "Enter"
                 },
-                "thought": "Press the Enter key to initiate the search for 'Headphones'.",
+                "thought": "Press the Enter key to initiate the search.",
                 "type": "KeyboardPress"
               }
             ],

diff --git a/playwright-demo/midscene_run/cache/online-order-zh.spec.ts(ai online order).json b/playwright-demo/midscene_run/cache/online-order-zh.spec.ts(ai online order).json
@@ -28,6 +28,14 @@
                 "thought": "Click the language switch button to open the language options.",
                 "type": "Tap"
               },
+              {
+                "locate": null,
+                "param": {
+                  "timeMs": 1000
+                },
+                "thought": "Wait for 1 second to ensure the language options are displayed.",
+                "type": "Sleep"
+              },
               {
                 "locate": null,
                 "param": null,
@@ -37,8 +45,8 @@
             ],
             "error": null,
             "furtherPlan": {
-              "whatHaveDone": "Clicked the language switch button to open the language options.",
-              "whatToDoNext": "Find the '中文' option in the dropdown list and click on it."
+              "whatHaveDone": "Click the language switch button and wait 1s",
+              "whatToDoNext": "find the '中文' option and click on it"
             },
             "taskWillBeAccomplished": false
           }
@@ -53,16 +61,16 @@
               "dpr": 1
             }
           },
-          "prompt": "Find the '中文' option in the dropdown list and click on it.",
+          "prompt": "find the '中文' option and click on it",
           "response": {
             "actions": [
               {
                 "locate": {
                   "id": "9aee8c53",
-                  "prompt": "the '中文' option in the dropdown list"
+                  "prompt": "the '中文' option in the language dropdown"
                 },
                 "param": null,
-                "thought": "Locate and click the '中文' option in the dropdown list.",
+                "thought": "Locate and click the '中文' option in the language dropdown menu.",
                 "type": "Tap"
               }
             ],
@@ -72,40 +80,6 @@
           }
         }
       ]
-    },
-    {
-      "prompt": "向下滚动一屏",
-      "tasks": [
-        {
-          "type": "plan",
-          "pageContext": {
-            "url": "https://heyteavivocity.meuu.online/home",
-            "size": {
-              "width": 400,
-              "height": 905,
-              "dpr": 1
-            }
-          },
-          "prompt": "向下滚动一屏",
-          "response": {
-            "actions": [
-              {
-                "locate": null,
-                "param": {
-                  "direction": "down",
-                  "scrollType": "once",
-                  "distance": null
-                },
-                "thought": "Scroll down one screen to view more content.",
-                "type": "Scroll"
-              }
-            ],
-            "error": null,
-            "furtherPlan": null,
-            "taskWillBeAccomplished": true
-          }
-        }
-      ]
     }
   ]
 }