WIP: 1.0

.github/workflows/ci.yml

@@ -7,6 +7,7 @@ on:
   pull_request:
     branches:
       - main
+      - "1.0"
 
 permissions:
   contents: read

.github/workflows/lint.yml

@@ -7,6 +7,7 @@ on:
   pull_request:
     branches:
       - main
+      - "1.0"
 
 permissions:
   contents: read

.gitignore

@@ -112,6 +112,7 @@ midscene_run/dump
 *.ignore.png
 extension_output
 .cursor
+apps/chrome-extension/web-ext-profile
 packages/android-playground/static/
 packages/ios-playground/static/
 packages/ios/static/

CONTRIBUTING.md

@@ -121,7 +121,7 @@ To change the AI-related code of this repository, you need to create a '.env 'fi
 
 ```
 OPENAI_API_KEY="your_token"
-MIDSCENE_MODEL_NAME="gpt-4o-2024-08-06"
+MIDSCENE_MODEL_NAME="qwen3-vl-plus"
 ```
 
 

README.md

@@ -81,7 +81,7 @@ Read more about [Choose a model](https://midscenejs.com/choose-a-model)
 Midscene will automatically plan the steps and execute them. It may be slower and heavily rely on the quality of the AI model.
 
 ```javascript
-await aiAction('click all the records one by one. If one record contains the text "completed", skip it');
+await aiAct('click all the records one by one. If one record contains the text "completed", skip it');
 ```
 
 ### Workflow Style
@@ -131,6 +131,7 @@ Community projects that extend Midscene.js capabilities:
 
 * [midscene-ios](https://github.com/lhuanyu/midscene-ios) - iOS automation support for Midscene
 * [Midscene-Python](https://github.com/Python51888/Midscene-Python) - Python SDK for Midscene automation
+* [midscene-java](https://github.com/Master-Frank/midscene-java) - Java SDK that brings Midscene automation features to JVM projects
 
 
 ## 📝 Credits

README.zh.md

@@ -81,7 +81,7 @@ Midscene.js 支持视觉语言模型，例如 `Qwen3-VL`、`Doubao-1.6-vision`
 Midscene 会自动规划步骤并执行。它可能较慢，并且深度依赖于 AI 模型的质量。
 
 ```javascript
-await aiAction('click all the records one by one. If one record contains the text "completed", skip it');
+await aiAct('click all the records one by one. If one record contains the text "completed", skip it');
 ```
 
 ### 工作流风格
@@ -133,6 +133,7 @@ for (const record of recordList) {
 
 * [midscene-ios](https://github.com/lhuanyu/midscene-ios) - iOS 设备自动化工具
 * [Midscene-Python](https://github.com/Python51888/Midscene-Python) - Python 版本的 Midscene SDK
+* [midscene-java](https://github.com/Master-Frank/midscene-java) - Java 版本的 Midscene SDK，便于在 JVM 项目中使用自动化能力
 
 ## 📝 致谢
 

apps/android-playground/package.json

@@ -27,12 +27,12 @@
     "socket.io-client": "4.8.1"
   },
   "devDependencies": {
-    "@rsbuild/core": "^1.3.22",
-    "@rsbuild/plugin-less": "^1.2.4",
-    "@rsbuild/plugin-node-polyfill": "1.3.0",
-    "@rsbuild/plugin-react": "^1.3.1",
-    "@rsbuild/plugin-svgr": "^1.1.1",
-    "@rsbuild/plugin-type-check": "1.2.3",
+    "@rsbuild/core": "^1.5.17",
+    "@rsbuild/plugin-less": "^1.5.0",
+    "@rsbuild/plugin-node-polyfill": "1.4.2",
+    "@rsbuild/plugin-react": "^1.4.1",
+    "@rsbuild/plugin-svgr": "^1.2.2",
+    "@rsbuild/plugin-type-check": "1.2.4",
     "@types/react": "^18.3.1",
     "@types/react-dom": "^18.3.1",
     "archiver": "^6.0.0",

apps/android-playground/src/App.tsx

@@ -1,6 +1,10 @@
 import './App.less';
-import { SCRCPY_SERVER_PORT } from '@midscene/shared/constants';
 import {
+  PLAYGROUND_SERVER_PORT,
+  SCRCPY_SERVER_PORT,
+} from '@midscene/shared/constants';
+import {
+  ScreenshotViewer,
   globalThemeConfig,
   safeOverrideAIConfig,
   useEnvConfig,
@@ -17,6 +21,13 @@ import ScrcpyPlayer, {
 
 const { Content } = Layout;
 
+// Helper function to detect if device is remote (IP:Port format)
+const isRemoteDevice = (deviceId: string | null): boolean => {
+  if (!deviceId) return false;
+  // Remote device format: IP:Port (e.g., 192.168.1.10:5555)
+  return /^\d+\.\d+\.\d+\.\d+:\d+$/.test(deviceId);
+};
+
 export default function App() {
   // Device and connection state - now simplified since device is pre-selected
   const [selectedDeviceId, setSelectedDeviceId] = useState<string | null>(null);
@@ -26,6 +37,7 @@ export default function App() {
     `http://localhost:${SCRCPY_SERVER_PORT}`,
   );
   const [isNarrowScreen, setIsNarrowScreen] = useState(false);
+  const [usePollingMode, setUsePollingMode] = useState(false);
 
   // Configuration state
   const { config } = useEnvConfig();
@@ -125,6 +137,11 @@ export default function App() {
     return () => window.removeEventListener('resize', handleResize);
   }, []);
 
+  // Detect if device is remote and switch to polling mode
+  useEffect(() => {
+    setUsePollingMode(isRemoteDevice(selectedDeviceId));
+  }, [selectedDeviceId]);
+
   return (
     <ConfigProvider theme={globalThemeConfig()}>
       {contextHolder}
@@ -157,11 +174,28 @@ export default function App() {
                   selectedDeviceId={selectedDeviceId}
                   scrcpyPlayerRef={scrcpyPlayerRef}
                 />
-                <ScrcpyPlayer
-                  ref={scrcpyPlayerRef}
-                  serverUrl={serverUrl}
-                  autoConnect={connectToDevice}
-                />
+                {!usePollingMode ? (
+                  <ScrcpyPlayer
+                    ref={scrcpyPlayerRef}
+                    serverUrl={serverUrl}
+                    autoConnect={connectToDevice}
+                  />
+                ) : (
+                  <ScreenshotViewer
+                    getScreenshot={() =>
+                      fetch(
+                        `http://localhost:${PLAYGROUND_SERVER_PORT}/screenshot`,
+                      ).then((r) => r.json())
+                    }
+                    getInterfaceInfo={() =>
+                      fetch(
+                        `http://localhost:${PLAYGROUND_SERVER_PORT}/interface-info`,
+                      ).then((r) => r.json())
+                    }
+                    serverOnline={true}
+                    isUserOperating={false}
+                  />
+                )}
               </div>
             </Panel>
           </PanelGroup>

apps/chrome-extension/package.json

@@ -5,7 +5,8 @@
   "type": "module",
   "scripts": {
     "build": "rsbuild build && npm run pack-extension",
-    "dev": "rsbuild dev --open",
+    "dev": "concurrently -k -n build,run \"rsbuild dev\" \"node scripts/wait-for-build.js && web-ext run --config web-ext-config.cjs --source-dir dist --target chromium\"",
+    "dev:simple": "rsbuild dev --open",
     "preview": "rsbuild preview",
     "pack-extension": "node scripts/pack-extension.js"
   },
@@ -18,7 +19,6 @@
     "@midscene/shared": "workspace:*",
     "@midscene/visualizer": "workspace:*",
     "@midscene/web": "workspace:*",
-
     "@types/file-saver": "2.0.7",
     "antd": "^5.21.6",
     "canvas-confetti": "1.9.3",
@@ -32,20 +32,22 @@
     "zustand": "4.5.2"
   },
   "devDependencies": {
-    "@rsbuild/core": "^1.3.22",
-    "@rsbuild/plugin-less": "^1.2.4",
-    "@rsbuild/plugin-node-polyfill": "1.3.0",
-    "@rsbuild/plugin-react": "^1.3.1",
-    "@rsbuild/plugin-svgr": "^1.1.1",
-    "@rsbuild/plugin-type-check": "1.2.3",
+    "@rsbuild/core": "^1.5.17",
+    "@rsbuild/plugin-less": "^1.5.0",
+    "@rsbuild/plugin-node-polyfill": "1.4.2",
+    "@rsbuild/plugin-react": "^1.4.1",
+    "@rsbuild/plugin-svgr": "^1.2.2",
+    "@rsbuild/plugin-type-check": "1.2.4",
     "@tailwindcss/postcss": "4.1.11",
     "@types/chrome": "0.0.279",
     "@types/react": "^18.3.1",
     "@types/react-dom": "^18.3.1",
     "archiver": "^6.0.0",
+    "concurrently": "^8.2.0",
     "less": "^4.2.0",
+    "openai": "6.3.0",
     "tailwindcss": "4.1.11",
     "typescript": "^5.8.3",
-    "openai": "4.81.0"
+    "web-ext": "9.0.0"
   }
 }

apps/chrome-extension/rsbuild.config.ts

@@ -11,7 +11,13 @@ export default defineConfig({
   tools: {
     rspack: {
       watchOptions: {
-        ignored: /\.git/,
+        ignored: [
+          '**/.git/**',
+          '**/web-ext-profile/**',
+          '**/extension_output/**',
+          'dist/**', // Only ignore THIS app's dist folder, not workspace packages
+          '**/node_modules/**',
+        ],
       },
     },
   },

apps/chrome-extension/scripts/wait-for-build.js

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const manifestPath = path.resolve(__dirname, '../dist/manifest.json');
+const indexHtmlPath = path.resolve(__dirname, '../dist/index.html');
+const indexJsPath = path.resolve(__dirname, '../dist/static/js/index.js');
+const popupJsPath = path.resolve(__dirname, '../dist/static/js/popup.js');
+const maxWaitTime = 60000; // 60 seconds
+const checkInterval = 500; // 500ms
+const stabilityWait = 1000; // Wait 1 second after detection to ensure files are stable
+
+let elapsed = 0;
+
+console.log('Waiting for initial build to complete...');
+
+const checkBuildComplete = () => {
+  const manifestExists = fs.existsSync(manifestPath);
+  const indexExists = fs.existsSync(indexHtmlPath);
+  const indexJsExists = fs.existsSync(indexJsPath);
+  const popupJsExists = fs.existsSync(popupJsPath);
+
+  if (manifestExists && indexExists && indexJsExists && popupJsExists) {
+    // Wait a bit more to ensure all files are written
+    console.log('Build files detected, waiting for stability...');
+    setTimeout(() => {
+      // Double check the files still exist
+      if (
+        fs.existsSync(manifestPath) &&
+        fs.existsSync(indexHtmlPath) &&
+        fs.existsSync(indexJsPath) &&
+        fs.existsSync(popupJsPath)
+      ) {
+        console.log('Build complete! Starting web-ext...');
+        process.exit(0);
+      } else {
+        console.log('Build files disappeared, continuing to wait...');
+        setTimeout(checkBuildComplete, checkInterval);
+      }
+    }, stabilityWait);
+    return;
+  }
+
+  elapsed += checkInterval;
+
+  if (elapsed >= maxWaitTime) {
+    console.error('Timeout waiting for build to complete');
+    process.exit(1);
+  }
+
+  setTimeout(checkBuildComplete, checkInterval);
+};
+
+checkBuildComplete();

apps/chrome-extension/src/extension/bridge/index.less

@@ -10,6 +10,58 @@
     line-height: 25px;
   }
 
+  .server-config-section {
+    margin-top: 12px;
+    margin-bottom: 20px;
+
+    .server-config-header {
+      display: flex;
+      align-items: center;
+      gap: 6px;
+      cursor: pointer;
+      user-select: none;
+      padding: 6px 0;
+      transition: opacity 0.2s ease;
+
+      &:hover {
+        opacity: 0.7;
+      }
+    }
+
+    .server-config-arrow {
+      font-size: 10px;
+      color: rgba(0, 0, 0, 0.45);
+      transition: transform 0.2s ease;
+      transform: rotate(-90deg);
+
+      &.expanded {
+        transform: rotate(0deg);
+      }
+    }
+
+    .server-config-title {
+      font-size: 13px;
+      color: rgba(0, 0, 0, 0.65);
+    }
+
+    .server-config-content {
+      margin-top: 12px;
+      padding-left: 16px;
+    }
+
+    .server-config-input {
+      width: 100%;
+    }
+
+    .server-config-hint {
+      display: block;
+      margin-top: 6px;
+      font-size: 12px;
+      color: rgba(0, 0, 0, 0.45);
+      line-height: 1.5;
+    }
+  }
+
   .middle-dialog-area {
     height: calc(100vh - 100px);
     overflow: hidden;
@@ -130,12 +182,6 @@
     font-style: italic;
   }
 
-  .bridge-log-container {
-  }
-
-  .bridge-log-item {
-  }
-
   .bottom-button-container {
     position: absolute;
     bottom: 16px;
@@ -251,15 +297,20 @@
 
     .bottom-status-btn {
       display: flex;
-      flex-direction: column;
+      flex-direction: row;
       align-items: center;
       justify-content: center;
-      position: relative;
+      margin-left: 12px;
 
       .stop-button {
         border: none;
         box-shadow: none;
         background: transparent;
+        display: flex;
+        align-items: center;
+        justify-content: center;
+        gap: 8px;
+        padding: 4px 8px;
 
         &::before {
           content: '';
@@ -268,7 +319,6 @@
           height: 10px;
           background-color: #000;
           border-radius: 2px;
-          vertical-align: middle;
         }
 
         &:hover,