From 429b3c996e76c38619f0a0a14022a91d3304d095 Mon Sep 17 00:00:00 2001 From: Ian Jennings Date: Tue, 2 Sep 2025 19:01:49 -0500 Subject: [PATCH 01/12] linux is back --- agent/index.js | 29 +++++++++---- agent/lib/commands.js | 65 ++++++++++++++++++++--------- agent/lib/redraw.js | 1 + agent/lib/sandbox.js | 2 + agent/lib/system.js | 7 +++- interfaces/cli/lib/base.js | 1 + interfaces/logger.js | 1 + schema.json | 5 ++- testdriver/lifecycle/postrun.yaml | 8 ---- testdriver/lifecycle/prerun.yaml | 11 ----- testdriver/lifecycle/provision.yaml | 12 +++++- 11 files changed, 92 insertions(+), 50 deletions(-) delete mode 100644 testdriver/lifecycle/postrun.yaml delete mode 100644 testdriver/lifecycle/prerun.yaml diff --git a/agent/index.js b/agent/index.js index 0b1d64dc..3e162a99 100755 --- a/agent/index.js +++ b/agent/index.js @@ -1761,15 +1761,24 @@ ${regression} return this.createNewSandbox(); }); - this.saveLastSandboxId(newSandbox.sandbox.instanceId); - let instance = await this.connectToSandboxDirect( - newSandbox.sandbox.instanceId, - true, // always persist by default - ); - this.instance = instance; - await this.renderSandbox(instance, headless); + console.log("New sandbox created:", newSandbox); + + let data = { + resolution: this.config.TD_RESOLUTION, + url: newSandbox.url, + }; + + const encodedData = encodeURIComponent(JSON.stringify(data)); + + // Use the debugger URL instead of the VNC URL + const urlToOpen = `${this.debuggerUrl}?data=${encodedData}`; + + this.emitter.emit(events.showWindow, urlToOpen); + await this.newSession(); await this.runLifecycle("provision"); + + console.log("provision run"); } async start() { @@ -1937,6 +1946,7 @@ Please check your network connection, TD_API_KEY, or the service status.`, async createNewSandbox() { const sandboxConfig = { type: "create", + os: "linux", resolution: this.config.TD_RESOLUTION, ci: this.config.CI, }; @@ -1949,7 +1959,12 @@ Please check your network connection, TD_API_KEY, or the service status.`, sandboxConfig.instanceType = this.sandboxInstance; } + console.log("sending create"); + let instance = await this.sandbox.send(sandboxConfig); + + console.log("instance created", instance); + return instance; } diff --git a/agent/lib/commands.js b/agent/lib/commands.js index 019245a9..9fe889a4 100644 --- a/agent/lib/commands.js +++ b/agent/lib/commands.js @@ -227,17 +227,31 @@ const createCommands = ( switch (direction) { case "up": if (method === "mouse") { - await sandbox.send({ type: "scroll", amount, direction }); + await sandbox.send({ + os: "linux", + type: "scroll", + amount, + direction, + }); } else { - await sandbox.send({ type: "press", keys: ["pageup"] }); + await sandbox.send({ os: "linux", type: "press", keys: ["pageup"] }); } await redraw.wait(2500); break; case "down": if (method === "mouse") { - await sandbox.send({ type: "scroll", amount, direction }); + await sandbox.send({ + os: "linux", + type: "scroll", + amount, + direction, + }); } else { - await sandbox.send({ type: "press", keys: ["pagedown"] }); + await sandbox.send({ + os: "linux", + type: "press", + keys: ["pagedown"], + }); } await redraw.wait(2500); break; @@ -284,7 +298,7 @@ const createCommands = ( x = parseInt(x); y = parseInt(y); - await sandbox.send({ type: "moveMouse", x, y }); + await sandbox.send({ os: "linux", type: "moveMouse", x, y }); emitter.emit(events.mouseMove, { x, y }); @@ -292,17 +306,21 @@ const createCommands = ( if (action !== "hover") { if (action === "click" || action === "left-click") { - await sandbox.send({ type: "leftClick" }); + await sandbox.send({ os: "linux", type: "leftClick" }); } else if (action === "right-click") { - await sandbox.send({ type: "rightClick" }); + await sandbox.send({ os: "linux", type: "rightClick" }); } else if (action === "middle-click") { - await sandbox.send({ type: "middleClick" }); + await sandbox.send({ os: "linux", type: "middleClick" }); } else if (action === "double-click") { - await sandbox.send({ type: "doubleClick" }); + await sandbox.send({ os: "linux", type: "doubleClick" }); } else if (action === "drag-start") { - await sandbox.send({ type: "mousePress", button: "left" }); + await sandbox.send({ os: "linux", type: "mousePress", button: "left" }); } else if (action === "drag-end") { - await sandbox.send({ type: "mouseRelease", button: "left" }); + await sandbox.send({ + os: "linux", + type: "mouseRelease", + button: "left", + }); } emitter.emit(events.mouseClick, { x, y, button, click, double }); @@ -319,7 +337,7 @@ const createCommands = ( x = parseInt(x); y = parseInt(y); - await sandbox.send({ type: "moveMouse", x, y }); + await sandbox.send({ os: "linux", type: "moveMouse", x, y }); await redraw.wait(2500); @@ -419,12 +437,13 @@ const createCommands = ( return true; }, // type a string + os: "linux", type: async (string, delay = 250) => { await redraw.start(); string = string.toString(); - await sandbox.send({ type: "write", text: string, delay }); + await sandbox.send({ os: "linux", type: "write", text: string, delay }); await redraw.wait(5000); return; }, @@ -434,7 +453,7 @@ const createCommands = ( await redraw.start(); // finally, press the keys - await sandbox.send({ type: "press", keys: inputKeys }); + await sandbox.send({ os: "linux", type: "press", keys: inputKeys }); await redraw.wait(5000); @@ -561,11 +580,15 @@ const createCommands = ( if (method === "keyboard") { try { - await sandbox.send({ type: "press", keys: ["f", "ctrl"] }); + await sandbox.send({ + os: "linux", + type: "press", + keys: ["f", "ctrl"], + }); await delay(1000); - await sandbox.send({ type: "write", text }); + await sandbox.send({ os: "linux", type: "write", text }); await redraw.wait(5000); - await sandbox.send({ type: "press", keys: ["escape"] }); + await sandbox.send({ os: "linux", type: "press", keys: ["escape"] }); } catch { throw new MatchError( "Could not find element using browser text search", @@ -689,6 +712,7 @@ const createCommands = ( await redraw.start(); await sandbox.send({ + os: "linux", type: "commands.focus-application", name, }); @@ -705,22 +729,25 @@ const createCommands = ( assert: async (assertion, async = false) => { return await assert(assertion, true, async); }, - exec: async (language, code, timeout, silent = false) => { + exec: async (language = "pwsh", code, timeout, silent = false) => { emitter.emit(events.log.narration, theme.dim(`calling exec...`), true); emitter.emit(events.log.log, code); let plat = system.platform(); - if (language == "pwsh") { + if (language == "pwsh" || language == "sh") { let result = null; result = await sandbox.send({ + os: "linux", type: "commands.run", command: code, timeout, }); + console.log("Exec result:", result); + if (result.out && result.out.returncode !== 0) { throw new MatchError( `Command failed with exit code ${result.out.returncode}: ${result.out.stderr}`, diff --git a/agent/lib/redraw.js b/agent/lib/redraw.js index 25fcce8b..ea929eac 100644 --- a/agent/lib/redraw.js +++ b/agent/lib/redraw.js @@ -69,6 +69,7 @@ const createRedraw = (emitter, system, sandbox) => { async function updateNetwork() { if (sandbox && sandbox.instanceSocketConnected) { let network = await sandbox.send({ + os: "linux", type: "system.network", }); parseNetworkStats( diff --git a/agent/lib/sandbox.js b/agent/lib/sandbox.js index 82ca9b3f..d856c064 100644 --- a/agent/lib/sandbox.js +++ b/agent/lib/sandbox.js @@ -51,6 +51,7 @@ const createSandbox = (emitter, analytics) => { async auth(apiKey) { let reply = await this.send({ type: "authenticate", + os: "linux", apiKey, }); @@ -64,6 +65,7 @@ const createSandbox = (emitter, analytics) => { async connect(sandboxId, persist = false) { let reply = await this.send({ type: "connect", + os: "linux", persist, sandboxId, }); diff --git a/agent/lib/system.js b/agent/lib/system.js index 9dbf7dcb..688b9a05 100644 --- a/agent/lib/system.js +++ b/agent/lib/system.js @@ -7,7 +7,10 @@ const { events } = require("../events.js"); const createSystem = (emitter, sandbox, config) => { const screenshot = async (options) => { - let { base64 } = await sandbox.send({ type: "system.screenshot" }); + let { base64 } = await sandbox.send({ + os: "linux", + type: "system.screenshot", + }); if (!base64) { console.error("Failed to take screenshot"); @@ -110,6 +113,7 @@ const createSystem = (emitter, sandbox, config) => { const activeWin = async () => { // Get Mouse Position from command line let result = await sandbox.send({ + os: "linux", type: "system.get-active-window", }); @@ -119,6 +123,7 @@ const createSystem = (emitter, sandbox, config) => { const getMousePosition = async () => { // Get Mouse Position from command line let result = await sandbox.send({ + os: "linux", type: "system.get-mouse-position", }); diff --git a/interfaces/cli/lib/base.js b/interfaces/cli/lib/base.js index 15289d9a..54409f31 100644 --- a/interfaces/cli/lib/base.js +++ b/interfaces/cli/lib/base.js @@ -42,6 +42,7 @@ class BaseCommand extends Command { message = JSON.stringify(message); } this.agent.sandbox.send({ + os: "linux", type: "output", output: Buffer.from(message).toString("base64"), }); diff --git a/interfaces/logger.js b/interfaces/logger.js index 6aef95ec..5d200d53 100644 --- a/interfaces/logger.js +++ b/interfaces/logger.js @@ -29,6 +29,7 @@ class CustomTransport extends Transport { } this.sandbox.send({ + os: "linux", type: "output", output: Buffer.from(message).toString("base64"), }); diff --git a/schema.json b/schema.json index 1989012d..bd1c3ef4 100644 --- a/schema.json +++ b/schema.json @@ -676,7 +676,8 @@ "type": "string", "enum": [ "js", - "pwsh" + "pwsh", + "sh" ] }, "code": { @@ -1042,4 +1043,4 @@ } } } -} \ No newline at end of file +} diff --git a/testdriver/lifecycle/postrun.yaml b/testdriver/lifecycle/postrun.yaml deleted file mode 100644 index e070afc8..00000000 --- a/testdriver/lifecycle/postrun.yaml +++ /dev/null @@ -1,8 +0,0 @@ -version: 5.1.1 -session: 67f00511acbd9ccac373edf7 -steps: - - prompt: stop dashcam - commands: - - command: exec - lang: pwsh - code: dashcam -t '${TD_THIS_FILE}' -p diff --git a/testdriver/lifecycle/prerun.yaml b/testdriver/lifecycle/prerun.yaml deleted file mode 100644 index cf762b50..00000000 --- a/testdriver/lifecycle/prerun.yaml +++ /dev/null @@ -1,11 +0,0 @@ -version: 6.0.0 -session: 67f00511acbd9ccac373edf7 -steps: - - prompt: start dashcam - commands: - - command: exec - lang: pwsh - code: dashcam track --name=TestDriver --type=application --pattern="C:\Users\testdriver\Documents\testdriver.log" - - command: exec - lang: pwsh - code: dashcam start diff --git a/testdriver/lifecycle/provision.yaml b/testdriver/lifecycle/provision.yaml index 5cb6ed6c..2a5772be 100644 --- a/testdriver/lifecycle/provision.yaml +++ b/testdriver/lifecycle/provision.yaml @@ -4,9 +4,17 @@ steps: - prompt: launch chrome commands: - command: exec - lang: pwsh + lang: sh code: | - Start-Process "C:\Program Files\Google\Chrome\Application\chrome.exe" -ArgumentList "--start-maximized --disable-infobars --disable-fre --no-default-browser-check --no-first-run --guest --load-extension=$(pwd)/node_modules/dashcam-chrome/build", "${TD_WEBSITE}" + google-chrome \ + --start-maximized \ + --disable-infobars \ + --disable-fre \ + --no-default-browser-check \ + --no-first-run \ + --guest \ + "${TD_WEBSITE}" \ + >/dev/null 2>&1 & - command: wait-for-text text: ${TD_WEBSITE} timeout: 60000 From dacba7987541b254b13f572cb988cead83853164 Mon Sep 17 00:00:00 2001 From: Ian Jennings Date: Fri, 17 Oct 2025 02:56:51 -0500 Subject: [PATCH 02/12] base64 encode --- agent/index.js | 2 +- debugger/index.html | 2 +- package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/agent/index.js b/agent/index.js index 3e162a99..bf50eab3 100755 --- a/agent/index.js +++ b/agent/index.js @@ -1768,7 +1768,7 @@ ${regression} url: newSandbox.url, }; - const encodedData = encodeURIComponent(JSON.stringify(data)); + const encodedData = Buffer.from(JSON.stringify(data)).toString('base64'); // Use the debugger URL instead of the VNC URL const urlToOpen = `${this.debuggerUrl}?data=${encodedData}`; diff --git a/debugger/index.html b/debugger/index.html index 21884ef4..22d1fe51 100644 --- a/debugger/index.html +++ b/debugger/index.html @@ -340,7 +340,7 @@ let parsedData; if (data) { try { - parsedData = JSON.parse(decodeURIComponent(data)); + parsedData = JSON.parse(atob(data)); console.log("Data from URL:", parsedData); // You can use parsedData here if needed } catch (error) { diff --git a/package.json b/package.json index 000e4df2..ee6d2fab 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "testdriverai", - "version": "6.0.26", + "version": "6.2.0", "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop", "main": "index.js", "bin": { From b131dcdf246c9f9f7fdbc8e49016278e33f1bc1c Mon Sep 17 00:00:00 2001 From: Ian Jennings Date: Fri, 17 Oct 2025 03:58:35 -0500 Subject: [PATCH 03/12] add credentialless to see if we can get this to load --- debugger/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debugger/index.html b/debugger/index.html index 22d1fe51..370501ce 100644 --- a/debugger/index.html +++ b/debugger/index.html @@ -330,7 +330,7 @@
Click to interact with VM
- +