ai sdk llm plugin

examples/package.json

@@ -3,7 +3,6 @@
   "name": "livekit-agents-examples",
   "type": "module",
   "scripts": {
-    "build": "tsc",
     "clean": "rm -rf dist",
     "clean:build": "pnpm clean && pnpm build",
     "lint": "eslint -f unix \"src/**/*.ts\""
@@ -14,16 +13,20 @@
     "typescript": "^5.0.0"
   },
   "dependencies": {
+    "@ai-sdk/mistral": "2.0.0-beta.9",
     "@livekit/agents": "workspace:*",
+    "@livekit/agents-plugin-aisdk": "workspace:*",
     "@livekit/agents-plugin-deepgram": "workspace:*",
     "@livekit/agents-plugin-elevenlabs": "workspace:*",
     "@livekit/agents-plugin-google": "workspace:*",
+    "@livekit/agents-plugin-livekit": "workspace:*",
     "@livekit/agents-plugin-openai": "workspace:*",
     "@livekit/agents-plugin-silero": "workspace:*",
-    "@livekit/agents-plugin-livekit": "workspace:*",
-    "livekit-server-sdk": "^2.9.2",
     "@livekit/rtc-node": "^0.13.11",
-    "zod": "^3.23.8"
+    "ai": "5.0.0-beta.32",
+    "dotenv": "^16.4.7",
+    "livekit-server-sdk": "^2.9.2",
+    "zod": "^3.25.59"
   },
   "version": null
 }

examples/src/ai_agent.ts

@@ -0,0 +1,156 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+// import { mistral } from '@ai-sdk/mistral';
+import { mistral } from '@ai-sdk/mistral';
+import {
+  type JobContext,
+  type JobProcess,
+  WorkerOptions,
+  cli,
+  defineAgent,
+  metrics,
+  voice,
+} from '@livekit/agents';
+import * as aisdk from '@livekit/agents-plugin-aisdk';
+import * as deepgram from '@livekit/agents-plugin-deepgram';
+import * as elevenlabs from '@livekit/agents-plugin-elevenlabs';
+import * as livekit from '@livekit/agents-plugin-livekit';
+import * as silero from '@livekit/agents-plugin-silero';
+import dotenv from 'dotenv';
+import { fileURLToPath } from 'node:url';
+import { z } from 'zod';
+
+dotenv.config();
+
+export default defineAgent({
+  prewarm: async (proc: JobProcess) => {
+    proc.userData.vad = await silero.VAD.load();
+  },
+  entry: async (ctx: JobContext) => {
+    const agent = new voice.Agent({
+      instructions:
+        "You are a helpful assistant called Karla, you can hear the user's message and respond to it.",
+    });
+
+    const vad = ctx.proc.userData.vad! as silero.VAD;
+
+    const session = new voice.AgentSession({
+      vad,
+      stt: new deepgram.STT(),
+      tts: new elevenlabs.TTS(),
+      llm: new aisdk.LLM({
+        model: mistral('mistral-large-latest'),
+        tools: {
+          createChannel: {
+            description: 'Create a new discord channel with the given name',
+            inputSchema: z.object({
+              channelName: z.string(),
+            }),
+            execute: async ({ channelName }: { channelName: string }) => {
+              try {
+                const result = await ctx.room.localParticipant?.performRpc({
+                  method: 'discord.room.create',
+                  payload: channelName,
+                  destinationIdentity: 'discord',
+                });
+                return `Channel ${channelName} created: ${result}`;
+              } catch (error: any) {
+                return `Error creating channel ${channelName}: ${error.message}`;
+              }
+            },
+          },
+          deleteChannel: {
+            description: 'Delete the given discord channel',
+            inputSchema: z.object({
+              channelName: z.string(),
+            }),
+            execute: async ({ channelName }: { channelName: string }) => {
+              try {
+                const result = await ctx.room.localParticipant?.performRpc({
+                  method: 'discord.room.delete',
+                  payload: channelName,
+                  destinationIdentity: 'discord',
+                });
+                return `Channel ${channelName} deleted: ${result}`;
+              } catch (error: any) {
+                return `Error deleting channel ${channelName}: ${error.message}`;
+              }
+            },
+          },
+          startWhip: {
+            description: 'Start up ingress whip video stream',
+            execute: async () => {
+              try {
+                const result = await ctx.room.localParticipant?.performRpc({
+                  method: 'whip.start',
+                  payload: '',
+                  destinationIdentity: 'discord',
+                });
+                return `Whip video stream started: ${result}`;
+              } catch (error: any) {
+                return `Error starting whip video stream: ${error.message}`;
+              }
+            },
+          },
+          stopWhip: {
+            description: 'Stop the ingress whip video stream',
+            execute: async () => {
+              try {
+                const result = await ctx.room.localParticipant?.performRpc({
+                  method: 'whip.stop',
+                  payload: '',
+                  destinationIdentity: 'discord',
+                });
+                return `Whip video stream stopped: ${result}`;
+              } catch (error: any) {
+                return `Error stopping whip video stream: ${error.message}`;
+              }
+            },
+          },
+          kickUserOut: {
+            description: 'Kick the given user out of the voice channel',
+            inputSchema: z.object({
+              userId: z.string(),
+            }),
+            execute: async ({ userId }: { userId: string }) => {
+              try {
+                const result = await ctx.room.localParticipant?.performRpc({
+                  method: 'discord.user.kick',
+                  payload: userId,
+                  destinationIdentity: 'discord',
+                });
+                return `User ${userId} kicked: ${result}`;
+              } catch (error: any) {
+                return `Error kicking user ${userId}: ${error.message}`;
+              }
+            },
+          },
+        },
+      }),
+
+      // to use realtime model, replace the stt, llm, tts and vad with the following
+      // llm: new openai.realtime.RealtimeModel(),
+      turnDetection: new livekit.turnDetector.EnglishModel(),
+    });
+
+    const usageCollector = new metrics.UsageCollector();
+
+    session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => {
+      metrics.logMetrics(ev.metrics);
+      usageCollector.collect(ev.metrics);
+    });
+
+    await session.start({
+      agent,
+      room: ctx.room,
+    });
+
+    // join the room when agent is ready
+    await ctx.connect();
+
+    session.say('Hello, how can I help you today?');
+  },
+});
+
+cli.runApp(new WorkerOptions({ agent: fileURLToPath(import.meta.url), agentName: 'LiveCord' }));

examples/src/dispatch.ts

@@ -0,0 +1,26 @@
+import dotenv from 'dotenv';
+import { AgentDispatchClient } from 'livekit-server-sdk';
+
+dotenv.config();
+
+const roomName = 'test-room';
+const agentName = 'LiveCord';
+
+async function createExplicitDispatch() {
+  const agentDispatchClient = new AgentDispatchClient(
+    process.env.LIVEKIT_URL!,
+    process.env.LIVEKIT_API_KEY!,
+    process.env.LIVEKIT_API_SECRET!,
+  );
+
+  // create a dispatch request for an agent named "test-agent" to join "my-room"
+  const dispatch = await agentDispatchClient.createDispatch(roomName, agentName, {
+    // metadata: '{"user_id": "12345"}',
+  });
+  console.log('created dispatch', dispatch);
+
+  const dispatches = await agentDispatchClient.listDispatch(roomName);
+  console.log(`there are ${dispatches.length} dispatches in ${roomName}`);
+}
+
+createExplicitDispatch();

examples/src/remove_agent.ts

@@ -0,0 +1,23 @@
+import dotenv from 'dotenv';
+import { AgentDispatchClient } from 'livekit-server-sdk';
+
+dotenv.config();
+
+const roomName = 'test-room';
+
+async function createExplicitDispatch() {
+  const agentDispatchClient = new AgentDispatchClient(
+    process.env.LIVEKIT_URL!,
+    process.env.LIVEKIT_API_KEY!,
+    process.env.LIVEKIT_API_SECRET!,
+  );
+
+  // create a dispatch request for an agent named "test-agent" to join "my-room"
+  const dispatches = await agentDispatchClient.listDispatch(roomName);
+
+  for (const dispatch of dispatches) {
+    await agentDispatchClient.deleteDispatch(dispatch.id, roomName);
+  }
+}
+
+createExplicitDispatch();

plugins/aisdk/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@livekit/agents-plugin-aisdk",
+  "version": "0.9.1",
+  "description": "AISDK plugin for LiveKit Node Agents",
+  "main": "dist/index.js",
+  "require": "dist/index.cjs",
+  "types": "dist/index.d.ts",
+  "exports": {
+    "import": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "require": {
+      "types": "./dist/index.d.cts",
+      "default": "./dist/index.cjs"
+    }
+  },
+  "author": "LiveKit",
+  "type": "module",
+  "repository": "[email protected]:livekit/agents-js.git",
+  "license": "Apache-2.0",
+  "files": [
+    "dist",
+    "src",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsup --onSuccess \"pnpm build:types\"",
+    "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js",
+    "clean": "rm -rf dist",
+    "clean:build": "pnpm clean && pnpm build",
+    "lint": "eslint -f unix \"src/**/*.{ts,js}\"",
+    "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
+    "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
+  },
+  "devDependencies": {
+    "@livekit/agents": "workspace:^x",
+    "@livekit/agents-plugin-silero": "workspace:^x",
+    "@livekit/agents-plugins-test": "workspace:^x",
+    "@livekit/rtc-node": "^0.13.12",
+    "@microsoft/api-extractor": "^7.35.0",
+    "@types/ws": "^8.5.10",
+    "tsup": "^8.3.5",
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "@livekit/mutex": "^1.1.1",
+    "@std/async": "npm:@jsr/std__async@^1.0.13",
+    "sharp": "^0.33.5",
+    "ws": "^8.16.0"
+  },
+  "peerDependencies": {
+    "@livekit/agents": "workspace:^x",
+    "@livekit/rtc-node": "^0.13.12",
+    "ai": "5.0.0-beta.32"
+  }
+}

plugins/aisdk/src/index.ts

@@ -0,0 +1,4 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export { LLM, LLMStream } from './llm.js';