From 213c4a23b3fa801abbbcbccbf5fa57aee10f3f39 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Tue, 29 Jul 2025 16:59:12 +0100 Subject: [PATCH 1/3] wip --- examples/package.json | 6 +- examples/src/ai_agent.ts | 71 +++++++ plugins/aisdk/friction_log.md | 0 plugins/aisdk/package.json | 57 ++++++ plugins/aisdk/src/index.ts | 5 + plugins/aisdk/src/llm.ts | 299 ++++++++++++++++++++++++++++ plugins/aisdk/src/models.ts | 136 +++++++++++++ plugins/aisdk/tsconfig.json | 15 ++ plugins/aisdk/tsup.config.ts | 7 + pnpm-lock.yaml | 354 ++++++++++++++-------------------- 10 files changed, 740 insertions(+), 210 deletions(-) create mode 100644 examples/src/ai_agent.ts create mode 100644 plugins/aisdk/friction_log.md create mode 100644 plugins/aisdk/package.json create mode 100644 plugins/aisdk/src/index.ts create mode 100644 plugins/aisdk/src/llm.ts create mode 100644 plugins/aisdk/src/models.ts create mode 100644 plugins/aisdk/tsconfig.json create mode 100644 plugins/aisdk/tsup.config.ts diff --git a/examples/package.json b/examples/package.json index a8768c62c..7a399a602 100644 --- a/examples/package.json +++ b/examples/package.json @@ -21,9 +21,13 @@ "@livekit/agents-plugin-openai": "workspace:*", "@livekit/agents-plugin-silero": "workspace:*", "@livekit/agents-plugin-livekit": "workspace:*", + "@livekit/agents-plugin-aisdk": "workspace:*", + "@ai-sdk/mistral": "^2.0.0-beta.10", + "@ai-sdk/openai": "^2.0.0-beta.10", "livekit-server-sdk": "^2.9.2", "@livekit/rtc-node": "^0.13.11", - "zod": "^3.23.8" + "zod": "^3.23.8", + "dotenv": "^16.4.7" }, "version": null } diff --git a/examples/src/ai_agent.ts b/examples/src/ai_agent.ts new file mode 100644 index 000000000..6347f5637 --- /dev/null +++ b/examples/src/ai_agent.ts @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: 2025 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// import { mistral } from '@ai-sdk/mistral'; +import { openai } from '@ai-sdk/openai'; +import { + type JobContext, + type JobProcess, + WorkerOptions, + cli, + defineAgent, + metrics, + voice, +} from '@livekit/agents'; +import * as aisdk from '@livekit/agents-plugin-aisdk'; +import * as deepgram from '@livekit/agents-plugin-deepgram'; +import * as elevenlabs from '@livekit/agents-plugin-elevenlabs'; +import * as livekit from '@livekit/agents-plugin-livekit'; +import * as silero from '@livekit/agents-plugin-silero'; +import dotenv from 'dotenv'; +import { fileURLToPath } from 'node:url'; + +dotenv.config(); + +export default defineAgent({ + prewarm: async (proc: JobProcess) => { + proc.userData.vad = await silero.VAD.load(); + }, + entry: async (ctx: JobContext) => { + const agent = new voice.Agent({ + instructions: + "You are a helpful assistant called Karla, you can hear the user's message and respond to it.", + }); + + const vad = ctx.proc.userData.vad! as silero.VAD; + + const session = new voice.AgentSession({ + vad, + stt: new deepgram.STT(), + tts: new elevenlabs.TTS(), + llm: new aisdk.LLM({ + model: openai('gpt-4o-mini'), + prompt: + "You are a helpful assistant called Anton, you can hear the user's message and respond to it.", + }), + + // to use realtime model, replace the stt, llm, tts and vad with the following + // llm: new openai.realtime.RealtimeModel(), + turnDetection: new livekit.turnDetector.EnglishModel(), + }); + + const usageCollector = new metrics.UsageCollector(); + + session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => { + metrics.logMetrics(ev.metrics); + usageCollector.collect(ev.metrics); + }); + + await session.start({ + agent, + room: ctx.room, + }); + + // join the room when agent is ready + await ctx.connect(); + + session.say('Hello, how can I help you today?'); + }, +}); + +cli.runApp(new WorkerOptions({ agent: fileURLToPath(import.meta.url) })); diff --git a/plugins/aisdk/friction_log.md b/plugins/aisdk/friction_log.md new file mode 100644 index 000000000..e69de29bb diff --git a/plugins/aisdk/package.json b/plugins/aisdk/package.json new file mode 100644 index 000000000..3b7a34eb8 --- /dev/null +++ b/plugins/aisdk/package.json @@ -0,0 +1,57 @@ +{ + "name": "@livekit/agents-plugin-aisdk", + "version": "0.9.1", + "description": "AISDK plugin for LiveKit Node Agents", + "main": "dist/index.js", + "require": "dist/index.cjs", + "types": "dist/index.d.ts", + "exports": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "author": "LiveKit", + "type": "module", + "repository": "git@github.com:livekit/agents-js.git", + "license": "Apache-2.0", + "files": [ + "dist", + "src", + "README.md" + ], + "scripts": { + "build": "tsup --onSuccess \"pnpm build:types\"", + "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js", + "clean": "rm -rf dist", + "clean:build": "pnpm clean && pnpm build", + "lint": "eslint -f unix \"src/**/*.{ts,js}\"", + "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript", + "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose" + }, + "devDependencies": { + "@livekit/agents": "workspace:^x", + "@livekit/agents-plugin-silero": "workspace:^x", + "@livekit/agents-plugins-test": "workspace:^x", + "@livekit/rtc-node": "^0.13.12", + "@microsoft/api-extractor": "^7.35.0", + "@types/ws": "^8.5.10", + "tsup": "^8.3.5", + "typescript": "^5.0.0" + }, + "dependencies": { + "@livekit/mutex": "^1.1.1", + "@std/async": "npm:@jsr/std__async@^1.0.13", + "sharp": "^0.33.5", + "ws": "^8.16.0" + }, + "peerDependencies": { + "@livekit/agents": "workspace:^x", + "@livekit/rtc-node": "^0.13.12", + "ai": "^5.0.0-beta.32" + } +} diff --git a/plugins/aisdk/src/index.ts b/plugins/aisdk/src/index.ts new file mode 100644 index 000000000..14f0150c7 --- /dev/null +++ b/plugins/aisdk/src/index.ts @@ -0,0 +1,5 @@ +// SPDX-FileCopyrightText: 2024 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +export * from './models.js'; +export { type LLMOptions, LLM, LLMStream } from './llm.js'; diff --git a/plugins/aisdk/src/llm.ts b/plugins/aisdk/src/llm.ts new file mode 100644 index 000000000..4ed5a4f7f --- /dev/null +++ b/plugins/aisdk/src/llm.ts @@ -0,0 +1,299 @@ +// SPDX-FileCopyrightText: 2024 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { llm, log } from '@livekit/agents'; +import { + type ModelMessage, + type ToolChoice, + type ToolSet, + type UserModelMessage, + streamText, +} from 'ai'; + +export class LLM extends llm.LLM { + #opts: Parameters[0]; + + constructor(...params: Parameters) { + super(); + this.#opts = params[0]; + } + + chat({ + chatCtx, + toolCtx, + toolChoice, + temperature, + n, + parallelToolCalls, + }: { + chatCtx: llm.ChatContext; + toolCtx?: llm.ToolContext | undefined; + toolChoice?: llm.ToolChoice; + temperature?: number | undefined; + n?: number | undefined; + parallelToolCalls?: boolean | undefined; + }): LLMStream { + temperature = temperature || this.#opts.temperature; + const convertedToolChoice = toolChoice + ? this.toLLMToolChoice(toolChoice) + : this.#opts.toolChoice; + + return new LLMStream( + this, + chatCtx, + toolCtx, + this.#opts, + parallelToolCalls, + temperature, + n, + convertedToolChoice, + ); + } + + private toLLMToolChoice(toolChoice: llm.ToolChoice): ToolChoice | undefined { + if (typeof toolChoice === 'string') { + // Direct mapping for string literals + switch (toolChoice) { + case 'auto': + case 'none': + case 'required': + return toolChoice; + default: + return 'auto'; // fallback + } + } + + if (typeof toolChoice === 'object' && toolChoice.type === 'function') { + // Convert agents framework function choice to AI SDK tool choice + return { + type: 'tool', + toolName: toolChoice.function.name, + }; + } + + return 'auto'; // fallback + } +} + +export class LLMStream extends llm.LLMStream { + // Current function call that we're waiting for full completion (args are streamed) + #toolCallId?: string; + #fncName?: string; + #fncRawArguments?: string; + #toolIndex?: number; + #logger = log(); + label = 'openai.LLMStream'; + + constructor( + llm: LLM, + chatCtx: llm.ChatContext, + toolCtx: llm.ToolContext | undefined, + opts: Parameters[0], + parallelToolCalls?: boolean, + temperature?: number, + n?: number, + toolChoice?: ToolChoice, + ) { + super(llm, chatCtx, toolCtx); + this.#run(opts, n, parallelToolCalls, temperature, toolChoice); + } + + async #run( + opts: Parameters[0], + n?: number, + parallelToolCalls?: boolean, + temperature?: number, + toolChoice?: ToolChoice, + ) { + // Convert tools to AI SDK format + const tools = this.toolCtx + ? Object.fromEntries( + Object.entries(this.toolCtx).map(([name, func]) => [ + name, + { + description: func.description, + parameters: func.parameters, + }, + ]), + ) + : undefined; + + try { + // Convert messages to AI SDK format + const messages: ModelMessage[] = this.chatCtx.items.map((item) => { + if (item.type === 'message') { + return { + role: item.role, + content: Array.isArray(item.content) + ? item.content.map((c) => + typeof c === 'string' + ? { type: 'text' as const, text: c } + : c.type === 'image_content' + ? { type: 'image' as const, image: c.image } + : { type: 'text' as const, text: String(c) }, + ) + : item.content, + } as ModelMessage; + } else if (item.type === 'function_call') { + return { + role: 'assistant', + content: '', + toolInvocations: [ + { + toolCallId: item.callId, + toolName: item.name, + args: JSON.parse(item.args), + }, + ], + } as ModelMessage; + } else if (item.type === 'function_call_output') { + return { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: item.callId, + toolName: item.name, + output: { + type: 'text', + value: item.output, + }, + }, + ], + } as ModelMessage; + } + + // Fallback + return { + role: 'user', + content: 'Invalid message type', + } as UserModelMessage; + }); + + const result = streamText({ + model: opts.model, + prompt: opts.prompt, + messages, + tools, + toolChoice, + temperature, + }); + + for await (const part of result.fullStream) { + if (this.abortController.signal.aborted) { + break; + } + switch (part.type) { + case 'start': { + // handle start of stream + break; + } + case 'start-step': { + // handle start of step + break; + } + case 'text-start': { + // handle text start + break; + } + case 'text-delta': { + // handle text delta here + this.queue.put({ + id: part.id, + delta: { + role: 'assistant', + content: part.text, + }, + }); + break; + } + case 'text-end': { + // handle text end + break; + } + case 'reasoning-start': { + // handle reasoning start + break; + } + case 'reasoning-delta': { + // handle reasoning delta here + break; + } + case 'reasoning-end': { + // handle reasoning end + break; + } + case 'source': { + // handle source here + break; + } + case 'file': { + // handle file here + break; + } + case 'tool-call': { + switch (part.toolName) { + case 'cityAttractions': { + // handle tool call here + break; + } + } + break; + } + case 'tool-input-start': { + // handle tool input start + break; + } + case 'tool-input-delta': { + // handle tool input delta + break; + } + case 'tool-input-end': { + // handle tool input end + break; + } + case 'tool-result': { + switch (part.toolName) { + case 'cityAttractions': { + // handle tool result here + break; + } + } + break; + } + case 'tool-error': { + // handle tool error + break; + } + case 'finish-step': { + // handle finish step + break; + } + case 'finish': { + // handle finish here + this.queue.put({ + id: 'finish', + usage: { + completionTokens: part.totalUsage.outputTokens || 0, + promptTokens: part.totalUsage.inputTokens || 0, + totalTokens: part.totalUsage.totalTokens || 0, + promptCachedTokens: part.totalUsage.cachedInputTokens || 0, + }, + }); + break; + } + case 'error': { + // handle error here + break; + } + case 'raw': { + // handle raw value + break; + } + } + } + } finally { + this.queue.close(); + } + } +} diff --git a/plugins/aisdk/src/models.ts b/plugins/aisdk/src/models.ts new file mode 100644 index 000000000..9c0778171 --- /dev/null +++ b/plugins/aisdk/src/models.ts @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: 2024 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +export type ChatModels = + | 'gpt-4o' + | 'gpt-4o-2024-05-13' + | 'gpt-4o-mini' + | 'gpt-4o-mini-2024-07-18' + | 'gpt-4-turbo' + | 'gpt-4-turbo-2024-04-09' + | 'gpt-4-turbo-preview' + | 'gpt-4-0125-preview' + | 'gpt-4-1106-preview' + | 'gpt-4-vision-preview' + | 'gpt-4-1106-vision-preview' + | 'gpt-4' + | 'gpt-4-0314' + | 'gpt-4-0613' + | 'gpt-4-32k' + | 'gpt-4-32k-0314' + | 'gpt-4-32k-0613' + | 'gpt-3.5-turbo' + | 'gpt-3.5-turbo-16k' + | 'gpt-3.5-turbo-0301' + | 'gpt-3.5-turbo-0613' + | 'gpt-3.5-turbo-1106' + | 'gpt-3.5-turbo-16k-0613'; + +export type WhisperModels = 'whisper-1'; + +export type TTSModels = 'tts-1' | 'tts-1-hd' | 'gpt-4o-mini-tts'; + +export type TTSVoices = + | 'alloy' + | 'ash' + | 'ballad' + | 'coral' + | 'echo' + | 'fable' + | 'nova' + | 'onyx' + | 'sage' + | 'shimmer' + | 'verse'; + +// adapters for OpenAI-compatible LLMs, TTSs, STTs + +export type TelnyxChatModels = + | 'meta-llama/Meta-Llama-3.1-8B-Instruct' + | 'meta-llama/Meta-Llama-3.1-70B-Instruct'; + +export type CerebrasChatModels = 'llama3.1-8b' | 'llama3.1-70b'; + +export type PerplexityChatModels = + | 'llama-3.1-sonar-small-128k-online' + | 'llama-3.1-sonar-small-128k-chat' + | 'llama-3.1-sonar-large-128k-online' + | 'llama-3.1-sonar-large-128k-chat' + | 'llama-3.1-8b-instruct' + | 'llama-3.1-70b-instruct'; + +export type GroqChatModels = + | 'llama-3.1-405b-reasoning' + | 'llama-3.1-70b-versatile' + | 'llama-3.1-8b-instant' + | 'llama-3.3-70b-versatile' + | 'llama3-groq-70b-8192-tool-use-preview' + | 'llama3-groq-8b-8192-tool-use-preview' + | 'llama-guard-3-8b' + | 'llama3-70b-8192' + | 'llama3-8b-8192' + | 'mixtral-8x7b-32768' + | 'gemma-7b-it' + | 'gemma2-9b-it'; + +export type GroqAudioModels = + | 'whisper-large-v3' + | 'distil-whisper-large-v3-en' + | 'whisper-large-v3-turbo'; + +export type DeepSeekChatModels = 'deepseek-coder' | 'deepseek-chat'; + +export type TogetherChatModels = + | 'garage-bAInd/Platypus2-70B-instruct' + | 'google/gemma-2-27b-it' + | 'google/gemma-2-9b-it' + | 'google/gemma-2b-it' + | 'google/gemma-7b-it' + | 'lmsys/vicuna-13b-v1.5' + | 'lmsys/vicuna-7b-v1.5' + | 'meta-llama/Llama-2-13b-chat-hf' + | 'meta-llama/Llama-2-70b-chat-hf' + | 'meta-llama/Llama-2-7b-chat-hf' + | 'meta-llama/Llama-3-70b-chat-hf' + | 'meta-llama/Llama-3-8b-chat-hf' + | 'meta-llama/Meta-Llama-3-70B-Instruct-Lite' + | 'meta-llama/Meta-Llama-3-70B-Instruct-Turbo' + | 'meta-llama/Meta-Llama-3-8B-Instruct-Lite' + | 'meta-llama/Meta-Llama-3-8B-Instruct-Turbo' + | 'meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo' + | 'meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo' + | 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo' + | 'mistralai/Mistral-7B-Instruct-v0.1' + | 'mistralai/Mistral-7B-Instruct-v0.2' + | 'mistralai/Mistral-7B-Instruct-v0.3' + | 'mistralai/Mixtral-8x22B-Instruct-v0.1' + | 'mistralai/Mixtral-8x7B-Instruct-v0.1' + | 'openchat/openchat-3.5-1210' + | 'snorkelai/Snorkel-Mistral-PairRM-DPO' + | 'teknium/OpenHermes-2-Mistral-7B' + | 'teknium/OpenHermes-2p5-Mistral-7B' + | 'togethercomputer/Llama-2-7B-32K-Instruct' + | 'togethercomputer/RedPajama-INCITE-7B-Chat' + | 'togethercomputer/RedPajama-INCITE-Chat-3B-v1' + | 'togethercomputer/StripedHyena-Nous-7B' + | 'togethercomputer/alpaca-7b' + | 'upstage/SOLAR-10.7B-Instruct-v1.0' + | 'zero-one-ai/Yi-34B-Chat'; + +export type OctoChatModels = + | 'meta-llama-3-70b-instruct' + | 'meta-llama-3.1-405b-instruct' + | 'meta-llama-3.1-70b-instruct' + | 'meta-llama-3.1-8b-instruct' + | 'mistral-7b-instruct' + | 'mixtral-8x7b-instruct' + | 'wizardlm-2-8x22bllamaguard-2-7b'; + +export type XAIChatModels = 'grok-2' | 'grok-2-mini' | 'grok-2-mini-public' | 'grok-2-public'; + +export type MetaChatModels = + | 'Llama-4-Scout-17B-16E-Instruct-FP8' + | 'Llama-4-Maverick-17B-128E-Instruct-FP8' + | 'Llama-3.3-70B-Instruct' + | 'Llama-3.3-8B-Instruct'; diff --git a/plugins/aisdk/tsconfig.json b/plugins/aisdk/tsconfig.json new file mode 100644 index 000000000..ba59073d9 --- /dev/null +++ b/plugins/aisdk/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.json", + "include": ["./src"], + "compilerOptions": { + // match output dir to input dir. e.g. dist/index instead of dist/src/index + "rootDir": "./src", + "declarationDir": "./dist", + "outDir": "./dist" + }, + "typedocOptions": { + "name": "plugins/agents-plugin-openai", + "entryPointStrategy": "resolve", + "entryPoints": ["src/index.ts"] + } +} diff --git a/plugins/aisdk/tsup.config.ts b/plugins/aisdk/tsup.config.ts new file mode 100644 index 000000000..8ca20961f --- /dev/null +++ b/plugins/aisdk/tsup.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'tsup'; + +import defaults from '../../tsup.config'; + +export default defineConfig({ + ...defaults, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e87b5f9cd..843111fa0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -156,9 +156,18 @@ importers: examples: dependencies: + '@ai-sdk/mistral': + specifier: ^2.0.0-beta.10 + version: 2.0.0-canary.19(zod@3.23.8) + '@ai-sdk/openai': + specifier: ^2.0.0-beta.10 + version: 2.0.0-canary.20(zod@3.23.8) '@livekit/agents': specifier: workspace:* version: link:../agents + '@livekit/agents-plugin-aisdk': + specifier: workspace:* + version: link:../plugins/aisdk '@livekit/agents-plugin-deepgram': specifier: workspace:* version: link:../plugins/deepgram @@ -180,6 +189,9 @@ importers: '@livekit/rtc-node': specifier: ^0.13.11 version: 0.13.13 + dotenv: + specifier: ^16.4.7 + version: 16.6.1 livekit-server-sdk: specifier: ^2.9.2 version: 2.9.2 @@ -197,6 +209,49 @@ importers: specifier: ^5.0.0 version: 5.4.5 + plugins/aisdk: + dependencies: + '@livekit/mutex': + specifier: ^1.1.1 + version: 1.1.1 + '@std/async': + specifier: npm:@jsr/std__async@^1.0.13 + version: '@jsr/std__async@1.0.13' + ai: + specifier: ^5.0.0-beta.32 + version: 5.0.0-canary.24(zod@3.23.8) + sharp: + specifier: ^0.33.5 + version: 0.33.5 + ws: + specifier: ^8.16.0 + version: 8.17.0 + devDependencies: + '@livekit/agents': + specifier: workspace:^x + version: link:../../agents + '@livekit/agents-plugin-silero': + specifier: workspace:^x + version: link:../silero + '@livekit/agents-plugins-test': + specifier: workspace:^x + version: link:../test + '@livekit/rtc-node': + specifier: ^0.13.12 + version: 0.13.13 + '@microsoft/api-extractor': + specifier: ^7.35.0 + version: 7.43.7(@types/node@22.15.30) + '@types/ws': + specifier: ^8.5.10 + version: 8.5.10 + tsup: + specifier: ^8.3.5 + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.19.2)(typescript@5.4.5) + typescript: + specifier: ^5.0.0 + version: 5.4.5 + plugins/cartesia: dependencies: ws: @@ -524,6 +579,28 @@ importers: packages: + '@ai-sdk/mistral@2.0.0-canary.19': + resolution: {integrity: sha512-OjGE7CiZDKJzy+h4u+NxJHnhs7psJqiXDpaT/TIjmuu4Mn5yZghyHAfdNidSzXsQS/Cwfvx2HnMO3sudZeccXw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.24.0 + + '@ai-sdk/openai@2.0.0-canary.20': + resolution: {integrity: sha512-AipaQeOz/nIPTtZLJaqG9sxf8zWqZ1UGLG1QOLhNpWwSBDXPVw5k0cWhLtReuZrL/ncKvL6BrGN9aEZLqcmWAg==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.24.0 + + '@ai-sdk/provider-utils@3.0.0-canary.19': + resolution: {integrity: sha512-4IJw6/wkWYLYfFYPvCs5go0L/sBRZsIRW1l/R6LniF4WjAH2+R4dMbESgBmzx+Z2+W+W6gFeK8dnQByn7vaA/w==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.23.8 + + '@ai-sdk/provider@2.0.0-canary.14': + resolution: {integrity: sha512-aN83hjdjDCyhkOdulwMsxmGb91owS+bCSe6FWg1TEwusNM35vv020nY//Gid/0NdIpVkZJGzAajgCWrnno2zzA==} + engines: {node: '>=18'} + '@ampproject/remapping@2.3.0': resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} engines: {node: '>=6.0.0'} @@ -1393,6 +1470,10 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@opentelemetry/api@1.9.0': + resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} + engines: {node: '>=8.0.0'} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -1431,41 +1512,21 @@ packages: '@protobufjs/utf8@1.1.0': resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} - '@rollup/rollup-android-arm-eabi@4.17.2': - resolution: {integrity: sha512-NM0jFxY8bB8QLkoKxIQeObCaDlJKewVlIEkuyYKm5An1tdVZ966w2+MPQ2l8LBZLjR+SgyV+nRkTIunzOYBMLQ==} - cpu: [arm] - os: [android] - '@rollup/rollup-android-arm-eabi@4.40.0': resolution: {integrity: sha512-+Fbls/diZ0RDerhE8kyC6hjADCXA1K4yVNlH0EYfd2XjyH0UGgzaQ8MlT0pCXAThfxv3QUAczHaL+qSv1E4/Cg==} cpu: [arm] os: [android] - '@rollup/rollup-android-arm64@4.17.2': - resolution: {integrity: sha512-yeX/Usk7daNIVwkq2uGoq2BYJKZY1JfyLTaHO/jaiSwi/lsf8fTFoQW/n6IdAsx5tx+iotu2zCJwz8MxI6D/Bw==} - cpu: [arm64] - os: [android] - '@rollup/rollup-android-arm64@4.40.0': resolution: {integrity: sha512-PPA6aEEsTPRz+/4xxAmaoWDqh67N7wFbgFUJGMnanCFs0TV99M0M8QhhaSCks+n6EbQoFvLQgYOGXxlMGQe/6w==} cpu: [arm64] os: [android] - '@rollup/rollup-darwin-arm64@4.17.2': - resolution: {integrity: sha512-kcMLpE6uCwls023+kknm71ug7MZOrtXo+y5p/tsg6jltpDtgQY1Eq5sGfHcQfb+lfuKwhBmEURDga9N0ol4YPw==} - cpu: [arm64] - os: [darwin] - '@rollup/rollup-darwin-arm64@4.40.0': resolution: {integrity: sha512-GwYOcOakYHdfnjjKwqpTGgn5a6cUX7+Ra2HeNj/GdXvO2VJOOXCiYYlRFU4CubFM67EhbmzLOmACKEfvp3J1kQ==} cpu: [arm64] os: [darwin] - '@rollup/rollup-darwin-x64@4.17.2': - resolution: {integrity: sha512-AtKwD0VEx0zWkL0ZjixEkp5tbNLzX+FCqGG1SvOu993HnSz4qDI6S4kGzubrEJAljpVkhRSlg5bzpV//E6ysTQ==} - cpu: [x64] - os: [darwin] - '@rollup/rollup-darwin-x64@4.40.0': resolution: {integrity: sha512-CoLEGJ+2eheqD9KBSxmma6ld01czS52Iw0e2qMZNpPDlf7Z9mj8xmMemxEucinev4LgHalDPczMyxzbq+Q+EtA==} cpu: [x64] @@ -1481,41 +1542,21 @@ packages: cpu: [x64] os: [freebsd] - '@rollup/rollup-linux-arm-gnueabihf@4.17.2': - resolution: {integrity: sha512-3reX2fUHqN7sffBNqmEyMQVj/CKhIHZd4y631duy0hZqI8Qoqf6lTtmAKvJFYa6bhU95B1D0WgzHkmTg33In0A==} - cpu: [arm] - os: [linux] - '@rollup/rollup-linux-arm-gnueabihf@4.40.0': resolution: {integrity: sha512-y/qUMOpJxBMy8xCXD++jeu8t7kzjlOCkoxxajL58G62PJGBZVl/Gwpm7JK9+YvlB701rcQTzjUZ1JgUoPTnoQA==} cpu: [arm] os: [linux] - '@rollup/rollup-linux-arm-musleabihf@4.17.2': - resolution: {integrity: sha512-uSqpsp91mheRgw96xtyAGP9FW5ChctTFEoXP0r5FAzj/3ZRv3Uxjtc7taRQSaQM/q85KEKjKsZuiZM3GyUivRg==} - cpu: [arm] - os: [linux] - '@rollup/rollup-linux-arm-musleabihf@4.40.0': resolution: {integrity: sha512-GoCsPibtVdJFPv/BOIvBKO/XmwZLwaNWdyD8TKlXuqp0veo2sHE+A/vpMQ5iSArRUz/uaoj4h5S6Pn0+PdhRjg==} cpu: [arm] os: [linux] - '@rollup/rollup-linux-arm64-gnu@4.17.2': - resolution: {integrity: sha512-EMMPHkiCRtE8Wdk3Qhtciq6BndLtstqZIroHiiGzB3C5LDJmIZcSzVtLRbwuXuUft1Cnv+9fxuDtDxz3k3EW2A==} - cpu: [arm64] - os: [linux] - '@rollup/rollup-linux-arm64-gnu@4.40.0': resolution: {integrity: sha512-L5ZLphTjjAD9leJzSLI7rr8fNqJMlGDKlazW2tX4IUF9P7R5TMQPElpH82Q7eNIDQnQlAyiNVfRPfP2vM5Avvg==} cpu: [arm64] os: [linux] - '@rollup/rollup-linux-arm64-musl@4.17.2': - resolution: {integrity: sha512-NMPylUUZ1i0z/xJUIx6VUhISZDRT+uTWpBcjdv0/zkp7b/bQDF+NfnfdzuTiB1G6HTodgoFa93hp0O1xl+/UbA==} - cpu: [arm64] - os: [linux] - '@rollup/rollup-linux-arm64-musl@4.40.0': resolution: {integrity: sha512-ATZvCRGCDtv1Y4gpDIXsS+wfFeFuLwVxyUBSLawjgXK2tRE6fnsQEkE4csQQYWlBlsFztRzCnBvWVfcae/1qxQ==} cpu: [arm64] @@ -1526,21 +1567,11 @@ packages: cpu: [loong64] os: [linux] - '@rollup/rollup-linux-powerpc64le-gnu@4.17.2': - resolution: {integrity: sha512-T19My13y8uYXPw/L/k0JYaX1fJKFT/PWdXiHr8mTbXWxjVF1t+8Xl31DgBBvEKclw+1b00Chg0hxE2O7bTG7GQ==} - cpu: [ppc64] - os: [linux] - '@rollup/rollup-linux-powerpc64le-gnu@4.40.0': resolution: {integrity: sha512-vgXfWmj0f3jAUvC7TZSU/m/cOE558ILWDzS7jBhiCAFpY2WEBn5jqgbqvmzlMjtp8KlLcBlXVD2mkTSEQE6Ixw==} cpu: [ppc64] os: [linux] - '@rollup/rollup-linux-riscv64-gnu@4.17.2': - resolution: {integrity: sha512-BOaNfthf3X3fOWAB+IJ9kxTgPmMqPPH5f5k2DcCsRrBIbWnaJCgX2ll77dV1TdSy9SaXTR5iDXRL8n7AnoP5cg==} - cpu: [riscv64] - os: [linux] - '@rollup/rollup-linux-riscv64-gnu@4.40.0': resolution: {integrity: sha512-uJkYTugqtPZBS3Z136arevt/FsKTF/J9dEMTX/cwR7lsAW4bShzI2R0pJVw+hcBTWF4dxVckYh72Hk3/hWNKvA==} cpu: [riscv64] @@ -1551,61 +1582,31 @@ packages: cpu: [riscv64] os: [linux] - '@rollup/rollup-linux-s390x-gnu@4.17.2': - resolution: {integrity: sha512-W0UP/x7bnn3xN2eYMql2T/+wpASLE5SjObXILTMPUBDB/Fg/FxC+gX4nvCfPBCbNhz51C+HcqQp2qQ4u25ok6g==} - cpu: [s390x] - os: [linux] - '@rollup/rollup-linux-s390x-gnu@4.40.0': resolution: {integrity: sha512-SpnYlAfKPOoVsQqmTFJ0usx0z84bzGOS9anAC0AZ3rdSo3snecihbhFTlJZ8XMwzqAcodjFU4+/SM311dqE5Sw==} cpu: [s390x] os: [linux] - '@rollup/rollup-linux-x64-gnu@4.17.2': - resolution: {integrity: sha512-Hy7pLwByUOuyaFC6mAr7m+oMC+V7qyifzs/nW2OJfC8H4hbCzOX07Ov0VFk/zP3kBsELWNFi7rJtgbKYsav9QQ==} - cpu: [x64] - os: [linux] - '@rollup/rollup-linux-x64-gnu@4.40.0': resolution: {integrity: sha512-RcDGMtqF9EFN8i2RYN2W+64CdHruJ5rPqrlYw+cgM3uOVPSsnAQps7cpjXe9be/yDp8UC7VLoCoKC8J3Kn2FkQ==} cpu: [x64] os: [linux] - '@rollup/rollup-linux-x64-musl@4.17.2': - resolution: {integrity: sha512-h1+yTWeYbRdAyJ/jMiVw0l6fOOm/0D1vNLui9iPuqgRGnXA0u21gAqOyB5iHjlM9MMfNOm9RHCQ7zLIzT0x11Q==} - cpu: [x64] - os: [linux] - '@rollup/rollup-linux-x64-musl@4.40.0': resolution: {integrity: sha512-HZvjpiUmSNx5zFgwtQAV1GaGazT2RWvqeDi0hV+AtC8unqqDSsaFjPxfsO6qPtKRRg25SisACWnJ37Yio8ttaw==} cpu: [x64] os: [linux] - '@rollup/rollup-win32-arm64-msvc@4.17.2': - resolution: {integrity: sha512-tmdtXMfKAjy5+IQsVtDiCfqbynAQE/TQRpWdVataHmhMb9DCoJxp9vLcCBjEQWMiUYxO1QprH/HbY9ragCEFLA==} - cpu: [arm64] - os: [win32] - '@rollup/rollup-win32-arm64-msvc@4.40.0': resolution: {integrity: sha512-UtZQQI5k/b8d7d3i9AZmA/t+Q4tk3hOC0tMOMSq2GlMYOfxbesxG4mJSeDp0EHs30N9bsfwUvs3zF4v/RzOeTQ==} cpu: [arm64] os: [win32] - '@rollup/rollup-win32-ia32-msvc@4.17.2': - resolution: {integrity: sha512-7II/QCSTAHuE5vdZaQEwJq2ZACkBpQDOmQsE6D6XUbnBHW8IAhm4eTufL6msLJorzrHDFv3CF8oCA/hSIRuZeQ==} - cpu: [ia32] - os: [win32] - '@rollup/rollup-win32-ia32-msvc@4.40.0': resolution: {integrity: sha512-+m03kvI2f5syIqHXCZLPVYplP8pQch9JHyXKZ3AGMKlg8dCyr2PKHjwRLiW53LTrN/Nc3EqHOKxUxzoSPdKddA==} cpu: [ia32] os: [win32] - '@rollup/rollup-win32-x64-msvc@4.17.2': - resolution: {integrity: sha512-TGGO7v7qOq4CYmSBVEYpI1Y5xDuCEnbVC5Vth8mOsW0gDSzxNrVERPc790IGHsrT2dQSimgMr9Ub3Y1Jci5/8w==} - cpu: [x64] - os: [win32] - '@rollup/rollup-win32-x64-msvc@4.40.0': resolution: {integrity: sha512-lpPE1cLfP5oPzVjKMx10pgBmKELQnFJXHgvtHCtuJWOv8MxqdEIMNtgHgBFf7Ea2/7EuVwa9fodWUfXAlXZLZQ==} cpu: [x64] @@ -1656,6 +1657,9 @@ packages: '@sinclair/typebox@0.27.8': resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==} + '@standard-schema/spec@1.0.0': + resolution: {integrity: sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA==} + '@trivago/prettier-plugin-sort-imports@4.3.0': resolution: {integrity: sha512-r3n0onD3BTOVUNPhR4lhVK4/pABGpbA7bW3eumZnYdKaHkf1qEC+Mag6DPbGNuuh0eG8AaYj+YqmVHSiGslaTQ==} peerDependencies: @@ -1674,9 +1678,6 @@ packages: '@types/deep-eql@4.0.2': resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} - '@types/estree@1.0.5': - resolution: {integrity: sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==} - '@types/estree@1.0.7': resolution: {integrity: sha512-w28IoSUCJpidD/TGviZwwMJckNESJZXFu7NBZ5YJ4mEUnNraUn9Pm8HSZm/jDF1pDWYKspWE7oVphigUPRakIQ==} @@ -1853,6 +1854,12 @@ packages: resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==} engines: {node: '>= 8.0.0'} + ai@5.0.0-canary.24: + resolution: {integrity: sha512-vqaMmM6XFwjz9mNjox9ehjkWFwXbSchhor5MiqgKZ1qRyoTvoYzAt6oCZwg5kN5jXNQ3rZVuyE8N3BbPbwma2Q==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.23.8 + ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} @@ -2201,15 +2208,6 @@ packages: supports-color: optional: true - debug@4.4.0: - resolution: {integrity: sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - debug@4.4.1: resolution: {integrity: sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==} engines: {node: '>=6.0'} @@ -2285,6 +2283,10 @@ packages: resolution: {integrity: sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==} engines: {node: '>=12'} + dotenv@16.6.1: + resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} + engines: {node: '>=12'} + dotenv@8.6.0: resolution: {integrity: sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==} engines: {node: '>=10'} @@ -2617,14 +2619,6 @@ packages: fastq@1.17.1: resolution: {integrity: sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==} - fdir@6.4.3: - resolution: {integrity: sha512-PMXmW2y1hDDfTSRc9gaXIuCCRpuoz3Kaz8cUelp3smouvfT632ozg2vrT6lJsHKKOF59YLbOGfAWGUcKEfRMQw==} - peerDependencies: - picomatch: ^3 || ^4 - peerDependenciesMeta: - picomatch: - optional: true - fdir@6.4.5: resolution: {integrity: sha512-4BG7puHpVsIYxZUbiUE3RqGloLaSSwzYie5jvasC4LWuBWzZawynvYouhjbQKw2JuIGYdm0DzIxl8iVidKlUEw==} peerDependencies: @@ -3802,11 +3796,6 @@ packages: resolution: {integrity: sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==} hasBin: true - rollup@4.17.2: - resolution: {integrity: sha512-/9ClTJPByC0U4zNLowV1tMBe8yMEAxewtR3cUNX5BoEpGH3dQEWpJLr6CLp0fPdYRF/fzVOgvDb1zXuakwF5kQ==} - engines: {node: '>=18.0.0', npm: '>=8.0.0'} - hasBin: true - rollup@4.40.0: resolution: {integrity: sha512-Noe455xmA96nnqH5piFtLobsGbCij7Tu+tb3c1vYjNbTkfzGqXqQXG3wJaYXkRZuQ0vEYN4bhwg7QnIrqB5B+w==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} @@ -4103,10 +4092,6 @@ packages: tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} - tinyglobby@0.2.12: - resolution: {integrity: sha512-qkf4trmKSIiMTs/E63cxH+ojC2unam7rJ0WrauAzpT3ECNTxGRMlaXxVbfxMUC/w0LaYk6jQ4y/nGR9uBO3tww==} - engines: {node: '>=12.0.0'} - tinyglobby@0.2.14: resolution: {integrity: sha512-tX5e7OM1HnYr2+a2C/4V0htOcSQcoSTH9KgJnVvNm5zm/cyEWKJ7j7YutsH9CxMdtOkkLFy2AHrMci9IM8IPZQ==} engines: {node: '>=12.0.0'} @@ -4593,6 +4578,29 @@ packages: snapshots: + '@ai-sdk/mistral@2.0.0-canary.19(zod@3.23.8)': + dependencies: + '@ai-sdk/provider': 2.0.0-canary.14 + '@ai-sdk/provider-utils': 3.0.0-canary.19(zod@3.23.8) + zod: 3.23.8 + + '@ai-sdk/openai@2.0.0-canary.20(zod@3.23.8)': + dependencies: + '@ai-sdk/provider': 2.0.0-canary.14 + '@ai-sdk/provider-utils': 3.0.0-canary.19(zod@3.23.8) + zod: 3.23.8 + + '@ai-sdk/provider-utils@3.0.0-canary.19(zod@3.23.8)': + dependencies: + '@ai-sdk/provider': 2.0.0-canary.14 + '@standard-schema/spec': 1.0.0 + zod: 3.23.8 + zod-to-json-schema: 3.24.6(zod@3.23.8) + + '@ai-sdk/provider@2.0.0-canary.14': + dependencies: + json-schema: 0.4.0 + '@ampproject/remapping@2.3.0': dependencies: '@jridgewell/gen-mapping': 0.3.5 @@ -4713,7 +4721,7 @@ snapshots: outdent: 0.5.0 prettier: 2.8.8 resolve-from: 5.0.0 - semver: 7.6.0 + semver: 7.6.3 '@changesets/assemble-release-plan@6.0.0': dependencies: @@ -4722,7 +4730,7 @@ snapshots: '@changesets/get-dependents-graph': 2.0.0 '@changesets/types': 6.0.0 '@manypkg/get-packages': 1.1.3 - semver: 7.6.0 + semver: 7.6.3 '@changesets/changelog-git@0.2.0': dependencies: @@ -4783,7 +4791,7 @@ snapshots: '@manypkg/get-packages': 1.1.3 chalk: 2.4.2 fs-extra: 7.0.1 - semver: 7.6.0 + semver: 7.6.3 '@changesets/get-github-info@0.5.2': dependencies: @@ -5087,7 +5095,7 @@ snapshots: '@eslint/eslintrc@2.1.4': dependencies: ajv: 6.12.6 - debug: 4.3.4 + debug: 4.4.1 espree: 9.6.1 globals: 13.24.0 ignore: 5.3.1 @@ -5128,7 +5136,7 @@ snapshots: '@humanwhocodes/config-array@0.11.14': dependencies: '@humanwhocodes/object-schema': 2.0.3 - debug: 4.3.4 + debug: 4.4.1 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -5366,6 +5374,8 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.17.1 + '@opentelemetry/api@1.9.0': {} + '@pkgjs/parseargs@0.11.0': optional: true @@ -5394,27 +5404,15 @@ snapshots: '@protobufjs/utf8@1.1.0': {} - '@rollup/rollup-android-arm-eabi@4.17.2': - optional: true - '@rollup/rollup-android-arm-eabi@4.40.0': optional: true - '@rollup/rollup-android-arm64@4.17.2': - optional: true - '@rollup/rollup-android-arm64@4.40.0': optional: true - '@rollup/rollup-darwin-arm64@4.17.2': - optional: true - '@rollup/rollup-darwin-arm64@4.40.0': optional: true - '@rollup/rollup-darwin-x64@4.17.2': - optional: true - '@rollup/rollup-darwin-x64@4.40.0': optional: true @@ -5424,81 +5422,45 @@ snapshots: '@rollup/rollup-freebsd-x64@4.40.0': optional: true - '@rollup/rollup-linux-arm-gnueabihf@4.17.2': - optional: true - '@rollup/rollup-linux-arm-gnueabihf@4.40.0': optional: true - '@rollup/rollup-linux-arm-musleabihf@4.17.2': - optional: true - '@rollup/rollup-linux-arm-musleabihf@4.40.0': optional: true - '@rollup/rollup-linux-arm64-gnu@4.17.2': - optional: true - '@rollup/rollup-linux-arm64-gnu@4.40.0': optional: true - '@rollup/rollup-linux-arm64-musl@4.17.2': - optional: true - '@rollup/rollup-linux-arm64-musl@4.40.0': optional: true '@rollup/rollup-linux-loongarch64-gnu@4.40.0': optional: true - '@rollup/rollup-linux-powerpc64le-gnu@4.17.2': - optional: true - '@rollup/rollup-linux-powerpc64le-gnu@4.40.0': optional: true - '@rollup/rollup-linux-riscv64-gnu@4.17.2': - optional: true - '@rollup/rollup-linux-riscv64-gnu@4.40.0': optional: true '@rollup/rollup-linux-riscv64-musl@4.40.0': optional: true - '@rollup/rollup-linux-s390x-gnu@4.17.2': - optional: true - '@rollup/rollup-linux-s390x-gnu@4.40.0': optional: true - '@rollup/rollup-linux-x64-gnu@4.17.2': - optional: true - '@rollup/rollup-linux-x64-gnu@4.40.0': optional: true - '@rollup/rollup-linux-x64-musl@4.17.2': - optional: true - '@rollup/rollup-linux-x64-musl@4.40.0': optional: true - '@rollup/rollup-win32-arm64-msvc@4.17.2': - optional: true - '@rollup/rollup-win32-arm64-msvc@4.40.0': optional: true - '@rollup/rollup-win32-ia32-msvc@4.17.2': - optional: true - '@rollup/rollup-win32-ia32-msvc@4.40.0': optional: true - '@rollup/rollup-win32-x64-msvc@4.17.2': - optional: true - '@rollup/rollup-win32-x64-msvc@4.40.0': optional: true @@ -5572,6 +5534,8 @@ snapshots: '@sinclair/typebox@0.27.8': {} + '@standard-schema/spec@1.0.0': {} + '@trivago/prettier-plugin-sort-imports@4.3.0(prettier@3.2.5)': dependencies: '@babel/generator': 7.17.7 @@ -5592,8 +5556,6 @@ snapshots: '@types/deep-eql@4.0.2': {} - '@types/estree@1.0.5': {} - '@types/estree@1.0.7': {} '@types/json-schema@7.0.15': {} @@ -5822,6 +5784,13 @@ snapshots: dependencies: humanize-ms: 1.2.1 + ai@5.0.0-canary.24(zod@3.23.8): + dependencies: + '@ai-sdk/provider': 2.0.0-canary.14 + '@ai-sdk/provider-utils': 3.0.0-canary.19(zod@3.23.8) + '@opentelemetry/api': 1.9.0 + zod: 3.23.8 + ajv@6.12.6: dependencies: fast-deep-equal: 3.1.3 @@ -6001,7 +5970,7 @@ snapshots: builtins@5.1.0: dependencies: - semver: 7.6.0 + semver: 7.6.3 bundle-require@5.1.0(esbuild@0.25.2): dependencies: @@ -6194,10 +6163,6 @@ snapshots: dependencies: ms: 2.1.2 - debug@4.4.0: - dependencies: - ms: 2.1.3 - debug@4.4.1: dependencies: ms: 2.1.3 @@ -6257,6 +6222,8 @@ snapshots: dotenv@16.0.3: {} + dotenv@16.6.1: {} + dotenv@8.6.0: {} eastasianwidth@0.2.0: {} @@ -6471,7 +6438,7 @@ snapshots: eslint-compat-utils@0.5.0(eslint@8.57.0): dependencies: eslint: 8.57.0 - semver: 7.6.0 + semver: 7.6.3 eslint-config-next@14.2.3(eslint@8.57.0)(typescript@5.4.5): dependencies: @@ -6517,7 +6484,7 @@ snapshots: eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0): dependencies: - debug: 4.3.4 + debug: 4.4.1 enhanced-resolve: 5.16.1 eslint: 8.57.0 eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) @@ -6735,7 +6702,7 @@ snapshots: estree-walker@3.0.3: dependencies: - '@types/estree': 1.0.5 + '@types/estree': 1.0.7 esutils@2.0.3: {} @@ -6795,10 +6762,6 @@ snapshots: dependencies: reusify: 1.0.4 - fdir@6.4.3(picomatch@4.0.2): - optionalDependencies: - picomatch: 4.0.2 - fdir@6.4.5(picomatch@4.0.2): optionalDependencies: picomatch: 4.0.2 @@ -7996,28 +7959,6 @@ snapshots: dependencies: glob: 10.4.5 - rollup@4.17.2: - dependencies: - '@types/estree': 1.0.5 - optionalDependencies: - '@rollup/rollup-android-arm-eabi': 4.17.2 - '@rollup/rollup-android-arm64': 4.17.2 - '@rollup/rollup-darwin-arm64': 4.17.2 - '@rollup/rollup-darwin-x64': 4.17.2 - '@rollup/rollup-linux-arm-gnueabihf': 4.17.2 - '@rollup/rollup-linux-arm-musleabihf': 4.17.2 - '@rollup/rollup-linux-arm64-gnu': 4.17.2 - '@rollup/rollup-linux-arm64-musl': 4.17.2 - '@rollup/rollup-linux-powerpc64le-gnu': 4.17.2 - '@rollup/rollup-linux-riscv64-gnu': 4.17.2 - '@rollup/rollup-linux-s390x-gnu': 4.17.2 - '@rollup/rollup-linux-x64-gnu': 4.17.2 - '@rollup/rollup-linux-x64-musl': 4.17.2 - '@rollup/rollup-win32-arm64-msvc': 4.17.2 - '@rollup/rollup-win32-ia32-msvc': 4.17.2 - '@rollup/rollup-win32-x64-msvc': 4.17.2 - fsevents: 2.3.3 - rollup@4.40.0: dependencies: '@types/estree': 1.0.7 @@ -8303,7 +8244,7 @@ snapshots: dependencies: '@jridgewell/gen-mapping': 0.3.5 commander: 4.1.1 - glob: 10.3.10 + glob: 10.4.5 lines-and-columns: 1.2.4 mz: 2.7.0 pirates: 4.0.6 @@ -8373,11 +8314,6 @@ snapshots: tinyexec@0.3.2: {} - tinyglobby@0.2.12: - dependencies: - fdir: 6.4.3(picomatch@4.0.2) - picomatch: 4.0.2 - tinyglobby@0.2.14: dependencies: fdir: 6.4.5(picomatch@4.0.2) @@ -8436,7 +8372,7 @@ snapshots: cac: 6.7.14 chokidar: 4.0.3 consola: 3.4.2 - debug: 4.4.0 + debug: 4.4.1 esbuild: 0.25.2 joycon: 3.1.1 picocolors: 1.1.1 @@ -8446,7 +8382,7 @@ snapshots: source-map: 0.8.0-beta.0 sucrase: 3.35.0 tinyexec: 0.3.2 - tinyglobby: 0.2.12 + tinyglobby: 0.2.14 tree-kill: 1.2.2 optionalDependencies: '@microsoft/api-extractor': 7.43.7(@types/node@22.15.30) @@ -8595,9 +8531,9 @@ snapshots: vite-node@1.6.0(@types/node@22.15.30): dependencies: cac: 6.7.14 - debug: 4.3.4 + debug: 4.4.1 pathe: 1.1.2 - picocolors: 1.0.1 + picocolors: 1.1.1 vite: 5.2.11(@types/node@22.15.30) transitivePeerDependencies: - '@types/node' @@ -8630,7 +8566,7 @@ snapshots: dependencies: esbuild: 0.20.2 postcss: 8.4.38 - rollup: 4.17.2 + rollup: 4.40.0 optionalDependencies: '@types/node': 22.15.30 fsevents: 2.3.3 From 0326be94f2be211585329f61ae6f938a79905b41 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Tue, 29 Jul 2025 19:44:46 +0100 Subject: [PATCH 2/3] working --- examples/package.json | 14 +-- examples/src/ai_agent.ts | 3 +- plugins/aisdk/package.json | 2 +- plugins/aisdk/src/index.ts | 3 +- plugins/aisdk/src/llm.ts | 172 ++++++------------------------------ plugins/aisdk/src/models.ts | 136 ---------------------------- 6 files changed, 36 insertions(+), 294 deletions(-) delete mode 100644 plugins/aisdk/src/models.ts diff --git a/examples/package.json b/examples/package.json index 7a399a602..7438139c7 100644 --- a/examples/package.json +++ b/examples/package.json @@ -14,20 +14,20 @@ "typescript": "^5.0.0" }, "dependencies": { + "@ai-sdk/openai": "2.0.0-beta.14", "@livekit/agents": "workspace:*", + "@livekit/agents-plugin-aisdk": "workspace:*", "@livekit/agents-plugin-deepgram": "workspace:*", "@livekit/agents-plugin-elevenlabs": "workspace:*", "@livekit/agents-plugin-google": "workspace:*", + "@livekit/agents-plugin-livekit": "workspace:*", "@livekit/agents-plugin-openai": "workspace:*", "@livekit/agents-plugin-silero": "workspace:*", - "@livekit/agents-plugin-livekit": "workspace:*", - "@livekit/agents-plugin-aisdk": "workspace:*", - "@ai-sdk/mistral": "^2.0.0-beta.10", - "@ai-sdk/openai": "^2.0.0-beta.10", - "livekit-server-sdk": "^2.9.2", "@livekit/rtc-node": "^0.13.11", - "zod": "^3.23.8", - "dotenv": "^16.4.7" + "ai": "5.0.0-beta.32", + "dotenv": "^16.4.7", + "livekit-server-sdk": "^2.9.2", + "zod": "^3.25.59" }, "version": null } diff --git a/examples/src/ai_agent.ts b/examples/src/ai_agent.ts index 6347f5637..e6bec5b22 100644 --- a/examples/src/ai_agent.ts +++ b/examples/src/ai_agent.ts @@ -40,8 +40,7 @@ export default defineAgent({ tts: new elevenlabs.TTS(), llm: new aisdk.LLM({ model: openai('gpt-4o-mini'), - prompt: - "You are a helpful assistant called Anton, you can hear the user's message and respond to it.", + tools: {}, }), // to use realtime model, replace the stt, llm, tts and vad with the following diff --git a/plugins/aisdk/package.json b/plugins/aisdk/package.json index 3b7a34eb8..1219eb101 100644 --- a/plugins/aisdk/package.json +++ b/plugins/aisdk/package.json @@ -52,6 +52,6 @@ "peerDependencies": { "@livekit/agents": "workspace:^x", "@livekit/rtc-node": "^0.13.12", - "ai": "^5.0.0-beta.32" + "ai": "5.0.0-beta.32" } } diff --git a/plugins/aisdk/src/index.ts b/plugins/aisdk/src/index.ts index 14f0150c7..164b9aa8e 100644 --- a/plugins/aisdk/src/index.ts +++ b/plugins/aisdk/src/index.ts @@ -1,5 +1,4 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -export * from './models.js'; -export { type LLMOptions, LLM, LLMStream } from './llm.js'; +export { LLM, LLMStream } from './llm.js'; diff --git a/plugins/aisdk/src/llm.ts b/plugins/aisdk/src/llm.ts index 4ed5a4f7f..47c259bdf 100644 --- a/plugins/aisdk/src/llm.ts +++ b/plugins/aisdk/src/llm.ts @@ -2,13 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 import { llm, log } from '@livekit/agents'; -import { - type ModelMessage, - type ToolChoice, - type ToolSet, - type UserModelMessage, - streamText, -} from 'ai'; +import { type ModelMessage, type ToolChoice, type ToolSet, streamText } from 'ai'; export class LLM extends llm.LLM { #opts: Parameters[0]; @@ -106,75 +100,36 @@ export class LLMStream extends llm.LLMStream { toolChoice?: ToolChoice, ) { // Convert tools to AI SDK format - const tools = this.toolCtx - ? Object.fromEntries( - Object.entries(this.toolCtx).map(([name, func]) => [ - name, - { - description: func.description, - parameters: func.parameters, - }, - ]), - ) - : undefined; try { // Convert messages to AI SDK format - const messages: ModelMessage[] = this.chatCtx.items.map((item) => { - if (item.type === 'message') { - return { - role: item.role, - content: Array.isArray(item.content) - ? item.content.map((c) => - typeof c === 'string' - ? { type: 'text' as const, text: c } - : c.type === 'image_content' - ? { type: 'image' as const, image: c.image } - : { type: 'text' as const, text: String(c) }, - ) - : item.content, - } as ModelMessage; - } else if (item.type === 'function_call') { - return { - role: 'assistant', - content: '', - toolInvocations: [ - { - toolCallId: item.callId, - toolName: item.name, - args: JSON.parse(item.args), - }, - ], - } as ModelMessage; - } else if (item.type === 'function_call_output') { - return { - role: 'tool', - content: [ - { - type: 'tool-result', - toolCallId: item.callId, - toolName: item.name, - output: { - type: 'text', - value: item.output, - }, - }, - ], - } as ModelMessage; - } + // @ts-ignore + const messages: ModelMessage[] = this.chatCtx.items + .map((item) => { + if (item.type === 'message') { + return { + role: item.role === 'developer' ? 'user' : item.role, + content: Array.isArray(item.content) + ? item.content.map((c) => (typeof c === 'string' ? c : '')).join('\n') + : item.content, + } satisfies ModelMessage; + } else if (item.type === 'function_call') { + return { + role: 'assistant', + content: '', + } satisfies ModelMessage; + } else if (item.type === 'function_call_output') { + return undefined; + } + }) + .filter((m) => m !== undefined); - // Fallback - return { - role: 'user', - content: 'Invalid message type', - } as UserModelMessage; - }); + console.log('calling stream text', messages); const result = streamText({ model: opts.model, - prompt: opts.prompt, messages, - tools, + tools: opts.tools, toolChoice, temperature, }); @@ -183,19 +138,8 @@ export class LLMStream extends llm.LLMStream { if (this.abortController.signal.aborted) { break; } + console.log('received part', part.type); switch (part.type) { - case 'start': { - // handle start of stream - break; - } - case 'start-step': { - // handle start of step - break; - } - case 'text-start': { - // handle text start - break; - } case 'text-delta': { // handle text delta here this.queue.put({ @@ -207,68 +151,7 @@ export class LLMStream extends llm.LLMStream { }); break; } - case 'text-end': { - // handle text end - break; - } - case 'reasoning-start': { - // handle reasoning start - break; - } - case 'reasoning-delta': { - // handle reasoning delta here - break; - } - case 'reasoning-end': { - // handle reasoning end - break; - } - case 'source': { - // handle source here - break; - } - case 'file': { - // handle file here - break; - } - case 'tool-call': { - switch (part.toolName) { - case 'cityAttractions': { - // handle tool call here - break; - } - } - break; - } - case 'tool-input-start': { - // handle tool input start - break; - } - case 'tool-input-delta': { - // handle tool input delta - break; - } - case 'tool-input-end': { - // handle tool input end - break; - } - case 'tool-result': { - switch (part.toolName) { - case 'cityAttractions': { - // handle tool result here - break; - } - } - break; - } - case 'tool-error': { - // handle tool error - break; - } - case 'finish-step': { - // handle finish step - break; - } + case 'finish': { // handle finish here this.queue.put({ @@ -284,10 +167,7 @@ export class LLMStream extends llm.LLMStream { } case 'error': { // handle error here - break; - } - case 'raw': { - // handle raw value + console.error('error', part.error); break; } } diff --git a/plugins/aisdk/src/models.ts b/plugins/aisdk/src/models.ts deleted file mode 100644 index 9c0778171..000000000 --- a/plugins/aisdk/src/models.ts +++ /dev/null @@ -1,136 +0,0 @@ -// SPDX-FileCopyrightText: 2024 LiveKit, Inc. -// -// SPDX-License-Identifier: Apache-2.0 - -export type ChatModels = - | 'gpt-4o' - | 'gpt-4o-2024-05-13' - | 'gpt-4o-mini' - | 'gpt-4o-mini-2024-07-18' - | 'gpt-4-turbo' - | 'gpt-4-turbo-2024-04-09' - | 'gpt-4-turbo-preview' - | 'gpt-4-0125-preview' - | 'gpt-4-1106-preview' - | 'gpt-4-vision-preview' - | 'gpt-4-1106-vision-preview' - | 'gpt-4' - | 'gpt-4-0314' - | 'gpt-4-0613' - | 'gpt-4-32k' - | 'gpt-4-32k-0314' - | 'gpt-4-32k-0613' - | 'gpt-3.5-turbo' - | 'gpt-3.5-turbo-16k' - | 'gpt-3.5-turbo-0301' - | 'gpt-3.5-turbo-0613' - | 'gpt-3.5-turbo-1106' - | 'gpt-3.5-turbo-16k-0613'; - -export type WhisperModels = 'whisper-1'; - -export type TTSModels = 'tts-1' | 'tts-1-hd' | 'gpt-4o-mini-tts'; - -export type TTSVoices = - | 'alloy' - | 'ash' - | 'ballad' - | 'coral' - | 'echo' - | 'fable' - | 'nova' - | 'onyx' - | 'sage' - | 'shimmer' - | 'verse'; - -// adapters for OpenAI-compatible LLMs, TTSs, STTs - -export type TelnyxChatModels = - | 'meta-llama/Meta-Llama-3.1-8B-Instruct' - | 'meta-llama/Meta-Llama-3.1-70B-Instruct'; - -export type CerebrasChatModels = 'llama3.1-8b' | 'llama3.1-70b'; - -export type PerplexityChatModels = - | 'llama-3.1-sonar-small-128k-online' - | 'llama-3.1-sonar-small-128k-chat' - | 'llama-3.1-sonar-large-128k-online' - | 'llama-3.1-sonar-large-128k-chat' - | 'llama-3.1-8b-instruct' - | 'llama-3.1-70b-instruct'; - -export type GroqChatModels = - | 'llama-3.1-405b-reasoning' - | 'llama-3.1-70b-versatile' - | 'llama-3.1-8b-instant' - | 'llama-3.3-70b-versatile' - | 'llama3-groq-70b-8192-tool-use-preview' - | 'llama3-groq-8b-8192-tool-use-preview' - | 'llama-guard-3-8b' - | 'llama3-70b-8192' - | 'llama3-8b-8192' - | 'mixtral-8x7b-32768' - | 'gemma-7b-it' - | 'gemma2-9b-it'; - -export type GroqAudioModels = - | 'whisper-large-v3' - | 'distil-whisper-large-v3-en' - | 'whisper-large-v3-turbo'; - -export type DeepSeekChatModels = 'deepseek-coder' | 'deepseek-chat'; - -export type TogetherChatModels = - | 'garage-bAInd/Platypus2-70B-instruct' - | 'google/gemma-2-27b-it' - | 'google/gemma-2-9b-it' - | 'google/gemma-2b-it' - | 'google/gemma-7b-it' - | 'lmsys/vicuna-13b-v1.5' - | 'lmsys/vicuna-7b-v1.5' - | 'meta-llama/Llama-2-13b-chat-hf' - | 'meta-llama/Llama-2-70b-chat-hf' - | 'meta-llama/Llama-2-7b-chat-hf' - | 'meta-llama/Llama-3-70b-chat-hf' - | 'meta-llama/Llama-3-8b-chat-hf' - | 'meta-llama/Meta-Llama-3-70B-Instruct-Lite' - | 'meta-llama/Meta-Llama-3-70B-Instruct-Turbo' - | 'meta-llama/Meta-Llama-3-8B-Instruct-Lite' - | 'meta-llama/Meta-Llama-3-8B-Instruct-Turbo' - | 'meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo' - | 'meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo' - | 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo' - | 'mistralai/Mistral-7B-Instruct-v0.1' - | 'mistralai/Mistral-7B-Instruct-v0.2' - | 'mistralai/Mistral-7B-Instruct-v0.3' - | 'mistralai/Mixtral-8x22B-Instruct-v0.1' - | 'mistralai/Mixtral-8x7B-Instruct-v0.1' - | 'openchat/openchat-3.5-1210' - | 'snorkelai/Snorkel-Mistral-PairRM-DPO' - | 'teknium/OpenHermes-2-Mistral-7B' - | 'teknium/OpenHermes-2p5-Mistral-7B' - | 'togethercomputer/Llama-2-7B-32K-Instruct' - | 'togethercomputer/RedPajama-INCITE-7B-Chat' - | 'togethercomputer/RedPajama-INCITE-Chat-3B-v1' - | 'togethercomputer/StripedHyena-Nous-7B' - | 'togethercomputer/alpaca-7b' - | 'upstage/SOLAR-10.7B-Instruct-v1.0' - | 'zero-one-ai/Yi-34B-Chat'; - -export type OctoChatModels = - | 'meta-llama-3-70b-instruct' - | 'meta-llama-3.1-405b-instruct' - | 'meta-llama-3.1-70b-instruct' - | 'meta-llama-3.1-8b-instruct' - | 'mistral-7b-instruct' - | 'mixtral-8x7b-instruct' - | 'wizardlm-2-8x22bllamaguard-2-7b'; - -export type XAIChatModels = 'grok-2' | 'grok-2-mini' | 'grok-2-mini-public' | 'grok-2-public'; - -export type MetaChatModels = - | 'Llama-4-Scout-17B-16E-Instruct-FP8' - | 'Llama-4-Maverick-17B-128E-Instruct-FP8' - | 'Llama-3.3-70B-Instruct' - | 'Llama-3.3-8B-Instruct'; From 2036a1edbb978caa9b7ac1752e5802b8bb713687 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 30 Jul 2025 13:15:34 +0100 Subject: [PATCH 3/3] bit more stable --- examples/package.json | 3 +- examples/src/ai_agent.ts | 94 +++++++++++++++++++++++++++++-- examples/src/dispatch.ts | 26 +++++++++ examples/src/remove_agent.ts | 23 ++++++++ plugins/aisdk/src/llm.ts | 24 ++++++++ pnpm-lock.yaml | 104 ++++++++++++++++++++--------------- tsconfig.json | 2 +- 7 files changed, 226 insertions(+), 50 deletions(-) create mode 100644 examples/src/dispatch.ts create mode 100644 examples/src/remove_agent.ts diff --git a/examples/package.json b/examples/package.json index 7438139c7..295120ab0 100644 --- a/examples/package.json +++ b/examples/package.json @@ -3,7 +3,6 @@ "name": "livekit-agents-examples", "type": "module", "scripts": { - "build": "tsc", "clean": "rm -rf dist", "clean:build": "pnpm clean && pnpm build", "lint": "eslint -f unix \"src/**/*.ts\"" @@ -14,7 +13,7 @@ "typescript": "^5.0.0" }, "dependencies": { - "@ai-sdk/openai": "2.0.0-beta.14", + "@ai-sdk/mistral": "2.0.0-beta.9", "@livekit/agents": "workspace:*", "@livekit/agents-plugin-aisdk": "workspace:*", "@livekit/agents-plugin-deepgram": "workspace:*", diff --git a/examples/src/ai_agent.ts b/examples/src/ai_agent.ts index e6bec5b22..ba3ff84bd 100644 --- a/examples/src/ai_agent.ts +++ b/examples/src/ai_agent.ts @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 // import { mistral } from '@ai-sdk/mistral'; -import { openai } from '@ai-sdk/openai'; +import { mistral } from '@ai-sdk/mistral'; import { type JobContext, type JobProcess, @@ -19,6 +19,7 @@ import * as livekit from '@livekit/agents-plugin-livekit'; import * as silero from '@livekit/agents-plugin-silero'; import dotenv from 'dotenv'; import { fileURLToPath } from 'node:url'; +import { z } from 'zod'; dotenv.config(); @@ -39,8 +40,93 @@ export default defineAgent({ stt: new deepgram.STT(), tts: new elevenlabs.TTS(), llm: new aisdk.LLM({ - model: openai('gpt-4o-mini'), - tools: {}, + model: mistral('mistral-large-latest'), + tools: { + createChannel: { + description: 'Create a new discord channel with the given name', + inputSchema: z.object({ + channelName: z.string(), + }), + execute: async ({ channelName }: { channelName: string }) => { + try { + const result = await ctx.room.localParticipant?.performRpc({ + method: 'discord.room.create', + payload: channelName, + destinationIdentity: 'discord', + }); + return `Channel ${channelName} created: ${result}`; + } catch (error: any) { + return `Error creating channel ${channelName}: ${error.message}`; + } + }, + }, + deleteChannel: { + description: 'Delete the given discord channel', + inputSchema: z.object({ + channelName: z.string(), + }), + execute: async ({ channelName }: { channelName: string }) => { + try { + const result = await ctx.room.localParticipant?.performRpc({ + method: 'discord.room.delete', + payload: channelName, + destinationIdentity: 'discord', + }); + return `Channel ${channelName} deleted: ${result}`; + } catch (error: any) { + return `Error deleting channel ${channelName}: ${error.message}`; + } + }, + }, + startWhip: { + description: 'Start up ingress whip video stream', + execute: async () => { + try { + const result = await ctx.room.localParticipant?.performRpc({ + method: 'whip.start', + payload: '', + destinationIdentity: 'discord', + }); + return `Whip video stream started: ${result}`; + } catch (error: any) { + return `Error starting whip video stream: ${error.message}`; + } + }, + }, + stopWhip: { + description: 'Stop the ingress whip video stream', + execute: async () => { + try { + const result = await ctx.room.localParticipant?.performRpc({ + method: 'whip.stop', + payload: '', + destinationIdentity: 'discord', + }); + return `Whip video stream stopped: ${result}`; + } catch (error: any) { + return `Error stopping whip video stream: ${error.message}`; + } + }, + }, + kickUserOut: { + description: 'Kick the given user out of the voice channel', + inputSchema: z.object({ + userId: z.string(), + }), + execute: async ({ userId }: { userId: string }) => { + try { + const result = await ctx.room.localParticipant?.performRpc({ + method: 'discord.user.kick', + payload: userId, + destinationIdentity: 'discord', + }); + return `User ${userId} kicked: ${result}`; + } catch (error: any) { + return `Error kicking user ${userId}: ${error.message}`; + } + }, + }, + }, }), // to use realtime model, replace the stt, llm, tts and vad with the following @@ -67,4 +153,4 @@ export default defineAgent({ }, }); -cli.runApp(new WorkerOptions({ agent: fileURLToPath(import.meta.url) })); +cli.runApp(new WorkerOptions({ agent: fileURLToPath(import.meta.url), agentName: 'LiveCord' })); diff --git a/examples/src/dispatch.ts b/examples/src/dispatch.ts new file mode 100644 index 000000000..902b105c7 --- /dev/null +++ b/examples/src/dispatch.ts @@ -0,0 +1,26 @@ +import dotenv from 'dotenv'; +import { AgentDispatchClient } from 'livekit-server-sdk'; + +dotenv.config(); + +const roomName = 'test-room'; +const agentName = 'LiveCord'; + +async function createExplicitDispatch() { + const agentDispatchClient = new AgentDispatchClient( + process.env.LIVEKIT_URL!, + process.env.LIVEKIT_API_KEY!, + process.env.LIVEKIT_API_SECRET!, + ); + + // create a dispatch request for an agent named "test-agent" to join "my-room" + const dispatch = await agentDispatchClient.createDispatch(roomName, agentName, { + // metadata: '{"user_id": "12345"}', + }); + console.log('created dispatch', dispatch); + + const dispatches = await agentDispatchClient.listDispatch(roomName); + console.log(`there are ${dispatches.length} dispatches in ${roomName}`); +} + +createExplicitDispatch(); diff --git a/examples/src/remove_agent.ts b/examples/src/remove_agent.ts new file mode 100644 index 000000000..1e82e64c1 --- /dev/null +++ b/examples/src/remove_agent.ts @@ -0,0 +1,23 @@ +import dotenv from 'dotenv'; +import { AgentDispatchClient } from 'livekit-server-sdk'; + +dotenv.config(); + +const roomName = 'test-room'; + +async function createExplicitDispatch() { + const agentDispatchClient = new AgentDispatchClient( + process.env.LIVEKIT_URL!, + process.env.LIVEKIT_API_KEY!, + process.env.LIVEKIT_API_SECRET!, + ); + + // create a dispatch request for an agent named "test-agent" to join "my-room" + const dispatches = await agentDispatchClient.listDispatch(roomName); + + for (const dispatch of dispatches) { + await agentDispatchClient.deleteDispatch(dispatch.id, roomName); + } +} + +createExplicitDispatch(); diff --git a/plugins/aisdk/src/llm.ts b/plugins/aisdk/src/llm.ts index 47c259bdf..8c882c972 100644 --- a/plugins/aisdk/src/llm.ts +++ b/plugins/aisdk/src/llm.ts @@ -152,6 +152,30 @@ export class LLMStream extends llm.LLMStream { break; } + case 'tool-call': { + // handle tool call here + this.queue.put({ + id: part.toolCallId, + delta: { + role: 'assistant', + content: `_Calling tool ${part.toolName} with input ${JSON.stringify(part.input)}_\n`, + }, + }); + break; + } + + case 'tool-result': { + // handle tool result here + this.queue.put({ + id: part.toolCallId, + delta: { + role: 'assistant', + content: part.output, + }, + }); + break; + } + case 'finish': { // handle finish here this.queue.put({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 843111fa0..1185c7b99 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -157,11 +157,8 @@ importers: examples: dependencies: '@ai-sdk/mistral': - specifier: ^2.0.0-beta.10 - version: 2.0.0-canary.19(zod@3.23.8) - '@ai-sdk/openai': - specifier: ^2.0.0-beta.10 - version: 2.0.0-canary.20(zod@3.23.8) + specifier: 2.0.0-beta.9 + version: 2.0.0-beta.9(zod@3.25.76) '@livekit/agents': specifier: workspace:* version: link:../agents @@ -189,6 +186,9 @@ importers: '@livekit/rtc-node': specifier: ^0.13.11 version: 0.13.13 + ai: + specifier: 5.0.0-beta.32 + version: 5.0.0-beta.32(zod@3.25.76) dotenv: specifier: ^16.4.7 version: 16.6.1 @@ -196,8 +196,8 @@ importers: specifier: ^2.9.2 version: 2.9.2 zod: - specifier: ^3.23.8 - version: 3.23.8 + specifier: ^3.25.59 + version: 3.25.76 devDependencies: '@types/node': specifier: ^22.5.5 @@ -218,8 +218,8 @@ importers: specifier: npm:@jsr/std__async@^1.0.13 version: '@jsr/std__async@1.0.13' ai: - specifier: ^5.0.0-beta.32 - version: 5.0.0-canary.24(zod@3.23.8) + specifier: 5.0.0-beta.32 + version: 5.0.0-beta.32(zod@3.25.76) sharp: specifier: ^0.33.5 version: 0.33.5 @@ -454,7 +454,7 @@ importers: version: '@jsr/std__async@1.0.13' openai: specifier: ^4.91.1 - version: 4.91.1(ws@8.17.0)(zod@3.23.8) + version: 4.91.1(ws@8.17.0)(zod@3.25.76) sharp: specifier: ^0.33.5 version: 0.33.5 @@ -579,26 +579,26 @@ importers: packages: - '@ai-sdk/mistral@2.0.0-canary.19': - resolution: {integrity: sha512-OjGE7CiZDKJzy+h4u+NxJHnhs7psJqiXDpaT/TIjmuu4Mn5yZghyHAfdNidSzXsQS/Cwfvx2HnMO3sudZeccXw==} + '@ai-sdk/gateway@1.0.0-beta.17': + resolution: {integrity: sha512-ikM6U0LHc2K4GLJNVK8diC1H16ewkaXumr3rMbYOaL1zOy8TmuxMSZk8Tj7rqiJTJPjRPmB70bTCMXrmUy30pw==} engines: {node: '>=18'} peerDependencies: - zod: ^3.24.0 + zod: ^3.25.76 || ^4 - '@ai-sdk/openai@2.0.0-canary.20': - resolution: {integrity: sha512-AipaQeOz/nIPTtZLJaqG9sxf8zWqZ1UGLG1QOLhNpWwSBDXPVw5k0cWhLtReuZrL/ncKvL6BrGN9aEZLqcmWAg==} + '@ai-sdk/mistral@2.0.0-beta.9': + resolution: {integrity: sha512-lv6pEjWT0FXh714AK3kbCUYjHkmaPohigUC+7YRV0spfKtduSUVB/lv4MFoan/e1uxomFfZNBe5eL8ofSJB1vA==} engines: {node: '>=18'} peerDependencies: - zod: ^3.24.0 + zod: ^3.25.76 || ^4 - '@ai-sdk/provider-utils@3.0.0-canary.19': - resolution: {integrity: sha512-4IJw6/wkWYLYfFYPvCs5go0L/sBRZsIRW1l/R6LniF4WjAH2+R4dMbESgBmzx+Z2+W+W6gFeK8dnQByn7vaA/w==} + '@ai-sdk/provider-utils@3.0.0-beta.8': + resolution: {integrity: sha512-N8H4HhqLRe7R635i33W2MkNuckGWjC5TdektsBpFUMFncqtZlNFJ8e3LV81Udk00ac8juX57bboUIi8APJ2V/g==} engines: {node: '>=18'} peerDependencies: - zod: ^3.23.8 + zod: ^3.25.76 || ^4 - '@ai-sdk/provider@2.0.0-canary.14': - resolution: {integrity: sha512-aN83hjdjDCyhkOdulwMsxmGb91owS+bCSe6FWg1TEwusNM35vv020nY//Gid/0NdIpVkZJGzAajgCWrnno2zzA==} + '@ai-sdk/provider@2.0.0-beta.1': + resolution: {integrity: sha512-Z8SPncMtS3RsoXITmT7NVwrAq6M44dmw0DoUOYJqNNtCu8iMWuxB8Nxsoqpa0uEEy9R1V1ZThJAXTYgjTUxl3w==} engines: {node: '>=18'} '@ampproject/remapping@2.3.0': @@ -1854,11 +1854,12 @@ packages: resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==} engines: {node: '>= 8.0.0'} - ai@5.0.0-canary.24: - resolution: {integrity: sha512-vqaMmM6XFwjz9mNjox9ehjkWFwXbSchhor5MiqgKZ1qRyoTvoYzAt6oCZwg5kN5jXNQ3rZVuyE8N3BbPbwma2Q==} + ai@5.0.0-beta.32: + resolution: {integrity: sha512-bi2ptrTwQ7N9yRLZWGdY5nc9l4wWFGTN2Oo0fR0bhw/dtVKZ9y75UrqILyLC4Xs4xDrtRdfjBIioLBSW36eyAQ==} engines: {node: '>=18'} + hasBin: true peerDependencies: - zod: ^3.23.8 + zod: ^3.25.76 || ^4 ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} @@ -2568,6 +2569,10 @@ packages: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} + eventsource-parser@3.0.3: + resolution: {integrity: sha512-nVpZkTMM9rF6AQ9gPJpFsNAMt48wIzB5TQgiTLdHiuO8XEDhUgZEhqKlZWXbIzo9VmJ/HvysHqEaVeD5v9TPvA==} + engines: {node: '>=20.0.0'} + execa@8.0.1: resolution: {integrity: sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==} engines: {node: '>=16.17'} @@ -4576,28 +4581,32 @@ packages: zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + snapshots: - '@ai-sdk/mistral@2.0.0-canary.19(zod@3.23.8)': + '@ai-sdk/gateway@1.0.0-beta.17(zod@3.25.76)': dependencies: - '@ai-sdk/provider': 2.0.0-canary.14 - '@ai-sdk/provider-utils': 3.0.0-canary.19(zod@3.23.8) - zod: 3.23.8 + '@ai-sdk/provider': 2.0.0-beta.1 + '@ai-sdk/provider-utils': 3.0.0-beta.8(zod@3.25.76) + zod: 3.25.76 - '@ai-sdk/openai@2.0.0-canary.20(zod@3.23.8)': + '@ai-sdk/mistral@2.0.0-beta.9(zod@3.25.76)': dependencies: - '@ai-sdk/provider': 2.0.0-canary.14 - '@ai-sdk/provider-utils': 3.0.0-canary.19(zod@3.23.8) - zod: 3.23.8 + '@ai-sdk/provider': 2.0.0-beta.1 + '@ai-sdk/provider-utils': 3.0.0-beta.8(zod@3.25.76) + zod: 3.25.76 - '@ai-sdk/provider-utils@3.0.0-canary.19(zod@3.23.8)': + '@ai-sdk/provider-utils@3.0.0-beta.8(zod@3.25.76)': dependencies: - '@ai-sdk/provider': 2.0.0-canary.14 + '@ai-sdk/provider': 2.0.0-beta.1 '@standard-schema/spec': 1.0.0 - zod: 3.23.8 - zod-to-json-schema: 3.24.6(zod@3.23.8) + eventsource-parser: 3.0.3 + zod: 3.25.76 + zod-to-json-schema: 3.24.6(zod@3.25.76) - '@ai-sdk/provider@2.0.0-canary.14': + '@ai-sdk/provider@2.0.0-beta.1': dependencies: json-schema: 0.4.0 @@ -5784,12 +5793,13 @@ snapshots: dependencies: humanize-ms: 1.2.1 - ai@5.0.0-canary.24(zod@3.23.8): + ai@5.0.0-beta.32(zod@3.25.76): dependencies: - '@ai-sdk/provider': 2.0.0-canary.14 - '@ai-sdk/provider-utils': 3.0.0-canary.19(zod@3.23.8) + '@ai-sdk/gateway': 1.0.0-beta.17(zod@3.25.76) + '@ai-sdk/provider': 2.0.0-beta.1 + '@ai-sdk/provider-utils': 3.0.0-beta.8(zod@3.25.76) '@opentelemetry/api': 1.9.0 - zod: 3.23.8 + zod: 3.25.76 ajv@6.12.6: dependencies: @@ -6710,6 +6720,8 @@ snapshots: events@3.3.0: {} + eventsource-parser@3.0.3: {} + execa@8.0.1: dependencies: cross-spawn: 7.0.3 @@ -7588,7 +7600,7 @@ snapshots: platform: 1.3.6 protobufjs: 7.4.0 - openai@4.91.1(ws@8.17.0)(zod@3.23.8): + openai@4.91.1(ws@8.17.0)(zod@3.25.76): dependencies: '@types/node': 18.19.64 '@types/node-fetch': 2.6.11 @@ -7599,7 +7611,7 @@ snapshots: node-fetch: 2.7.0 optionalDependencies: ws: 8.17.0 - zod: 3.23.8 + zod: 3.25.76 transitivePeerDependencies: - encoding @@ -8817,4 +8829,10 @@ snapshots: dependencies: zod: 3.23.8 + zod-to-json-schema@3.24.6(zod@3.25.76): + dependencies: + zod: 3.25.76 + zod@3.23.8: {} + + zod@3.25.76: {} diff --git a/tsconfig.json b/tsconfig.json index e5ade081e..c9b3dd568 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "rootDir": ".", "baseUrl": ".", "target": "es2022", - "types": ["node"], + "types": ["@types/node"], "module": "node16", "declaration": true, "declarationMap": true,