--wip-- [skip ci]

Author: thomasyuill-livekit

app-config.ts

@@ -1,9 +1,10 @@
 import type { AppConfig } from './lib/types';
 
 export const APP_CONFIG_DEFAULTS: AppConfig = {
+  sandboxId: undefined,
+  agentName: undefined,
+
   companyName: 'LiveKit',
-  pageTitle: 'LiveKit Embed',
-  pageDescription: 'A web embed connected to an agent, built with LiveKit',
 
   supportsChatInput: true,
   supportsVideoInput: true,
@@ -14,5 +15,4 @@ export const APP_CONFIG_DEFAULTS: AppConfig = {
   accent: '#002cf2',
   logoDark: '/lk-logo-dark.svg',
   accentDark: '#1fd5f9',
-  startButtonText: 'Start call',
 };

app/api/connection-details/route.ts

@@ -1,5 +1,6 @@
 import { NextResponse } from 'next/server';
 import { AccessToken, type AccessTokenOptions, type VideoGrant } from 'livekit-server-sdk';
+import { RoomConfiguration } from '@livekit/protocol';
 
 // NOTE: you are expected to define the following environment variables in `.env.local`:
 const API_KEY = process.env.LIVEKIT_API_KEY;
@@ -16,7 +17,7 @@ export type ConnectionDetails = {
   participantToken: string;
 };
 
-export async function GET() {
+export async function POST(req: Request) {
   try {
     if (LIVEKIT_URL === undefined) {
       throw new Error('LIVEKIT_URL is not defined');
@@ -28,13 +29,19 @@ export async function GET() {
       throw new Error('LIVEKIT_API_SECRET is not defined');
     }
 
+    // Parse agent configuration from request body
+    const body = await req.json();
+    const agentName: string = body?.room_config?.agents?.[0]?.agent_name;
+
     // Generate participant token
     const participantName = 'user';
     const participantIdentity = `voice_assistant_user_${Math.floor(Math.random() * 10_000)}`;
     const roomName = `voice_assistant_room_${Math.floor(Math.random() * 10_000)}`;
+
     const participantToken = await createParticipantToken(
       { identity: participantIdentity, name: participantName },
-      roomName
+      roomName,
+      agentName
     );
 
     // Return connection details
@@ -56,7 +63,11 @@ export async function GET() {
   }
 }
 
-function createParticipantToken(userInfo: AccessTokenOptions, roomName: string) {
+function createParticipantToken(
+  userInfo: AccessTokenOptions,
+  roomName: string,
+  agentName?: string
+): Promise<string> {
   const at = new AccessToken(API_KEY, API_SECRET, {
     ...userInfo,
     ttl: '15m',
@@ -69,5 +80,12 @@ function createParticipantToken(userInfo: AccessTokenOptions, roomName: string)
     canSubscribe: true,
   };
   at.addGrant(grant);
+
+  if (agentName) {
+    at.roomConfig = new RoomConfiguration({
+      agents: [{ agentName }],
+    });
+  }
+
   return at.toJwt();
 }

components/embed-iframe/agent-client.tsx

@@ -21,7 +21,7 @@ interface AppProps {
 function EmbedAgentClient({ appConfig }: AppProps) {
   const room = useMemo(() => new Room(), []);
   const [sessionStarted, setSessionStarted] = useState(false);
-  const { connectionDetails, refreshConnectionDetails } = useConnectionDetails();
+  const { connectionDetails, refreshConnectionDetails } = useConnectionDetails(appConfig);
 
   const [currentError, setCurrentError] = useState<EmbedErrorDetails | null>(null);
 

components/embed-popup/action-bar.tsx

@@ -1,40 +1,215 @@
-import { MicrophoneToggle } from '@/components/embed-popup/microphone-toggle';
+'use client';
+
+import * as React from 'react';
+import { useCallback } from 'react';
+import { Track } from 'livekit-client';
+import { BarVisualizer, useRemoteParticipants } from '@livekit/components-react';
+import { ChatTextIcon } from '@phosphor-icons/react/dist/ssr';
 import { ChatInput } from '@/components/livekit/chat/chat-input';
-import { useAgentControlBar } from '@/hooks/use-agent-control-bar';
+import { DeviceSelect } from '@/components/livekit/device-select';
+import { TrackToggle } from '@/components/livekit/track-toggle';
+import { Toggle } from '@/components/ui/toggle';
+import { UseAgentControlBarProps, useAgentControlBar } from '@/hooks/use-agent-control-bar';
+import { AppConfig } from '@/lib/types';
+import { cn } from '@/lib/utils';
 
-interface ActionBarProps {
-  onSend: (message: string) => void;
+export interface AgentControlBarProps
+  extends React.HTMLAttributes<HTMLDivElement>,
+    UseAgentControlBarProps {
+  capabilities: Pick<AppConfig, 'supportsChatInput' | 'supportsVideoInput' | 'supportsScreenShare'>;
+  onChatOpenChange?: (open: boolean) => void;
+  onSendMessage?: (message: string) => Promise<void>;
+  onDeviceError?: (error: { source: Track.Source; error: Error }) => void;
 }
 
-export function ActionBar({ onSend }: ActionBarProps) {
+/**
+ * A control bar specifically designed for voice assistant interfaces
+ */
+export function ActionBar({
+  controls,
+  saveUserChoices = true,
+  capabilities,
+  className,
+  onSendMessage,
+  onChatOpenChange,
+  onDeviceError,
+  ...props
+}: AgentControlBarProps) {
+  const participants = useRemoteParticipants();
+  const [chatOpen, setChatOpen] = React.useState(false);
+  const [isSendingMessage, setIsSendingMessage] = React.useState(false);
+
+  const isAgentAvailable = participants.some((p) => p.isAgent);
+  const isInputDisabled = !chatOpen || !isAgentAvailable || isSendingMessage;
+
   const {
     micTrackRef,
-    // FIXME: how do I explicitly ensure only the microphone channel is used?
     visibleControls,
+    cameraToggle,
     microphoneToggle,
+    screenShareToggle,
     handleAudioDeviceChange,
+    handleVideoDeviceChange,
   } = useAgentControlBar({
-    controls: { microphone: true },
-    saveUserChoices: true,
+    controls,
+    saveUserChoices,
   });
 
+  const handleSendMessage = async (message: string) => {
+    setIsSendingMessage(true);
+    try {
+      await onSendMessage?.(message);
+    } finally {
+      setIsSendingMessage(false);
+    }
+  };
+
+  React.useEffect(() => {
+    onChatOpenChange?.(chatOpen);
+  }, [chatOpen, onChatOpenChange]);
+
+  const onMicrophoneDeviceSelectError = useCallback(
+    (error: Error) => {
+      onDeviceError?.({ source: Track.Source.Microphone, error });
+    },
+    [onDeviceError]
+  );
+  const onCameraDeviceSelectError = useCallback(
+    (error: Error) => {
+      onDeviceError?.({ source: Track.Source.Camera, error });
+    },
+    [onDeviceError]
+  );
+
   return (
     <div
       aria-label="Voice assistant controls"
-      className="bg-bg1 border-separator1 relative z-20 mx-1 flex h-12 shrink-0 grow-0 items-center gap-1 rounded-full border px-1 drop-shadow-md"
+      className={cn(
+        'bg-background border-separator1 dark:border-separator1 relative z-20 mx-2 mb-1 flex flex-col rounded-[24px] border p-1 drop-shadow-md',
+        className
+      )}
+      {...props}
     >
-      <div className="flex gap-1">
-        {visibleControls.microphone && (
-          <MicrophoneToggle
-            micTrackRef={micTrackRef}
-            microphoneToggle={microphoneToggle}
-            handleAudioDeviceChange={handleAudioDeviceChange}
-          />
-        )}
-        {/* FIXME: do I need to handle the other channels here? */}
-      </div>
+      {capabilities.supportsChatInput && (
+        <div
+          inert={!chatOpen}
+          className={cn(
+            'relative overflow-hidden transition-[height] duration-300 ease-out',
+            chatOpen ? 'h-[46px]' : 'h-0'
+          )}
+        >
+          <div
+            className={cn(
+              'absolute inset-x-0 top-0 flex h-9 w-full transition-opacity duration-150 ease-linear',
+              chatOpen ? 'opacity-100 delay-150' : 'opacity-0'
+            )}
+          >
+            <ChatInput onSend={handleSendMessage} disabled={isInputDisabled} className="w-full" />
+          </div>
+          <hr className="border-bg2 absolute inset-x-0 bottom-0 my-1 w-full" />
+        </div>
+      )}
 
-      <ChatInput className="w-0 shrink-1 grow-1" onSend={onSend} />
+      <div className="flex flex-row justify-between gap-1">
+        <div className="flex gap-1">
+          {visibleControls.microphone && (
+            <div className="flex items-center gap-0">
+              <TrackToggle
+                variant="primary"
+                source={Track.Source.Microphone}
+                pressed={microphoneToggle.enabled}
+                disabled={microphoneToggle.pending}
+                onPressedChange={microphoneToggle.toggle}
+                className="peer/track group/track relative w-auto pr-3 pl-3 has-[+_*]:rounded-r-none has-[+_*]:border-r-0 has-[+_*]:pr-2"
+              >
+                <BarVisualizer
+                  barCount={3}
+                  trackRef={micTrackRef}
+                  options={{ minHeight: 5 }}
+                  className="flex h-full w-auto items-center justify-center gap-0.5"
+                >
+                  <span
+                    className={cn([
+                      'h-full w-0.5 origin-center rounded-2xl',
+                      'group-data-[state=on]/track:bg-fg1 group-data-[state=off]/track:bg-destructive-foreground',
+                      'data-lk-muted:bg-muted',
+                    ])}
+                  ></span>
+                </BarVisualizer>
+              </TrackToggle>
+              <DeviceSelect
+                size="sm"
+                kind="audioinput"
+                requestPermissions={false}
+                onMediaDeviceError={onMicrophoneDeviceSelectError}
+                onActiveDeviceChange={handleAudioDeviceChange}
+                className={cn([
+                  'pl-2',
+                  'peer-data-[state=off]/track:text-destructive-foreground',
+                  'hover:text-fg1 focus:text-fg1',
+                  'hover:peer-data-[state=off]/track:text-destructive-foreground focus:peer-data-[state=off]/track:text-destructive-foreground',
+                  'hidden rounded-l-none md:block',
+                ])}
+              />
+            </div>
+          )}
+
+          {capabilities.supportsVideoInput && visibleControls.camera && (
+            <div className="flex items-center gap-0">
+              <TrackToggle
+                variant="primary"
+                source={Track.Source.Camera}
+                pressed={cameraToggle.enabled}
+                pending={cameraToggle.pending}
+                disabled={cameraToggle.pending}
+                onPressedChange={cameraToggle.toggle}
+                className="peer/track relative w-auto pr-3 pl-3 disabled:opacity-100 has-[+_*]:rounded-r-none has-[+_*]:border-r-0 has-[+_*]:pr-2"
+              />
+              <DeviceSelect
+                size="sm"
+                kind="videoinput"
+                requestPermissions={false}
+                onMediaDeviceError={onCameraDeviceSelectError}
+                onActiveDeviceChange={handleVideoDeviceChange}
+                className={cn([
+                  'pl-2',
+                  'peer-data-[state=off]/track:text-destructive-foreground',
+                  'hover:text-fg1 focus:text-fg1',
+                  'hover:peer-data-[state=off]/track:text-destructive-foreground focus:peer-data-[state=off]/track:text-destructive-foreground',
+                  'rounded-l-none',
+                ])}
+              />
+            </div>
+          )}
+        </div>
+        <div className="flex gap-1">
+          {capabilities.supportsScreenShare && visibleControls.screenShare && (
+            <div className="flex items-center gap-0">
+              <TrackToggle
+                variant="secondary"
+                source={Track.Source.ScreenShare}
+                pressed={screenShareToggle.enabled}
+                disabled={screenShareToggle.pending}
+                onPressedChange={screenShareToggle.toggle}
+                className="relative w-auto"
+              />
+            </div>
+          )}
+
+          {visibleControls.chat && (
+            <Toggle
+              variant="secondary"
+              aria-label="Toggle chat"
+              pressed={chatOpen}
+              onPressedChange={setChatOpen}
+              disabled={!isAgentAvailable}
+              className="aspect-square h-full"
+            >
+              <ChatTextIcon weight="bold" />
+            </Toggle>
+          )}
+        </div>
+      </div>
     </div>
   );
 }

components/embed-popup/agent-client.tsx

@@ -21,7 +21,8 @@ function AgentClient({ appConfig }: EmbedFixedAgentClientProps) {
   const room = useMemo(() => new Room(), []);
   const [popupOpen, setPopupOpen] = useState(false);
   const [error, setError] = useState<EmbedErrorDetails | null>(null);
-  const { connectionDetails, refreshConnectionDetails } = useConnectionDetails();
+  const { connectionDetails, refreshConnectionDetails, existingOrRefreshConnectionDetails } =
+    useConnectionDetails(appConfig);
 
   const handleTogglePopup = () => {
     if (isAnimating.current) {
@@ -79,24 +80,32 @@ function AgentClient({ appConfig }: EmbedFixedAgentClientProps) {
     }
 
     const connect = async () => {
-      try {
-        await room.connect(connectionDetails.serverUrl, connectionDetails.participantToken);
-        await room.localParticipant.setMicrophoneEnabled(true, undefined, {
+      Promise.all([
+        room.localParticipant.setMicrophoneEnabled(true, undefined, {
           preConnectBuffer: appConfig.isPreConnectBufferEnabled,
-        });
-      } catch (error: unknown) {
+        }),
+        existingOrRefreshConnectionDetails().then((connectionDetails) =>
+          room.connect(connectionDetails.serverUrl, connectionDetails.participantToken)
+        ),
+      ]).catch((error) => {
         if (error instanceof Error) {
           console.error('Error connecting to agent:', error);
           setError({
             title: 'There was an error connecting to the agent',
             description: `${error.name}: ${error.message}`,
           });
         }
-      }
+      });
     };
 
     connect();
-  }, [room, popupOpen, connectionDetails, appConfig.isPreConnectBufferEnabled]);
+  }, [
+    room,
+    popupOpen,
+    connectionDetails,
+    existingOrRefreshConnectionDetails,
+    appConfig.isPreConnectBufferEnabled,
+  ]);
 
   return (
     <RoomContext.Provider value={room}>
@@ -129,6 +138,7 @@ function AgentClient({ appConfig }: EmbedFixedAgentClientProps) {
             <ErrorMessage error={error} />
             {!error && (
               <PopupViewMotion
+                appConfig={appConfig}
                 initial={{ opacity: 1 }}
                 animate={{ opacity: error === null ? 1 : 0 }}
                 transition={{