feat: data copilot text to sql

.gitignore

@@ -10,4 +10,6 @@ scripts/scaffold.yaml
 node_modules
 .prettierrc
 **/.env*
-!.env.dist
+!.env.dist
+database/Dockerfile.flyway
+database/flyway_migrate.sh

database/migrate.sh

@@ -2,7 +2,14 @@
 set -ex
 set +o history
 
-# Grab all command line arguments to pass them into Docker, or default to "migrate".
+# Check if first argument is --host-network
+DOCKER_NETWORK=""
+if [ "$1" = "--host-network" ]; then
+    DOCKER_NETWORK="--network host"
+    shift  # Remove --host-network from arguments
+fi
+
+# Grab remaining command line arguments to pass them into Docker, or default to "migrate".
 if [ $# -eq 0 ]; then
     FLYWAY_COMMAND=("migrate")
 else
@@ -11,7 +18,7 @@ fi
 
 echo "Running Flyway command: ${FLYWAY_COMMAND[@]} on jdbc:postgresql://${PGHOST}:${PGPORT}/${PGDATABASE}"
 
-docker run --rm \
+docker run --rm ${DOCKER_NETWORK} \
   -v "$(pwd)/migrations:/tmp/migrations" \
   flyway/flyway:latest-alpine \
   -locations="filesystem:/tmp/migrations" \

database/migrations/V1756973486__addConversationIdToChatResponses.sql

@@ -0,0 +1,8 @@
+ALTER TABLE chat_responses 
+ADD COLUMN conversation_id UUID DEFAULT gen_random_uuid();
+
+-- Create index for efficient conversation queries
+CREATE INDEX idx_chat_responses_conversation_id ON chat_responses(conversation_id);
+
+-- Create index for efficient conversation + timestamp queries  
+CREATE INDEX idx_chat_responses_conversation_created_at ON chat_responses(conversation_id, created_at);

database/migrations/V1756973487__alterChatResponsesRouterResponseCheck.sql

@@ -0,0 +1,9 @@
+-- Drop the existing check constraint
+ALTER TABLE chat_responses DROP CONSTRAINT chat_responses_router_response_check;
+
+UPDATE chat_responses SET router_response = 'create_query' 
+WHERE router_response = 'text-to-sql';
+
+-- Add the new check constraint with 'create_query' instead of 'text-to-sql'
+ALTER TABLE chat_responses ADD CONSTRAINT chat_responses_router_response_check 
+    CHECK (router_response IN ('pipes', 'create_query', 'stop'));

frontend/app/components/shared/modules/copilot/components/copilot-modal.vue

@@ -78,11 +78,11 @@ const isModalOpen = computed({
   }
 })
 
-const handleDataUpdate = (id: string, data: MessageData[], routerReasoning?: string) => {
+const handleDataUpdate = (id: string, data: MessageData[], conversationId?: string) => {
   resultData.value.push({
     id,
     data,
-    routerReasoning
+    conversationId
   });
 
 

frontend/app/components/shared/modules/copilot/components/copilot-sidebar.vue

@@ -133,7 +133,7 @@ const emit = defineEmits<{
   (e: 'update:selectedResult', value: string): void;
   (e: 'update:isLoading', value: boolean): void;
   (e: 'update:error', value: string): void;
-  (e: 'update:data', id: string, value: MessageData[], routerReasoning?: string): void;
+  (e: 'update:data', id: string, value: MessageData[], conversationId?: string): void;
 }>();
 
 const { copilotDefaults, selectedResultId, selectedWidgetKey } = storeToRefs(useCopilotStore());
@@ -147,6 +147,7 @@ const input = ref('')
 const streamingStatus = ref('')
 const error = ref('')
 const messages = ref<Array<AIMessage>>([]) // tempData as AIMessage
+const conversationId = ref<string | undefined>(undefined)
 const isEmptyMessages = computed(() => messages.value.length === 0)
 
 const isLoading = computed<boolean>({
@@ -183,35 +184,40 @@ const callChatApi = async (userMessage: string) => {
       const response = await copilotApiService.callChatStream(
         messages.value, 
         copilotDefaults.value.project, 
-        selectedWidgetKey.value, 
-        copilotDefaults.value.params)
+        selectedWidgetKey.value,
+        copilotDefaults.value.params,
+        conversationId.value)
 
       // Handle the streaming response
-      await copilotApiService.handleStreamingResponse(response, messages.value, (status) => {
-        streamingStatus.value = status;
-      }, (message, index) => {
-        if (index === -1) {
-          messages.value.push(message);
-        } else {
+      const returnedConversationId = await copilotApiService.handleStreamingResponse(
+        response, messages.value, (status) => {
+          streamingStatus.value = status;
+        }, (message, index) => {
+          if (index === -1) {
+            messages.value.push(message);
+          } else {
           messages.value[index] = message;
         }
 
         if (message.data) {
-          // Find router reasoning from the latest router-status message in the conversation
-          const routerReasoning = messages.value
-            .slice()
-            .reverse()
-            .find(msg => msg.type === 'router-status' && msg.routerReasoning)
-            ?.routerReasoning;
-
-          emit('update:data', message.id, message.data, routerReasoning);
+          // Pass the current conversation ID instead of extracting routerReasoning
+          emit('update:data', message.id, message.data, conversationId.value);
           selectedResultId.value = message.id;
         }
         scrollToEnd();
-      }, () => {
+      }, (receivedConversationId) => {
         isLoading.value = false;
         streamingStatus.value = '';
+        // Store the conversationId for subsequent calls
+        if (receivedConversationId) {
+          conversationId.value = receivedConversationId;
+        }
       });
+
+      // Also capture conversationId from the return value as backup
+      if (returnedConversationId && !conversationId.value) {
+        conversationId.value = returnedConversationId;
+      }
     }
   } catch (err) {
     console.error('Failed to send message:', err)
@@ -241,7 +247,13 @@ const selectResult = (id: string) => {
   selectedResultId.value = id;
 }
 
-watch(copilotDefaults, (newDefaults) => {
+watch(copilotDefaults, (newDefaults, oldDefaults) => {
+  // Clear conversation when widget changes
+  if (oldDefaults && newDefaults.widget !== oldDefaults.widget) {
+    conversationId.value = undefined;
+    messages.value = [];
+  }
+
   if (newDefaults.question) {
     callChatApi(newDefaults.question);
   }

frontend/app/components/shared/modules/copilot/components/results/chart-results.vue

@@ -78,7 +78,7 @@ const props = defineProps<{
   config: Config | null,
   isSnapshotModalOpen: boolean,
   chartErrorType?: ChartErrorType,
-  routerReasoning?: string
+  conversationId?: string
 }>()
 
 const isSnapshotModalOpen = computed({
@@ -120,7 +120,7 @@ const generateChart = async () => {
 
   isLoading.value = true;
 
-  const response = await copilotApiService.callChartApi(props.data, props.routerReasoning);
+  const response = await copilotApiService.callChartApi(props.data, props.conversationId);
   const data = await response.json();
 
   if (data.config && data.success && data.dataMapping) {

frontend/app/components/shared/modules/copilot/components/results/results-section.vue

@@ -47,7 +47,7 @@ SPDX-License-Identifier: MIT
               :config="selectedResultConfig"
               :is-snapshot-modal-open="isSnapshotModalOpen"
               :chart-error-type="selectedResultChartErrorType"
-              :router-reasoning="selectedResultRouterReasoning"
+              :conversation-id="selectedResultConversationId"
               @update:config="handleConfigUpdate"
               @update:is-loading="handleChartLoading"
               @update:is-snapshot-modal-open="isSnapshotModalOpen = $event"
@@ -117,8 +117,8 @@ const selectedResultChartErrorType = computed(() => {
   return resultData.value.find(result => result.id === selectedResultId.value)?.chartErrorType;
 })
 
-const selectedResultRouterReasoning = computed(() => {
-  return resultData.value.find(result => result.id === selectedResultId.value)?.routerReasoning;
+const selectedResultConversationId = computed(() => {
+  return resultData.value.find(result => result.id === selectedResultId.value)?.conversationId;
 })
 
 const isEmpty = computed(() => {

frontend/app/components/shared/modules/copilot/store/copilot.api.service.ts

@@ -17,7 +17,7 @@ import type { Project } from '~~/types/project'
 export const tempData = testData3 as AIMessage[]
 class CopilotApiService {
   // Generate unique ID for messages
-  generateId = () => Date.now().toString(36) + Math.random().toString(36).substr(2)
+  generateId = () => Date.now().toString(36) + Math.random().toString(36).substring(2);
 
   generateTextMessage = (
     message: string,
@@ -42,6 +42,7 @@ class CopilotApiService {
     project: Project,
     pipe?: string,
     parameters?: CopilotParams,
+    conversationId?: string,
   ): Promise<Response> {
     // Prepare the request body with the correct format
     const requestBody = {
@@ -50,9 +51,10 @@ class CopilotApiService {
         content: m.content,
       })),
       pipe,
-      segmentId: project?.id,
+      projectSlug: project?.slug,
       projectName: project?.name,
       parameters,
+      conversationId
     }
     // Send streaming request
     const response = await fetch('/api/chat/stream', {
@@ -70,12 +72,15 @@ class CopilotApiService {
     return response
   }
 
-  async callChartApi(sampleData: MessageData[], routerReasoning?: string): Promise<Response> {
+  async callChartApi(
+    sampleData: MessageData[],
+    conversationId?: string,
+  ): Promise<Response> {
     // Prepare the request body with the correct format
     const requestBody = {
       results: sampleData,
       userQuery: 'Generate a chart for this data',
-      routerReasoning,
+      conversationId,
     }
 
     // Send streaming request
@@ -121,8 +126,8 @@ class CopilotApiService {
     messages: Array<AIMessage>,
     statusCallBack: (status: string) => void,
     messageCallBack: (message: AIMessage, index: number) => void,
-    completionCallBack: () => void,
-  ) {
+    completionCallBack: (conversationId?: string) => void
+  ): Promise<string | undefined> {
     const reader = response.body?.getReader()
     const decoder = new TextDecoder()
 
@@ -132,6 +137,7 @@ class CopilotApiService {
 
     let assistantContent = ''
     let assistantMessageId: string | null = null
+    let conversationId: string | undefined = undefined
     let lineBuffer = '' // Buffer to accumulate partial lines
 
     try {
@@ -177,13 +183,18 @@ class CopilotApiService {
           if (result) {
             assistantContent = result.assistantContent
             assistantMessageId = result.assistantMessageId
+            if (result.conversationId) {
+              conversationId = result.conversationId
+            }
           }
         }
       }
     } finally {
       reader.releaseLock()
-      completionCallBack()
+      completionCallBack(conversationId);
     }
+
+    return conversationId
   }
 
   private processCompleteLine(
@@ -192,8 +203,8 @@ class CopilotApiService {
     assistantContent: string,
     messages: Array<AIMessage>,
     statusCallBack: (status: string) => void,
-    messageCallBack: (message: AIMessage, index: number) => void,
-  ): { assistantMessageId: string | null; assistantContent: string } | null {
+    messageCallBack: (message: AIMessage, index: number) => void
+  ): { assistantMessageId: string | null; assistantContent: string; conversationId?: string } | null {
     try {
       // Parse AI SDK data stream format: "prefix:data"
       const colonIndex = line.indexOf(':')
@@ -206,7 +217,9 @@ class CopilotApiService {
 
       // Handle different stream prefixes
       if (prefix === '2') {
-        assistantMessageId = null
+        assistantMessageId = null;
+        let capturedConversationId: string | undefined = undefined;
+
         // Custom data events from your backend (like router-status)
         const dataArray = JSON.parse(dataString)
         for (const data of dataArray) {
@@ -242,30 +255,35 @@ class CopilotApiService {
               statusCallBack('Tool execution completed')
             }
 
+            // Capture conversationId from chat-response-id for return
+            if (data.type === 'chat-response-id' && data.conversationId) {
+              capturedConversationId = data.conversationId;
+            }
+
             const content = data.type === 'chat-response-id' ? data.id : data.explanation
 
             // Create assistant message if it doesn't exist yet
             if (!assistantMessageId) {
               assistantMessageId = this.generateId()
             }
 
-            messageCallBack(
-              {
-                id: assistantMessageId,
-                role: 'assistant',
-                type: data.type,
-                status: data.status,
-                sql: data.sql,
-                data: data.data,
-                content,
-                explanation: data.explanation,
-                instructions: data.instructions,
-                timestamp: Date.now(),
-              },
-              -1,
-            )
-          }
+            messageCallBack({
+              id: assistantMessageId,
+              role: 'assistant',
+              type: data.type,
+              status: data.status,
+              sql: data.sql, 
+              data: data.data,
+              content,
+              explanation: data.explanation,
+              instructions: data.instructions,
+              conversationId: data.conversationId,
+              timestamp: Date.now()
+            }, -1);
+          } 
         }
+
+        return { assistantMessageId, assistantContent, conversationId: capturedConversationId }
       } else if (prefix === '0') {
         // Text delta from streamText (streaming text content)
         const textDelta = JSON.parse(dataString)