feat: keep history in short answer XBlock

Author: ArturGaspar

ai_eval/coding_ai_eval.py

@@ -8,7 +8,7 @@
 from web_fragments.fragment import Fragment
 from xblock.core import XBlock
 from xblock.exceptions import JsonHandlerError
-from xblock.fields import Dict, Scope, String
+from xblock.fields import Dict, List, Scope, String
 from xblock.validation import ValidationMessage
 
 from .base import AIEvalXBlock
@@ -83,10 +83,13 @@ class CodingAIEvalXBlock(AIEvalXBlock):
         scope=Scope.settings,
     )
 
-    messages = Dict(
+    # XXX: deprecated
+    messages = Dict(scope=Scope.user_state)
+
+    sessions = List(
         help=_("Dictionary with messages"),
         scope=Scope.user_state,
-        default={USER_RESPONSE: "", AI_EVALUATION: "", CODE_EXEC_RESULT: {}},
+        default=[{USER_RESPONSE: "", AI_EVALUATION: "", CODE_EXEC_RESULT: {}}],
     )
 
     editable_fields = AIEvalXBlock.editable_fields + (
@@ -96,6 +99,13 @@ class CodingAIEvalXBlock(AIEvalXBlock):
         "language",
     )
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.messages:
+            self.sessions = [self.messages]
+            self.messages = {}
+            self.save()
+
     def resource_string(self, path):
         """Handy helper for getting resources from our kit."""
         data = pkg_resources.resource_string(__name__, path)
@@ -129,9 +139,9 @@ def student_view(self, context=None):
         js_data = {
             "monaco_html": monaco_html,
             "question": self.question,
-            "code": self.messages[USER_RESPONSE],
-            "ai_evaluation": self.messages[AI_EVALUATION],
-            "code_exec_result": self.messages[CODE_EXEC_RESULT],
+            "code": self.sessions[-1][USER_RESPONSE],
+            "ai_evaluation": self.sessions[-1][AI_EVALUATION],
+            "code_exec_result": self.sessions[-1][CODE_EXEC_RESULT],
             "marked_html": marked_html,
             "language": self.language,
         }
@@ -234,9 +244,9 @@ def get_response(self, data, suffix=""):  # pylint: disable=unused-argument
             raise JsonHandlerError(500, "A probem occured. Please retry.") from e
 
         if response:
-            self.messages[USER_RESPONSE] = data["code"]
-            self.messages[AI_EVALUATION] = response
-            self.messages[CODE_EXEC_RESULT] = {
+            self.sessions[-1][USER_RESPONSE] = data["code"]
+            self.sessions[-1][AI_EVALUATION] = response
+            self.sessions[-1][CODE_EXEC_RESULT] = {
                 "stdout": data["stdout"],
                 "stderr": data["stderr"],
             }
@@ -258,7 +268,11 @@ def reset_handler(self, data, suffix=""):  # pylint: disable=unused-argument
         """
         Reset the Xblock.
         """
-        self.messages = {USER_RESPONSE: "", AI_EVALUATION: "", CODE_EXEC_RESULT: {}}
+        self.sessions.append({
+            USER_RESPONSE: "",
+            AI_EVALUATION: "",
+            CODE_EXEC_RESULT: {},
+        })
         return {"message": "reset successful."}
 
     @XBlock.json_handler

ai_eval/export.py

@@ -38,9 +38,7 @@
 class DataExportXBlock(XBlock):
     icon_class = "problem"
     display_name = String(
-        display_name=_("(Display name)"),
-        help=_("Title to display"),
-        default=_("Data export"),
+        default=_("AI XBlocks data export"),
         scope=Scope.settings
     )
     active_export_task_id = String(
@@ -129,7 +127,6 @@ def raise_error(self, code, message):
         self.last_export_result = {
             'error': message,
         }
-        self.display_data = None
         raise JsonHandlerError(code, message)
 
     @XBlock.json_handler
@@ -142,9 +139,18 @@ def delete_export(self, data, suffix=''):
         return self._get_status()
 
     def _delete_export(self):
+        if not self.last_export_result or 'error' in self.last_export_result:
+            return
+        filename = self.last_export_result['report_filename']
+        from lms.djangoapps.instructor_task.models import ReportStore
+        report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
+        course_key = getattr(self.scope_ids.usage_id, 'course_key', None)
+        path = report_store.path_to(course_key, filename)
+        try:
+            report_store.storage.delete(path)
+        except NotImplementedError:
+            pass
         self.last_export_result = None
-        self.display_data = None
-        self.active_export_task_id = ''
 
     @XBlock.json_handler
     def start_export(self, data, suffix=''):

ai_eval/multiagent.py

@@ -284,11 +284,17 @@ class MultiAgentAIEvalXBlock(AIEvalXBlock):
         default=False,
     )
 
+    # XXX: Deprecated.
     chat_history = List(
         scope=Scope.user_state,
         default=[],
     )
 
+    sessions = List(
+        scope=Scope.user_state,
+        default=[[]],
+    )
+
     editable_fields = AIEvalXBlock.editable_fields + (
         "scenario_data",
         "character_data",
@@ -302,6 +308,13 @@ class MultiAgentAIEvalXBlock(AIEvalXBlock):
         "blacklist",
     )
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.chat_history:
+            self.sessions = [self.chat_history]
+            self.chat_history = []
+            self.save()
+
     def studio_view(self, context):
         """
         Render a form for editing this XBlock
@@ -352,7 +365,7 @@ def _llm_input(self, prompt, user_input):
         # history with an LLM completion, with each message having a "role"
         # of "user" or "assistant".
         for message in itertools.chain(initial_messages,
-                                       self.chat_history,
+                                       self.sessions[-1],
                                        [user_message]):
             if message["role"] == "assistant":
                 agent = message["extra"].get("role") or ""
@@ -503,7 +516,7 @@ def student_view(self, context=None):
                 main_data = self._get_character_data(main_name)
                 break
         js_data = {
-            "messages": self.chat_history,
+            "messages": self.sessions[-1],
             "main_character_agent": main_agent,
             "main_character_data": {
                 "name": main_data.get("name", main_name),
@@ -625,10 +638,10 @@ def get_response(self, data, suffix=""):  # pylint: disable=unused-argument
             character_data = character_data.copy()
             character_data.setdefault("name", character_name)
 
-        self.chat_history.append({"role": "user", "content": user_input})
+        self.sessions[-1].append({"role": "user", "content": user_input})
         extra = {"is_evaluator": is_evaluator, "role": agent,
                  "character_data": character_data}
-        self.chat_history.append({"role": "assistant", "content": message,
+        self.sessions[-1].append({"role": "assistant", "content": message,
                                   "extra": extra})
         return {
             "message": message,
@@ -646,6 +659,6 @@ def reset(self, data, suffix=""):
         """Reset the chat history."""
         if not self.allow_reset:
             raise JsonHandlerError(403, "Reset is disabled.")
-        self.chat_history = []
+        self.sessions.append([])
         self.finished = False
         return {}

ai_eval/shortanswer.py

@@ -27,8 +27,6 @@ class ShortAnswerAIEvalXBlock(AIEvalXBlock):
     Short Answer Xblock.
     """
 
-    USER_KEY = "USER"
-    LLM_KEY = "LLM"
     ATTACHMENT_PARALLEL_DOWNLOADS = 5
 
     display_name = String(
@@ -84,19 +82,23 @@ class ShortAnswerAIEvalXBlock(AIEvalXBlock):
         default=False,
     )
 
-    messages = Dict(
-        help=_("Dictionary with chat messages"),
-        scope=Scope.user_state,
-        default={USER_KEY: [], LLM_KEY: []},
-    )
-
     attachment_urls = List(
         display_name=_("Attachment URLs"),
         help=_("Attachments to include with the evaluation prompt"),
         scope=Scope.settings,
         resettable_editor=False,
     )
 
+    # XXX: Deprecated.
+    messages = Dict(
+        scope=Scope.user_state,
+    )
+
+    sessions = List(
+        scope=Scope.user_state,
+        default=[[]],
+    )
+
     editable_fields = AIEvalXBlock.editable_fields + (
         "question",
         "evaluation_prompt",
@@ -106,6 +108,22 @@ class ShortAnswerAIEvalXBlock(AIEvalXBlock):
         "attachment_urls",
     )
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.messages:
+            for user_msg, assistant_msg in zip(self.messages["USER"],
+                                               self.messages["LLM"]):
+                self.sessions[-1].append({
+                    "source": "user",
+                    "content": user_msg or ".",
+                })
+                self.sessions[-1].append({
+                    "source": "llm",
+                    "content": assistant_msg,
+                })
+            self.messages = {}
+            self.save()
+
     def validate_field_data(self, validation, data):
         """
         Validate fields
@@ -155,7 +173,7 @@ def student_view(self, context=None):
 
         js_data = {
             "question": self.question,
-            "messages": self.messages,
+            "messages": self.sessions[-1],
             "max_responses": self.max_responses,
             "marked_html": marked_html,
         }
@@ -224,25 +242,36 @@ def get_response(self, data, suffix=""):  # pylint: disable=unused-argument
         # add previous messages
         # the first AI role is 'system' which defines the LLM's personnality and behavior.
         # subsequent roles are 'assistant' and 'user'
-        for user_msg, assistant_msg in zip(self.messages[self.USER_KEY],
-                                           self.messages[self.LLM_KEY]):
-            messages.append({"content": user_msg or ".", "role": "user"})
-            messages.append({"content": assistant_msg, "role": "assistant"})
+        for message in self.sessions[-1]:
+            if message["source"] == "user":
+                role = "user"
+            else:
+                role = "assistant"
+            messages.append({
+                "role": role,
+                "content": message["content"] or ".",
+            })
         messages.append({"role": "user", "content": user_submission})
 
         try:
-            text = self.get_llm_response(messages, tag=current_tag)
+            response = self.get_llm_response(messages, tag=current_tag)
         except Exception as e:
             logger.error(
                 f"Failed while making LLM request using model {self.model}. Error: {e}",
                 exc_info=True,
             )
             raise JsonHandlerError(500, "A probem occured. Please retry.") from e
 
-        if text:
-            self.messages[self.USER_KEY].append(user_submission)
-            self.messages[self.LLM_KEY].append(text)
-            return {"response": text}
+        if response:
+            self.sessions[-1].append({
+                "source": "user",
+                "content": user_submission,
+            })
+            self.sessions[-1].append({
+                "source": "llm",
+                "content": response,
+            })
+            return {"response": response}
 
         raise JsonHandlerError(500, "A probem occured. The LLM sent an empty response.")
 
@@ -253,8 +282,8 @@ def reset(self, data, suffix=""):
         """
         if not self.allow_reset:
             raise JsonHandlerError(403, "Reset is disabled.")
-        self.messages = {self.USER_KEY: [], self.LLM_KEY: []}
         self.thread_map = {}
+        self.sessions.append([])
         return {}
 
     @staticmethod

ai_eval/static/js/src/shortanswer.js

@@ -8,12 +8,18 @@ function ShortAnswerAIEvalXBlock(runtime, element, data) {
 
   const handleInit = function() {
     $("#question-text", element).html(MarkdownToHTML(data.question));
-    for (var i = 0; i < data.messages.USER.length; i++) {
-      this.insertUserMessage(data.messages.USER[i]);
-      this.insertAIMessage(formatAIMessage(data.messages.LLM[i]));
+    var userMessageCount = 0;
+    for (var i = 0; i < data.messages.length; i++) {
+      var message = data.messages[i];
+      if (message.source == "user") {
+        userMessageCount++;
+        this.insertUserMessage(message.content);
+      } else if (message.source == "llm") {
+        this.insertAIMessage(formatAIMessage(message.content));
+      }
     }
-    this.enableInput(data.messages.USER.length < data.max_responses);
-    this.enableReset(data.messages.USER.length > 0);
+    this.enableInput(userMessageCount < data.max_responses);
+    this.enableReset(userMessageCount > 0);
   };
 
   const handleResponse = function(response) {