feat: keep history in short answer XBlock

Author: ArturGaspar

ai_eval/coding_ai_eval.py

@@ -8,7 +8,7 @@
 from web_fragments.fragment import Fragment
 from xblock.core import XBlock
 from xblock.exceptions import JsonHandlerError
-from xblock.fields import Dict, Scope, String
+from xblock.fields import Dict, List, Scope, String
 from xblock.validation import ValidationMessage
 
 from .base import AIEvalXBlock
@@ -84,10 +84,13 @@ class CodingAIEvalXBlock(AIEvalXBlock):
         scope=Scope.settings,
     )
 
-    messages = Dict(
+    # XXX: deprecated
+    messages = Dict(scope=Scope.user_state)
+
+    sessions = List(
         help=_("Dictionary with messages"),
         scope=Scope.user_state,
-        default={USER_RESPONSE: "", AI_EVALUATION: "", CODE_EXEC_RESULT: {}},
+        default=[{USER_RESPONSE: "", AI_EVALUATION: "", CODE_EXEC_RESULT: {}}],
     )
 
     editable_fields = AIEvalXBlock.editable_fields + (
@@ -97,6 +100,13 @@ class CodingAIEvalXBlock(AIEvalXBlock):
         "language",
     )
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.messages:
+            self.sessions = [self.messages]
+            self.messages = {}
+            self.save()
+
     def resource_string(self, path):
         """Handy helper for getting resources from our kit."""
         data = pkg_resources.resource_string(__name__, path)
@@ -130,9 +140,9 @@ def student_view(self, context=None):
         js_data = {
             "monaco_html": monaco_html,
             "question": self.question,
-            "code": self.messages[USER_RESPONSE],
-            "ai_evaluation": self.messages[AI_EVALUATION],
-            "code_exec_result": self.messages[CODE_EXEC_RESULT],
+            "code": self.sessions[-1][USER_RESPONSE],
+            "ai_evaluation": self.sessions[-1][AI_EVALUATION],
+            "code_exec_result": self.sessions[-1][CODE_EXEC_RESULT],
             "marked_html": marked_html,
             "language": self.language,
         }
@@ -228,9 +238,9 @@ def get_response(self, data, suffix=""):  # pylint: disable=unused-argument
             raise JsonHandlerError(500, "A probem occured. Please retry.") from e
 
         if response:
-            self.messages[USER_RESPONSE] = data["code"]
-            self.messages[AI_EVALUATION] = response
-            self.messages[CODE_EXEC_RESULT] = {
+            self.sessions[-1][USER_RESPONSE] = data["code"]
+            self.sessions[-1][AI_EVALUATION] = response
+            self.sessions[-1][CODE_EXEC_RESULT] = {
                 "stdout": data["stdout"],
                 "stderr": data["stderr"],
             }
@@ -253,7 +263,11 @@ def reset_handler(self, data, suffix=""):  # pylint: disable=unused-argument
         """
         Reset the Xblock.
         """
-        self.messages = {USER_RESPONSE: "", AI_EVALUATION: "", CODE_EXEC_RESULT: {}}
+        self.sessions.append({
+            USER_RESPONSE: "",
+            AI_EVALUATION: "",
+            CODE_EXEC_RESULT: {},
+        })
         return {"message": "reset successful."}
 
     @XBlock.json_handler

ai_eval/export.py

@@ -38,9 +38,7 @@
 class DataExportXBlock(XBlock):
     icon_class = "problem"
     display_name = String(
-        display_name=_("(Display name)"),
-        help=_("Title to display"),
-        default=_("Data export"),
+        default=_("AI XBlocks data export"),
         scope=Scope.settings
     )
     active_export_task_id = String(
@@ -129,7 +127,6 @@ def raise_error(self, code, message):
         self.last_export_result = {
             'error': message,
         }
-        self.display_data = None
         raise JsonHandlerError(code, message)
 
     @XBlock.json_handler
@@ -142,9 +139,18 @@ def delete_export(self, data, suffix=''):
         return self._get_status()
 
     def _delete_export(self):
+        if not self.last_export_result or 'error' in self.last_export_result:
+            return
+        filename = self.last_export_result['report_filename']
+        from lms.djangoapps.instructor_task.models import ReportStore
+        report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
+        course_key = getattr(self.scope_ids.usage_id, 'course_key', None)
+        path = report_store.path_to(course_key, filename)
+        try:
+            report_store.storage.delete(path)
+        except NotImplementedError:
+            pass
         self.last_export_result = None
-        self.display_data = None
-        self.active_export_task_id = ''
 
     @XBlock.json_handler
     def start_export(self, data, suffix=''):

ai_eval/multiagent.py

@@ -284,11 +284,17 @@ class MultiAgentAIEvalXBlock(AIEvalXBlock):
         default=False,
     )
 
+    # XXX: Deprecated.
     chat_history = List(
         scope=Scope.user_state,
         default=[],
     )
 
+    sessions = List(
+        scope=Scope.user_state,
+        default=[[]],
+    )
+
     editable_fields = AIEvalXBlock.editable_fields + (
         "scenario_data",
         "character_data",
@@ -302,6 +308,13 @@ class MultiAgentAIEvalXBlock(AIEvalXBlock):
         "blacklist",
     )
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.chat_history:
+            self.sessions = [self.chat_history]
+            self.chat_history = []
+            self.save()
+
     def studio_view(self, context):
         """
         Render a form for editing this XBlock
@@ -352,7 +365,7 @@ def _llm_input(self, prompt, user_input):
         # history with an LLM completion, with each message having a "role"
         # of "user" or "assistant".
         for message in itertools.chain(initial_messages,
-                                       self.chat_history,
+                                       self.sessions[-1],
                                        [user_message]):
             if message["role"] == "assistant":
                 agent = message["extra"].get("role") or ""
@@ -503,7 +516,7 @@ def student_view(self, context=None):
                 main_data = self._get_character_data(main_name)
                 break
         js_data = {
-            "messages": self.chat_history,
+            "messages": self.sessions[-1],
             "main_character_agent": main_agent,
             "main_character_data": {
                 "name": main_data.get("name", main_name),
@@ -625,10 +638,10 @@ def get_response(self, data, suffix=""):  # pylint: disable=unused-argument
             character_data = character_data.copy()
             character_data.setdefault("name", character_name)
 
-        self.chat_history.append({"role": "user", "content": user_input})
+        self.sessions[-1].append({"role": "user", "content": user_input})
         extra = {"is_evaluator": is_evaluator, "role": agent,
                  "character_data": character_data}
-        self.chat_history.append({"role": "assistant", "content": message,
+        self.sessions[-1].append({"role": "assistant", "content": message,
                                   "extra": extra})
         return {
             "message": message,
@@ -646,6 +659,6 @@ def reset(self, data, suffix=""):
         """Reset the chat history."""
         if not self.allow_reset:
             raise JsonHandlerError(403, "Reset is disabled.")
-        self.chat_history = []
+        self.sessions.append([])
         self.finished = False
         return {}

ai_eval/shortanswer.py

@@ -27,9 +27,6 @@ class ShortAnswerAIEvalXBlock(AIEvalXBlock):
 
     ATTACHMENT_PARALLEL_DOWNLOADS = 5
 
-    USER_KEY = "USER"
-    LLM_KEY = "LLM"
-
     display_name = String(
         display_name=_("Display Name"),
         help=_("Name of the component in the studio"),
@@ -90,10 +87,14 @@ class ShortAnswerAIEvalXBlock(AIEvalXBlock):
         resettable_editor=False,
     )
 
+    # XXX: Deprecated.
     messages = Dict(
-        help=_("Dictionary with chat messages"),
         scope=Scope.user_state,
-        default={USER_KEY: [], LLM_KEY: []},
+    )
+
+    sessions = List(
+        scope=Scope.user_state,
+        default=[[]],
     )
 
     editable_fields = AIEvalXBlock.editable_fields + (
@@ -105,6 +106,22 @@ class ShortAnswerAIEvalXBlock(AIEvalXBlock):
         "attachment_urls",
     )
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.messages:
+            for user_msg, assistant_msg in zip(self.messages["USER"],
+                                               self.messages["LLM"]):
+                self.sessions[-1].append({
+                    "source": "user",
+                    "content": user_msg or ".",
+                })
+                self.sessions[-1].append({
+                    "source": "llm",
+                    "content": assistant_msg,
+                })
+            self.messages = {}
+            self.save()
+
     def validate_field_data(self, validation, data):
         """
         Validate fields
@@ -154,7 +171,7 @@ def student_view(self, context=None):
 
         js_data = {
             "question": self.question,
-            "messages": self.messages,
+            "messages": self.sessions[-1],
             "max_responses": self.max_responses,
             "marked_html": marked_html,
         }
@@ -202,10 +219,15 @@ def get_response(self, data, suffix=""):  # pylint: disable=unused-argument
         # add previous messages
         # the first AI role is 'system' which defines the LLM's personnality and behavior.
         # subsequent roles are 'assistant' and 'user'
-        for user_msg, assistant_msg in zip(self.messages[self.USER_KEY],
-                                           self.messages[self.LLM_KEY]):
-            messages.append({"content": user_msg or ".", "role": "user"})
-            messages.append({"content": assistant_msg, "role": "assistant"})
+        for message in self.sessions[-1]:
+            if message["source"] == "user":
+                role = "user"
+            else:
+                role = "assistant"
+            messages.append({
+                "role": role,
+                "content": message["content"] or ".",
+            })
         messages.append({"role": "user", "content": user_submission})
 
         try:
@@ -218,8 +240,14 @@ def get_response(self, data, suffix=""):  # pylint: disable=unused-argument
             raise JsonHandlerError(500, "A probem occured. Please retry.") from e
 
         if response:
-            self.messages[self.USER_KEY].append(user_submission)
-            self.messages[self.LLM_KEY].append(response)
+            self.sessions[-1].append({
+                "source": "user",
+                "content": user_submission,
+            })
+            self.sessions[-1].append({
+                "source": "llm",
+                "content": response,
+            })
             return {"response": response}
 
         raise JsonHandlerError(500, "A probem occured. The LLM sent an empty response.")
@@ -231,7 +259,7 @@ def reset(self, data, suffix=""):
         """
         if not self.allow_reset:
             raise JsonHandlerError(403, "Reset is disabled.")
-        self.messages = {self.USER_KEY: [], self.LLM_KEY: []}
+        self.sessions.append([])
         return {}
 
     @staticmethod

ai_eval/static/js/src/shortanswer.js

@@ -8,12 +8,18 @@ function ShortAnswerAIEvalXBlock(runtime, element, data) {
 
   const handleInit = function() {
     $("#question-text", element).html(MarkdownToHTML(data.question));
-    for (var i = 0; i < data.messages.USER.length; i++) {
-      this.insertUserMessage(data.messages.USER[i]);
-      this.insertAIMessage(formatAIMessage(data.messages.LLM[i]));
+    var userMessageCount = 0;
+    for (var i = 0; i < data.messages.length; i++) {
+      var message = data.messages[i];
+      if (message.source == "user") {
+        userMessageCount++;
+        this.insertUserMessage(message.content);
+      } else if (message.source == "llm") {
+        this.insertAIMessage(message.content);
+      }
     }
-    this.enableInput(data.messages.USER.length < data.max_responses);
-    this.enableReset(data.messages.USER.length > 0);
+    this.enableInput(userMessageCount < data.max_responses);
+    this.enableReset(userMessageCount > 0);
   };
 
   const handleResponse = function(response) {

ai_eval/tasks.py

@@ -9,10 +9,20 @@
 from django.contrib.auth.models import User
 from xblock.fields import Scope
 
-from . import ShortAnswerAIEvalXBlock
+from . import (
+    CodingAIEvalXBlock,
+    MultiAgentAIEvalXBlock,
+    ShortAnswerAIEvalXBlock,
+)
 
 logger = get_task_logger(__name__)
 
+_SUPPORTED_BLOCKS = [
+    CodingAIEvalXBlock,
+    MultiAgentAIEvalXBlock,
+    ShortAnswerAIEvalXBlock,
+]
+
 
 @shared_task()
 def export_data(course_id_str):
@@ -59,7 +69,7 @@ def _extract_all_data(course_id):
 
     store = modulestore()
     for block in store.get_items(course_id):
-        if isinstance(block, ShortAnswerAIEvalXBlock):
+        if isinstance(block, _SUPPORTED_BLOCKS):
             yield from _extract_data(block)
 
 
@@ -76,25 +86,29 @@ def _extract_data(block):
         data.add_blocks_to_cache([block])
 
         try:
-            messages = data.get(DjangoKeyValueStore.Key(
+            sessions = data.get(DjangoKeyValueStore.Key(
                 scope=Scope.user_state,
                 user_id=user.id,
                 block_scope_id=block.location,
-                field_name='messages'
+                field_name='sessions'
             ))
         except KeyError:
             continue
 
-        conversation = 1
-        for user_message, llm_message in zip(messages['USER'], messages['LLM']):
-            yield (section_name, subsection_name, unit_name,
-                   str(block.location), block.display_name,
-                   user.username, user.email or "", conversation,
-                   "user", user_message)
-            yield (section_name, subsection_name, unit_name,
-                   str(block.location), block.display_name,
-                   user.username, user.email or "", conversation,
-                   "llm", llm_message)
+        for i, conversation in enumerate(sessions, start=1):
+            for message in conversation:
+                yield (
+                    section_name,
+                    subsection_name,
+                    unit_name,
+                    str(block.location),
+                    block.display_name,
+                    user.username,
+                    user.email or "",
+                    i,
+                    message["source"],
+                    message["content"]
+                )
 
 
 def _get_context(block):
@@ -110,4 +124,4 @@ def _get_context(block):
     section_name = block_names_by_type.get('chapter', '')
     subsection_name = block_names_by_type.get('sequential', '')
     unit_name = block_names_by_type.get('vertical', '')
-    return section_name, subsection_name, unit_name
+    return section_name, subsection_name, unit_name