Skip to content

Commit b736f41

Browse files
committed
feat: AI XBlock conversation history export
1 parent b5bd1c3 commit b736f41

File tree

14 files changed

+695
-120
lines changed

14 files changed

+695
-120
lines changed

ai_eval/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@
55
from .shortanswer import ShortAnswerAIEvalXBlock
66
from .coding_ai_eval import CodingAIEvalXBlock
77
from .multiagent import MultiAgentAIEvalXBlock
8+
from .export import DataExportXBlock

ai_eval/export.py

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
#
2+
# Copyright (c) 2014-2015 Harvard, edX & OpenCraft
3+
#
4+
# This software's license gives you freedom; you can copy, convey,
5+
# propagate, redistribute and/or modify this program under the terms of
6+
# the GNU Affero General Public License (AGPL) as published by the Free
7+
# Software Foundation (FSF), either version 3 of the License, or (at your
8+
# option) any later version of the AGPL published by the FSF.
9+
#
10+
# This program is distributed in the hope that it will be useful, but
11+
# WITHOUT ANY WARRANTY; without even the implied warranty of
12+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
13+
# General Public License for more details.
14+
#
15+
# You should have received a copy of the GNU Affero General Public License
16+
# along with this program in a file in the toplevel directory called
17+
# "AGPLv3". If not, see <http://www.gnu.org/licenses/>.
18+
#
19+
"""
20+
Instructor Tool: An XBlock for instructors to export student answers from a course.
21+
22+
All processing is done offline.
23+
"""
24+
import json
25+
import pkg_resources
26+
27+
from django.utils.translation import gettext_noop as _
28+
from web_fragments.fragment import Fragment
29+
from xblock.core import XBlock
30+
from xblock.exceptions import JsonHandlerError
31+
from xblock.fields import Dict, Scope, String
32+
from xblock.utils.resources import ResourceLoader
33+
34+
loader = ResourceLoader(__name__)
35+
36+
37+
@XBlock.wants('user')
38+
class DataExportXBlock(XBlock):
39+
icon_class = "problem"
40+
display_name = String(
41+
display_name=_("(Display name)"),
42+
help=_("Title to display"),
43+
default=_("Data export"),
44+
scope=Scope.settings
45+
)
46+
active_export_task_id = String(
47+
# The UUID of the celery AsyncResult for the most recent export,
48+
# IF we are sill waiting for it to finish
49+
default="",
50+
scope=Scope.user_state,
51+
)
52+
last_export_result = Dict(
53+
# The info dict returned by the most recent successful export.
54+
# If the export failed, it will have an "error" key set.
55+
default=None,
56+
scope=Scope.user_state,
57+
)
58+
59+
has_author_view = True
60+
61+
def author_view(self, context=None):
62+
""" Studio View """
63+
# Warn the user that this block will only work from the LMS. (Since the CMS uses
64+
# different celery queues; our task listener is waiting for tasks on the LMS queue)
65+
return Fragment('<p>Data Export Block</p><p>This block only works from the LMS.</p>')
66+
67+
def studio_view(self, context=None):
68+
""" View for editing Instructor Tool block in Studio. """
69+
# Display friendly message explaining that the block is not editable.
70+
return Fragment('<p>This is a preconfigured block. It is not editable.</p>')
71+
72+
def check_pending_export(self):
73+
"""
74+
If we're waiting for an export, see if it has finished, and if so, get the result.
75+
"""
76+
from .tasks import export_data as export_data_task
77+
if self.active_export_task_id:
78+
async_result = export_data_task.AsyncResult(self.active_export_task_id)
79+
if async_result.ready():
80+
self._save_result(async_result)
81+
82+
def _save_result(self, task_result):
83+
""" Given an AsyncResult or EagerResult, save it. """
84+
self.active_export_task_id = ''
85+
if task_result.successful():
86+
self.last_export_result = task_result.result
87+
else:
88+
self.last_export_result = {'error': str(task_result.result)}
89+
90+
def resource_string(self, path):
91+
"""Handy helper for getting resources from our kit."""
92+
data = pkg_resources.resource_string(__name__, path)
93+
return data.decode("utf8")
94+
95+
def student_view(self, context=None):
96+
""" Normal View """
97+
if not self.user_is_staff():
98+
return Fragment('<p>This interface can only be used by course staff.</p>')
99+
100+
html = loader.render_django_template('templates/export.html')
101+
frag = Fragment(html)
102+
frag.add_javascript(self.resource_string("static/js/src/export.js"))
103+
frag.initialize_js('DataExportXBlock')
104+
return frag
105+
106+
@property
107+
def download_url_for_last_report(self):
108+
""" Get the URL for the last report, if any """
109+
# Unfortunately this is a bit inefficient due to the ReportStore API
110+
if not self.last_export_result or 'error' in self.last_export_result:
111+
return None
112+
from lms.djangoapps.instructor_task.models import ReportStore
113+
report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
114+
course_key = getattr(self.scope_ids.usage_id, 'course_key', None)
115+
return dict(report_store.links_for(course_key)).get(self.last_export_result['report_filename'])
116+
117+
def _get_status(self):
118+
self.check_pending_export()
119+
return {
120+
'export_pending': bool(self.active_export_task_id),
121+
'last_export_result': self.last_export_result,
122+
'download_url': self.download_url_for_last_report,
123+
}
124+
125+
def raise_error(self, code, message):
126+
"""
127+
Raises an error and marks the block with a simulated failed task dict.
128+
"""
129+
self.last_export_result = {
130+
'error': message,
131+
}
132+
self.display_data = None
133+
raise JsonHandlerError(code, message)
134+
135+
@XBlock.json_handler
136+
def get_status(self, data, suffix=''):
137+
return self._get_status()
138+
139+
@XBlock.json_handler
140+
def delete_export(self, data, suffix=''):
141+
self._delete_export()
142+
return self._get_status()
143+
144+
def _delete_export(self):
145+
self.last_export_result = None
146+
self.display_data = None
147+
self.active_export_task_id = ''
148+
149+
@XBlock.json_handler
150+
def start_export(self, data, suffix=''):
151+
""" Start a new asynchronous export """
152+
if not self.user_is_staff():
153+
raise JsonHandlerError(403, "Permission denied.")
154+
155+
# Launch task
156+
from .tasks import export_data as export_data_task
157+
self._delete_export()
158+
# Make sure we nail down our state before sending off an asynchronous task.
159+
self.save()
160+
async_result = export_data_task.delay(
161+
# course_id not available in workbench.
162+
str(getattr(self.runtime, 'course_id', 'course_id')),
163+
)
164+
if async_result.ready():
165+
# In development mode, the task may have executed synchronously.
166+
# Store the result now, because we won't be able to retrieve it later :-/
167+
if async_result.successful():
168+
# Make sure the result can be represented as JSON, since the non-eager celery
169+
# requires that
170+
json.dumps(async_result.result)
171+
self._save_result(async_result)
172+
else:
173+
# The task is running asynchronously. Store the result ID so we can query its progress:
174+
self.active_export_task_id = async_result.id
175+
return self._get_status()
176+
177+
@XBlock.json_handler
178+
def cancel_export(self, request, suffix=''):
179+
from .tasks import export_data as export_data_task
180+
if self.active_export_task_id:
181+
async_result = export_data_task.AsyncResult(self.active_export_task_id)
182+
async_result.revoke()
183+
self._delete_export()
184+
185+
def _get_user_attr(self, attr):
186+
"""Get an attribute of the current user."""
187+
user_service = self.runtime.service(self, 'user')
188+
if user_service:
189+
# May be None when creating bok choy test fixtures
190+
return user_service.get_current_user().opt_attrs.get(attr)
191+
return None
192+
193+
def user_is_staff(self):
194+
"""Return a Boolean value indicating whether the current user is a member of staff."""
195+
return self._get_user_attr('edx-platform.user_is_staff')

ai_eval/static/js/src/export.js

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
function DataExportXBlock(runtime, element, data) {
2+
"use strict";
3+
4+
var status;
5+
6+
const getStatusUrl = runtime.handlerUrl(element, "get_status");
7+
const startExportUrl = runtime.handlerUrl(element, "start_export");
8+
9+
const $results = $("#results-wrapper", element);
10+
const $refresh = $(".data-export-refresh", $results);
11+
const $status = $(".data-export-status", $results);
12+
const $download = $(".data-export-download", $results);
13+
const $start = $(".data-export-start", $results);
14+
15+
const updateStatus = function(newStatus) {
16+
status = newStatus;
17+
if (newStatus.export_pending) {
18+
$start.hide();
19+
$download.hide();
20+
$download.removeAttr("href");
21+
$status.text("Running...");
22+
} else if (newStatus.download_url !== null) {
23+
$start.show();
24+
$download.attr("href", newStatus.download_url);
25+
$download.show();
26+
$status.text("Ready.");
27+
} else if (newStatus.last_export_result.error) {
28+
$start.show();
29+
$download.hide();
30+
$download.removeAttr("href");
31+
$status.text("Error.");
32+
} else {
33+
$start.show();
34+
$download.hide();
35+
$download.removeAttr("href");
36+
$status.text("Idle.");
37+
}
38+
}
39+
40+
const getStatus = function() {
41+
$status.text("...");
42+
$.ajax({
43+
type: 'POST',
44+
url: getStatusUrl,
45+
data: JSON.stringify({}),
46+
success: updateStatus,
47+
error: function() {
48+
alert(gettext("An error has occurred."));
49+
},
50+
});
51+
}
52+
$refresh.on("click", getStatus);
53+
54+
$start.on('click', function() {
55+
$status.text("...");
56+
$.ajax({
57+
type: 'POST',
58+
url: startExportUrl,
59+
data: JSON.stringify({}),
60+
success: updateStatus,
61+
error: function() {
62+
alert(gettext("An error has occurred."));
63+
},
64+
});
65+
});
66+
67+
getStatus();
68+
}

ai_eval/tasks.py

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
"""
2+
Celery task for student messages export.
3+
"""
4+
import itertools
5+
import time
6+
7+
from celery import shared_task
8+
from celery.utils.log import get_task_logger
9+
from django.contrib.auth.models import User
10+
from xblock.fields import Scope
11+
12+
from . import ShortAnswerAIEvalXBlock
13+
14+
logger = get_task_logger(__name__)
15+
16+
17+
@shared_task()
18+
def export_data(course_id_str):
19+
"""
20+
Exports chat logs from all supported XBlocks.
21+
"""
22+
from common.djangoapps.util.file import course_filename_prefix_generator
23+
from lms.djangoapps.instructor_task.models import ReportStore
24+
from opaque_keys.edx.keys import CourseKey
25+
26+
start_timestamp = time.time()
27+
28+
course_id = CourseKey.from_string(course_id_str)
29+
30+
logger.debug("Beginning data export")
31+
32+
header = ("Section", "Subsection", "Unit", "Location",
33+
"Display Name", "Username", "User E-mail", "Conversation",
34+
"Source", "Message")
35+
36+
rows = itertools.chain([header], _extract_all_data(course_id))
37+
38+
report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
39+
40+
timestamp = time.strftime("%Y-%m-%d-%H%M%S", time.gmtime(start_timestamp))
41+
filename = "ai_eval_history-{course_prefix}-{timestamp_str}.csv".format(
42+
course_prefix=course_filename_prefix_generator(course_id),
43+
timestamp_str=timestamp
44+
)
45+
report_store.store_rows(course_id, filename, rows)
46+
47+
generation_time_s = time.time() - start_timestamp
48+
logger.debug(f"Done data export - took {generation_time_s} seconds")
49+
50+
return {
51+
"report_filename": filename,
52+
"start_timestamp": start_timestamp,
53+
"generation_time_s": generation_time_s,
54+
}
55+
56+
57+
def _extract_all_data(course_id):
58+
from xmodule.modulestore.django import modulestore
59+
60+
store = modulestore()
61+
for block in store.get_items(course_id):
62+
if isinstance(block, ShortAnswerAIEvalXBlock):
63+
yield from _extract_data(block)
64+
65+
66+
def _extract_data(block):
67+
from common.djangoapps.student.models import CourseEnrollment
68+
from lms.djangoapps.courseware.model_data import (
69+
DjangoKeyValueStore,
70+
FieldDataCache,
71+
)
72+
73+
section_name, subsection_name, unit_name = _get_context(block)
74+
75+
for user in CourseEnrollment.objects.users_enrolled_in(str(block.course_id)):
76+
data = FieldDataCache([], block.course_id, user)
77+
data.add_blocks_to_cache([block])
78+
79+
try:
80+
messages = data.get(DjangoKeyValueStore.Key(
81+
scope=Scope.user_state,
82+
user_id=user.id,
83+
block_scope_id=block.location,
84+
field_name='messages'
85+
))
86+
except KeyError:
87+
continue
88+
89+
conversation = 1
90+
for user_message, llm_message in zip(messages['USER'], messages['LLM']):
91+
yield (section_name, subsection_name, unit_name,
92+
str(block.location), block.display_name,
93+
user.username, user.email or "", conversation,
94+
"user", user_message)
95+
yield (section_name, subsection_name, unit_name,
96+
str(block.location), block.display_name,
97+
user.username, user.email or "", conversation,
98+
"llm", llm_message)
99+
100+
101+
def _get_context(block):
102+
"""
103+
Return section, subsection, and unit names for `block`.
104+
"""
105+
block_names_by_type = {}
106+
block_iter = block
107+
while block_iter:
108+
block_iter_type = block_iter.scope_ids.block_type
109+
block_names_by_type[block_iter_type] = block_iter.display_name_with_default
110+
block_iter = block_iter.get_parent() if block_iter.parent else None
111+
section_name = block_names_by_type.get('chapter', '')
112+
subsection_name = block_names_by_type.get('sequential', '')
113+
unit_name = block_names_by_type.get('vertical', '')
114+
return section_name, subsection_name, unit_name

ai_eval/templates/export.html

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<h4>Export tool</h4>
2+
3+
<div id="results-wrapper">
4+
<div class="data-export-status"></div>
5+
6+
<div class="data-export-actions">
7+
<button class="data-export-refresh">Refresh</button>
8+
<a class="button btn data-export-download" target="_blank" style="line-height: normal;">Download</a>
9+
<button class="data-export-start">Start export</button>
10+
</div>
11+
</div>

0 commit comments

Comments
 (0)