Exercise: Migration

.vscode/launch.json

@@ -15,7 +15,11 @@
       "pathMappings": [
         {
           "localRoot": "${workspaceFolder}/bakery-src/scripts/",
-          "remoteRoot": "/workspace/enki/venv/lib/python3.8/site-packages/bakery_scripts"
+          "remoteRoot": "/usr/local/lib/python3.10/dist-packages/bakery_scripts"
+        },
+        {
+          "localRoot": "${workspaceFolder}/nebuchadnezzar/nebu",
+          "remoteRoot": "/usr/local/lib/python3.10/dist-packages/nebu"
         }
       ],
       "justMyCode": false

dockerfiles/steps/step-prebake.bash

@@ -99,6 +99,7 @@ if [[ $LOCAL_ATTIC_DIR != '' ]]; then
     for file in $files; do
         node --unhandled-rejections=strict "${JS_EXTRA_VARS[@]}"  "$JS_UTILS_STUFF_ROOT/bin/bakery-helper" add-sourcemap-info "$file" "$file"
     done
+    echo "XML files annotated successfully!"
     popd > /dev/null
 fi
 

nebuchadnezzar/nebu/cli/assemble.py

@@ -2,48 +2,37 @@
 
 import click
 
-from ._common import common_params
+from ..formatters import (assemble_collection, fetch_insert_includes,
+                          interactive_callback_factory, resolve_module_links,
+                          update_ids)
 from ..models.book_part import BookPart
-from ..formatters import (
-    fetch_insert_includes,
-    resolve_module_links,
-    update_ids,
-    assemble_collection,
-    exercise_callback_factory,
-)
 from ..xml_utils import fix_namespaces
-
+from ._common import common_params
 
 ASSEMBLED_FILENAME = "collection.assembled.xhtml"
-DEFAULT_EXERCISES_HOST = "exercises.openstax.org"
+DEFAULT_INTERACTIVES_PATH = "interactives"
 
 
-def create_exercise_factories(exercise_host, token):
-    exercise_match_urls = (
+def create_interactive_factories(interactives_root):
+    exercise_match_paths = (
         (
-            "#ost/api/ex/",
-            "https://{}/api/exercises?q=tag:{{itemCode}}".format(
-                exercise_host
-            ),
-        ),
-        (
-            "#exercise/",
-            "https://{}/api/exercises?q=nickname:{{itemCode}}".format(
-                exercise_host
+            "{INTERACTIVES_ROOT}",
+            "{}{{itemCode}}".format(
+                interactives_root
             ),
         ),
     )
     return [
-        exercise_callback_factory(exercise_match, exercise_url, token=token)
-        for exercise_match, exercise_url in exercise_match_urls
+        interactive_callback_factory(exercise_match, exercise_path)
+        for exercise_match, exercise_path in exercise_match_paths
     ]
 
 
 def collection_to_assembled_xhtml(
-    collection, docs_by_id, docs_by_uuid, input_dir, token, exercise_host
+    collection, docs_by_id, docs_by_uuid, input_dir, interactives_path
 ):
     page_uuids = list(docs_by_uuid.keys())
-    includes = create_exercise_factories(exercise_host, token)
+    includes = create_interactive_factories(interactives_path)
     # Use docs_by_uuid.values to ensure each document is only used one time
     for document in docs_by_uuid.values():
         # Step 1: Rewrite module links
@@ -65,15 +54,12 @@ def collection_to_assembled_xhtml(
 @click.argument("input-dir", type=click.Path(exists=True))
 @click.argument("output-dir", type=click.Path())
 @click.option(
-    "--exercise-token", help="Token for including answers in exercises"
-)
-@click.option(
-    "--exercise-host",
-    default=DEFAULT_EXERCISES_HOST,
-    help="Default {}".format(DEFAULT_EXERCISES_HOST),
+    "--interactives-path",
+    default=DEFAULT_INTERACTIVES_PATH,
+    help="Default {}".format(DEFAULT_INTERACTIVES_PATH),
 )
 @click.pass_context
-def assemble(ctx, input_dir, output_dir, exercise_token, exercise_host):
+def assemble(ctx, input_dir, output_dir, interactives_path):
     """Assembles litezip structure data into a single-page-html file.
 
     This also stores the intermediary results alongside the resulting
@@ -103,8 +89,7 @@ def assemble(ctx, input_dir, output_dir, exercise_token, exercise_host):
         docs_by_id,
         docs_by_uuid,
         input_dir,
-        exercise_token,
-        exercise_host,
+        interactives_path
     )
     output_assembled_xhtml.write_bytes(assembled_xhtml)
 

nebuchadnezzar/nebu/formatters.py

@@ -5,28 +5,22 @@
 # Public License version 3 (AGPLv3).
 # See LICENCE.txt for details.
 # ###
+import os
 import asyncio
 import logging
 from copy import copy
 from functools import lru_cache
+import json
 
 import jinja2
 import lxml.html
 from lxml import etree
 
-import requests
-from requests.exceptions import RequestException
-import backoff
-
+from .async_job_queue import AsyncJobQueue
 from .converters import cnxml_abstract_to_html
-from .xml_utils import (
-    HTML_DOCUMENT_NAMESPACES,
-    xpath_html,
-    etree_from_str,
-    squash_xml_to_text,
-)
 from .templates.exercise_template import EXERCISE_TEMPLATE
-from .async_job_queue import AsyncJobQueue
+from .xml_utils import (HTML_DOCUMENT_NAMESPACES, etree_from_str,
+                        squash_xml_to_text, xpath_html)
 
 logger = logging.getLogger("nebu")
 
@@ -43,7 +37,7 @@ def etree_to_content(etree_, strip_root_node=False):
 
 
 def fetch_insert_includes(root_elem, page_uuids, includes, threads=20):
-    async def async_exercise_fetching():
+    async def async_interactive_fetching():
         loop = asyncio.get_running_loop()
         for match, proc in includes:
             job_queue = AsyncJobQueue(threads)
@@ -58,7 +52,7 @@ async def async_exercise_fetching():
                     "\n###### NEXT ERROR ######\n".join(job_queue.errors)
                 )
 
-    asyncio.run(async_exercise_fetching())
+    asyncio.run(async_interactive_fetching())
 
 
 def update_ids(document):
@@ -245,13 +239,13 @@ def recursive_build(subcol, elem):
     return root
 
 
-def exercise_callback_factory(match, url_template, token=None):
+def interactive_callback_factory(match, url_template):
     """Create a callback function to replace an exercise by fetching from
     a server."""
 
     def _annotate_exercise(elem, exercise, page_uuids):
         """Annotate exercise based upon tag data"""
-        tags = exercise["items"][0].get("tags")
+        tags = exercise["metadata"]["tags"]
         if not tags:
             return
 
@@ -330,40 +324,95 @@ def _annotate_exercise(elem, exercise, page_uuids):
         )
         assert feature_element is not None, assert_msg
 
-        exercise["items"][0]["required_context"] = {}
-        exercise["items"][0]["required_context"]["module"] = target_module
-        exercise["items"][0]["required_context"]["feature"] = feature
-        exercise["items"][0]["required_context"]["ref"] = target_ref
-
-    @backoff.on_exception(
-        backoff.expo,
-        RequestException,
-        max_time=60 * 15,
-        giveup=lambda e: (
-            # Give up if the status code is something like 404, 403, etc.
-            isinstance(e, RequestException) and
-            e.response.status_code in range(400, 500)
-        ),
-        jitter=backoff.full_jitter,
-        raise_on_giveup=True
-    )
+        exercise["required_context"] = {}
+        exercise["required_context"]["module"] = target_module
+        exercise["required_context"]["feature"] = feature
+        exercise["required_context"]["ref"] = target_ref
+
+    def _recursive_merge(interactive, private):
+        if private is None or len(private) == 0:
+            return interactive
+        merged = interactive.copy()
+        for key, value in private.items():
+            if key in merged:
+                if isinstance(merged[key], dict) and isinstance(value, dict):
+                    merged[key] = _recursive_merge(merged[key], value)
+                elif isinstance(merged[key], list) and isinstance(value, list) and len(merged[key]) == len(value):
+                    merged[key] = [_recursive_merge(
+                        merged_item, value_item) for merged_item, value_item in zip(merged[key], value)]
+            else:
+                merged[key] = value
+        return merged
+
+    def _load_interactive_data(interactive_path, private_path):
+        metadata_path= os.path.join(interactive_path, "metadata.json")
+        content_path= os.path.join(interactive_path, "content.json")
+        h5p_path =os.path.join(interactive_path, "h5p.json")
+        private_metadata_path= os.path.join(private_path, "metadata.json")
+        private_content_path= os.path.join(private_path, "content.json")
+        if not os.path.exists(metadata_path) or not os.path.exists(content_path) or not os.path.exists(h5p_path):
+            logging.error("MISSING INTERACTIVE DATA: {}".format(interactive_path))
+            return None
+        exercise ={}
+        exercise['h5p'] = json.loads(open(h5p_path).read())
+        exercise['content'] = _recursive_merge(json.loads(open(content_path).read()), json.loads(open(private_content_path).read()) if os.path.exists(private_content_path) else None )
+        exercise['metadata'] = _recursive_merge(json.loads(open(metadata_path).read()), json.loads(open(private_metadata_path).read()) if os.path.exists(private_metadata_path) else None )
+        return _parse_exercise_content(exercise)
+
+    def _format_exercise_content(exercise):
+        formats = []
+        library = exercise["h5p"]["mainLibrary"]
+        if exercise['metadata']['is_free_response_supported']:
+            formats.append("free-response")
+
+        if library == "H5P.MultiChoice":
+            formats.append("multiple-choice")
+            question ={
+                "id": 1,
+                "html": exercise["content"]["question"],
+                "stimulus": exercise["metadata"]["questions"][0]['stimulus'],
+                "answers" : [],
+                "formats": formats,
+                "is_answer_order_important": not exercise['content']['behaviour']['randomAnswers'],
+                "collaborator_solutions": []
+            }
+            for i, a in enumerate(exercise['content']['answers']):
+                question['answers'].append({
+                    "id": i,
+                    "html": a['text'],
+                    "correctness": a['correct'],
+                    "feedback": a['tipsAndFeedback']['chosenFeedback']
+
+                })
+
+            for a in exercise['metadata']['collaborator_solutions'][0]:
+                question['collaborator_solutions'].append({
+                        "html": a["content"],
+                        "type": a["solution_type"]
+                })
+            exercise['questions'] = [question]
+        else:
+            logging.error("UNSUPPORTED EXERCISE TYPE: {}".format(library))
+        return exercise
+
+    def _parse_exercise_content(exercise):
+        assert exercise["content"] is not None, "Exercise content is None"
+        assert exercise["h5p"] is not None and exercise["h5p"]["mainLibrary"] is not None, "Exercise h5p library undefined"
+        _format_exercise_content(exercise)
+        return exercise
+
     def _replace_exercises(elem, page_uuids):
         item_code = elem.get("href")[len(match):]
-        url = url_template.format(itemCode=item_code)
+        interactive_path = f"{os.getenv('IO_FETCHED')}/{url_template.format(itemCode=item_code)}"
+        private_path = f"{os.getenv('IO_FETCHED')}/private/{url_template.format(itemCode=item_code)}"
         exercise_class = elem.get("class")
-        if token:
-            headers = {"Authorization": "Bearer {}".format(token)}
-            res = requests.get(url, headers=headers)
-        else:
-            res = requests.get(url)
 
-        assert res
         # grab the json exercise, run it through Jinja2 template,
         # replace element w/ it
-        exercise = res.json()
+        exercise = _load_interactive_data(interactive_path, private_path)
 
-        if exercise["total_count"] == 0:
-            logger.warning("MISSING EXERCISE: {}".format(url))
+        if exercise is None:
+            logger.warning("MISSING EXERCISE: {}".format(interactive_path))
 
             XHTML = "{{{}}}".format(HTML_DOCUMENT_NAMESPACES["xhtml"])
             missing = etree.Element(
@@ -374,8 +423,8 @@ def _replace_exercises(elem, page_uuids):
             missing.text = "MISSING EXERCISE: tag:{}".format(item_code)
             nodes = [missing]
         else:
-            exercise["items"][0]["url"] = url
-            exercise["items"][0]["class"] = exercise_class
+            exercise["url"] = interactive_path
+            exercise["class"] = exercise_class
             _annotate_exercise(elem, exercise, page_uuids)
 
             html = render_exercise(exercise)
@@ -394,10 +443,8 @@ def _replace_exercises(elem, page_uuids):
 
 
 def render_exercise(exercise):
-    assert len(exercise["items"]) == 1, 'Exercise "items" array is nonsingular'
-    exercise_content = exercise["items"][0]
-
-    return EXERCISE_TEMPLATE.render(data=exercise_content)
+    assert len(exercise["questions"]) > 0, 'No exercise found!'
+    return EXERCISE_TEMPLATE.render(data=exercise)
 
 
 HTML_DOCUMENT = """\

nebuchadnezzar/nebu/templates/exercise_template.py

@@ -4,11 +4,9 @@
 <div
     data-type="injected-exercise"
     class="{{ data.class }}"
-    data-injected-from-nickname="{{ data.nickname }}"
-    data-injected-from-version="{{ data.version }}"
+    data-injected-from-nickname="{{ data.metadata.nickname }}"
     data-injected-from-url="{{ data.url }}"
-    data-tags="{{ ' '.join(data.tags) }}"
-    data-is-vocab="{{ data.is_vocab | lower }}">
+    data-tags="{{ ' '.join(data.metadata.tags) }}">
     {% if data.required_context -%}
     <div data-type="exercise-context"
         data-context-module="{{ data.required_context.module }}"
@@ -17,8 +15,8 @@
         href="#{{ data.required_context.ref }}">[link]</a>
     </div>
     {% endif -%}
-    {% if data.stimulus_html -%}
-    <div data-type="exercise-stimulus">{{ data.stimulus_html }}</div>
+    {% if data.metadata.stimulus -%}
+    <div data-type="exercise-stimulus">{{ data.metadata.stimulus }}</div>
     {% endif -%}
     {% for question in data.questions -%}
     <div
@@ -27,28 +25,23 @@
         data-formats="{{ ' '.join(question.formats) }}"
         {% if question.id %}data-id="{{ question.id }}"{% endif %}
         >
-        {% if question.stimulus_html -%}
-        <div data-type="question-stimulus">{{ question.stimulus_html }}</div>
+        {% if question.stimulus -%}
+        <div data-type="question-stimulus">{{ question.stimulus }}</div>
         {% endif -%}
-        <div data-type="question-stem">{{ question.stem_html }}</div>
+        <div data-type="question-stem">{{ question.html }}</div>
         {% if question.answers -%}
         <ol data-type="question-answers" type="a">
             {% for option in question.answers -%}
             <li data-type="question-answer" data-id="{{ option.id }}" data-correctness="{{ option.correctness }}">
-                <div data-type="answer-content">{{ option.content_html }}</div>
-                {% if option.feedback_html %}<div data-type="answer-feedback">{{ option.feedback_html }}</div>{% endif %}
+                <div data-type="answer-content">{{ option.html }}</div>
+                {% if option.feedback %}<div data-type="answer-feedback">{{ option.feedback }}</div>{% endif %}
             </li>
             {% endfor -%}
         </ol>
         {% endif -%}
         {% for solution in question.collaborator_solutions -%}
-        <div data-type="question-solution" data-solution-source="collaborator" data-solution-type="{{ solution.solution_type }}">
-            {{ solution.content_html }}
-        </div>
-        {% endfor -%}
-        {% for solution in question.community_solutions -%}
-        <div data-type="question-solution" data-solution-source="community" data-solution-type="{{ solution.solution_type }}">
-            {{ solution.content_html }}
+        <div data-type="question-solution" data-solution-source="collaborator" data-solution-type="{{ solution.type }}">
+            {{ solution.html }}
         </div>
         {% endfor -%}
     </div>