diff --git a/.vscode/launch.json b/.vscode/launch.json index a1b9ea74..0ef087fd 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -15,7 +15,11 @@ "pathMappings": [ { "localRoot": "${workspaceFolder}/bakery-src/scripts/", - "remoteRoot": "/workspace/enki/venv/lib/python3.8/site-packages/bakery_scripts" + "remoteRoot": "/usr/local/lib/python3.10/dist-packages/bakery_scripts" + }, + { + "localRoot": "${workspaceFolder}/nebuchadnezzar/nebu", + "remoteRoot": "/usr/local/lib/python3.10/dist-packages/nebu" } ], "justMyCode": false diff --git a/dockerfiles/steps/step-prebake.bash b/dockerfiles/steps/step-prebake.bash index 99114bcc..609b0b70 100644 --- a/dockerfiles/steps/step-prebake.bash +++ b/dockerfiles/steps/step-prebake.bash @@ -99,6 +99,7 @@ if [[ $LOCAL_ATTIC_DIR != '' ]]; then for file in $files; do node --unhandled-rejections=strict "${JS_EXTRA_VARS[@]}" "$JS_UTILS_STUFF_ROOT/bin/bakery-helper" add-sourcemap-info "$file" "$file" done + echo "XML files annotated successfully!" popd > /dev/null fi diff --git a/nebuchadnezzar/nebu/cli/assemble.py b/nebuchadnezzar/nebu/cli/assemble.py index 34aef820..7013fa00 100644 --- a/nebuchadnezzar/nebu/cli/assemble.py +++ b/nebuchadnezzar/nebu/cli/assemble.py @@ -2,48 +2,37 @@ import click -from ._common import common_params +from ..formatters import (assemble_collection, fetch_insert_includes, + interactive_callback_factory, resolve_module_links, + update_ids) from ..models.book_part import BookPart -from ..formatters import ( - fetch_insert_includes, - resolve_module_links, - update_ids, - assemble_collection, - exercise_callback_factory, -) from ..xml_utils import fix_namespaces - +from ._common import common_params ASSEMBLED_FILENAME = "collection.assembled.xhtml" -DEFAULT_EXERCISES_HOST = "exercises.openstax.org" +DEFAULT_INTERACTIVES_PATH = "interactives" -def create_exercise_factories(exercise_host, token): - exercise_match_urls = ( +def create_interactive_factories(interactives_root): + exercise_match_paths = ( ( - "#ost/api/ex/", - "https://{}/api/exercises?q=tag:{{itemCode}}".format( - exercise_host - ), - ), - ( - "#exercise/", - "https://{}/api/exercises?q=nickname:{{itemCode}}".format( - exercise_host + "{INTERACTIVES_ROOT}", + "{}{{itemCode}}".format( + interactives_root ), ), ) return [ - exercise_callback_factory(exercise_match, exercise_url, token=token) - for exercise_match, exercise_url in exercise_match_urls + interactive_callback_factory(exercise_match, exercise_path) + for exercise_match, exercise_path in exercise_match_paths ] def collection_to_assembled_xhtml( - collection, docs_by_id, docs_by_uuid, input_dir, token, exercise_host + collection, docs_by_id, docs_by_uuid, input_dir, interactives_path ): page_uuids = list(docs_by_uuid.keys()) - includes = create_exercise_factories(exercise_host, token) + includes = create_interactive_factories(interactives_path) # Use docs_by_uuid.values to ensure each document is only used one time for document in docs_by_uuid.values(): # Step 1: Rewrite module links @@ -65,15 +54,12 @@ def collection_to_assembled_xhtml( @click.argument("input-dir", type=click.Path(exists=True)) @click.argument("output-dir", type=click.Path()) @click.option( - "--exercise-token", help="Token for including answers in exercises" -) -@click.option( - "--exercise-host", - default=DEFAULT_EXERCISES_HOST, - help="Default {}".format(DEFAULT_EXERCISES_HOST), + "--interactives-path", + default=DEFAULT_INTERACTIVES_PATH, + help="Default {}".format(DEFAULT_INTERACTIVES_PATH), ) @click.pass_context -def assemble(ctx, input_dir, output_dir, exercise_token, exercise_host): +def assemble(ctx, input_dir, output_dir, interactives_path): """Assembles litezip structure data into a single-page-html file. This also stores the intermediary results alongside the resulting @@ -103,8 +89,7 @@ def assemble(ctx, input_dir, output_dir, exercise_token, exercise_host): docs_by_id, docs_by_uuid, input_dir, - exercise_token, - exercise_host, + interactives_path ) output_assembled_xhtml.write_bytes(assembled_xhtml) diff --git a/nebuchadnezzar/nebu/formatters.py b/nebuchadnezzar/nebu/formatters.py index 5616e428..08b73e1e 100644 --- a/nebuchadnezzar/nebu/formatters.py +++ b/nebuchadnezzar/nebu/formatters.py @@ -5,28 +5,22 @@ # Public License version 3 (AGPLv3). # See LICENCE.txt for details. # ### +import os import asyncio import logging from copy import copy from functools import lru_cache +import json import jinja2 import lxml.html from lxml import etree -import requests -from requests.exceptions import RequestException -import backoff - +from .async_job_queue import AsyncJobQueue from .converters import cnxml_abstract_to_html -from .xml_utils import ( - HTML_DOCUMENT_NAMESPACES, - xpath_html, - etree_from_str, - squash_xml_to_text, -) from .templates.exercise_template import EXERCISE_TEMPLATE -from .async_job_queue import AsyncJobQueue +from .xml_utils import (HTML_DOCUMENT_NAMESPACES, etree_from_str, + squash_xml_to_text, xpath_html) logger = logging.getLogger("nebu") @@ -43,7 +37,7 @@ def etree_to_content(etree_, strip_root_node=False): def fetch_insert_includes(root_elem, page_uuids, includes, threads=20): - async def async_exercise_fetching(): + async def async_interactive_fetching(): loop = asyncio.get_running_loop() for match, proc in includes: job_queue = AsyncJobQueue(threads) @@ -58,7 +52,7 @@ async def async_exercise_fetching(): "\n###### NEXT ERROR ######\n".join(job_queue.errors) ) - asyncio.run(async_exercise_fetching()) + asyncio.run(async_interactive_fetching()) def update_ids(document): @@ -245,13 +239,13 @@ def recursive_build(subcol, elem): return root -def exercise_callback_factory(match, url_template, token=None): +def interactive_callback_factory(match, url_template): """Create a callback function to replace an exercise by fetching from a server.""" def _annotate_exercise(elem, exercise, page_uuids): """Annotate exercise based upon tag data""" - tags = exercise["items"][0].get("tags") + tags = exercise["metadata"]["tags"] if not tags: return @@ -330,40 +324,95 @@ def _annotate_exercise(elem, exercise, page_uuids): ) assert feature_element is not None, assert_msg - exercise["items"][0]["required_context"] = {} - exercise["items"][0]["required_context"]["module"] = target_module - exercise["items"][0]["required_context"]["feature"] = feature - exercise["items"][0]["required_context"]["ref"] = target_ref - - @backoff.on_exception( - backoff.expo, - RequestException, - max_time=60 * 15, - giveup=lambda e: ( - # Give up if the status code is something like 404, 403, etc. - isinstance(e, RequestException) and - e.response.status_code in range(400, 500) - ), - jitter=backoff.full_jitter, - raise_on_giveup=True - ) + exercise["required_context"] = {} + exercise["required_context"]["module"] = target_module + exercise["required_context"]["feature"] = feature + exercise["required_context"]["ref"] = target_ref + + def _recursive_merge(interactive, private): + if private is None or len(private) == 0: + return interactive + merged = interactive.copy() + for key, value in private.items(): + if key in merged: + if isinstance(merged[key], dict) and isinstance(value, dict): + merged[key] = _recursive_merge(merged[key], value) + elif isinstance(merged[key], list) and isinstance(value, list) and len(merged[key]) == len(value): + merged[key] = [_recursive_merge( + merged_item, value_item) for merged_item, value_item in zip(merged[key], value)] + else: + merged[key] = value + return merged + + def _load_interactive_data(interactive_path, private_path): + metadata_path= os.path.join(interactive_path, "metadata.json") + content_path= os.path.join(interactive_path, "content.json") + h5p_path =os.path.join(interactive_path, "h5p.json") + private_metadata_path= os.path.join(private_path, "metadata.json") + private_content_path= os.path.join(private_path, "content.json") + if not os.path.exists(metadata_path) or not os.path.exists(content_path) or not os.path.exists(h5p_path): + logging.error("MISSING INTERACTIVE DATA: {}".format(interactive_path)) + return None + exercise ={} + exercise['h5p'] = json.loads(open(h5p_path).read()) + exercise['content'] = _recursive_merge(json.loads(open(content_path).read()), json.loads(open(private_content_path).read()) if os.path.exists(private_content_path) else None ) + exercise['metadata'] = _recursive_merge(json.loads(open(metadata_path).read()), json.loads(open(private_metadata_path).read()) if os.path.exists(private_metadata_path) else None ) + return _parse_exercise_content(exercise) + + def _format_exercise_content(exercise): + formats = [] + library = exercise["h5p"]["mainLibrary"] + if exercise['metadata']['is_free_response_supported']: + formats.append("free-response") + + if library == "H5P.MultiChoice": + formats.append("multiple-choice") + question ={ + "id": 1, + "html": exercise["content"]["question"], + "stimulus": exercise["metadata"]["questions"][0]['stimulus'], + "answers" : [], + "formats": formats, + "is_answer_order_important": not exercise['content']['behaviour']['randomAnswers'], + "collaborator_solutions": [] + } + for i, a in enumerate(exercise['content']['answers']): + question['answers'].append({ + "id": i, + "html": a['text'], + "correctness": a['correct'], + "feedback": a['tipsAndFeedback']['chosenFeedback'] + + }) + + for a in exercise['metadata']['collaborator_solutions'][0]: + question['collaborator_solutions'].append({ + "html": a["content"], + "type": a["solution_type"] + }) + exercise['questions'] = [question] + else: + logging.error("UNSUPPORTED EXERCISE TYPE: {}".format(library)) + return exercise + + def _parse_exercise_content(exercise): + assert exercise["content"] is not None, "Exercise content is None" + assert exercise["h5p"] is not None and exercise["h5p"]["mainLibrary"] is not None, "Exercise h5p library undefined" + _format_exercise_content(exercise) + return exercise + def _replace_exercises(elem, page_uuids): item_code = elem.get("href")[len(match):] - url = url_template.format(itemCode=item_code) + interactive_path = f"{os.getenv('IO_FETCHED')}/{url_template.format(itemCode=item_code)}" + private_path = f"{os.getenv('IO_FETCHED')}/private/{url_template.format(itemCode=item_code)}" exercise_class = elem.get("class") - if token: - headers = {"Authorization": "Bearer {}".format(token)} - res = requests.get(url, headers=headers) - else: - res = requests.get(url) - assert res # grab the json exercise, run it through Jinja2 template, # replace element w/ it - exercise = res.json() + exercise = _load_interactive_data(interactive_path, private_path) - if exercise["total_count"] == 0: - logger.warning("MISSING EXERCISE: {}".format(url)) + if exercise is None: + logger.warning("MISSING EXERCISE: {}".format(interactive_path)) XHTML = "{{{}}}".format(HTML_DOCUMENT_NAMESPACES["xhtml"]) missing = etree.Element( @@ -374,8 +423,8 @@ def _replace_exercises(elem, page_uuids): missing.text = "MISSING EXERCISE: tag:{}".format(item_code) nodes = [missing] else: - exercise["items"][0]["url"] = url - exercise["items"][0]["class"] = exercise_class + exercise["url"] = interactive_path + exercise["class"] = exercise_class _annotate_exercise(elem, exercise, page_uuids) html = render_exercise(exercise) @@ -394,10 +443,8 @@ def _replace_exercises(elem, page_uuids): def render_exercise(exercise): - assert len(exercise["items"]) == 1, 'Exercise "items" array is nonsingular' - exercise_content = exercise["items"][0] - - return EXERCISE_TEMPLATE.render(data=exercise_content) + assert len(exercise["questions"]) > 0, 'No exercise found!' + return EXERCISE_TEMPLATE.render(data=exercise) HTML_DOCUMENT = """\ diff --git a/nebuchadnezzar/nebu/templates/exercise_template.py b/nebuchadnezzar/nebu/templates/exercise_template.py index 64532644..f072df04 100644 --- a/nebuchadnezzar/nebu/templates/exercise_template.py +++ b/nebuchadnezzar/nebu/templates/exercise_template.py @@ -4,11 +4,9 @@
+ data-tags="{{ ' '.join(data.metadata.tags) }}"> {% if data.required_context -%}
[link]
{% endif -%} - {% if data.stimulus_html -%} -
{{ data.stimulus_html }}
+ {% if data.metadata.stimulus -%} +
{{ data.metadata.stimulus }}
{% endif -%} {% for question in data.questions -%}
- {% if question.stimulus_html -%} -
{{ question.stimulus_html }}
+ {% if question.stimulus -%} +
{{ question.stimulus }}
{% endif -%} -
{{ question.stem_html }}
+
{{ question.html }}
{% if question.answers -%}
    {% for option in question.answers -%}
  1. -
    {{ option.content_html }}
    - {% if option.feedback_html %}
    {{ option.feedback_html }}
    {% endif %} +
    {{ option.html }}
    + {% if option.feedback %}
    {{ option.feedback }}
    {% endif %}
    2. {% endfor -%}
{% endif -%} {% for solution in question.collaborator_solutions -%} -
- {{ solution.content_html }} -
- {% endfor -%} - {% for solution in question.community_solutions -%} -
- {{ solution.content_html }} +
+ {{ solution.html }}
{% endfor -%}