From 157c5c02e89d527a35d8bad078eb5daedad09737 Mon Sep 17 00:00:00 2001 From: Eric Ihli Date: Tue, 13 Sep 2022 21:16:11 -0700 Subject: [PATCH 1/9] Add ability to link to chapter/section To link to a chapter/section: [anchor text](@chapter or section title) This change adds a slot to the weaver to store a map of chapter/section titles to their IDs. Then when we're weaving, we search for these links and replace them with their [anchor text](#c:id). This commit is a work-in-progress that I want to at least get out to start bouncing ideas around. --- dev.lit | 25 ++++++ parse.lisp | 3 + tests/basic/basic.lit | 7 +- utils.lisp | 50 +++++++++++ weave.lisp | 197 ++++++++++++++++++++++++++++-------------- 5 files changed, 216 insertions(+), 66 deletions(-) create mode 100644 dev.lit diff --git a/dev.lit b/dev.lit new file mode 100644 index 0000000..6628822 --- /dev/null +++ b/dev.lit @@ -0,0 +1,25 @@ +# My test lit file + +Preamble + +@toc + +## Foobar + +Section 1: foobar. +What follows is foobar.lisp. + +--- /foobar.lisp +(+ 2 2) +@{foobaz} +--- + +## Foobazs + +And this is the content of the [Foobaz](@Foobazs) section. + +Here's a ref. @{foobaz} What's it do? + +--- foobaz +(format nil (* 2 2)) +--- diff --git a/parse.lisp b/parse.lisp index 301b61c..f174ef6 100644 --- a/parse.lisp +++ b/parse.lisp @@ -37,6 +37,9 @@ (:REGISTER (:GREEDY-REPETITION 1 NIL (:INVERTED-CHAR-CLASS #\}))) #\}))) +;; To link to an section or chapter +(defparameter *anchor-pattern* "\\[.*?\\]\\(@(.*?)\\)") + (defun parse-refs (line) (let ((parts (ppcre:split *ref-pattern* line :with-registers-p t))) (mapcar-indexed (lambda (string i) diff --git a/tests/basic/basic.lit b/tests/basic/basic.lit index 88fd2ec..a4f70d1 100644 --- a/tests/basic/basic.lit +++ b/tests/basic/basic.lit @@ -87,8 +87,11 @@ Lastly, there are some boring headers to include. #include --- -We can also reference blocks in prose. -Note that @{algorithms} is `O(n)`. +We can reference blocks in prose. +Note that @{algorithms} is `O(n)`. + +We can also link to chapters and sections. +This link leads back up to [the Including blocks](@Including blocks) section. ## Formatting and whitespace diff --git a/utils.lisp b/utils.lisp index 708192f..6226de3 100644 --- a/utils.lisp +++ b/utils.lisp @@ -82,3 +82,53 @@ (define-condition user-error (simple-error) ()) +(defun singlep (l) + (= 1 (length l))) + +(defmacro comment (&rest body) + (declare (ignore body)) + nil) + +;; The original idea for this was that I wanted +;; a way to link using a minimal unique prefix. +;; +;; If my section heading is something long, +;; I don't want to have to type out the entire +;; Click [here](@Some long section heading that is easy to typo) to read more! +;; +;; So I was going to use a trie to lookup the Chapter/Section ID given +;; a title prefix. If it ends up being unique, we're good to go. If it's +;; not unique, we can pick one and leave a warning. +;; +;; I haven't yet gotten close to finishing that implementation. So it +;; doesn't make much sense to have this data structure right now. +;; But oh well. Whatever. It works for an MVP of the chapter/section linking stuff. +(defun trie-assoc (key alist) + (assoc key alist :test #'equal)) + +(defun trie-put (alist key val) + (if (singlep key) + (acons (car key) (acons :terminal val '()) alist) + (let ((child (trie-assoc (car key) alist))) + (if child + (progn + (rplacd child (trie-put (cdr child) (cdr key) val)) + alist) + (acons (car key) (trie-put '() (cdr key) val) alist))))) + +(defun trie-get (alist key) + (if (singlep key) + (trie-assoc (car key) alist) + (trie-get (cdr (trie-assoc (car key) alist)) (cdr key)))) + +(comment + (trie-put '() '(1 2) 3) + ; => ((1 (2 (:TERMINAL . 3)))) + (let ((trie (trie-put + (trie-put + (trie-put '() '(2) 3) + '(1) 2) + '(1 2) 3))) + trie) + ; => ((1 (2 (:TERMINAL . 3)) (:TERMINAL . 2)) (2 (:TERMINAL . 3))) + ) diff --git a/weave.lisp b/weave.lisp index 48f9aff..ae1b538 100644 --- a/weave.lisp +++ b/weave.lisp @@ -22,9 +22,39 @@ ; One-to-one correspondence between .lit and .html files. ; The body of an .html file should look the same regardless of whether the .lit is in another context (like a book). -(defstruct weaver +(defun get-section (weaver section) + (trie-get (weaver-section-map weaver) section)) + +(defun add-section (weaver section anchor) + (setf (weaver-section-map weaver) + (trie-put (weaver-section-map weaver) section anchor))) + +(comment + (let* ((file-defs (parse-lit-files '("dev.lit"))) + (weaver (make-weaver-default file-defs))) + (add-section weaver '("Some Chapter") "c2") + (add-section weaver '("Some Chapter" "some-section") "s2:1") + (trie-get (weaver-section-map weaver) '("some-chapter")) + (weaver-section-map weaver)) + ; => (("Some Chapter" + ; ("some-section" (:TERMINAL . "s2:1")) + ; (:TERMINAL . "c2"))) + ) + +(defstruct weaver (def-table (make-hash-table :test #'equal) :type hash-table) (use-table (make-hash-table :test #'equal) :type hash-table) + ;; section-map is used to map chapter/section text to their respective + ;; anchor ids, like #s2:4 + ;; + ;; This lets you create markdown links like this: + ;; [some section or chapter](@some section or chapter) + ;; Which will be woven to (for example): + ;; [some section or chapter](#s2:4) + ;; + ;; Right now, it doesn't work across files in a book. + ;; Heck... it barely works at all. + (section-map '() :type list) (initial-def-table (make-hash-table :test #'equal) :type hash-table) (toc nil :type list) @@ -197,71 +227,110 @@ (weave-uses weaver def)) (write-line "")) +;; This is not elegant. +(defun replace-sections-with-id-anchors (trie line) + (let ((r "") + (s 0) + (e 0)) + (ppcre:do-scans (start end reg-starts reg-ends *anchor-pattern* line) + (let ((title (subseq line (aref reg-starts 0) (aref reg-ends 0)))) + (setf + r + (format + nil + "~a~a" + r + (concatenate + 'string + (subseq line s (- (aref reg-starts 0) 1)) + (format nil "#~a" (cdr (assoc :terminal (cdr (trie-get trie (list title))))))))) + (setf s (aref reg-ends 0)) + (setf e (aref reg-ends 0)))) + (setf r (format nil "~a~a" r (subseq line e))) + r)) + +(comment + (let ((trie (trie-put '() '("some-chapter") "c1"))) + (replace-sections-with-id-anchors + trie + "foo [some chapter](@some-chapter) bar [other](@some-chapter) buz")) + ; => "foo [some chapter](#c1) bar [other](#c1) buz" + ) + (defun weave-prose-line (weaver line def) (loop for expr in line do - (cond ((stringp expr) (write-string expr)) - ((commandp expr) - (case (first expr) - (:INCLUDE (weave-include - (second expr) - (textblockdef-file def) - weaver - nil)) - (:TITLE - (setf (weaver-title weaver) - (second expr))) - (:C - ; Also can act as a title - (when (null (weaver-title weaver)) - (setf (weaver-title weaver) - (second expr))) - (incf (weaver-chapter-counter weaver)) - (setf (weaver-section-counter weaver) -1) - (format t "

~a

~%" - (second expr) - (chapter-id (weaver-chapter-counter weaver)))) - (:S - (incf (weaver-section-counter weaver)) - (format t "

~a. ~a

~%" - (+ (weaver-section-counter weaver) 1) - (second expr) - (section-id - (weaver-section-counter weaver) - (weaver-chapter-counter weaver)))) - (:CODE_TYPE - (let* ((args (split-whitespace (second expr))) - (language (first args)) - (extension (subseq (second args) 1))) - (setf (gethash extension (weaver-code-type-table weaver)) language) - (push extension (weaver-used-extensions weaver)))) - (:COMMENT_TYPE nil) - (:ADD_CSS nil) - (:OVERWRITE_CSS nil) - (:COLORSCHEME nil) - (:ERROR_FORMAT nil) - (:MATHBLOCK - ; Put tags in block so it displays tex nicely without JS. - (setf (weaver-used-math weaver) t) - (write-string "
") - (when (not (equal (second expr) "displaymath")) - (format t "\\begin{~a}" (second expr))) - (write-separated-list (third expr) #\newline *standard-output*) - (when (not (equal (second expr) "displaymath")) - (format t "\\end{~a}" (second expr))) - (write-string "
")) - (:MATH - ; Use backticks to prevent markdown from formatting tex, - ; for example treating _ as emphasis. - (setf (weaver-used-math weaver) t) - (format t "`~a`" - (second expr))) - (:TOC (weave-toc - (weaver-toc weaver) - (textblockdef-file def))) - (otherwise (error 'user-error - :format-control "unknown prose command ~S" - :format-arguments (first expr))))) - (t (error "unknown structure ~s" expr))))) + (cond ((stringp expr) + (if (ppcre:scan *anchor-pattern* expr) + (write-string + (replace-sections-with-id-anchors + (weaver-section-map weaver) + expr)) + (write-string expr))) + ((commandp expr) + (case (first expr) + (:INCLUDE (weave-include + (second expr) ; title of block + (textblockdef-file def) + weaver + nil)) + (:TITLE + (setf (weaver-title weaver) + (second expr))) + (:C + ; Also can act as a title + (when (null (weaver-title weaver)) + (setf (weaver-title weaver) + (second expr))) + (incf (weaver-chapter-counter weaver)) + (setf (weaver-section-counter weaver) -1) + (format t "

~a

~%" + (second expr) + (chapter-id (weaver-chapter-counter weaver))) + (add-section weaver (list (second expr)) (chapter-id (weaver-chapter-counter weaver)))) + (:S + (incf (weaver-section-counter weaver)) + (format t "

~a. ~a

~%" + (+ (weaver-section-counter weaver) 1) + (second expr) + (section-id + (weaver-section-counter weaver) + (weaver-chapter-counter weaver))) + (add-section weaver (list (second expr)) (section-id (weaver-section-counter weaver) + (weaver-chapter-counter weaver)))) + (:CODE_TYPE + (let* ((args (split-whitespace (second expr))) + (language (first args)) + (extension (subseq (second args) 1))) + (setf (gethash extension (weaver-code-type-table weaver)) language) + (push extension (weaver-used-extensions weaver)))) + (:COMMENT_TYPE nil) + (:ADD_CSS nil) + (:OVERWRITE_CSS nil) + (:COLORSCHEME nil) + (:ERROR_FORMAT nil) + (:MATHBLOCK + ; Put tags in block so it displays tex nicely without JS. + (setf (weaver-used-math weaver) t) + (write-string "
") + (when (not (equal (second expr) "displaymath")) + (format t "\\begin{~a}" (second expr))) + (write-separated-list (third expr) #\newline *standard-output*) + (when (not (equal (second expr) "displaymath")) + (format t "\\end{~a}" (second expr))) + (write-string "
")) + (:MATH + ; Use backticks to prevent markdown from formatting tex, + ; for example treating _ as emphasis. + (setf (weaver-used-math weaver) t) + (format t "`~a`" + (second expr))) + (:TOC (weave-toc + (weaver-toc weaver) + (textblockdef-file def))) + (otherwise (error 'user-error + :format-control "unknown prose command ~S" + :format-arguments (first expr))))) + (t (error "unknown structure ~s" expr))))) (defun weave-prosedef (weaver def) (let ((block (textblockdef-block def)) From 638e3a4b49c3e2587ac0bc74ac7d2ffa6d3264a9 Mon Sep 17 00:00:00 2001 From: Eric Ihli Date: Sun, 18 Sep 2022 22:40:29 -0700 Subject: [PATCH 2/9] Revert --- dev.lit | 25 ------ parse.lisp | 3 - tests/basic/basic.lit | 7 +- utils.lisp | 50 ----------- weave.lisp | 197 ++++++++++++++---------------------------- 5 files changed, 66 insertions(+), 216 deletions(-) delete mode 100644 dev.lit diff --git a/dev.lit b/dev.lit deleted file mode 100644 index 6628822..0000000 --- a/dev.lit +++ /dev/null @@ -1,25 +0,0 @@ -# My test lit file - -Preamble - -@toc - -## Foobar - -Section 1: foobar. -What follows is foobar.lisp. - ---- /foobar.lisp -(+ 2 2) -@{foobaz} ---- - -## Foobazs - -And this is the content of the [Foobaz](@Foobazs) section. - -Here's a ref. @{foobaz} What's it do? - ---- foobaz -(format nil (* 2 2)) ---- diff --git a/parse.lisp b/parse.lisp index f174ef6..301b61c 100644 --- a/parse.lisp +++ b/parse.lisp @@ -37,9 +37,6 @@ (:REGISTER (:GREEDY-REPETITION 1 NIL (:INVERTED-CHAR-CLASS #\}))) #\}))) -;; To link to an section or chapter -(defparameter *anchor-pattern* "\\[.*?\\]\\(@(.*?)\\)") - (defun parse-refs (line) (let ((parts (ppcre:split *ref-pattern* line :with-registers-p t))) (mapcar-indexed (lambda (string i) diff --git a/tests/basic/basic.lit b/tests/basic/basic.lit index a4f70d1..88fd2ec 100644 --- a/tests/basic/basic.lit +++ b/tests/basic/basic.lit @@ -87,11 +87,8 @@ Lastly, there are some boring headers to include. #include --- -We can reference blocks in prose. -Note that @{algorithms} is `O(n)`. - -We can also link to chapters and sections. -This link leads back up to [the Including blocks](@Including blocks) section. +We can also reference blocks in prose. +Note that @{algorithms} is `O(n)`. ## Formatting and whitespace diff --git a/utils.lisp b/utils.lisp index 6226de3..708192f 100644 --- a/utils.lisp +++ b/utils.lisp @@ -82,53 +82,3 @@ (define-condition user-error (simple-error) ()) -(defun singlep (l) - (= 1 (length l))) - -(defmacro comment (&rest body) - (declare (ignore body)) - nil) - -;; The original idea for this was that I wanted -;; a way to link using a minimal unique prefix. -;; -;; If my section heading is something long, -;; I don't want to have to type out the entire -;; Click [here](@Some long section heading that is easy to typo) to read more! -;; -;; So I was going to use a trie to lookup the Chapter/Section ID given -;; a title prefix. If it ends up being unique, we're good to go. If it's -;; not unique, we can pick one and leave a warning. -;; -;; I haven't yet gotten close to finishing that implementation. So it -;; doesn't make much sense to have this data structure right now. -;; But oh well. Whatever. It works for an MVP of the chapter/section linking stuff. -(defun trie-assoc (key alist) - (assoc key alist :test #'equal)) - -(defun trie-put (alist key val) - (if (singlep key) - (acons (car key) (acons :terminal val '()) alist) - (let ((child (trie-assoc (car key) alist))) - (if child - (progn - (rplacd child (trie-put (cdr child) (cdr key) val)) - alist) - (acons (car key) (trie-put '() (cdr key) val) alist))))) - -(defun trie-get (alist key) - (if (singlep key) - (trie-assoc (car key) alist) - (trie-get (cdr (trie-assoc (car key) alist)) (cdr key)))) - -(comment - (trie-put '() '(1 2) 3) - ; => ((1 (2 (:TERMINAL . 3)))) - (let ((trie (trie-put - (trie-put - (trie-put '() '(2) 3) - '(1) 2) - '(1 2) 3))) - trie) - ; => ((1 (2 (:TERMINAL . 3)) (:TERMINAL . 2)) (2 (:TERMINAL . 3))) - ) diff --git a/weave.lisp b/weave.lisp index ae1b538..48f9aff 100644 --- a/weave.lisp +++ b/weave.lisp @@ -22,39 +22,9 @@ ; One-to-one correspondence between .lit and .html files. ; The body of an .html file should look the same regardless of whether the .lit is in another context (like a book). -(defun get-section (weaver section) - (trie-get (weaver-section-map weaver) section)) - -(defun add-section (weaver section anchor) - (setf (weaver-section-map weaver) - (trie-put (weaver-section-map weaver) section anchor))) - -(comment - (let* ((file-defs (parse-lit-files '("dev.lit"))) - (weaver (make-weaver-default file-defs))) - (add-section weaver '("Some Chapter") "c2") - (add-section weaver '("Some Chapter" "some-section") "s2:1") - (trie-get (weaver-section-map weaver) '("some-chapter")) - (weaver-section-map weaver)) - ; => (("Some Chapter" - ; ("some-section" (:TERMINAL . "s2:1")) - ; (:TERMINAL . "c2"))) - ) - -(defstruct weaver +(defstruct weaver (def-table (make-hash-table :test #'equal) :type hash-table) (use-table (make-hash-table :test #'equal) :type hash-table) - ;; section-map is used to map chapter/section text to their respective - ;; anchor ids, like #s2:4 - ;; - ;; This lets you create markdown links like this: - ;; [some section or chapter](@some section or chapter) - ;; Which will be woven to (for example): - ;; [some section or chapter](#s2:4) - ;; - ;; Right now, it doesn't work across files in a book. - ;; Heck... it barely works at all. - (section-map '() :type list) (initial-def-table (make-hash-table :test #'equal) :type hash-table) (toc nil :type list) @@ -227,110 +197,71 @@ (weave-uses weaver def)) (write-line "")) -;; This is not elegant. -(defun replace-sections-with-id-anchors (trie line) - (let ((r "") - (s 0) - (e 0)) - (ppcre:do-scans (start end reg-starts reg-ends *anchor-pattern* line) - (let ((title (subseq line (aref reg-starts 0) (aref reg-ends 0)))) - (setf - r - (format - nil - "~a~a" - r - (concatenate - 'string - (subseq line s (- (aref reg-starts 0) 1)) - (format nil "#~a" (cdr (assoc :terminal (cdr (trie-get trie (list title))))))))) - (setf s (aref reg-ends 0)) - (setf e (aref reg-ends 0)))) - (setf r (format nil "~a~a" r (subseq line e))) - r)) - -(comment - (let ((trie (trie-put '() '("some-chapter") "c1"))) - (replace-sections-with-id-anchors - trie - "foo [some chapter](@some-chapter) bar [other](@some-chapter) buz")) - ; => "foo [some chapter](#c1) bar [other](#c1) buz" - ) - (defun weave-prose-line (weaver line def) (loop for expr in line do - (cond ((stringp expr) - (if (ppcre:scan *anchor-pattern* expr) - (write-string - (replace-sections-with-id-anchors - (weaver-section-map weaver) - expr)) - (write-string expr))) - ((commandp expr) - (case (first expr) - (:INCLUDE (weave-include - (second expr) ; title of block - (textblockdef-file def) - weaver - nil)) - (:TITLE - (setf (weaver-title weaver) - (second expr))) - (:C - ; Also can act as a title - (when (null (weaver-title weaver)) - (setf (weaver-title weaver) - (second expr))) - (incf (weaver-chapter-counter weaver)) - (setf (weaver-section-counter weaver) -1) - (format t "

~a

~%" - (second expr) - (chapter-id (weaver-chapter-counter weaver))) - (add-section weaver (list (second expr)) (chapter-id (weaver-chapter-counter weaver)))) - (:S - (incf (weaver-section-counter weaver)) - (format t "

~a. ~a

~%" - (+ (weaver-section-counter weaver) 1) - (second expr) - (section-id - (weaver-section-counter weaver) - (weaver-chapter-counter weaver))) - (add-section weaver (list (second expr)) (section-id (weaver-section-counter weaver) - (weaver-chapter-counter weaver)))) - (:CODE_TYPE - (let* ((args (split-whitespace (second expr))) - (language (first args)) - (extension (subseq (second args) 1))) - (setf (gethash extension (weaver-code-type-table weaver)) language) - (push extension (weaver-used-extensions weaver)))) - (:COMMENT_TYPE nil) - (:ADD_CSS nil) - (:OVERWRITE_CSS nil) - (:COLORSCHEME nil) - (:ERROR_FORMAT nil) - (:MATHBLOCK - ; Put tags in block so it displays tex nicely without JS. - (setf (weaver-used-math weaver) t) - (write-string "
") - (when (not (equal (second expr) "displaymath")) - (format t "\\begin{~a}" (second expr))) - (write-separated-list (third expr) #\newline *standard-output*) - (when (not (equal (second expr) "displaymath")) - (format t "\\end{~a}" (second expr))) - (write-string "
")) - (:MATH - ; Use backticks to prevent markdown from formatting tex, - ; for example treating _ as emphasis. - (setf (weaver-used-math weaver) t) - (format t "`~a`" - (second expr))) - (:TOC (weave-toc - (weaver-toc weaver) - (textblockdef-file def))) - (otherwise (error 'user-error - :format-control "unknown prose command ~S" - :format-arguments (first expr))))) - (t (error "unknown structure ~s" expr))))) + (cond ((stringp expr) (write-string expr)) + ((commandp expr) + (case (first expr) + (:INCLUDE (weave-include + (second expr) + (textblockdef-file def) + weaver + nil)) + (:TITLE + (setf (weaver-title weaver) + (second expr))) + (:C + ; Also can act as a title + (when (null (weaver-title weaver)) + (setf (weaver-title weaver) + (second expr))) + (incf (weaver-chapter-counter weaver)) + (setf (weaver-section-counter weaver) -1) + (format t "

~a

~%" + (second expr) + (chapter-id (weaver-chapter-counter weaver)))) + (:S + (incf (weaver-section-counter weaver)) + (format t "

~a. ~a

~%" + (+ (weaver-section-counter weaver) 1) + (second expr) + (section-id + (weaver-section-counter weaver) + (weaver-chapter-counter weaver)))) + (:CODE_TYPE + (let* ((args (split-whitespace (second expr))) + (language (first args)) + (extension (subseq (second args) 1))) + (setf (gethash extension (weaver-code-type-table weaver)) language) + (push extension (weaver-used-extensions weaver)))) + (:COMMENT_TYPE nil) + (:ADD_CSS nil) + (:OVERWRITE_CSS nil) + (:COLORSCHEME nil) + (:ERROR_FORMAT nil) + (:MATHBLOCK + ; Put tags in block so it displays tex nicely without JS. + (setf (weaver-used-math weaver) t) + (write-string "
") + (when (not (equal (second expr) "displaymath")) + (format t "\\begin{~a}" (second expr))) + (write-separated-list (third expr) #\newline *standard-output*) + (when (not (equal (second expr) "displaymath")) + (format t "\\end{~a}" (second expr))) + (write-string "
")) + (:MATH + ; Use backticks to prevent markdown from formatting tex, + ; for example treating _ as emphasis. + (setf (weaver-used-math weaver) t) + (format t "`~a`" + (second expr))) + (:TOC (weave-toc + (weaver-toc weaver) + (textblockdef-file def))) + (otherwise (error 'user-error + :format-control "unknown prose command ~S" + :format-arguments (first expr))))) + (t (error "unknown structure ~s" expr))))) (defun weave-prosedef (weaver def) (let ((block (textblockdef-block def)) From 6dbbef9e2223875d5352871f38c07bba6bbc5dbe Mon Sep 17 00:00:00 2001 From: Eric Ihli Date: Mon, 19 Sep 2022 22:42:02 -0700 Subject: [PATCH 3/9] Refactor to parse to :ANCHOR --- dev.lit | 25 +++++++++++++ parse.lisp | 108 +++++++++++++++++++++++++++++++++++++++++++++-------- utils.lisp | 4 ++ weave.lisp | 9 +++++ 4 files changed, 130 insertions(+), 16 deletions(-) create mode 100644 dev.lit diff --git a/dev.lit b/dev.lit new file mode 100644 index 0000000..6628822 --- /dev/null +++ b/dev.lit @@ -0,0 +1,25 @@ +# My test lit file + +Preamble + +@toc + +## Foobar + +Section 1: foobar. +What follows is foobar.lisp. + +--- /foobar.lisp +(+ 2 2) +@{foobaz} +--- + +## Foobazs + +And this is the content of the [Foobaz](@Foobazs) section. + +Here's a ref. @{foobaz} What's it do? + +--- foobaz +(format nil (* 2 2)) +--- diff --git a/parse.lisp b/parse.lisp index 301b61c..e3fb00a 100644 --- a/parse.lisp +++ b/parse.lisp @@ -30,10 +30,30 @@ :format-control "unknown modifier ~s" :format-arguments x)))) +(defparameter *anchor-pattern* + (ppcre:create-scanner '(:SEQUENCE "@{" (:GREEDY-REPETITION 1 2 #\#) + (:GREEDY-REPETITION 1 NIL :WHITESPACE-CHAR-CLASS) + (:REGISTER (:NON-GREEDY-REPETITION 0 NIL :EVERYTHING)) #\}))) + +(defun parse-anchor (line) + (let ((parts (ppcre:split *anchor-pattern* line :with-registers-p t))) + (mapcar-indexed (lambda (string i) + (if (evenp i) + (ppcre:regex-replace-all "@@({[^}]+})" string "@\\1") + (list :ANCHOR string))) + parts))) + +(comment + (let ((line "Foobar @{# Baz} @{buz} @@{fizz}")) + (parse-anchor line)) + ; => ("Foobar " (:ANCHOR "Baz") " @{buz} @{fizz}") + ) + (defparameter *ref-pattern* (ppcre:create-scanner '(:SEQUENCE (:NEGATIVE-LOOKBEHIND #\@) "@{" + (:negative-lookahead #\#) (:REGISTER (:GREEDY-REPETITION 1 NIL (:INVERTED-CHAR-CLASS #\}))) #\}))) @@ -65,7 +85,6 @@ (:REGISTER (:GREEDY-REPETITION 1 nil #\#)) (:GREEDY-REPETITION 1 nil :WHITESPACE-CHAR-CLASS)))) - (defparameter *math-inline-pattern* (ppcre:create-scanner '(:SEQUENCE #\\ @@ -111,27 +130,84 @@ (push (subseq line start) expr) (nreverse expr))) +(defun parse-repeatedly (parsers line) + "Parses line with each parser. +Line starts off as a string. After the first parse, it will be a list of regular text and parsed segments. + +Example: + \"Some text @{some ref}\" + will turn into + (\"Some text\" (:INCLUDE \"some ref\")). + +The subsequent parsers will be mapped over the result of the first parse." + (cond + ((null line) nil) + ((null parsers) line) + ((stringp line) + (parse-repeatedly + (cdr parsers) + (funcall (car parsers) line))) + ((symbolp (car line)) (list line)) + (t (alexandria-2:mappend + (lambda (l) + (parse-repeatedly parsers l)) + line)))) + +(comment + (parse-repeatedly (list #'parse-anchor #'parse-refs #'parse-math-text) + "Foobar @{# Baz} \\begin{math}n + m\\end{math} buz @{fizz}") + ; => ("Foobar " (:ANCHOR "Baz") " " (:MATH "n + m") " buz " (:INCLUDE "fizz")) + ) + (defun parse-prose-line (line) (or - (multiple-value-bind (match groups) - (ppcre:scan-to-strings *heading-pattern* line) - (if match - (list (case (length (aref groups 0)) - (1 (list :C (subseq line (length match)))) - (2 (list :S (subseq line (length match)))) - (otherwise line))) - nil)) - (multiple-value-bind (match groups) - (ppcre:scan-to-strings *command-pattern* line) - (if match - (list (list (intern (string-upcase (aref groups 0)) :KEYWORD) - (subseq line (length match)))) - nil)) + (multiple-value-bind (match groups) + (ppcre:scan-to-strings *heading-pattern* line) + (if match + (list (case (length (aref groups 0)) + (1 (list :C (subseq line (length match)))) + (2 (list :S (subseq line (length match)))) + (otherwise line))) + nil)) + (multiple-value-bind (match groups) + (ppcre:scan-to-strings *command-pattern* line) + (if match + (list (list (intern (string-upcase (aref groups 0)) :KEYWORD) + (subseq line (length match)))) + nil)) + ;; Leaving this commented out while in PR review so that it's easy to try + ;; back and forth. + (comment (alexandria-2:mappend (lambda (expr) (if (stringp expr) (parse-math-text expr) (list expr))) - (parse-refs line)))) + (parse-refs line))) + (parse-repeatedly + (list #'parse-refs #'parse-math-text #'parse-anchor) + line))) + +(comment + (parse-prose-line "\\n") + ; => ("\\n") + (parse-prose-line "") + ; => NIL + (parse-prose-line "Foobar @{# Baz} \\begin{math}n + m\\end{math} buz @{fizz}") + ; => ("Foobar " (:ANCHOR "Baz") " " (:MATH "n + m") " buz " (:INCLUDE "fizz")) + (parse-prose-line "Foobar @{fizz} \\begin{math}n + m\\end{math} buz @{# Baz}") + ; => ("Foobar " (:INCLUDE "fizz") " " (:MATH "n + m") " buz " (:ANCHOR "Baz")) + (parse-prose-line "# Some heading @{with a ref}") + ; => ((:C "Some heading @{with a ref}")) + (mapcar #'parse-prose-line + '("# Foobar" + "@{bazz}" + "" + "@{# Foobar}")) + ; => (((:C "Foobar")) ("" (:INCLUDE "bazz")) NIL ("@{# Foobar}")) <- original + ; => (((:C "Foobar")) ((:INCLUDE "bazz")) NIL ("" (:ANCHOR "Foobar"))) <- new + ; Not sure why this (:INCLUDE...) is different between the two. + ; I think it will still work. And everything else looks the same. + ) (defparameter *block-start-pattern* (ppcre:create-scanner '(:SEQUENCE :START-ANCHOR "---"))) diff --git a/utils.lisp b/utils.lisp index 708192f..4ffe572 100644 --- a/utils.lisp +++ b/utils.lisp @@ -82,3 +82,7 @@ (define-condition user-error (simple-error) ()) +(defmacro comment (&rest body) + (declare (ignore body)) + nil) + diff --git a/weave.lisp b/weave.lisp index 48f9aff..2881b12 100644 --- a/weave.lisp +++ b/weave.lisp @@ -35,6 +35,14 @@ (used-extensions nil :type list) (used-math nil :type boolean)) +(comment + (let* ((file-defs (parse-lit-files '("dev.lit" "scratch.lit"))) + (weaver (make-weaver-default file-defs))) + (weaver-toc weaver)) + ; => ((:FILE "dev.lit" (:C "My test lit file" (:S "Foobar") (:S "Foobazs"))) + ; (:FILE "scratch.lit" (:C "My scratch lit file" (:S "Scratch")))) + ) + (defun lit-page-filename (filename) (concatenate 'string (uiop:split-name-type filename) @@ -258,6 +266,7 @@ (:TOC (weave-toc (weaver-toc weaver) (textblockdef-file def))) + (:ANCHOR (write-string ":anchor")) (otherwise (error 'user-error :format-control "unknown prose command ~S" :format-arguments (first expr))))) From 768a5b6617b1dd6b25915768cbedd037d38061c0 Mon Sep 17 00:00:00 2001 From: Eric Ihli Date: Sat, 1 Oct 2022 10:58:46 -0700 Subject: [PATCH 4/9] Update parse-anchor to include Chap/Section Detect # vs ## and use the list (:ANCHOR (:C "Ch Title")) or (:ANCHOR (:S "Section Title")) depending on which. --- dev.lit | 2 ++ parse.lisp | 71 ++++++++++++++++++++++++++++++++++++++++++------------ weave.lisp | 3 ++- 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/dev.lit b/dev.lit index 6628822..0609421 100644 --- a/dev.lit +++ b/dev.lit @@ -23,3 +23,5 @@ Here's a ref. @{foobaz} What's it do? --- foobaz (format nil (* 2 2)) --- + +# Section 2 diff --git a/parse.lisp b/parse.lisp index e3fb00a..9bc5309 100644 --- a/parse.lisp +++ b/parse.lisp @@ -30,23 +30,47 @@ :format-control "unknown modifier ~s" :format-arguments x)))) +(comment + (ppcre:parse-string + "@\{(#{1,2}\\s.*?)\}") + ; => (:SEQUENCE "@{" + ; (:REGISTER + ; (:SEQUENCE (:GREEDY-REPETITION 1 2 #\#) #\s + ; (:NON-GREEDY-REPETITION 0 NIL :EVERYTHING))) + ; #\}) + ) + (defparameter *anchor-pattern* - (ppcre:create-scanner '(:SEQUENCE "@{" (:GREEDY-REPETITION 1 2 #\#) - (:GREEDY-REPETITION 1 NIL :WHITESPACE-CHAR-CLASS) - (:REGISTER (:NON-GREEDY-REPETITION 0 NIL :EVERYTHING)) #\}))) + (ppcre:create-scanner '(:SEQUENCE "@{" + (:REGISTER + (:SEQUENCE (:GREEDY-REPETITION 1 2 #\#) :WHITESPACE-CHAR-CLASS + (:NON-GREEDY-REPETITION 0 NIL :EVERYTHING))) + #\}) + )) (defun parse-anchor (line) (let ((parts (ppcre:split *anchor-pattern* line :with-registers-p t))) (mapcar-indexed (lambda (string i) (if (evenp i) - (ppcre:regex-replace-all "@@({[^}]+})" string "@\\1") - (list :ANCHOR string))) + ;; Not sure what to do write here. Should `parse-anchor' have + ;; the duplicated escape logic? + ;; I'm leaning towards duplicate the same escape logic from `parse-ref' to here. + (progn + string + ;; or... + (ppcre:regex-replace-all "@@({[^}]+})" string "@\\1")) + (list :ANCHOR (if (eql (char string 1) #\#) + (list :S (ppcre:regex-replace "##\\s+" string "")) + (list :C (ppcre:regex-replace "#\\s+" string "")))))) parts))) + (comment - (let ((line "Foobar @{# Baz} @{buz} @@{fizz}")) + ;; How should escaping work if we go this route of parse-anchor+parse-ref and parse-repeatedly? + (let ((line "Foobar @{# Baz} @{## Biz} @{buz} @@{fizz}")) (parse-anchor line)) - ; => ("Foobar " (:ANCHOR "Baz") " @{buz} @{fizz}") + ; => ("Foobar " (:ANCHOR (:C "Baz")) " " (:ANCHOR (:S "Biz")) " @{buz} @@{fizz}") + ; => ("Foobar " (:ANCHOR (:C "Baz")) " " (:ANCHOR (:S "Biz")) " @{buz} @{fizz}") ) (defparameter *ref-pattern* @@ -135,11 +159,18 @@ Line starts off as a string. After the first parse, it will be a list of regular text and parsed segments. Example: - \"Some text @{some ref}\" + \"Some @{# some chapter} text @{some ref}\" will turn into - (\"Some text\" (:INCLUDE \"some ref\")). + (\"Some \" (:ANCHOR \"#some chapter\") \"text \" (:INCLUDE \"some ref\")). -The subsequent parsers will be mapped over the result of the first parse." +The subsequent parsers will be mapped over the result of the first parse. + +NOTE: +There's at least one issue with this. +`parse-refs' treats the double `@@' as an escape sequence. +Instead of turning `@@{# foo}' into `@' it turns it into `@{# foo}'. +So if we first `parse-refs' and turn `@@{# foo}' into `@{# foo}' and then run `parse-anchors' after that +then we're bypassing our escape mechanism." (cond ((null line) nil) ((null parsers) line) @@ -188,14 +219,16 @@ The subsequent parsers will be mapped over the result of the first parse." line))) (comment + ;; Testing out the `parse-repeatedly' behavior. + (parse-prose-line "\\n") ; => ("\\n") (parse-prose-line "") ; => NIL (parse-prose-line "Foobar @{# Baz} \\begin{math}n + m\\end{math} buz @{fizz}") - ; => ("Foobar " (:ANCHOR "Baz") " " (:MATH "n + m") " buz " (:INCLUDE "fizz")) + ; => ("Foobar " (:ANCHOR (:C "Baz")) " " (:MATH "n + m") " buz " (:INCLUDE "fizz")) (parse-prose-line "Foobar @{fizz} \\begin{math}n + m\\end{math} buz @{# Baz}") - ; => ("Foobar " (:INCLUDE "fizz") " " (:MATH "n + m") " buz " (:ANCHOR "Baz")) + ; => ("Foobar " (:INCLUDE "fizz") " " (:MATH "n + m") " buz " (:ANCHOR (:C "Baz"))) (parse-prose-line "# Some heading @{with a ref}") ; => ((:C "Some heading @{with a ref}")) (mapcar #'parse-prose-line @@ -203,10 +236,7 @@ The subsequent parsers will be mapped over the result of the first parse." "@{bazz}" "" "@{# Foobar}")) - ; => (((:C "Foobar")) ("" (:INCLUDE "bazz")) NIL ("@{# Foobar}")) <- original - ; => (((:C "Foobar")) ((:INCLUDE "bazz")) NIL ("" (:ANCHOR "Foobar"))) <- new - ; Not sure why this (:INCLUDE...) is different between the two. - ; I think it will still work. And everything else looks the same. + ; => (((:C "Foobar")) ((:INCLUDE "bazz")) NIL ("" (:ANCHOR (:C "Foobar")))) ) (defparameter *block-start-pattern* @@ -263,6 +293,15 @@ The subsequent parsers will be mapped over the result of the first parse." (textblock-lines (textblockdef-block def))) (go TEXT))) +(comment + ;; Just want to get a feel for what the def-table looks like + (let* ((file-defs (parse-lit-files '("dev.lit" "scratch.lit"))) + (weaver (make-weaver-default file-defs))) + (let ((defs (weaver-def-table weaver))) + (maphash (lambda (k v) + (format t "~a ~a~%" k v)) + defs))) + ) (defparameter *math-block-pattern* (ppcre:create-scanner diff --git a/weave.lisp b/weave.lisp index 94672c1..09ec4fa 100644 --- a/weave.lisp +++ b/weave.lisp @@ -38,7 +38,8 @@ (comment (let* ((file-defs (parse-lit-files '("dev.lit" "scratch.lit"))) (weaver (make-weaver-default file-defs))) - (weaver-toc weaver)) + weaver) + ; => ((:FILE "dev.lit" (:C "My test lit file" (:S "Foobar") (:S "Foobazs"))) ; (:FILE "scratch.lit" (:C "My scratch lit file" (:S "Scratch")))) ) From b3f977c46c663f0074b1c9400b09eb5e78ebacc7 Mon Sep 17 00:00:00 2001 From: Eric Ihli Date: Sat, 26 Nov 2022 21:35:26 -0800 Subject: [PATCH 5/9] indentation --- command-line.lisp | 12 ++++++------ weave.lisp | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/command-line.lisp b/command-line.lisp index a3ee2c2..6b198eb 100644 --- a/command-line.lisp +++ b/command-line.lisp @@ -60,10 +60,10 @@ (when (getf options :help) (opts:describe - :prefix "Literate programming system. Write code to be read by humans, not machines." - :usage-of "srcweave" - :suffix "Created by Justin Meiners (2022)" - :args "LITFILE") + :prefix "Literate programming system. Write code to be read by humans, not machines." + :usage-of "srcweave" + :suffix "Created by Justin Meiners (2022)" + :args "LITFILE") (opts:exit 0)) (when (null free-args) @@ -85,13 +85,13 @@ (weave-path (getf options :weave)) (tangle-path (getf options :tangle))) - (when tangle-path + (when tangle-path (format t "TANGLE~%") (tangle (alexandria-2:mappend #'cdr file-defs) tangle-path :ignore-dates ignore-dates) - (format t "DONE~%")) + (format t "DONE~%")) (when weave-path (format t "WEAVE~%") (weave file-defs diff --git a/weave.lisp b/weave.lisp index 09ec4fa..e83948b 100644 --- a/weave.lisp +++ b/weave.lisp @@ -270,7 +270,8 @@ (:TOC (weave-toc (weaver-toc weaver) (textblockdef-file def))) - (:ANCHOR (write-string ":anchor")) + (:ANCHOR (let ((ref (cadr expr))) + )) ; These commands are from Zach's Literate. ; We treat them as warnings instead of errors to make migration easier. From 23b9cb11f405b170d875ecc14c9920c174551cd4 Mon Sep 17 00:00:00 2001 From: Eric Ihli Date: Sat, 10 Dec 2022 16:37:16 -0800 Subject: [PATCH 6/9] Add function to map sections to href --- dev.lit | 15 +++++++++++++++ parse.lisp | 13 ++++++++++--- toc.lisp | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/dev.lit b/dev.lit index 0609421..c1b50d0 100644 --- a/dev.lit +++ b/dev.lit @@ -18,10 +18,25 @@ What follows is foobar.lisp. And this is the content of the [Foobaz](@Foobazs) section. + Here's a ref. @{foobaz} What's it do? +@{scratch-thing} + +--- scratch-things +(format "duplicate") +--- + --- foobaz (format nil (* 2 2)) --- # Section 2 + +Here's a link to the scratch.lit code block. + +@{scratch-thing} + +And here's a link to a scratch.lit section. + +@@{scratch.lit @{scratch.c}} diff --git a/parse.lisp b/parse.lisp index cd0d449..596d007 100644 --- a/parse.lisp +++ b/parse.lisp @@ -302,9 +302,16 @@ then we're bypassing our escape mechanism." (let* ((file-defs (parse-lit-files '("dev.lit" "scratch.lit"))) (weaver (make-weaver-default file-defs))) (let ((defs (weaver-def-table weaver))) - (maphash (lambda (k v) - (format t "~a ~a~%" k v)) - defs))) + (progn + (maphash (lambda (k v) + (format t "~a ~a~%" k v) + ) + defs) + (maphash + (lambda (k v) + (format t "~a: ~a~%" k v)) + (create-global-toc-linkmap (create-global-toc file-defs)))))) + ) (defparameter *math-block-pattern* diff --git a/toc.lisp b/toc.lisp index 5791018..5afa9c6 100644 --- a/toc.lisp +++ b/toc.lisp @@ -49,6 +49,42 @@ (textblockdef-create-toc (cdr pair))))) file-def-pairs)) +(defun create-global-toc-linkmap (toc) + (do ((linkmap (make-hash-table)) + (file (car toc) (car (cdr toc))) + (toc toc (cdr toc))) + ((null file) linkmap) + (do ((chapters (cddr file) (cdr chapters)) + (chapter (caddr file) (car (cdr chapters))) + (chapter-counter 0 (incf chapter-counter))) + ((null chapter)) + (let ((link (format nil "~a#c~a" (lit-page-filename (cadr file)) chapter-counter))) + (setf (gethash (cadr (subseq chapter 0 2)) linkmap) link) + (do ((sections (cddr chapter) (cdr sections)) + (section (caddr chapter) (car (cdr sections))) + (section-counter 0 (incf section-counter))) + ((null section)) + (let ((link (format nil "~a#s~a:~a" + (lit-page-filename (cadr file)) + chapter-counter + section-counter))) + (setf (gethash (cadr section) linkmap) link))))))) + +(comment + (let* ((file-defs (parse-lit-files '("dev.lit" "scratch.lit")))) + (maphash + (lambda (k v) + (format t "~a: ~a~%" k v)) + (create-global-toc-linkmap (create-global-toc file-defs)))) + +; My test lit file: dev.html#c0 +; Foobar: dev.html#s0:0 +; Foobazs: dev.html#s0:1 +; Section 2: dev.html#c1 +; My scratch lit file: scratch.html#c0 +; Scratch: scratch.html#s0:0 +; => NIL + ) (defun weave-toc-section (name file chapter-counter section-counter) (format t "
  • ~a
  • " From 6d57dbbcf1fa29ea50175d7c49c4fcd2028d57c8 Mon Sep 17 00:00:00 2001 From: Eric Ihli Date: Sat, 10 Dec 2022 17:36:28 -0800 Subject: [PATCH 7/9] Weave anchor --- dev.lit | 6 ++- toc.lisp | 13 +++-- weave.lisp | 145 +++++++++++++++++++++++++++++------------------------ 3 files changed, 93 insertions(+), 71 deletions(-) diff --git a/dev.lit b/dev.lit index c1b50d0..06db6e8 100644 --- a/dev.lit +++ b/dev.lit @@ -39,4 +39,8 @@ Here's a link to the scratch.lit code block. And here's a link to a scratch.lit section. -@@{scratch.lit @{scratch.c}} +@{## Scratch} + +And here's a link to a scratch.lit chapter. + +@{# My scratch lit file} diff --git a/toc.lisp b/toc.lisp index 5afa9c6..b0c1ac4 100644 --- a/toc.lisp +++ b/toc.lisp @@ -50,7 +50,7 @@ file-def-pairs)) (defun create-global-toc-linkmap (toc) - (do ((linkmap (make-hash-table)) + (do ((linkmap (make-hash-table :test 'equal)) (file (car toc) (car (cdr toc))) (toc toc (cdr toc))) ((null file) linkmap) @@ -71,11 +71,14 @@ (setf (gethash (cadr section) linkmap) link))))))) (comment - (let* ((file-defs (parse-lit-files '("dev.lit" "scratch.lit")))) + (let* ((file-defs (parse-lit-files '("dev.lit" "scratch.lit"))) + (linkmap (create-global-toc-linkmap (create-global-toc file-defs))) + (res nil)) (maphash - (lambda (k v) - (format t "~a: ~a~%" k v)) - (create-global-toc-linkmap (create-global-toc file-defs)))) + (lambda (k _) + (setf res (cons k res))) + linkmap) + (list res (gethash "Foobar" linkmap))) ; My test lit file: dev.html#c0 ; Foobar: dev.html#s0:0 diff --git a/weave.lisp b/weave.lisp index 13d14ec..a5d5103 100644 --- a/weave.lisp +++ b/weave.lisp @@ -215,73 +215,80 @@ (write-line "")) (defun weave-prose-line (weaver line def) - (loop for expr in line do - (cond ((stringp expr) (write-string expr)) - ((commandp expr) - (case (first expr) - (:INCLUDE (weave-include - (second expr) - (textblockdef-file def) - weaver - nil)) - (:TITLE + (let ((linkmap (create-global-toc-linkmap (weaver-toc weaver)))) + (loop for expr in line do + (cond ((stringp expr) (write-string expr)) + ((commandp expr) + (case (first expr) + (:INCLUDE (weave-include + (second expr) + (textblockdef-file def) + weaver + nil)) + (:TITLE + (setf (weaver-title weaver) + (second expr))) + (:C + ; Also can act as a title + (when (null (weaver-title weaver)) (setf (weaver-title weaver) (second expr))) - (:C - ; Also can act as a title - (when (null (weaver-title weaver)) - (setf (weaver-title weaver) - (second expr))) - (incf (weaver-chapter-counter weaver)) - (setf (weaver-section-counter weaver) -1) - (format t "

    ~a

    ~%" - (second expr) - (chapter-id (weaver-chapter-counter weaver)))) - (:S - (incf (weaver-section-counter weaver)) - (format t "

    ~a. ~a

    ~%" - (+ (weaver-section-counter weaver) 1) - (second expr) - (section-id - (weaver-section-counter weaver) - (weaver-chapter-counter weaver)))) - (:CODE_TYPE - (let* ((args (split-whitespace (second expr))) - (language (first args)) - (extension (subseq (second args) 1))) - (setf (gethash extension (weaver-code-type-table weaver)) language) - (push extension (weaver-used-extensions weaver)))) - (:MATHBLOCK - ; Put tags in block so it displays tex nicely without JS. - (setf (weaver-used-math weaver) t) - (write-string "
    ") - (when (not (equal (second expr) "displaymath")) - (format t "\\begin{~a}" (second expr))) - (write-separated-list (third expr) #\newline *standard-output*) - (when (not (equal (second expr) "displaymath")) - (format t "\\end{~a}" (second expr))) - (write-string "
    ")) - (:MATH - ; Use backticks to prevent markdown from formatting tex, - ; for example treating _ as emphasis. - (setf (weaver-used-math weaver) t) - (format t "`~a`" - (second expr))) - (:TOC (weave-toc - (weaver-toc weaver) - (textblockdef-file def))) - (:ANCHOR (let ((ref (cadr expr))) - )) - - ; These commands are from Zach's Literate. - ; We treat them as warnings instead of errors to make migration easier. - ; It's possible they may be useful for us in the future. - ((:COMMENT_TYPE :ADD_CSS :OVERWRITE_CSS :COLORSCHEME :ERROR_FORMAT) - (warn "deprecated Literate prose command ~s. ignored." (first expr))) - (otherwise (error 'user-error - :format-control "unknown prose command ~S" - :format-arguments (first expr))))) - (t (error "unknown structure ~s" expr))))) + (incf (weaver-chapter-counter weaver)) + (setf (weaver-section-counter weaver) -1) + (format t "

    ~a

    ~%" + (second expr) + (chapter-id (weaver-chapter-counter weaver)))) + (:S + (incf (weaver-section-counter weaver)) + (format t "

    ~a. ~a

    ~%" + (+ (weaver-section-counter weaver) 1) + (second expr) + (section-id + (weaver-section-counter weaver) + (weaver-chapter-counter weaver)))) + (:CODE_TYPE + (let* ((args (split-whitespace (second expr))) + (language (first args)) + (extension (subseq (second args) 1))) + (setf (gethash extension (weaver-code-type-table weaver)) language) + (push extension (weaver-used-extensions weaver)))) + (:MATHBLOCK + ; Put tags in block so it displays tex nicely without JS. + (setf (weaver-used-math weaver) t) + (write-string "
    ") + (when (not (equal (second expr) "displaymath")) + (format t "\\begin{~a}" (second expr))) + (write-separated-list (third expr) #\newline *standard-output*) + (when (not (equal (second expr) "displaymath")) + (format t "\\end{~a}" (second expr))) + (write-string "
    ")) + (:MATH + ; Use backticks to prevent markdown from formatting tex, + ; for example treating _ as emphasis. + (setf (weaver-used-math weaver) t) + (format t "`~a`" + (second expr))) + (:TOC (weave-toc + (weaver-toc weaver) + (textblockdef-file def))) + (:ANCHOR + (let ((ref (cadadr expr))) + (write-string + (format + nil + "~a" + (gethash ref linkmap) + ref)))) + + ; These commands are from Zach's Literate. + ; We treat them as warnings instead of errors to make migration easier. + ; It's possible they may be useful for us in the future. + ((:COMMENT_TYPE :ADD_CSS :OVERWRITE_CSS :COLORSCHEME :ERROR_FORMAT) + (warn "deprecated Literate prose command ~s. ignored." (first expr))) + (otherwise (error 'user-error + :format-control "unknown prose command ~S" + :format-arguments (first expr))))) + (t (error "unknown structure ~s" expr)))))) (defun weave-prosedef (weaver def) (let ((block (textblockdef-block def)) @@ -291,6 +298,14 @@ (weave-prose-line weaver line def) (write-line "")))) +(comment + (let* ((file-defs (parse-lit-files '("dev.lit" "scratch.lit"))) + (weaver (make-weaver-default file-defs)) + (prosedef (nth 6 (cdar file-defs)))) + (weave-prosedef weaver prosedef)) + + ) + (defun weave-blocks (weaver source-defs) (dolist (def source-defs) (when (textblockdef-weavable def) From 0d4a798c6789215777b4cbdced963d29bc8f2f3c Mon Sep 17 00:00:00 2001 From: Eric Ihli Date: Sun, 11 Dec 2022 10:24:06 -0800 Subject: [PATCH 8/9] Fixes escape in code blocks --- parse.lisp | 165 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 103 insertions(+), 62 deletions(-) diff --git a/parse.lisp b/parse.lisp index 596d007..5f84153 100644 --- a/parse.lisp +++ b/parse.lisp @@ -34,35 +34,53 @@ :format-control "unknown modifier ~s" :format-arguments x)))) +;; The *anchor-pattern* and *ref-pattern* have negative lookbehinds to detect +;; and ignore `@@' so that you can weave `@@{some-reference}' into +;; `@{some-reference}'. +;; +;; So, `parse-anchor' and `parse-ref' will perform no action on those since they +;; won't match. +;; +;;`parse-escapes' must run *after* `parse-anchor' and `parse-ref'. If it runs +;; before, then it will translate `@@{some-ref}' to `@{some-ref}' which will +;; then get translated to `(:INCLUDE "some-ref")'. +(defparameter *escape-pattern* + (ppcre:create-scanner "@@({[^}]+})")) + +(defun parse-escapes (line) + (let ((parts (ppcre:split *escape-pattern* line :with-registers-p t))) + (mapcar-indexed + (lambda (string i) + (if (evenp i) + string + (format nil "@~a" string))) + parts))) + (comment - (ppcre:parse-string - "@\{(#{1,2}\\s.*?)\}") - ; => (:SEQUENCE "@{" - ; (:REGISTER - ; (:SEQUENCE (:GREEDY-REPETITION 1 2 #\#) #\s - ; (:NON-GREEDY-REPETITION 0 NIL :EVERYTHING))) - ; #\}) + (parse-escapes "Foobar @{# Baz} @@{# Buzz}") + ; => ("Foobar @{# Baz} " "@{# Buzz}") ) (defparameter *anchor-pattern* - (ppcre:create-scanner '(:SEQUENCE "@{" + (ppcre:create-scanner '(:SEQUENCE + (:NEGATIVE-LOOKBEHIND #\@) + "@{" (:REGISTER (:SEQUENCE (:GREEDY-REPETITION 1 2 #\#) :WHITESPACE-CHAR-CLASS (:NON-GREEDY-REPETITION 0 NIL :EVERYTHING))) #\}) - )) + ) + "This pattern matches @{# Some Chapter} and @{## Some Section}. +It doesn't match the escaped @@{# Some Chapter}.") (defun parse-anchor (line) + "Searches line for `*anchor-pattern*' and returns +(:ANCHOR (:C \"Some Chapter\")) for @{# Some Chapter} +and (:ANCHOR (:S \"Some Section\")) for @{## Some Section}." (let ((parts (ppcre:split *anchor-pattern* line :with-registers-p t))) (mapcar-indexed (lambda (string i) (if (evenp i) - ;; Not sure what to do write here. Should `parse-anchor' have - ;; the duplicated escape logic? - ;; I'm leaning towards duplicate the same escape logic from `parse-ref' to here. - (progn - string - ;; or... - (ppcre:regex-replace-all "@@({[^}]+})" string "@\\1")) + string (list :ANCHOR (if (eql (char string 1) #\#) (list :S (ppcre:regex-replace "##\\s+" string "")) (list :C (ppcre:regex-replace "#\\s+" string "")))))) @@ -70,18 +88,17 @@ (comment - ;; How should escaping work if we go this route of parse-anchor+parse-ref and parse-repeatedly? - (let ((line "Foobar @{# Baz} @{## Biz} @{buz} @@{fizz}")) + (let ((line "Foobar @{# Baz} @{## Biz} @@{# Boz} @{buz} @@{fizz}")) (parse-anchor line)) - ; => ("Foobar " (:ANCHOR (:C "Baz")) " " (:ANCHOR (:S "Biz")) " @{buz} @@{fizz}") - ; => ("Foobar " (:ANCHOR (:C "Baz")) " " (:ANCHOR (:S "Biz")) " @{buz} @{fizz}") + ; => ("Foobar " (:ANCHOR (:C "Baz")) " " (:ANCHOR (:S "Biz")) + ; " @@{# Boz} @{buz} @@{fizz}") ) (defparameter *ref-pattern* (ppcre:create-scanner '(:SEQUENCE (:NEGATIVE-LOOKBEHIND #\@) "@{" - (:negative-lookahead #\#) + (:NEGATIVE-LOOKAHEAD #\#) (:REGISTER (:GREEDY-REPETITION 1 NIL (:INVERTED-CHAR-CLASS #\}))) #\}))) @@ -89,7 +106,7 @@ (let ((parts (ppcre:split *ref-pattern* line :with-registers-p t))) (mapcar-indexed (lambda (string i) (if (evenp i) - (ppcre:regex-replace-all "@@({[^}]+})" string "@\\1") + string (list :INCLUDE string))) parts))) @@ -171,10 +188,11 @@ The subsequent parsers will be mapped over the result of the first parse. NOTE: There's at least one issue with this. -`parse-refs' treats the double `@@' as an escape sequence. -Instead of turning `@@{# foo}' into `@' it turns it into `@{# foo}'. -So if we first `parse-refs' and turn `@@{# foo}' into `@{# foo}' and then run `parse-anchors' after that -then we're bypassing our escape mechanism." +`parse-escapes' handles the double `@@' as an escape sequence. +It truns `@@{# foo}' into `@{# foo}'. +So if we first `parse-escapes' and turn `@@{# foo}' into `@{# foo}' and then run `parse-anchors' after that +then we're bypassing our escape mechanism. +So, `parse-escapes' must be after `parse-refs' and `parse-anchors' in the list of parsers." (cond ((null line) nil) ((null parsers) line) @@ -189,9 +207,12 @@ then we're bypassing our escape mechanism." line)))) (comment - (parse-repeatedly (list #'parse-anchor #'parse-refs #'parse-math-text) - "Foobar @{# Baz} \\begin{math}n + m\\end{math} buz @{fizz}") - ; => ("Foobar " (:ANCHOR "Baz") " " (:MATH "n + m") " buz " (:INCLUDE "fizz")) + (parse-repeatedly (list #'parse-anchor #'parse-refs #'parse-math-text #'parse-escapes) + "Foobar @{# Baz} @@{# Buzz} \\begin{math}n + m\\end{math} buz @{fizz}") + ; => ("Foobar " (:ANCHOR (:C "Baz")) " @{# Buzz} " (:MATH "n + m") " buz " + ; (:INCLUDE "fizz")) + (parse-repeatedly (list #'parse-anchor #'parse-escapes) + "(defvar foo @@{baz}") ) (defun parse-prose-line (line) @@ -219,12 +240,11 @@ then we're bypassing our escape mechanism." (list expr))) (parse-refs line))) (parse-repeatedly - (list #'parse-refs #'parse-math-text #'parse-anchor) + (list #'parse-refs #'parse-math-text #'parse-anchor #'parse-escapes) line))) (comment - ;; Testing out the `parse-repeatedly' behavior. - + ;; Some examples to getting a feel for behavior. (parse-prose-line "\\n") ; => ("\\n") (parse-prose-line "") @@ -266,38 +286,59 @@ then we're bypassing our escape mechanism." (defun read-code-block (line n stream) (prog ((def nil)) - (multiple-value-bind (title operator modifiers) - (parse-block-start line) - - (when (null title) - (error 'user-error - :format-control "block is missing title on line: ~s" - :format-arguments (list n))) - - (setf def (make-textblockdef :line-number n - :kind :CODE - :title title - :operation (if (null operator) :DEFINE (first operator)) - :modifiers (if (is-filename title) - (cons :FILE modifiers) - modifiers) ))) - - TEXT - (setf line (strip-line (read-line stream nil))) - (incf n) - (when (null line) - (error 'user-error - :format-control "unexpected end of file in code block: ~s" - :format-arguments (list (textblockdef-title def)))) - - (when (ppcre:scan *block-start-pattern* line) - (return (values def line n))) - - (vector-push-extend (parse-refs line) - (textblock-lines (textblockdef-block def))) - (go TEXT))) + (multiple-value-bind (title operator modifiers) + (parse-block-start line) + + (when (null title) + (error 'user-error + :format-control "block is missing title on line: ~s" + :format-arguments (list n))) + + (setf def (make-textblockdef :line-number n + :kind :CODE + :title title + :operation (if (null operator) :DEFINE (first operator)) + :modifiers (if (is-filename title) + (cons :FILE modifiers) + modifiers) ))) + + TEXT + (setf line (strip-line (read-line stream nil))) + (incf n) + (when (null line) + (error 'user-error + :format-control "unexpected end of file in code block: ~s" + :format-arguments (list (textblockdef-title def)))) + + (when (ppcre:scan *block-start-pattern* line) + (return (values def line n))) + + (vector-push-extend (parse-repeatedly (list #'parse-refs #'parse-escapes) line) + (textblock-lines (textblockdef-block def))) + (go TEXT))) (comment + (let ((s (make-string-output-stream))) + (format s "(defvar includes-regex @@{escaped include}~%") + (format s "---~%") + (format s "~%") + (read-code-block "--- foo" 0 (make-string-input-stream (get-output-stream-string s)))) + +; => #S(TEXTBLOCKDEF +; :TITLE "foo" +; :BLOCK #S(TEXTBLOCK +; :LINES #(("(defvar includes-regex " "@{escaped include}")) +; :MODIFY-DATE 0) +; :KIND :CODE +; :LINE-NUMBER 0 +; :FILE NIL +; :INDEX 0 +; :OPERATION :DEFINE +; :MODIFIERS NIL +; :LANGUAGE "text") +; "---" +; 2 + ;; Just want to get a feel for what the def-table looks like (let* ((file-defs (parse-lit-files '("dev.lit" "scratch.lit"))) (weaver (make-weaver-default file-defs))) From cb4e960053c0450f95cb96846175c7061fd38f9c Mon Sep 17 00:00:00 2001 From: Eric Ihli Date: Sun, 11 Dec 2022 10:33:26 -0800 Subject: [PATCH 9/9] Update filepaths --- dev.lit => dev/dev.lit | 0 dev/scratch.lit | 12 ++++++++++++ parse.lisp | 2 +- toc.lisp | 22 +++++++++++----------- weave.lisp | 8 ++------ 5 files changed, 26 insertions(+), 18 deletions(-) rename dev.lit => dev/dev.lit (100%) create mode 100644 dev/scratch.lit diff --git a/dev.lit b/dev/dev.lit similarity index 100% rename from dev.lit rename to dev/dev.lit diff --git a/dev/scratch.lit b/dev/scratch.lit new file mode 100644 index 0000000..89a6260 --- /dev/null +++ b/dev/scratch.lit @@ -0,0 +1,12 @@ +# My scratch lit file + +## Scratch + +--- scratch.c +(format nil (* 2 2)) +@{scratch-thing} +--- + +--- scratch-thing +(+ 9 9) +--- diff --git a/parse.lisp b/parse.lisp index 5f84153..eb8701f 100644 --- a/parse.lisp +++ b/parse.lisp @@ -340,7 +340,7 @@ So, `parse-escapes' must be after `parse-refs' and `parse-anchors' in the list o ; 2 ;; Just want to get a feel for what the def-table looks like - (let* ((file-defs (parse-lit-files '("dev.lit" "scratch.lit"))) + (let* ((file-defs (parse-lit-files '("dev/dev.lit" "dev/scratch.lit"))) (weaver (make-weaver-default file-defs))) (let ((defs (weaver-def-table weaver))) (progn diff --git a/toc.lisp b/toc.lisp index b0c1ac4..a45f0d2 100644 --- a/toc.lisp +++ b/toc.lisp @@ -50,6 +50,10 @@ file-def-pairs)) (defun create-global-toc-linkmap (toc) + "Creates a map of chapter/section names to the href when woven into a book file.html#s0:1. +The current implementation has the downside that subsequent chapter/section names +will overwrite antecedent chapter/section entries in the map. Users will just have +to be aware of this and make their chapter/section names unique." (do ((linkmap (make-hash-table :test 'equal)) (file (car toc) (car (cdr toc))) (toc toc (cdr toc))) @@ -71,22 +75,18 @@ (setf (gethash (cadr section) linkmap) link))))))) (comment - (let* ((file-defs (parse-lit-files '("dev.lit" "scratch.lit"))) + (let* ((file-defs (parse-lit-files '("dev/dev.lit" "dev/scratch.lit"))) (linkmap (create-global-toc-linkmap (create-global-toc file-defs))) (res nil)) (maphash - (lambda (k _) - (setf res (cons k res))) + (lambda (k v) + (setf res (cons (list k v) res))) linkmap) - (list res (gethash "Foobar" linkmap))) + res) -; My test lit file: dev.html#c0 -; Foobar: dev.html#s0:0 -; Foobazs: dev.html#s0:1 -; Section 2: dev.html#c1 -; My scratch lit file: scratch.html#c0 -; Scratch: scratch.html#s0:0 -; => NIL + ; => (("Scratch" "scratch.html#s0:0") ("My scratch lit file" "scratch.html#c0") + ; ("Section 2" "dev.html#c1") ("Foobazs" "dev.html#s0:1") + ; ("Foobar" "dev.html#s0:0") ("My test lit file" "dev.html#c0")) ) (defun weave-toc-section (name file chapter-counter section-counter) diff --git a/weave.lisp b/weave.lisp index a5d5103..cd176ba 100644 --- a/weave.lisp +++ b/weave.lisp @@ -36,12 +36,9 @@ (used-math nil :type boolean)) (comment - (let* ((file-defs (parse-lit-files '("dev.lit" "scratch.lit"))) + (let* ((file-defs (parse-lit-files '("dev/dev.lit" "dev/scratch.lit"))) (weaver (make-weaver-default file-defs))) weaver) - - ; => ((:FILE "dev.lit" (:C "My test lit file" (:S "Foobar") (:S "Foobazs"))) - ; (:FILE "scratch.lit" (:C "My scratch lit file" (:S "Scratch")))) ) (defun lit-page-filename (filename) @@ -299,11 +296,10 @@ (write-line "")))) (comment - (let* ((file-defs (parse-lit-files '("dev.lit" "scratch.lit"))) + (let* ((file-defs (parse-lit-files '("dev/dev.lit" "dev/scratch.lit"))) (weaver (make-weaver-default file-defs)) (prosedef (nth 6 (cdar file-defs)))) (weave-prosedef weaver prosedef)) - ) (defun weave-blocks (weaver source-defs)