Force re-read of newly added pages to build

This generates a couple of build warnings, because some references are
missing on the first pass. This is expected, but we should either
silence these errors or defer resolution of these references until later.

This also doesn't include badges for similar reasons: because the references
in the badges can't be resolved in the first pass, the badges are not
linked to any pages.

Author: melissawm

docs/index.rst

@@ -27,3 +27,7 @@ Check out the `list of projects that use this extension <https://github.com/meli
    :titlesonly:
 
    _tags/tagsindex.md
+
+Indices and tables
+==================
+* :ref:`tags-index`

src/sphinx_tags/__init__.py

@@ -8,6 +8,9 @@
 from sphinx.errors import ExtensionError
 from sphinx.util.rst import textwidth
 
+# from sphinx.addnodes import pending_xref
+from sphinx.builders import Builder
+
 from pathlib import Path
 from docutils import nodes
 import os
@@ -71,22 +74,18 @@ def run(self):
                     for tag in ",".join(self.content).split(",")
                 ]
             )
+
         # Remove empty elements from page_tags
         # (can happen after _normalize_tag())
         page_tags = list(filter(None, page_tags))
-
-        # Leftover code from previous attempt:
-        #settings = frontend.get_default_settings(Parser)
-        #document = utils.new_document("tags", settings)
-        #tags = Parser().parse(tagline, document)
-        #print(f"Tags: {tags}")
-
         global_tags = self.env.get_domain("tags")
         # Append this document to the list of documents containing this tag
         for tag in page_tags:
             global_tags.add_tag(tag, self.env.docname)
 
         tag_dir = Path(self.env.app.srcdir) / self.env.app.config.tags_output_dir
+
+        # sphinx.addnodes.pending_xref(rawsource="", *children, **attributes)
         result = nodes.paragraph()
         result["classes"] = ["tags"]
         result += nodes.inline(text=f"{self.env.app.config.tags_intro_text} ")
@@ -126,6 +125,7 @@ def _get_plaintext_node(
     ) -> List[nodes.Node]:
         """Get a plaintext reference link for the given tag"""
         link = relative_tag_dir / f"{file_basename}.html"
+        # return pending_xref(refuri=str(link), text=tag)
         return nodes.reference(refuri=str(link), text=tag)
 
     def _get_badge_node(
@@ -139,7 +139,7 @@ def _get_badge_node(
         text_nodes, messages = self.state.inline_text("", self.lineno)
 
         # Ref paths always use forward slashes, even on Windows
-        tag_ref = f"{tag} <{relative_tag_dir.as_posix()}/{file_basename}>"
+        tag_ref = f"{tag} <{relative_tag_dir.as_posix()}/{file_basename}.html>"
         tag_color = self._get_tag_color(tag)
         tag_badge = XRefBadgeRole(tag_color)
         return tag_badge(
@@ -161,12 +161,14 @@ def _get_tag_color(self, tag: str) -> str:
         return "primary"
 
 
-class PagesIndex(Index):
-    """A custom index that creates a pages matrix."""
+class TagsIndex(Index):
+    """A custom index that creates a page with all tags and corresponding pages
+    where they are defined.
+    """
 
-    name = "tagpage"
-    localname = "Page Index"
-    shortname = "TagPage"
+    name = "index"
+    localname = "All tags"
+    shortname = "TagsIndex"
 
     def generate(
         self, docnames: Iterable[str] | None = None
@@ -177,14 +179,12 @@ def generate(
         pages = sorted(self.domain.get_objects(), key=lambda page: page[0])
 
         # name, subtype, docname, anchor, extra, qualifier, description
-
         for _name, dispname, typ, docname, anchor, _priority in pages:
             content[dispname[0].lower()].append(
                 (dispname, 0, docname, anchor, docname, "", typ)
             )
 
         # convert the dict to the sorted list of tuples expected
-
         return sorted(content.items()), True
 
 
@@ -199,7 +199,7 @@ class TagsDomain(Domain):
         "tags": TagLinks,
     }
 
-    indices = [PagesIndex]
+    indices = [TagsIndex]
 
     # The values defined in initial_data will be copied to
     # env.domaindata[domain_name] as the initial data of the domain, and domain
@@ -211,7 +211,7 @@ class TagsDomain(Domain):
     }
 
     def get_full_qualified_name(self, node):
-        print(f"Node: {node}")
+        # print(f"Node: {node}")
         return f"tags.{node.arguments[0]}"
 
     def get_objects(self):
@@ -230,21 +230,35 @@ def add_tag(self, tagname, page):
         # Add this tag to the global list of tags
         # name, dispname, type, docname, anchor, priority
         self.data["tags"].append((tagname, tagname, "Tag", page, anchor, 0))
+        # Create pages for each tag
+        create_file(
+            (tagname, self.data["entries"][tagname]),
+            # self.app.config.tags_extension,
+            ".md",
+            # Path(self.app.config.tags_output_dir),
+            Path("_tags"),
+            # self.app.srcdir,
+            Path("docs"),
+            # self.app.config.tags_page_title,
+            "My tags",
+            # self.app.config.tags_page_header,
+            "With this tag",
+            # self.app.config.tags_intro_text
+            "Tags:",
+        )
 
     def add_tagpage(self, docname, tags):
         """Add a new page of tags to domain"""
-        name = f"tagpage.{docname}"
-        anchor = f"tagpage-{docname}"
+        name = f"index.{docname}"
+        anchor = f"index-{docname}"
 
-        print(f"{self.data['tags']=}")
         # name, dispname, type, docname, anchor, priority
         self.data["pages"].append(
-            (name, docname, "TagPage", self.env.docname, anchor, 0)
+            (name, docname, "TagsIndex", self.env.docname, anchor, 0)
         )
 
 
 def create_file(
-    app,
     tag: tuple,
     extension: List[str],
     tags_output_dir: Path,
@@ -278,17 +292,11 @@ def create_file(
         the words after which the pages with the tag are listed (e.g. "With this tag: Hello World")
     tag_intro_text: str
         the words after which the tags of a given page are listed (e.g. "Tags: programming, python")
-
-        self.tags = []
-        if tagblock:
-            self.tags = [_normalize_display_tag(tag) for tag in tagblock if tag]
-
     """
 
-    name = tag[0]
-    file_basename = _normalize_tag(tag[0], dashes=True)
-
     # Get sorted file paths for tag pages, relative to /docs/_tags
+    file_basename = _normalize_tag(tag[0], dashes=True)
+    name = _normalize_display_tag(tag[0])
     tag_page_paths = sorted([os.path.relpath(i, srcdir) for i in tag[1]])
     ref_label = f"sphx_tag_{file_basename}"
 
@@ -355,10 +363,6 @@ def tagpage(tags, outdir, title, extension, tags_index_head):
     This page contains a list of all available tags.
 
     """
-
-    print(f"Tags: {tags=}")
-    print(f"outdir: {outdir=}")
-
     if "md" in extension:
         content = []
         content.append("(tagoverview)=")
@@ -371,9 +375,7 @@ def tagpage(tags, outdir, title, extension, tags_index_head):
         content.append(f"caption: {tags_index_head}")
         content.append("maxdepth: 1")
         content.append("---")
-        for name, pages in tags.items():
-            file_basename = _normalize_tag(name, dashes=True)
-            content.append(f"{name} ({len(pages)}) <{file_basename}>")
+        content.append("PLACEHOLDER")
         content.append("```")
         content.append("")
         filename = os.path.join(outdir, "tagsindex.md")
@@ -391,61 +393,77 @@ def tagpage(tags, outdir, title, extension, tags_index_head):
         content.append(f"    :caption: {tags_index_head}")
         content.append("    :maxdepth: 1")
         content.append("")
-        for name, pages in tags.items():
-            file_basename = _normalize_tag(name, dashes=True)
-            content.append(f"    {name} ({len(pages)}) <{file_basename}.rst>")
+        content.append("")  # placeholder
         content.append("")
         filename = os.path.join(outdir, "tagsindex.rst")
 
     with open(filename, "w", encoding="utf8") as f:
         f.write("\n".join(content))
 
 
-def update_tags(app):
+def update_tags(app, env):
     """Update tags according to pages found"""
     if app.config.tags_create_tags:
         tags_output_dir = Path(app.config.tags_output_dir)
-
-        if not os.path.exists(os.path.join(app.srcdir, tags_output_dir)):
-            os.makedirs(os.path.join(app.srcdir, tags_output_dir))
-
-        for file in os.listdir(os.path.join(app.srcdir, tags_output_dir)):
-            if file.endswith("md") or file.endswith("rst"):
-                os.remove(os.path.join(app.srcdir, tags_output_dir, file))
-
         # Create pages for each tag
         global_tags = app.env.get_domain("tags").data["entries"]
-        logger.info(f"Global tags: {global_tags=}", color="green")
-
-        for tag in global_tags.items():
-            create_file(
-                app,
-                tag,
-                app.config.tags_extension,
-                tags_output_dir,
-                app.srcdir,
-                app.config.tags_page_title,
-                app.config.tags_page_header,
-                app.config.tags_intro_text,
+        # Inject these into found_docs
+        for docname, tags in global_tags.items():
+            env.found_docs.add(docname)
+        # Inject these into the tagsindex
+        tags_output_dir = Path(app.config.tags_output_dir)
+        outdir = os.path.join(app.srcdir, tags_output_dir)
+        newcontent = []
+        if "md" in app.config.tags_extension:
+            filename = os.path.join(outdir, "tagsindex.md")
+            taglist = "{name} ({len}) <{file_basename}>"
+        else:
+            filename = os.path.join(outdir, "tagsindex.rst")
+            taglist = "    {name} ({len}) <{file_basename}.rst>"
+
+        for name, pages in global_tags.items():
+            file_basename = _normalize_tag(name, dashes=True)
+            newcontent.append(
+                taglist.format(name=name, len=len(pages), file_basename=file_basename)
             )
+        with open(filename, "r", encoding="utf8") as f:
+            content = f.read()
+        with open(filename, "w", encoding="utf8") as f:
+            f.write(content.replace("PLACEHOLDER", "\n".join(newcontent)))
+    else:
+        logger.info(
+            "Tags were not created (tags_create_tags=False in conf.py)", color="white"
+        )
+    logger.info("Tags updated", color="white")
+
+    # Re-read files, create doctrees and add to env.found_docs and return
+    # iterable of docnames to re-read for env-get-updated
+    app.builder.read()
+    return env.found_docs
+
+
+def prepare_tags(app):
+    if app.config.tags_create_tags:
+        tags_output_dir = Path(app.config.tags_output_dir)
+
+        if not os.path.exists(os.path.join(app.srcdir, tags_output_dir)):
+            os.makedirs(os.path.join(app.srcdir, tags_output_dir))
 
         # Create tags overview page
         tagpage(
-            global_tags,
+            [],
             os.path.join(app.srcdir, tags_output_dir),
             app.config.tags_overview_title,
             app.config.tags_extension,
             app.config.tags_index_head,
         )
-        logger.info("Tags updated", color="white")
+
+        logger.info("Tags output dir created", color="green")
     else:
         logger.info(
-            "Tags were not created (tags_create_tags=False in conf.py)", color="white"
+            "Tags were not created (tags_create_tags=False in conf.py)", color="red"
         )
 
-    # Return iterable of docnames to re-read
-    return os.listdir(os.path.join(app.srcdir, tags_output_dir))
-
 
 def setup(app):
     """Setup for Sphinx."""
@@ -473,10 +491,10 @@ def setup(app):
         "html",
     )
 
-    # Update tags
     # Tags should be updated after sphinx-gallery is generated, on
     # builder-inited
-    app.connect("builder-inited", update_tags)
+    app.connect("builder-inited", prepare_tags)
+    app.connect("env-get-updated", update_tags)
     app.add_directive("tags", TagLinks)
     app.add_domain(TagsDomain)