Add manually curated non-symbol nodes in navigator

PR #757 enabled DocC to include manually curated non-symbol nodes in the
topics section, irrespective of the node language. This allows curating
articles across documentation in different languages, even though the
article's language is considered to be "Swift". The change was not
propagated to the navigator, resulting in the references being present
in the topics section but not in the sidebar. This patch adds these
references to the navigator. It also ensures that the logic is tested
in the same way as the topics section logic to maintain consistency.

rdar://160284853

Author: snprajwal

Sources/SwiftDocC/Indexing/Navigator/NavigatorIndex.swift

@@ -466,6 +466,17 @@ extension NavigatorIndex {
             self.fragment = fragment
             self.languageIdentifier = languageIdentifier
         }
+
+        /// Compare an identifier with another one, ignoring the identifier language.
+        ///
+        /// Used when curating cross-language references in multi-language frameworks.
+        ///
+        /// - Parameter other: The other identifier to compare with.
+        func isEquivalentIgnoringLanguage(to other: Identifier) -> Bool {
+            return self.bundleIdentifier == other.bundleIdentifier &&
+                   self.path == other.path &&
+                   self.fragment == other.fragment
+        }
     }
 
     /**
@@ -949,7 +960,20 @@ extension NavigatorIndex {
                     let (nodeID, parent) = nodesMultiCurated[index]
                     let placeholders = identifierToChildren[nodeID]!
                     for reference in placeholders {
-                        if let child = identifierToNode[reference] {
+                        var child = identifierToNode[reference]
+                        // If no node is found for the reference, look for nodes that match the identifier ignoring the language.
+                        // This is valid in case of references to nodes that are not symbols. Since the identifier is assumed to
+                        // be of the same language mask as the parent node, references to articles or frameworks will be missed
+                        // during the node lookup, even though they are valid. If a node with the same identifier in a different
+                        // language is found within the same bundle, that node is used for the reference.
+                        if child == nil {
+                            child = identifierToNode
+                           .first { identifier, node in
+                               identifier.isEquivalentIgnoringLanguage(to: reference)
+                               && PageType.init(rawValue: node.item.pageType)?.isSymbolKind == false
+                            }?.value
+                        }
+                        if let child = child {
                             parent.add(child: child)
                             pendingUncuratedReferences.remove(reference)
                             if !multiCurated.keys.contains(reference) && reference.fragment == nil {
@@ -970,7 +994,20 @@ extension NavigatorIndex {
             for (nodeIdentifier, placeholders) in identifierToChildren {
                 for reference in placeholders {
                     let parent = identifierToNode[nodeIdentifier]!
-                    if let child = identifierToNode[reference] {
+                    var child = identifierToNode[reference]
+                    // If no node is found for the reference, look for nodes that match the identifier ignoring the language.
+                    // This is valid in case of references to nodes that are not symbols. Since the identifier is assumed to
+                    // be of the same language mask as the parent node, references to articles or frameworks will be missed
+                    // during the node lookup, even though they are valid. If a node with the same identifier in a different
+                    // language is found within the same bundle, that node is used for the reference.
+                    if child == nil {
+                        child = identifierToNode
+                       .first { identifier, node in
+                           identifier.isEquivalentIgnoringLanguage(to: reference)
+                           && PageType.init(rawValue: node.item.pageType)?.isSymbolKind == false
+                        }?.value
+                    }
+                    if let child = child {
                         let needsCopy = multiCurated[reference] != nil
                         parent.add(child: (needsCopy) ? child.copy() : child)
                         pendingUncuratedReferences.remove(reference)
@@ -1001,14 +1038,11 @@ extension NavigatorIndex {
                 // page types as symbol nodes on the assumption that an unknown page type is a
                 // symbol kind added in a future version of Swift-DocC.
                 // Finally, don't add external references to the root; if they are not referenced within the navigation tree, they should be dropped altogether.
-                if let node = identifierToNode[nodeID], PageType(rawValue: node.item.pageType)?.isSymbolKind == false , !node.item.isExternal {
+                if let node = identifierToNode[nodeID], PageType(rawValue: node.item.pageType)?.isSymbolKind == false, !node.item.isExternal {
 
                     // If an uncurated page has been curated in another language, don't add it to the top-level.
                     if curatedReferences.contains(where: { curatedNodeID in
-                        // Compare all the identifier's properties for equality, except for its language.
-                        curatedNodeID.bundleIdentifier == nodeID.bundleIdentifier
-                            && curatedNodeID.path == nodeID.path
-                            && curatedNodeID.fragment == nodeID.fragment
+                        curatedNodeID.isEquivalentIgnoringLanguage(to: nodeID)
                     }) {
                         continue
                     }

Tests/SwiftDocCTests/Indexing/ExternalRenderNodeTests.swift

@@ -202,10 +202,11 @@ class ExternalRenderNodeTests: XCTestCase {
         // Verify that the curated external links are part of the index.
         let swiftExternalNodes = renderIndex.interfaceLanguages["swift"]?.first { $0.path == "/documentation/mixedlanguageframework" }?.children?.filter { $0.path?.contains("/path/to/external") ?? false } ?? []
         let occExternalNodes = renderIndex.interfaceLanguages["occ"]?.first { $0.path == "/documentation/mixedlanguageframework" }?.children?.filter { $0.path?.contains("/path/to/external") ?? false } ?? []
-        XCTAssertEqual(swiftExternalNodes.count, 2)
-        XCTAssertEqual(occExternalNodes.count, 2)
-        XCTAssertEqual(swiftExternalNodes.map(\.title), ["SwiftArticle", "SwiftSymbol"])
-        XCTAssertEqual(occExternalNodes.map(\.title), ["ObjCArticle", "ObjCSymbol"])
+        XCTAssertEqual(swiftExternalNodes.count, 3)
+        XCTAssertEqual(occExternalNodes.count, 3)
+        // Articles should be curated all the time irrespective of their language
+        XCTAssertEqual(swiftExternalNodes.map(\.title), ["SwiftArticle", "SwiftSymbol", "ObjCArticle"])
+        XCTAssertEqual(occExternalNodes.map(\.title), ["SwiftArticle", "ObjCArticle", "ObjCSymbol"])
         XCTAssert(swiftExternalNodes.allSatisfy(\.isExternal))
         XCTAssert(occExternalNodes.allSatisfy(\.isExternal))
     }
@@ -263,9 +264,10 @@ class ExternalRenderNodeTests: XCTestCase {
         let swiftExternalNodes = renderIndex.interfaceLanguages["swift"]?.first { $0.path == "/documentation/mixedlanguageframework" }?.children?.filter { $0.path?.contains("/path/to/external") ?? false } ?? []
         let occExternalNodes = renderIndex.interfaceLanguages["occ"]?.first { $0.path == "/documentation/mixedlanguageframework" }?.children?.filter { $0.path?.contains("/path/to/external") ?? false } ?? []
         XCTAssertEqual(swiftExternalNodes.count, 1)
-        XCTAssertEqual(occExternalNodes.count, 1)
         XCTAssertEqual(swiftExternalNodes.map(\.title), ["SwiftArticle"])
-        XCTAssertEqual(occExternalNodes.map(\.title), ["ObjCSymbol"])
+        // Articles should be curated all the time irrespective of their language
+        XCTAssertEqual(occExternalNodes.count, 2)
+        XCTAssertEqual(occExternalNodes.map(\.title), ["SwiftArticle", "ObjCSymbol"])
         XCTAssert(swiftExternalNodes.allSatisfy(\.isExternal))
         XCTAssert(occExternalNodes.allSatisfy(\.isExternal))
     }

Tests/SwiftDocCTests/Indexing/NavigatorIndexTests.swift

@@ -2008,6 +2008,157 @@ Root
 
         return navigatorIndex
     }
+
+    func testCrossLanguageCurationBehavior() async throws {
+        let catalog = Folder(name: "CrossLanguageTest.docc", content: [
+            InfoPlist(identifier: "org.swift.docc.crosslanguagetest"),
+            // Symbol graph with both Swift and Objective-C symbols
+            JSONFile(name: "CrossLanguageFramework.symbols.json", content: makeSymbolGraph(
+                moduleName: "CrossLanguageFramework",
+                symbols: [
+                    .init(
+                        identifier: .init(precise: "swift-class-id", interfaceLanguage: SourceLanguage.swift.id),
+                        names: .init(title: "SwiftClass", navigator: [.init(kind: .identifier, spelling: "SwiftClass", preciseIdentifier: nil)], subHeading: nil, prose: nil),
+                        pathComponents: ["SwiftClass"],
+                        docComment: nil,
+                        accessLevel: .public,
+                        kind: .init(parsedIdentifier: .class, displayName: "Class"),
+                        mixins: [:]
+                    ),
+                    .init(
+                        identifier: .init(precise: "objc-class-id", interfaceLanguage: SourceLanguage.objectiveC.id),
+                        names: .init(title: "ObjCClass", navigator: [.init(kind: .identifier, spelling: "ObjCClass", preciseIdentifier: nil)], subHeading: nil, prose: nil),
+                        pathComponents: ["ObjCClass"],
+                        docComment: nil,
+                        accessLevel: .public,
+                        kind: .init(parsedIdentifier: .class, displayName: "Class"),
+                        mixins: [:]
+                    )
+                ],
+                relationships: []
+            )),
+            TextFile(name: "CrossLanguageArticle.md", utf8Content: """
+            # Cross Language Article
+
+            This is a general article.
+            """),
+            TextFile(name: "SwiftOnlyArticle.md", utf8Content: """
+            # Swift Only Article
+
+            @Metadata {
+               @SupportedLanguage(swift)
+            }
+
+            This is a Swift-only article.
+            """),
+            TextFile(name: "ObjCOnlyArticle.md", utf8Content: """
+            # Objective-C Only Article
+
+            @Metadata {
+               @SupportedLanguage(objc)
+            }
+
+            This article is only available in Objective-C.
+            """),
+            // Swift page that curates ObjC articles and symbols
+            TextFile(name: "SwiftContainer.md", utf8Content: """
+            # Swift Container
+
+            @Metadata {
+               @SupportedLanguage(swift)
+            }
+
+            This page curates articles and attempts to curate Objective-C symbols (which should be dropped).
+
+            ## Topics
+
+            ### Articles
+            - <doc:CrossLanguageArticle>
+            - <doc:ObjCOnlyArticle>
+
+            ### Swift Symbols
+            - ``SwiftClass``
+
+            ### Objective-C Symbols
+            - ``ObjCClass``
+            """),
+            // ObjC page that curates Swift articles and symbols
+            TextFile(name: "ObjCContainer.md", utf8Content: """
+            # Objective-C Container
+
+            @Metadata {
+               @SupportedLanguage(objc)
+            }
+
+            This page curates articles and attempts to curate Swift symbols (which should be dropped).
+
+            ## Topics
+
+            ### Articles
+            - <doc:CrossLanguageArticle>
+            - <doc:SwiftOnlyArticle>
+
+            ### Objective-C Symbols
+            - ``ObjCClass``
+
+            ### Swift Symbols
+            - ``SwiftClass``
+            """)
+        ])
+
+        let (_, context) = try await loadBundle(catalog: catalog)
+        let bundle = try XCTUnwrap(context.bundle)
+
+        XCTAssert(context.problems.isEmpty, "Failed to unwrap bundle: \(context.problems.map(\.diagnostic.summary))")
+
+        // Generate navigator index
+        let targetURL = try createTemporaryDirectory()
+        let builder = NavigatorIndex.Builder(outputURL: targetURL, bundleIdentifier: "org.swift.docc.crosslanguagetest", groupByLanguage: true)
+        builder.setup()
+
+        let renderContext = RenderContext(documentationContext: context, bundle: bundle)
+        let converter = DocumentationContextConverter(bundle: bundle, context: context, renderContext: renderContext)
+
+        for identifier in context.knownPages {
+            let entity = try context.entity(with: identifier)
+            let renderNode = try XCTUnwrap(converter.renderNode(for: entity))
+            try builder.index(renderNode: renderNode)
+        }
+
+        builder.finalize()
+        let navigatorIndex = try XCTUnwrap(builder.navigatorIndex)
+
+        let swiftNode = try XCTUnwrap(navigatorIndex.navigatorTree.root.children.first { $0.item.title == "Swift" })
+        let swiftContainer = try XCTUnwrap(search(node: swiftNode) { $0.item.title == "Swift Container" })
+        let swiftContainerChildren = Set(swiftContainer.children.map { $0.item.title })
+        // Articles should be curated across languages
+        XCTAssertTrue(swiftContainerChildren.contains("Cross Language Article"),
+                     "Cross-language article should be curated in Swift container")
+        XCTAssertTrue(swiftContainerChildren.contains("Objective-C Only Article"),
+                     "Objective-C-only article should be curated in Swift container")
+
+        // Swift symbols should be curated under Swift
+        XCTAssertTrue(swiftContainerChildren.contains("SwiftClass"),
+                     "Swift symbol should be curated in Swift container")
+        // Objective-C symbols should NOT be curated under Swift
+        XCTAssertFalse(swiftContainerChildren.contains("ObjCClass"),
+                      "Objective-C symbol should NOT be curated in Swift container")
+
+        // Same checks as above for Objective-C
+        let objcNode = try XCTUnwrap(navigatorIndex.navigatorTree.root.children.first { $0.item.title == "Objective-C" })
+        let objcContainer = try XCTUnwrap(search(node: objcNode) { $0.item.title == "Objective-C Container" })
+        let objcContainerChildren = Set(objcContainer.children.map { $0.item.title })
+
+        XCTAssertTrue(objcContainerChildren.contains("Cross Language Article"),
+                     "Cross-language article should be curated in Objective-C container")
+        XCTAssertTrue(objcContainerChildren.contains("Swift Only Article"),
+                     "Swift-only article should be curated in Objective-C container")
+
+        XCTAssertTrue(objcContainerChildren.contains("ObjCClass"),
+                     "Objective-C symbol should be curated in Objective-C container")
+        XCTAssertFalse(objcContainerChildren.contains("SwiftClass"),
+                      "Swift symbol should NOT be curated in Objective-C container")
+    }
 }
 
 /// This function compares two nodes to ensure their data is equal.