Fix linker filelist parsing when discovering tests

Take the format into account when parsing response files, and consistently track it when we read and write them in various places throughout the build system

Author: owenv

Sources/SWBCore/PlannedTaskAction.swift

@@ -349,7 +349,7 @@ public protocol TaskActionCreationDelegate
     func createProcessSDKImportsTaskAction() -> any PlannedTaskAction
     func createValidateDependenciesTaskAction() -> any PlannedTaskAction
     func createObjectLibraryAssemblerTaskAction() -> any PlannedTaskAction
-    func createLinkerTaskAction(expandResponseFiles: Bool) -> any PlannedTaskAction
+    func createLinkerTaskAction(expandResponseFiles: Bool, responseFileFormat: ResponseFileFormat) -> any PlannedTaskAction
 }
 
 extension TaskActionCreationDelegate {

Sources/SWBCore/Settings/BuiltinMacros.swift

@@ -1188,7 +1188,8 @@ public final class BuiltinMacros {
     public static let _WRAPPER_PARENT_PATH = BuiltinMacros.declareStringMacro("_WRAPPER_PARENT_PATH")
     public static let _WRAPPER_RESOURCES_DIR = BuiltinMacros.declareStringMacro("_WRAPPER_RESOURCES_DIR")
     public static let __INPUT_FILE_LIST_PATH__ = BuiltinMacros.declarePathMacro("__INPUT_FILE_LIST_PATH__")
-    public static let LINKER_FILE_LIST_FORMAT = BuiltinMacros.declareEnumMacro("LINKER_FILE_LIST_FORMAT") as EnumMacroDeclaration<LinkerFileListFormat>
+    public static let LINKER_FILE_LIST_FORMAT = BuiltinMacros.declareEnumMacro("LINKER_FILE_LIST_FORMAT") as EnumMacroDeclaration<ResponseFileFormat>
+    public static let LINKER_RESPONSE_FILE_FORMAT = BuiltinMacros.declareEnumMacro("LINKER_RESPONSE_FILE_FORMAT") as EnumMacroDeclaration<ResponseFileFormat>
     public static let SWIFT_RESPONSE_FILE_PATH = BuiltinMacros.declarePathMacro("SWIFT_RESPONSE_FILE_PATH")
     public static let __ARCHS__ = BuiltinMacros.declareStringListMacro("__ARCHS__")
 
@@ -2441,6 +2442,7 @@ public final class BuiltinMacros {
         _WRAPPER_RESOURCES_DIR,
         __INPUT_FILE_LIST_PATH__,
         LINKER_FILE_LIST_FORMAT,
+        LINKER_RESPONSE_FILE_FORMAT,
         __ARCHS__,
         __SWIFT_MODULE_ONLY_ARCHS__,
         arch,
@@ -2898,12 +2900,8 @@ public enum StripStyle: String, Equatable, Hashable, EnumerationMacroType {
     case debugging
 }
 
-public enum LinkerFileListFormat: String, Equatable, Hashable, EnumerationMacroType {
-    public static let defaultValue = Self.unescapedNewlineSeparated
-
-    case unescapedNewlineSeparated
-    case unixShellQuotedNewlineSeparated
-    case windowsShellQuotedNewlineSeparated
+extension ResponseFileFormat: EnumerationMacroType {
+    public static let defaultValue = ResponseFileFormat.unixShellQuotedNewlineSeparated
 }
 
 public enum MergedBinaryType: String, Equatable, Hashable, EnumerationMacroType {

Sources/SWBCore/SpecImplementations/LinkerSpec.swift

@@ -139,20 +139,7 @@ open class LinkerSpec : CommandLineToolSpec, @unchecked Sendable {
     }
 
     public func inputFileListContents(_ cbc: CommandBuildContext) -> ByteString {
-        let contents = OutputByteStream()
-        for input in cbc.inputs {
-            switch cbc.scope.evaluate(BuiltinMacros.LINKER_FILE_LIST_FORMAT) {
-            case .unescapedNewlineSeparated:
-                contents <<< input.absolutePath.strWithPosixSlashes <<< "\n"
-            case .unixShellQuotedNewlineSeparated:
-                let escaper = UNIXShellCommandCodec(encodingStrategy: .singleQuotes, encodingBehavior: .argumentsOnly)
-                contents <<< escaper.encode([input.absolutePath.strWithPosixSlashes]) <<< "\n"
-            case .windowsShellQuotedNewlineSeparated:
-                let escaper = WindowsProcessArgumentsCodec()
-                contents <<< escaper.encode([input.absolutePath.strWithPosixSlashes]) <<< "\n"
-            }
-        }
-        return contents.bytes
+        return ByteString(encodingAsUTF8: ResponseFiles.responseFileContents(args: cbc.inputs.map { $0.absolutePath.strWithPosixSlashes }, format: cbc.scope.evaluate(BuiltinMacros.LINKER_FILE_LIST_FORMAT)))
     }
 
     open override func constructTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) async {

Sources/SWBCore/SpecImplementations/PropertyDomainSpec.swift

@@ -117,7 +117,9 @@ private final class EnumBuildOptionType : BuildOptionType {
         case "SWIFT_API_DIGESTER_MODE":
             return try namespace.declareEnumMacro(name) as EnumMacroDeclaration<SwiftAPIDigesterMode>
         case "LINKER_FILE_LIST_FORMAT":
-            return try namespace.declareEnumMacro(name) as EnumMacroDeclaration<LinkerFileListFormat>
+            return try namespace.declareEnumMacro(name) as EnumMacroDeclaration<ResponseFileFormat>
+        case "LINKER_RESPONSE_FILE_FORMAT":
+            return try namespace.declareEnumMacro(name) as EnumMacroDeclaration<ResponseFileFormat>
         case "DOCC_MINIMUM_ACCESS_LEVEL":
             return try namespace.declareEnumMacro(name) as EnumMacroDeclaration<DoccMinimumAccessLevel>
         default:

Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift

@@ -235,7 +235,7 @@ public struct ClangSourceFileIndexingInfo: SourceFileIndexingInfo {
                 // Skip
             } else if arg.starts(with: ByteString(unicodeScalarLiteral: "@")),
                       let attachmentPath = responseFileMapping[Path(arg.asString.dropFirst())],
-                      let responseFileArgs = try? ResponseFiles.expandResponseFiles(["@\(attachmentPath.str)"], fileSystem: localFS, relativeTo: workingDir) {
+                      let responseFileArgs = try? ResponseFiles.expandResponseFiles(["@\(attachmentPath.str)"], fileSystem: localFS, relativeTo: workingDir, format: .unixShellQuotedSpaceSeparated) {
                 result.append(contentsOf: responseFileArgs.map { ByteString(encodingAsUTF8: $0) })
             } else {
                 result.append(arg)
@@ -719,7 +719,7 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
             ctx.add(string: self.identifier)
 
             let responseFilePath = scope.evaluate(BuiltinMacros.PER_ARCH_OBJECT_FILE_DIR).join("\(ctx.signature.asString)-common-args.resp")
-            let attachmentPath = producer.writeFileSpec.constructFileTasks(CommandBuildContext(producer: producer, scope: scope, inputs: [], output: responseFilePath), delegate, contents: ByteString(encodingAsUTF8: ResponseFiles.responseFileContents(args: responseFileCommandLine)), permissions: nil, logContents: true, preparesForIndexing: true, additionalTaskOrderingOptions: [.immediate, .ignorePhaseOrdering])
+            let attachmentPath = producer.writeFileSpec.constructFileTasks(CommandBuildContext(producer: producer, scope: scope, inputs: [], output: responseFilePath), delegate, contents: ByteString(encodingAsUTF8: ResponseFiles.responseFileContents(args: responseFileCommandLine, format: .unixShellQuotedSpaceSeparated)), permissions: nil, logContents: true, preparesForIndexing: true, additionalTaskOrderingOptions: [.immediate, .ignorePhaseOrdering])
 
             return ConstantFlags(flags: regularCommandLine + ["@\(responseFilePath.str)"], headerSearchPaths: headerSearchPaths, inputs: [responseFilePath], responseFileMapping: [responseFilePath: attachmentPath])
         } else {

Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift

@@ -1428,7 +1428,8 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec
 
     override public func createTaskAction(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) -> (any PlannedTaskAction)? {
         let useResponseFile = cbc.scope.evaluate(BuiltinMacros.CLANG_USE_RESPONSE_FILE)
-        return delegate.taskActionCreationDelegate.createLinkerTaskAction(expandResponseFiles: !useResponseFile)
+        let responseFileFormat = cbc.scope.evaluate(BuiltinMacros.LINKER_RESPONSE_FILE_FORMAT)
+        return delegate.taskActionCreationDelegate.createLinkerTaskAction(expandResponseFiles: !useResponseFile, responseFileFormat: responseFileFormat)
     }
 
     override public func discoveredCommandLineToolSpecInfo(_ producer: any CommandProducer, _ scope: MacroEvaluationScope, _ delegate: any CoreClientTargetDiagnosticProducingDelegate) async -> (any DiscoveredCommandLineToolSpecInfo)? {
@@ -1645,7 +1646,8 @@ public final class LibtoolLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @u
 
     override public func createTaskAction(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) -> (any PlannedTaskAction)? {
         let useResponseFile = cbc.scope.evaluate(BuiltinMacros.LIBTOOL_USE_RESPONSE_FILE)
-        return delegate.taskActionCreationDelegate.createLinkerTaskAction(expandResponseFiles: !useResponseFile)
+        let responseFileFormat = cbc.scope.evaluate(BuiltinMacros.LINKER_RESPONSE_FILE_FORMAT)
+        return delegate.taskActionCreationDelegate.createLinkerTaskAction(expandResponseFiles: !useResponseFile, responseFileFormat: responseFileFormat)
     }
 
     override public func constructLinkerTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, libraries: [LibrarySpecifier], usedTools: [CommandLineToolSpec: Set<FileTypeSpec>]) async {

Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift

@@ -373,8 +373,9 @@ public struct SwiftDriverPayload: Serializable, TaskPayload, Encodable {
     public let linkerResponseFilePath: Path?
     public let dependencyFilteringRootPath: Path?
     public let verifyScannerDependencies: Bool
+    public let linkerResponseFileFormat: ResponseFileFormat
 
-    internal init(uniqueID: String, compilerLocation: LibSwiftDriver.CompilerLocation, moduleName: String, outputPrefix: String, tempDirPath: Path, explicitModulesTempDirPath: Path, variant: String, architecture: String, eagerCompilationEnabled: Bool, explicitModulesEnabled: Bool, commandLine: [String], ruleInfo: [String], isUsingWholeModuleOptimization: Bool, casOptions: CASOptions?, reportRequiredTargetDependencies: BooleanWarningLevel, linkerResponseFilePath: Path?, dependencyFilteringRootPath: Path?, verifyScannerDependencies: Bool) {
+    internal init(uniqueID: String, compilerLocation: LibSwiftDriver.CompilerLocation, moduleName: String, outputPrefix: String, tempDirPath: Path, explicitModulesTempDirPath: Path, variant: String, architecture: String, eagerCompilationEnabled: Bool, explicitModulesEnabled: Bool, commandLine: [String], ruleInfo: [String], isUsingWholeModuleOptimization: Bool, casOptions: CASOptions?, reportRequiredTargetDependencies: BooleanWarningLevel, linkerResponseFilePath: Path?, linkerResponseFileFormat: ResponseFileFormat, dependencyFilteringRootPath: Path?, verifyScannerDependencies: Bool) {
         self.uniqueID = uniqueID
         self.compilerLocation = compilerLocation
         self.moduleName = moduleName
@@ -391,12 +392,13 @@ public struct SwiftDriverPayload: Serializable, TaskPayload, Encodable {
         self.casOptions = casOptions
         self.reportRequiredTargetDependencies = reportRequiredTargetDependencies
         self.linkerResponseFilePath = linkerResponseFilePath
+        self.linkerResponseFileFormat = linkerResponseFileFormat
         self.dependencyFilteringRootPath = dependencyFilteringRootPath
         self.verifyScannerDependencies = verifyScannerDependencies
     }
 
     public init(from deserializer: any Deserializer) throws {
-        try deserializer.beginAggregate(18)
+        try deserializer.beginAggregate(19)
         self.uniqueID = try deserializer.deserialize()
         self.compilerLocation = try deserializer.deserialize()
         self.moduleName = try deserializer.deserialize()
@@ -413,12 +415,13 @@ public struct SwiftDriverPayload: Serializable, TaskPayload, Encodable {
         self.casOptions = try deserializer.deserialize()
         self.reportRequiredTargetDependencies = try deserializer.deserialize()
         self.linkerResponseFilePath = try deserializer.deserialize()
+        self.linkerResponseFileFormat = try deserializer.deserialize()
         self.dependencyFilteringRootPath = try deserializer.deserialize()
         self.verifyScannerDependencies = try deserializer.deserialize()
     }
 
     public func serialize<T>(to serializer: T) where T : Serializer {
-        serializer.serializeAggregate(18) {
+        serializer.serializeAggregate(19) {
             serializer.serialize(self.uniqueID)
             serializer.serialize(self.compilerLocation)
             serializer.serialize(self.moduleName)
@@ -435,6 +438,7 @@ public struct SwiftDriverPayload: Serializable, TaskPayload, Encodable {
             serializer.serialize(self.casOptions)
             serializer.serialize(self.reportRequiredTargetDependencies)
             serializer.serialize(self.linkerResponseFilePath)
+            serializer.serialize(self.linkerResponseFileFormat)
             serializer.serialize(self.dependencyFilteringRootPath)
             serializer.serialize(self.verifyScannerDependencies)
         }
@@ -2560,7 +2564,7 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi
             let explicitModuleBuildEnabled = await swiftExplicitModuleBuildEnabled(cbc.producer, cbc.scope, delegate)
             let verifyScannerDependencies = explicitModuleBuildEnabled && cbc.scope.evaluate(BuiltinMacros.SWIFT_DEPENDENCY_REGISTRATION_MODE) == .verifySwiftDependencyScanner
 
-            return SwiftDriverPayload(uniqueID: uniqueID, compilerLocation: compilerLocation, moduleName: scope.evaluate(BuiltinMacros.SWIFT_MODULE_NAME), outputPrefix: scope.evaluate(BuiltinMacros.TARGET_NAME) + compilationMode.moduleBaseNameSuffix, tempDirPath: tempDirPath, explicitModulesTempDirPath: explicitModulesTempDirPath, variant: variant, architecture: arch, eagerCompilationEnabled: eagerCompilationEnabled(args: args, scope: scope, compilationMode: compilationMode, isUsingWholeModuleOptimization: isUsingWholeModuleOptimization), explicitModulesEnabled: explicitModuleBuildEnabled, commandLine: commandLine, ruleInfo: ruleInfo, isUsingWholeModuleOptimization: isUsingWholeModuleOptimization, casOptions: casOptions, reportRequiredTargetDependencies: scope.evaluate(BuiltinMacros.DIAGNOSE_MISSING_TARGET_DEPENDENCIES), linkerResponseFilePath: linkerResponseFilePath, dependencyFilteringRootPath: cbc.producer.sdk?.path, verifyScannerDependencies: verifyScannerDependencies)
+            return SwiftDriverPayload(uniqueID: uniqueID, compilerLocation: compilerLocation, moduleName: scope.evaluate(BuiltinMacros.SWIFT_MODULE_NAME), outputPrefix: scope.evaluate(BuiltinMacros.TARGET_NAME) + compilationMode.moduleBaseNameSuffix, tempDirPath: tempDirPath, explicitModulesTempDirPath: explicitModulesTempDirPath, variant: variant, architecture: arch, eagerCompilationEnabled: eagerCompilationEnabled(args: args, scope: scope, compilationMode: compilationMode, isUsingWholeModuleOptimization: isUsingWholeModuleOptimization), explicitModulesEnabled: explicitModuleBuildEnabled, commandLine: commandLine, ruleInfo: ruleInfo, isUsingWholeModuleOptimization: isUsingWholeModuleOptimization, casOptions: casOptions, reportRequiredTargetDependencies: scope.evaluate(BuiltinMacros.DIAGNOSE_MISSING_TARGET_DEPENDENCIES), linkerResponseFilePath: linkerResponseFilePath, linkerResponseFileFormat: cbc.scope.evaluate(BuiltinMacros.LINKER_RESPONSE_FILE_FORMAT), dependencyFilteringRootPath: cbc.producer.sdk?.path, verifyScannerDependencies: verifyScannerDependencies)
         }
 
         func constructSwiftResponseFileTask(path: Path) {

Sources/SWBGenericUnixPlatform/Specs/UnixLd.xcspec

@@ -186,10 +186,22 @@
                 Type = Enumeration;
                 Values = (
                     unescapedNewlineSeparated,
+                    unixShellQuotedSpaceSeparated,
                     unixShellQuotedNewlineSeparated,
                     windowsShellQuotedNewlineSeparated,
                 );
                 DefaultValue = unixShellQuotedNewlineSeparated;
+            },
+            {
+                Name = "LINKER_RESPONSE_FILE_FORMAT";
+                Type = Enumeration;
+                Values = (
+                    unescapedNewlineSeparated,
+                    unixShellQuotedSpaceSeparated,
+                    unixShellQuotedNewlineSeparated,
+                    windowsShellQuotedNewlineSeparated,
+                );
+                DefaultValue = unixShellQuotedSpaceSeparated;
             }
         );
     },

Sources/SWBGenericUnixPlatform/Specs/UnixLibtool.xcspec

@@ -77,6 +77,17 @@
                     windowsShellQuotedNewlineSeparated,
                 );
                 DefaultValue = unixShellQuotedNewlineSeparated;
+            },
+            {
+                Name = "LINKER_RESPONSE_FILE_FORMAT";
+                Type = Enumeration;
+                Values = (
+                    unescapedNewlineSeparated,
+                    unixShellQuotedSpaceSeparated,
+                    unixShellQuotedNewlineSeparated,
+                    windowsShellQuotedNewlineSeparated,
+                );
+                DefaultValue = unixShellQuotedSpaceSeparated;
             }
         );
     },

Sources/SWBTaskExecution/BuildDescriptionManager.swift

@@ -904,8 +904,8 @@ extension BuildSystemTaskPlanningDelegate: TaskActionCreationDelegate {
         return ObjectLibraryAssemblerTaskAction()
     }
 
-    func createLinkerTaskAction(expandResponseFiles: Bool) -> any PlannedTaskAction {
-        return LinkerTaskAction(expandResponseFiles: expandResponseFiles)
+    func createLinkerTaskAction(expandResponseFiles: Bool, responseFileFormat: ResponseFileFormat) -> any PlannedTaskAction {
+        return LinkerTaskAction(expandResponseFiles: expandResponseFiles, responseFileFormat: responseFileFormat)
     }
 }