From 692dd44fbb0c4a52ee9437465fb06e9f2402d447 Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Thu, 10 Apr 2025 12:40:53 -0700 Subject: [PATCH 1/6] Adapt QNX support to the introduction of LINKER_DRIVER --- Sources/SWBCore/Settings/BuiltinMacros.swift | 1 + Sources/SWBCore/SpecImplementations/ProductTypes.swift | 4 ++++ Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift | 2 ++ Sources/SWBQNXPlatform/Plugin.swift | 2 ++ Sources/SWBUniversalPlatform/Specs/Ld.xcspec | 2 +- 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index ccde9c38..73e6209e 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -2670,6 +2670,7 @@ public enum LinkerDriverChoice: String, Equatable, Hashable, EnumerationMacroTyp case clang case swiftc + case qcc } /// Enumeration macro type for the value of the `INFOPLIST_KEY_LSApplicationCategoryType` build setting. diff --git a/Sources/SWBCore/SpecImplementations/ProductTypes.swift b/Sources/SWBCore/SpecImplementations/ProductTypes.swift index 7cb6a1d5..cf4c755a 100644 --- a/Sources/SWBCore/SpecImplementations/ProductTypes.swift +++ b/Sources/SWBCore/SpecImplementations/ProductTypes.swift @@ -275,6 +275,8 @@ public class ProductTypeSpec : Spec, SpecType, @unchecked Sendable { args += ["-compatibility_version", compatibilityVersion] case .swiftc: args += ["-Xlinker", "-compatibility_version", "-Xlinker", compatibilityVersion] + case .qcc: + break } } @@ -285,6 +287,8 @@ public class ProductTypeSpec : Spec, SpecType, @unchecked Sendable { args += ["-current_version", currentVersion] case .swiftc: args += ["-Xlinker", "-current_version", "-Xlinker", currentVersion] + case .qcc: + break } } } diff --git a/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift b/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift index 8dd603b1..b7085f04 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift @@ -240,6 +240,8 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec return cbc.producer.hostOperatingSystem.imageFormat.executableName(basename: "clang") case .swiftc: return cbc.producer.hostOperatingSystem.imageFormat.executableName(basename: "swiftc") + case .qcc: + return cbc.producer.hostOperatingSystem.imageFormat.executableName(basename: "qcc") } } diff --git a/Sources/SWBQNXPlatform/Plugin.swift b/Sources/SWBQNXPlatform/Plugin.swift index 89d1177c..2f7cb6d9 100644 --- a/Sources/SWBQNXPlatform/Plugin.swift +++ b/Sources/SWBQNXPlatform/Plugin.swift @@ -105,6 +105,8 @@ struct QNXSDKRegistryExtension: SDKRegistryExtension { "ARCH_NAME_x86_64": .plString("x86_64"), "ARCH_NAME_aarch64": .plString("aarch64le"), + + "LINKER_DRIVER": "qcc", ] return [(qnxSdk.sysroot, qnxPlatform, [ diff --git a/Sources/SWBUniversalPlatform/Specs/Ld.xcspec b/Sources/SWBUniversalPlatform/Specs/Ld.xcspec index d30e08ad..3dcc7f47 100644 --- a/Sources/SWBUniversalPlatform/Specs/Ld.xcspec +++ b/Sources/SWBUniversalPlatform/Specs/Ld.xcspec @@ -175,7 +175,7 @@ { Name = "LD_OPTIMIZATION_LEVEL"; Type = String; - Condition = "$(LINKER_DRIVER) == clang"; + Condition = "$(LINKER_DRIVER) == clang || $(LINKER_DRIVER) == qcc"; DefaultValue = "$(GCC_OPTIMIZATION_LEVEL)"; "CommandLinePrefixFlag" = "-O"; }, From ffb12656d8f11a1506893fdbbae8b84264aec0cb Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Wed, 30 Apr 2025 11:29:02 -0700 Subject: [PATCH 2/6] Remove more blocking_sync calls This just removes a bunch of calls that can trivially become async. --- Sources/SWBBuildSystem/BuildOperation.swift | 6 +++--- .../SWBCore/LibSwiftDriver/LibSwiftDriver.swift | 6 +++--- .../TaskActions/SwiftDriverTaskAction.swift | 2 +- Sources/SWBUtil/SWBDispatch.swift | 14 ++++++++++++++ 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/Sources/SWBBuildSystem/BuildOperation.swift b/Sources/SWBBuildSystem/BuildOperation.swift index 49673a9f..9efc58ee 100644 --- a/Sources/SWBBuildSystem/BuildOperation.swift +++ b/Sources/SWBBuildSystem/BuildOperation.swift @@ -1254,7 +1254,7 @@ private class InProcessCommand: SWBLLBuild.ExternalCommand, SWBLLBuild.ExternalD // Get the current output delegate from the adaptor. // // FIXME: This should never fail (since we are executing), but we have seen a crash here with that assumption. For now we are defensive until the source can be tracked down: Diagnose unexpected missing output delegate from: Crash in InProcessCommand.execute() - guard let outputDelegate = adaptor.getActiveOutputDelegate(command) else { + guard let outputDelegate = await adaptor.getActiveOutputDelegate(command) else { return .failed } @@ -1604,9 +1604,9 @@ internal final class OperationSystemAdaptor: SWBLLBuild.BuildSystemDelegate, Act /// Get the active output delegate for an executing command. /// /// - returns: The active delegate, or nil if not found. - func getActiveOutputDelegate(_ command: Command) -> (any TaskOutputDelegate)? { + func getActiveOutputDelegate(_ command: Command) async -> (any TaskOutputDelegate)? { // FIXME: This is a very bad idea, doing a sync against the response queue is introducing artificial latency when an in-process command needs to wait for the response queue to flush. However, we also can't simply move to a decoupled lock, because we don't want the command to start reporting output before it has been fully reported as having started. We need to move in-process task to another model. - return queue.blocking_sync { + return await queue.sync { self.commandOutputDelegates[command] } } diff --git a/Sources/SWBCore/LibSwiftDriver/LibSwiftDriver.swift b/Sources/SWBCore/LibSwiftDriver/LibSwiftDriver.swift index fa323660..bd6cd847 100644 --- a/Sources/SWBCore/LibSwiftDriver/LibSwiftDriver.swift +++ b/Sources/SWBCore/LibSwiftDriver/LibSwiftDriver.swift @@ -238,9 +238,9 @@ public final class SwiftModuleDependencyGraph: SwiftGlobalExplicitDependencyGrap return fileDependencies } - public func queryTransitiveDependencyModuleNames(for key: String) throws -> [String] { - let graph = try registryQueue.blocking_sync { - guard let driver = registry[key] else { + public func queryTransitiveDependencyModuleNames(for key: String) async throws -> [String] { + let graph = try await registryQueue.sync { + guard let driver = self.registry[key] else { throw StubError.error("Unable to find jobs for key \(key). Be sure to plan the build ahead of fetching results.") } return driver.intermoduleDependencyGraph diff --git a/Sources/SWBTaskExecution/TaskActions/SwiftDriverTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/SwiftDriverTaskAction.swift index 864b1f72..512673c1 100644 --- a/Sources/SWBTaskExecution/TaskActions/SwiftDriverTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/SwiftDriverTaskAction.swift @@ -94,7 +94,7 @@ final public class SwiftDriverTaskAction: TaskAction, BuildValueValidatingTaskAc } if driverPayload.reportRequiredTargetDependencies != .no && driverPayload.explicitModulesEnabled, let target = task.forTarget { - let dependencyModuleNames = try dependencyGraph.queryTransitiveDependencyModuleNames(for: driverPayload.uniqueID) + let dependencyModuleNames = try await dependencyGraph.queryTransitiveDependencyModuleNames(for: driverPayload.uniqueID) for dependencyModuleName in dependencyModuleNames { if let targetDependencies = dynamicExecutionDelegate.operationContext.definingTargetsByModuleName[dependencyModuleName] { for targetDependency in targetDependencies { diff --git a/Sources/SWBUtil/SWBDispatch.swift b/Sources/SWBUtil/SWBDispatch.swift index 4845461f..26aad751 100644 --- a/Sources/SWBUtil/SWBDispatch.swift +++ b/Sources/SWBUtil/SWBDispatch.swift @@ -295,6 +295,20 @@ public final class SWBQueue: Sendable { } } + /// Submits a block object for execution and returns after that block finishes executing. + /// - note: This implementation won't block the calling thread, unlike the synchronous overload of ``sync()``. + public func sync(qos: SWBQoS = .unspecified, flags: DispatchWorkItemFlags = [], execute block: @Sendable @escaping () throws -> T) async throws -> T { + try await withCheckedThrowingContinuation { continuation in + queue.async(qos: qos.dispatchQoS, flags: flags.dispatchFlags) { + do { + continuation.resume(returning: try block()) + } catch { + continuation.resume(throwing: error) + } + } + } + } + public func async(group: SWBDispatchGroup? = nil, qos: SWBQoS = .unspecified, execute body: @escaping @Sendable () -> Void) { return queue.async(group: group?.group, qos: qos.dispatchQoS, execute: body) } From 07130103472e82cb4768cdb701f3b63f89d6cfab Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Thu, 1 May 2025 01:28:47 -0700 Subject: [PATCH 3/6] Remove blocking_sync usages from TaskProducerContext These just need simple synchronization; use a mutex instead. This also plugs a couple occurrences of unsynchronized access. --- Sources/SWBCore/OnDemandResources.swift | 2 +- .../TaskProducers/TaskProducer.swift | 297 ++++++++++-------- 2 files changed, 162 insertions(+), 137 deletions(-) diff --git a/Sources/SWBCore/OnDemandResources.swift b/Sources/SWBCore/OnDemandResources.swift index c009e726..a3eef722 100644 --- a/Sources/SWBCore/OnDemandResources.swift +++ b/Sources/SWBCore/OnDemandResources.swift @@ -18,7 +18,7 @@ public import SWBMacro public typealias ODRTagSet = Set -public struct ODRAssetPackInfo { +public struct ODRAssetPackInfo: Sendable { public var identifier: String public var tags: ODRTagSet public var path: Path diff --git a/Sources/SWBTaskConstruction/TaskProducers/TaskProducer.swift b/Sources/SWBTaskConstruction/TaskProducers/TaskProducer.swift index bc5e5a7b..edfe281f 100644 --- a/Sources/SWBTaskConstruction/TaskProducers/TaskProducer.swift +++ b/Sources/SWBTaskConstruction/TaskProducers/TaskProducer.swift @@ -15,6 +15,7 @@ public import SWBCore import struct SWBProtocol.BuildOperationTaskEnded public import Foundation public import SWBMacro +import Synchronization /// A `TaskProducer` has two distinct phases that are used to create the necessary planning work. enum TaskProducerPhase { @@ -113,78 +114,101 @@ public class TaskProducerContext: StaleFileRemovalContext, BuildFileResolution let onDemandResourcesEnabled: Bool let onDemandResourcesInitialInstallTags: Set let onDemandResourcesPrefetchOrder: [String] - private var onDemandResourcesAssetPacks: [ODRTagSet: ODRAssetPackInfo] = [:] - private var onDemandResourcesAssetPackSubPaths: [String: Set] = [:] /// The registry used for spec data caches. - public var specDataCaches = Registry() + public let specDataCaches = Registry() - /// The list of generated source files produced in this target. - private var _generatedSourceFiles: [Path] = [] + /// Whether a task planned by this producer has requested frontend command line emission. + var emitFrontendCommandLines: Bool - /// The list of generated info plist additions produced in this target. - private var _generatedInfoPlistContents: [Path] = [] + private struct State: Sendable { + fileprivate var onDemandResourcesAssetPacks: [ODRTagSet: ODRAssetPackInfo] = [:] + fileprivate var onDemandResourcesAssetPackSubPaths: [String: Set] = [:] - private var _generatedPrivacyContentFilePaths: [Path] = [] + /// The list of generated source files produced in this target. + fileprivate var _generatedSourceFiles: [Path] = [] - /// The list of generated TBD files produced in this target. - /// - /// This is currently only ever done by Swift. - private var _generatedTBDFiles: [String: [Path]] = [:] + /// The list of generated info plist additions produced in this target. + fileprivate var _generatedInfoPlistContents: [Path] = [] - /// Whether a task planned by this producer has requested frontend command line emission. - var emitFrontendCommandLines: Bool + fileprivate var _generatedPrivacyContentFilePaths: [Path] = [] - /// The map of architecture names to generated Swift Objective-C interface header files produced in this target. - /// - /// This is currently only ever done by Swift. - private var _generatedSwiftObjectiveCHeaderFiles: [String: Path] = [:] + /// The list of generated TBD files produced in this target. + /// + /// This is currently only ever done by Swift. + fileprivate var _generatedTBDFiles: [String: [Path]] = [:] - /// The map of architecture names to generated Swift compile-time value metadata files produced in this target. - /// Only ever done by Swift. - private var _generatedGeneratedSwiftConstMetadataFiles: [String: [Path]] = [:] + /// The map of architecture names to generated Swift Objective-C interface header files produced in this target. + /// + /// This is currently only ever done by Swift. + fileprivate var _generatedSwiftObjectiveCHeaderFiles: [String: Path] = [:] - /// Virtual output nodes for shell script build phases that don't have any declared outputs. - private var _shellScriptVirtualOutputs: [PlannedVirtualNode] = [] + /// The map of architecture names to generated Swift compile-time value metadata files produced in this target. + /// Only ever done by Swift. + fileprivate var _generatedGeneratedSwiftConstMetadataFiles: [String: [Path]] = [:] - /// The outputs of the tasks for this target prior to deferred task production. - private var _outputsOfMainTaskProducers: [any PlannedNode] = [] + /// Virtual output nodes for shell script build phases that don't have any declared outputs. + fileprivate var _shellScriptVirtualOutputs: [PlannedVirtualNode] = [] + + /// The outputs of the tasks for this target prior to deferred task production. + fileprivate var _outputsOfMainTaskProducers: [any PlannedNode] = [] - /// The map of top-level binaries path, keyed by variant. - private var _producedBinaryPaths: [String: Path] = [:] + /// The map of top-level binaries path, keyed by variant. + fileprivate var _producedBinaryPaths: [String: Path] = [:] - /// The map of dSYM paths, keyed by variant. - private var _producedDSYMPaths: [String: Path] = [:] + /// The map of dSYM paths, keyed by variant. + fileprivate var _producedDSYMPaths: [String: Path] = [:] - /// The list of deferred task production blocks. - private var _deferredProducers: [() async -> [any PlannedTask]] = [] + /// The list of deferred task production blocks. + fileprivate var _deferredProducers: [() async -> [any PlannedTask]] = [] - /// Whether we have transitioned to processing deferred tasks. - private var _inDeferredMode = false + /// Whether we have transitioned to processing deferred tasks. + fileprivate var _inDeferredMode = false - /// Map of the files which are copied during the build, used for mapping diagnostics. - private var _copiedPathMap: [String: Set] = [:] + /// Map of the files which are copied during the build, used for mapping diagnostics. + fileprivate var _copiedPathMap: [String: Set] = [:] - /// The set of additional inputs for codesigning. These are tracked explicitly on the codesign task and are captured during the `.planning` phase. - private var _additionalCodeSignInputs: OrderedSet = [] + /// The set of additional inputs for codesigning. These are tracked explicitly on the codesign task and are captured during the `.planning` phase. + fileprivate var _additionalCodeSignInputs: OrderedSet = [] + + /// Notes generated by the internal state of the task producer context. These are harvested by the `BuildPlan` once all task producers have been run. + /// + /// This is an `OrderedSet` because the context is shared among all task producers for a target, and multiple producers could cause the same note to be emitted + fileprivate(set) var notes = OrderedSet() + + /// Warnings generated by the internal state of the task producer context. These are harvested by the `BuildPlan` once all task producers have been run. + /// + /// This is an `OrderedSet` because the context is shared among all task producers for a target, and multiple producers could cause the same warning to be emitted + fileprivate(set) var warnings = OrderedSet() + + /// Errors generated by the internal state of the task producer context. These are harvested by the `BuildPlan` once all task producers have been run. + /// + /// This is an `OrderedSet` because the context is shared among all task producers for a target, and multiple producers could cause the same error to be emitted . + fileprivate(set) var errors = OrderedSet() + } /// Notes generated by the internal state of the task producer context. These are harvested by the `BuildPlan` once all task producers have been run. /// /// This is an `OrderedSet` because the context is shared among all task producers for a target, and multiple producers could cause the same note to be emitted - private(set) var notes = OrderedSet() + var notes: OrderedSet { + state.withLock { $0.notes } + } /// Warnings generated by the internal state of the task producer context. These are harvested by the `BuildPlan` once all task producers have been run. /// /// This is an `OrderedSet` because the context is shared among all task producers for a target, and multiple producers could cause the same warning to be emitted - private(set) var warnings = OrderedSet() + var warnings: OrderedSet { + state.withLock { $0.warnings } + } /// Errors generated by the internal state of the task producer context. These are harvested by the `BuildPlan` once all task producers have been run. /// /// This is an `OrderedSet` because the context is shared among all task producers for a target, and multiple producers could cause the same error to be emitted . - private(set) var errors = OrderedSet() + var errors: OrderedSet { + state.withLock { $0.errors } + } - /// The queue used to serialize concurrent operations. - let queue: SWBQueue + private let state = SWBMutex(.init()) // MARK: Bound Tool Specs. @@ -194,8 +218,8 @@ public class TaskProducerContext: StaleFileRemovalContext, BuildFileResolution case .success(let toolSpec): return toolSpec case .failure(let error): - return queue.blocking_sync { - errors.append("\(error)") + return state.withLock { state in + state.errors.append("\(error)") return nil } } @@ -283,7 +307,6 @@ public class TaskProducerContext: StaleFileRemovalContext, BuildFileResolution let settings = configuredTarget.map(globalProductPlan.getTargetSettings) ?? globalProductPlan.getWorkspaceSettings() self.settings = settings self.delegate = delegate - self.queue = SWBQueue(label: "SWBTaskConstruction.TaskProducerContext.queue", qos: globalProductPlan.planRequest.buildRequest.qos, autoreleaseFrequency: .workItem) // Construct a build ruleset from the built-in system rules (which may be different for different platforms), and for any custom rules from the target. // @@ -423,190 +446,188 @@ public class TaskProducerContext: StaleFileRemovalContext, BuildFileResolution /// Get the list of source files generated for this target. var generatedSourceFiles: [Path] { - return queue.blocking_sync { - assert(_inDeferredMode) - return _generatedSourceFiles + return state.withLock { state in + assert(state._inDeferredMode) + return state._generatedSourceFiles } } /// Add a source file generated by this target. func addGeneratedSourceFile(_ path: Path) { - queue.blocking_sync { - assert(!_inDeferredMode) - _generatedSourceFiles.append(path) + state.withLock { state in + assert(!state._inDeferredMode) + state._generatedSourceFiles.append(path) } } /// Get the list of info plist additions generated for this target. var generatedInfoPlistContents: [Path] { - return queue.blocking_sync { - assert(_inDeferredMode) - return _generatedInfoPlistContents + return state.withLock { state in + assert(state._inDeferredMode) + return state._generatedInfoPlistContents } } var generatedPrivacyContentFilePaths: [Path] { - return queue.blocking_sync { - assert(_inDeferredMode) - return _generatedPrivacyContentFilePaths + return state.withLock { state in + assert(state._inDeferredMode) + return state._generatedPrivacyContentFilePaths } } /// Add an info plist addition generated by this target. func addGeneratedInfoPlistContent(_ path: Path) { - queue.blocking_sync { - assert(!_inDeferredMode) - _generatedInfoPlistContents.append(path) + state.withLock { state in + assert(!state._inDeferredMode) + state._generatedInfoPlistContents.append(path) } } func addPrivacyContentPlistContent(_ path: Path) { - queue.blocking_sync { - assert(!_inDeferredMode) - _generatedPrivacyContentFilePaths.append(path) + state.withLock { state in + assert(!state._inDeferredMode) + state._generatedPrivacyContentFilePaths.append(path) } } /// Get the produced binary path for the given variant, if any. func producedBinary(forVariant variant: String) -> Path? { - return queue.blocking_sync { - assert(_inDeferredMode) - return _producedBinaryPaths[variant] + return state.withLock { state in + assert(state._inDeferredMode) + return state._producedBinaryPaths[variant] } } /// Add a produced binary path for the given variant. func addProducedBinary(path: Path, forVariant variant: String) { - queue.blocking_sync { - assert(!_inDeferredMode) - assert(_producedBinaryPaths[variant] == nil || _producedBinaryPaths[variant] == path) - _producedBinaryPaths[variant] = path + state.withLock { state in + assert(!state._inDeferredMode) + assert(state._producedBinaryPaths[variant] == nil || state._producedBinaryPaths[variant] == path) + state._producedBinaryPaths[variant] = path } } /// Get the produced dSYM path for the given variant, if any. func producedDSYM(forVariant variant: String) -> Path? { - return queue.blocking_sync { - assert(_inDeferredMode) - return _producedDSYMPaths[variant] + return state.withLock { state in + assert(state._inDeferredMode) + return state._producedDSYMPaths[variant] } } /// Add a produced dSYM path for the given variant. func addProducedDSYM(path: Path, forVariant variant: String) { - queue.blocking_sync { - assert(!_inDeferredMode) - assert(_producedDSYMPaths[variant] == nil || _producedDSYMPaths[variant] == path) - _producedDSYMPaths[variant] = path + state.withLock { state in + assert(!state._inDeferredMode) + assert(state._producedDSYMPaths[variant] == nil || state._producedDSYMPaths[variant] == path) + state._producedDSYMPaths[variant] = path } } /// Add a file that was copied. func addCopiedPath(src: String, dst: String) { - // This is async because we only need to read copied path map - // after running all of the deferred task producers. - queue.async { - assert(!self._inDeferredMode) - self._copiedPathMap[dst, default: []].insert(src) + state.withLock { state in + assert(!state._inDeferredMode) + state._copiedPathMap[dst, default: []].insert(src) } } /// Get the map of the files which will be copied. func copiedPathMap() -> [String: Set] { - return queue.blocking_sync { - assert(_inDeferredMode) - return _copiedPathMap + return state.withLock { state in + assert(state._inDeferredMode) + return state._copiedPathMap } } /// Get the product custom TBD paths. func generatedTBDFiles(forVariant variant: String) -> [Path] { - return queue.blocking_sync { - assert(_inDeferredMode) - return _generatedTBDFiles[variant] ?? [] + return state.withLock { state in + assert(state._inDeferredMode) + return state._generatedTBDFiles[variant] ?? [] } } /// Add a produced binary path for the given variant. func addGeneratedTBDFile(_ path: Path, forVariant variant: String) { - queue.blocking_sync { - assert(!_inDeferredMode) - _generatedTBDFiles[variant, default: []].append(path) + state.withLock { state in + assert(!state._inDeferredMode) + state._generatedTBDFiles[variant, default: []].append(path) } } /// Get the product generated Swift Objective-C interface header files. func generatedSwiftObjectiveCHeaderFiles() -> [String: Path] { - return queue.blocking_sync { - //assert(_inDeferredMode) - return _generatedSwiftObjectiveCHeaderFiles + return state.withLock { state in + //assert(state._inDeferredMode) + return state._generatedSwiftObjectiveCHeaderFiles } } /// Add a generated Swift Objective-C interface header file. func addGeneratedSwiftObjectiveCHeaderFile(_ path: Path, architecture: String) { - queue.blocking_sync { - assert(!_inDeferredMode) - _generatedSwiftObjectiveCHeaderFiles[architecture] = path + state.withLock { state in + assert(!state._inDeferredMode) + state._generatedSwiftObjectiveCHeaderFiles[architecture] = path } } /// Get the product generated Swift Objective-C interface header files. public func generatedSwiftConstMetadataFiles() -> [String: [Path]] { - return queue.blocking_sync { - assert(_inDeferredMode) - return _generatedGeneratedSwiftConstMetadataFiles + return state.withLock { state in + assert(state._inDeferredMode) + return state._generatedGeneratedSwiftConstMetadataFiles } } /// Add a generated Swift supplementary const metadata file. func addGeneratedSwiftConstMetadataFile(_ path: Path, architecture: String) { - queue.blocking_sync { - assert(!_inDeferredMode) - if _generatedGeneratedSwiftConstMetadataFiles[architecture] != nil { - _generatedGeneratedSwiftConstMetadataFiles[architecture]?.append(path) + state.withLock { state in + assert(!state._inDeferredMode) + if state._generatedGeneratedSwiftConstMetadataFiles[architecture] != nil { + state._generatedGeneratedSwiftConstMetadataFiles[architecture]?.append(path) } else { - _generatedGeneratedSwiftConstMetadataFiles[architecture] = [path] + state._generatedGeneratedSwiftConstMetadataFiles[architecture] = [path] } } } /// Virtual output nodes for shell script build phases that don't have any declared outputs. func shellScriptVirtualOutputs() -> [PlannedVirtualNode] { - return queue.blocking_sync { - assert(_inDeferredMode) - return _shellScriptVirtualOutputs + return state.withLock { state in + assert(state._inDeferredMode) + return state._shellScriptVirtualOutputs } } func addShellScriptVirtualOutput(_ virtualOutput: PlannedVirtualNode) { - queue.async { - assert(!self._inDeferredMode) - self._shellScriptVirtualOutputs.append(virtualOutput) + state.withLock { state in + assert(!state._inDeferredMode) + state._shellScriptVirtualOutputs.append(virtualOutput) } } /// The outputs of the tasks for this target prior to deferred task production. var outputsOfMainTaskProducers: [any PlannedNode] { get { - queue.blocking_sync { - assert(_inDeferredMode) - return _outputsOfMainTaskProducers + state.withLock { state in + assert(state._inDeferredMode) + return state._outputsOfMainTaskProducers } } set { - queue.async { - assert(!self._inDeferredMode) - self._outputsOfMainTaskProducers = newValue + state.withLock { state in + assert(!state._inDeferredMode) + state._outputsOfMainTaskProducers = newValue } } } /// Add a deferred task production block. public func addDeferredProducer(_ body: @escaping () async -> [any PlannedTask]) { - queue.blocking_sync { - assert(!_inDeferredMode) - _deferredProducers.append(body) + state.withLock { state in + assert(!state._inDeferredMode) + state._deferredProducers.append(body) } } @@ -614,11 +635,11 @@ public class TaskProducerContext: StaleFileRemovalContext, BuildFileResolution /// /// We model this as a "take" operation to ensure that any potential reference cycles through the producers are automatically discarded. func takeDeferredProducers() -> [() async -> [any PlannedTask]] { - return queue.blocking_sync { - assert(!_inDeferredMode) - _inDeferredMode = true - let result = _deferredProducers - _deferredProducers.removeAll() + return state.withLock { state in + assert(!state._inDeferredMode) + state._inDeferredMode = true + let result = state._deferredProducers + state._deferredProducers.removeAll() return result } } @@ -632,11 +653,13 @@ public class TaskProducerContext: StaleFileRemovalContext, BuildFileResolution // A feature guard to provide a fallback mechanism, primarily to alleviate risk. guard scope.evaluate(BuiltinMacros.ENABLE_ADDITIONAL_CODESIGN_INPUT_TRACKING) else { return } - queue.blocking_sync { - assert(!_inDeferredMode) - // This is a bit unfortunate, but to prevent workspace diagnostic issues downstream, silently ignore any missing items, it's necessary to allow callers to ignore files that do not actually exist on disk. - if alwaysAdd || fs.exists(path) { - _additionalCodeSignInputs.append(path) + // This is a bit unfortunate, but to prevent workspace diagnostic issues downstream, silently ignore any missing items, it's necessary to allow callers to ignore files that do not actually exist on disk. + let exists = fs.exists(path) + + state.withLock { state in + assert(!state._inDeferredMode) + if alwaysAdd || exists { + state._additionalCodeSignInputs.append(path) } } } @@ -681,7 +704,9 @@ public class TaskProducerContext: StaleFileRemovalContext, BuildFileResolution var additionalCodeSignInputs: OrderedSet { // The data from this collection should only be used after the planning phase has been completed. Doing so before can lead to incorrect assumptions due to the un-ordered nature of task producers. assert(phase == .taskGeneration) - return _additionalCodeSignInputs + return state.withLock { state in + return state._additionalCodeSignInputs + } } // FIXME: This is something of a hack. Uses in the ProductPostprocessingTaskProducer say this should be expressed instead on a check against a provisional task of the product, but as of this writing the future of provisional tasks is unclear. @@ -921,12 +946,12 @@ public class TaskProducerContext: StaleFileRemovalContext, BuildFileResolution } package var allOnDemandResourcesAssetPacks: [ODRAssetPackInfo] { - return queue.blocking_sync { Array(onDemandResourcesAssetPacks.values) } + return state.withLock { state in Array(state.onDemandResourcesAssetPacks.values) } } public func onDemandResourcesAssetPack(for tags: ODRTagSet) -> ODRAssetPackInfo? { guard onDemandResourcesEnabled else { return nil } - if let r = (queue.blocking_sync { onDemandResourcesAssetPacks[tags] }) { return r } + if let r = (state.withLock { state in state.onDemandResourcesAssetPacks[tags] }) { return r } let maxPriority = tags.lazy.compactMap { self.onDemandResourcesAssetTagPriority(tag: $0) }.max() let productBundleIdentifier = settings.globalScope.evaluate(BuiltinMacros.PRODUCT_BUNDLE_IDENTIFIER) @@ -936,7 +961,7 @@ public class TaskProducerContext: StaleFileRemovalContext, BuildFileResolution } let info = ODRAssetPackInfo(tags: tags, priority: maxPriority, productBundleIdentifier: productBundleIdentifier, settings.globalScope) - queue.blocking_sync { onDemandResourcesAssetPacks[tags] = info } + state.withLock { state in state.onDemandResourcesAssetPacks[tags] = info } return info } @@ -948,11 +973,11 @@ public class TaskProducerContext: StaleFileRemovalContext, BuildFileResolution } package func didProduceAssetPackSubPath(_ assetPack: ODRAssetPackInfo, _ subPath: String) { - _ = queue.blocking_sync { onDemandResourcesAssetPackSubPaths[assetPack.identifier, default: Set()].insert(subPath) } + _ = state.withLock { state in state.onDemandResourcesAssetPackSubPaths[assetPack.identifier, default: Set()].insert(subPath) } } package var allOnDemandResourcesAssetPackSubPaths: [String: Set] { - return queue.blocking_sync { onDemandResourcesAssetPackSubPaths } + return state.withLock { state in state.onDemandResourcesAssetPackSubPaths } } private func onDemandResourcesAssetTagPriority(tag: String) -> Double? { From 44c58d015f86b011f83458dd2fe15c058a76f3d9 Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Thu, 1 May 2025 02:21:04 -0700 Subject: [PATCH 4/6] workspace --- .../SwiftPM+SwiftBuild.xcworkspace/contents.xcworkspacedata | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Utilities/SwiftPM+SwiftBuild.xcworkspace/contents.xcworkspacedata b/Utilities/SwiftPM+SwiftBuild.xcworkspace/contents.xcworkspacedata index 06dfbdf6..c7761141 100644 --- a/Utilities/SwiftPM+SwiftBuild.xcworkspace/contents.xcworkspacedata +++ b/Utilities/SwiftPM+SwiftBuild.xcworkspace/contents.xcworkspacedata @@ -5,6 +5,9 @@ location = "container:../../swift-build"> + location = "container:../../swift-llbuild"> + + From fbdbca8ef364d1236c4ff4b569fb1a23bb4b1335 Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Fri, 2 May 2025 23:13:46 -0700 Subject: [PATCH 5/6] Simplify implementation of Dispatch-async bridge Stress testing this approach over use of a long-lived DispatchIO channel appears to resolve the nondeterministic failures with the file descriptor being destroyed, and is easier to reason about. Closes #21 --- Sources/SWBUtil/Dispatch+Async.swift | 69 ++++------------------------ Sources/SWBUtil/SWBDispatch.swift | 27 ----------- 2 files changed, 10 insertions(+), 86 deletions(-) diff --git a/Sources/SWBUtil/Dispatch+Async.swift b/Sources/SWBUtil/Dispatch+Async.swift index bf70a86d..42415395 100644 --- a/Sources/SWBUtil/Dispatch+Async.swift +++ b/Sources/SWBUtil/Dispatch+Async.swift @@ -68,22 +68,9 @@ extension AsyncThrowingStream where Element == UInt8, Failure == any Error { @available(visionOS, deprecated: 2.0, message: "Use the AsyncSequence-returning overload.") public static func _dataStream(reading fileDescriptor: DispatchFD, on queue: SWBQueue) -> AsyncThrowingStream { AsyncThrowingStream { continuation in - let newFD: DispatchFD - do { - newFD = try fileDescriptor._duplicate() - } catch { - continuation.finish(throwing: error) - return - } - - let io = SWBDispatchIO.stream(fileDescriptor: newFD, queue: queue) { error in - do { - try newFD._close() - if error != 0 { - continuation.finish(throwing: POSIXError(error, context: "dataStream(reading: \(fileDescriptor))#1")) - } - } catch { - continuation.finish(throwing: error) + let io = SWBDispatchIO.stream(fileDescriptor: fileDescriptor, queue: queue) { error in + if error != 0 { + continuation.finish(throwing: POSIXError(error, context: "dataStream(reading: \(fileDescriptor))#1")) } } io.setLimit(lowWater: 0) @@ -120,51 +107,15 @@ extension AsyncThrowingStream where Element == UInt8, Failure == any Error { extension AsyncSequence where Element == UInt8, Failure == any Error { /// Returns an async stream which reads bytes from the specified file descriptor. Unlike `FileHandle.bytes`, it does not block the caller. public static func dataStream(reading fileDescriptor: DispatchFD, on queue: SWBQueue) -> any AsyncSequence { - AsyncThrowingStream { continuation in - let newFD: DispatchFD - do { - newFD = try fileDescriptor._duplicate() - } catch { - continuation.finish(throwing: error) - return - } - - let io = SWBDispatchIO.stream(fileDescriptor: newFD, queue: queue) { error in - do { - try newFD._close() - if error != 0 { - let context = "dataStream(reading: \(fileDescriptor) \"\(Result { try fileDescriptor._filePath() })\")#1" - continuation.finish(throwing: POSIXError(error, context: context)) - } - } catch { - continuation.finish(throwing: error) - } - } - io.setLimit(lowWater: 0) - io.setLimit(highWater: 4096) - - continuation.onTermination = { termination in - if case .cancelled = termination { - io.close(flags: .stop) - } else { - io.close() - } - } - - io.read(offset: 0, length: .max, queue: queue) { done, data, error in - guard error == 0 else { - let context = "dataStream(reading: \(fileDescriptor) \"\(Result { try fileDescriptor._filePath() })\")#2" - continuation.finish(throwing: POSIXError(error, context: context)) - return - } - - let data = data ?? .empty - continuation.yield(data) - - if done { - continuation.finish() + AsyncThrowingStream { + while !Task.isCancelled { + let chunk = try await fileDescriptor.readChunk(upToLength: 4096) + if chunk.isEmpty { + return nil } + return chunk } + throw CancellationError() }.flattened } } diff --git a/Sources/SWBUtil/SWBDispatch.swift b/Sources/SWBUtil/SWBDispatch.swift index 26aad751..0aa561b2 100644 --- a/Sources/SWBUtil/SWBDispatch.swift +++ b/Sources/SWBUtil/SWBDispatch.swift @@ -49,33 +49,6 @@ public struct DispatchFD { rawValue = fileHandle.fileDescriptor #endif } - - internal func _duplicate() throws -> DispatchFD { - #if os(Windows) - return self - #else - return try DispatchFD(fileDescriptor: FileDescriptor(rawValue: rawValue).duplicate()) - #endif - } - - internal func _close() throws { - #if !os(Windows) - try FileDescriptor(rawValue: rawValue).close() - #endif - } - - // Only exists to help debug a rare concurrency issue where the file descriptor goes invalid - internal func _filePath() throws -> String { - #if canImport(Darwin) - var buffer = [CChar](repeating: 0, count: Int(MAXPATHLEN)) - if fcntl(rawValue, F_GETPATH, &buffer) == -1 { - throw POSIXError(errno, "fcntl", String(rawValue), "F_GETPATH") - } - return String(cString: buffer) - #else - return String() - #endif - } } // @unchecked: rdar://130051790 (DispatchData should be Sendable) From fed524ea27ec06441b553856911111c17cf331c5 Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Thu, 1 May 2025 18:36:41 -0700 Subject: [PATCH 6/6] [DO NOT MERGE] testing test crashes --- .github/workflows/dispatch.sh | 25 +++++++++++++++++++++++++ .github/workflows/pull_request.yml | 16 ++++++++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) create mode 100755 .github/workflows/dispatch.sh diff --git a/.github/workflows/dispatch.sh b/.github/workflows/dispatch.sh new file mode 100755 index 00000000..75a4e54f --- /dev/null +++ b/.github/workflows/dispatch.sh @@ -0,0 +1,25 @@ +#!/bin/bash +apt-get update -y +apt-get install -y cmake ninja-build +cd /tmp && mkdir build +git clone https://github.com/apple/swift-corelibs-libdispatch +cd /tmp/swift-corelibs-libdispatch +#git checkout swift-6.0-RELEASE +cd /tmp +cmake -G Ninja \ + /tmp/swift-corelibs-libdispatch \ + -B /tmp/build \ + -DCMAKE_C_FLAGS=-fno-omit-frame-pointer \ + -DCMAKE_CXX_FLAGS=-fno-omit-frame-pointer \ + -DCMAKE_REQUIRED_DEFINITIONS=-D_GNU_SOURCE \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DENABLE_SWIFT=YES \ + -DCMAKE_C_COMPILER=/usr/bin/clang \ + -DCMAKE_CXX_COMPILER=/usr/bin/clang++ \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_INSTALL_LIBDIR=lib +cd /tmp/build && ninja install +rm -rf /tmp/build /tmp/swift-corelibs-libdispatch +apt-get remove -y cmake ninja-build +apt-get autoremove -y +rm -r /var/lib/apt/lists/* diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 47ce72a9..d27453cf 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -13,7 +13,7 @@ jobs: name: Test uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main with: - linux_os_versions: '["noble", "jammy", "focal", "rhel-ubi9"]' + linux_os_versions: '["noble"]' linux_pre_build_command: | if command -v apt-get >/dev/null 2>&1 ; then # bookworm, noble, jammy, focal apt-get update -y @@ -23,6 +23,10 @@ jobs: # Debug symbols apt-get install -y libc6-dbg + + apt-get install -y valgrind + + ./.github/workflows/dispatch.sh elif command -v dnf >/dev/null 2>&1 ; then # rhel-ubi9 dnf update -y @@ -31,6 +35,8 @@ jobs: # Debug symbols dnf debuginfo-install -y glibc + + dnf install -y valgrind elif command -v yum >/dev/null 2>&1 ; then # amazonlinux2 yum update -y @@ -40,10 +46,12 @@ jobs: # Debug symbols yum install -y yum-utils debuginfo-install -y glibc + + yum install -y valgrind fi - linux_build_command: 'swift test --no-parallel' - linux_swift_versions: '["nightly-main", "nightly-6.2"]' - windows_swift_versions: '["nightly-main"]' + linux_build_command: 'swift build --build-tests -Xswiftc -sanitize=thread -Xcc -fsanitize=thread && ASAN_OPTIONS=detect_leaks=0 ./.build/debug/SwiftBuildPackageTests.xctest --testing-library swift-testing' + linux_swift_versions: '["nightly-main"]' + windows_swift_versions: '[]' windows_build_command: 'swift test --no-parallel' cmake-smoke-test: name: cmake-smoke-test