From 58bcc1e97dd111ba28415037303209cbae383f9c Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Wed, 14 Oct 2026 01:57:03 +0000 Subject: [PATCH 1/2] sdk: explain GitHub toolchain download failures Fetch the Darwin toolset and OpenAppleMacros with retries and an error that names the GitHub URL so a failed toolchain download is not mistaken for a broken Xcode.xip. --- Sources/XToolSupport/SDKBuilder.swift | 39 +++-- Sources/XToolSupport/SDKGitHubDownload.swift | 136 ++++++++++++++++ Tests/XToolTests/SDKGitHubDownloadTests.swift | 151 ++++++++++++++++++ 3 files changed, 306 insertions(+), 20 deletions(-) create mode 100644 Sources/XToolSupport/SDKGitHubDownload.swift create mode 100644 Tests/XToolTests/SDKGitHubDownloadTests.swift diff --git a/Sources/XToolSupport/SDKBuilder.swift b/Sources/XToolSupport/SDKBuilder.swift index 09af2dd7..90293489 100644 --- a/Sources/XToolSupport/SDKBuilder.swift +++ b/Sources/XToolSupport/SDKBuilder.swift @@ -1,5 +1,4 @@ import Foundation -import Dependencies import libunxip import Subprocess import XKit // HTTPClient, stdoutSafe @@ -105,8 +104,9 @@ struct SDKBuilder { withIntermediateDirectories: true ) - // TODO: parallelize these three steps - // we need to synchronize progress reporting though + // Download GitHub artifacts before unxip so a toolchain fetch failure + // is reported immediately and is not mistaken for a corrupt Xcode.xip. + // TODO: parallelize these three steps (need synchronized progress). try await installMacros(in: output) @@ -242,13 +242,13 @@ struct SDKBuilder { withIntermediateDirectories: false ) - @Dependency(\.httpClient) var httpClient - let url = URL(string: """ - https://github.com/xtool-org/darwin-tools-linux-llvm/releases/download/\ - v\(Self.darwinToolsVersion)/toolset-\(arch.rawValue).tar.gz - """)! - let (response, body) = try await httpClient.send(HTTPRequest(url: url)) - guard response.status == 200, let body else { throw Console.Error("Could not fetch toolset") } + let artifact = SDKGitHubArtifact.toolset(arch: arch) + let body: HTTPBody + do { + body = try await SDKGitHubDownloader().body(for: artifact) + } catch let error as SDKGitHubDownloadError { + throw Console.Error(error.description) + } let length: Int64? = switch body.length { case .known(let known): known case .unknown: nil @@ -273,7 +273,7 @@ struct SDKBuilder { guard let length else { continue } let progress = Int(Double(totalWritten) / Double(length) * 100) - print("\r[Downloading toolset] \(progress)%", terminator: "") + print("\r[\(artifact.progressLabel)] \(progress)%", terminator: "") fflush(stdoutSafe) } } @@ -386,14 +386,13 @@ struct SDKBuilder { } private func installMacros(in output: URL) async throws { - @Dependency(\.httpClient) var httpClient - let url = URL(string: """ - https://github.com/xtool-org/OpenAppleMacros/releases/download/\ - v\(Self.oamVersion)/OpenAppleMacrosServer-\(arch.rawValue) - """)! - - let (response, body) = try await httpClient.send(HTTPRequest(url: url)) - guard response.status == 200, let body else { throw Console.Error("Could not fetch toolset") } + let artifact = SDKGitHubArtifact.openAppleMacros(arch: arch) + let body: HTTPBody + do { + body = try await SDKGitHubDownloader().body(for: artifact) + } catch let error as SDKGitHubDownloadError { + throw Console.Error(error.description) + } let length: Int64? = switch body.length { case .known(let known): known @@ -417,7 +416,7 @@ struct SDKBuilder { if let length { let progress = Int(Double(written) / Double(length) * 100) - print("\r[Downloading OpenAppleMacros] \(progress)%", terminator: "") + print("\r[\(artifact.progressLabel)] \(progress)%", terminator: "") fflush(stdoutSafe) } } diff --git a/Sources/XToolSupport/SDKGitHubDownload.swift b/Sources/XToolSupport/SDKGitHubDownload.swift new file mode 100644 index 00000000..35aad59d --- /dev/null +++ b/Sources/XToolSupport/SDKGitHubDownload.swift @@ -0,0 +1,136 @@ +import Foundation +import Dependencies +import HTTPTypesFoundation +import XKit + +struct SDKGitHubArtifact: Equatable, Sendable { + var name: String + var progressLabel: String + var url: URL + + static func toolset( + arch: SDKBuilder.Arch, + version: String = SDKBuilder.darwinToolsVersion + ) -> Self { + Self( + name: "Darwin toolset", + progressLabel: "Downloading toolset", + url: URL(string: """ + https://github.com/xtool-org/darwin-tools-linux-llvm/releases/download/\ + v\(version)/toolset-\(arch.rawValue).tar.gz + """)! + ) + } + + static func openAppleMacros( + arch: SDKBuilder.Arch, + version: String = SDKBuilder.oamVersion + ) -> Self { + Self( + name: "OpenAppleMacros", + progressLabel: "Downloading OpenAppleMacros", + url: URL(string: """ + https://github.com/xtool-org/OpenAppleMacros/releases/download/\ + v\(version)/OpenAppleMacrosServer-\(arch.rawValue) + """)! + ) + } +} + +enum SDKGitHubDownloadError: Error, Equatable, CustomStringConvertible, LocalizedError { + case httpStatus(SDKGitHubArtifact, Int) + case emptyBody(SDKGitHubArtifact) + case network(SDKGitHubArtifact, String) + + var artifact: SDKGitHubArtifact { + switch self { + case .httpStatus(let artifact, _), .emptyBody(let artifact), .network(let artifact, _): + artifact + } + } + + var isRetryable: Bool { + switch self { + case .httpStatus(_, let status): + status == 429 || (500...599).contains(status) + case .emptyBody: + false + case .network: + true + } + } + + var description: String { + let detail: String + switch self { + case .httpStatus(_, let status): + detail = "GitHub responded with HTTP \(status)." + case .emptyBody: + detail = "GitHub returned an empty response body." + case .network(_, let message): + detail = "Network error: \(message)" + } + return """ + Failed to download the \(artifact.name) from GitHub. + + This error is not caused by your Xcode.xip. xtool fetches the Darwin \ + toolchain separately from: + \(artifact.url) + + \(detail) + + Check your network connection and GitHub availability, then retry \ + `xtool sdk install`. You do not need to re-download Xcode. + """ + } + + var errorDescription: String? { description } +} + +struct SDKGitHubDownloader: Sendable { + var maxAttempts: Int + var retryDelay: Duration + var sleep: @Sendable (Duration) async throws -> Void + + init( + maxAttempts: Int = 3, + retryDelay: Duration = .milliseconds(250), + sleep: @escaping @Sendable (Duration) async throws -> Void = { try await Task.sleep(for: $0) } + ) { + self.maxAttempts = maxAttempts + self.retryDelay = retryDelay + self.sleep = sleep + } + + func body(for artifact: SDKGitHubArtifact) async throws -> HTTPBody { + var lastError: SDKGitHubDownloadError? + for attempt in 1...maxAttempts { + do { + return try await fetch(artifact) + } catch let error as SDKGitHubDownloadError { + lastError = error + guard error.isRetryable, attempt < maxAttempts else { throw error } + try await sleep(retryDelay) + } + } + throw lastError! + } + + private func fetch(_ artifact: SDKGitHubArtifact) async throws -> HTTPBody { + @Dependency(\.httpClient) var httpClient + let response: HTTPResponse + let body: HTTPBody? + do { + (response, body) = try await httpClient.send(HTTPRequest(url: artifact.url)) + } catch { + throw SDKGitHubDownloadError.network(artifact, String(describing: error)) + } + guard response.status == .ok else { + throw SDKGitHubDownloadError.httpStatus(artifact, response.status.code) + } + guard let body else { + throw SDKGitHubDownloadError.emptyBody(artifact) + } + return body + } +} diff --git a/Tests/XToolTests/SDKGitHubDownloadTests.swift b/Tests/XToolTests/SDKGitHubDownloadTests.swift new file mode 100644 index 00000000..2a892dbf --- /dev/null +++ b/Tests/XToolTests/SDKGitHubDownloadTests.swift @@ -0,0 +1,151 @@ +import Foundation +import Testing +import Dependencies +import HTTPTypes +import OpenAPIRuntime +import XKit +@testable import XToolSupport + +private struct ScriptedHTTPClient: HTTPClientProtocol, ClientTransport { + var asOpenAPITransport: ClientTransport { self } + var handler: @Sendable (HTTPRequest) async throws -> (HTTPResponse, HTTPBody?) + + func send( + _ request: HTTPRequest, + body: HTTPBody?, + baseURL: URL, + operationID: String + ) async throws -> (HTTPResponse, HTTPBody?) { + try await handler(request) + } + + func withEphemeralClient( + perform: (any HTTPClientProtocol) async throws -> T + ) async throws -> T { + try await perform(self) + } + + func makeWebSocket(url: URL) async throws -> any WebSocketSession { + struct Unused: Error {} + throw Unused() + } +} + +private final class AttemptBox: @unchecked Sendable { + var count = 0 +} + +@Test func sdkGitHubArtifactURLsMatchPublishedReleases() { + let toolset = SDKGitHubArtifact.toolset(arch: .x86_64, version: "1.1.0") + #expect(toolset.name == "Darwin toolset") + #expect(toolset.url.absoluteString == """ + https://github.com/xtool-org/darwin-tools-linux-llvm/releases/download/v1.1.0/toolset-x86_64.tar.gz + """) + + let macros = SDKGitHubArtifact.openAppleMacros(arch: .aarch64, version: "1.3.0") + #expect(macros.name == "OpenAppleMacros") + #expect(macros.url.absoluteString == """ + https://github.com/xtool-org/OpenAppleMacros/releases/download/v1.3.0/OpenAppleMacrosServer-aarch64 + """) +} + +@Test func sdkGitHubDownloadExplainsGitHubFailureIsNotTheXIP() { + let artifact = SDKGitHubArtifact.toolset(arch: .x86_64) + let error = SDKGitHubDownloadError.httpStatus(artifact, 502) + #expect(error.description.contains("not caused by your Xcode.xip")) + #expect(error.description.contains("Darwin toolset")) + #expect(error.description.contains(artifact.url.absoluteString)) + #expect(error.description.contains("HTTP 502")) + #expect(error.description.contains("You do not need to re-download Xcode")) + #expect(error.isRetryable) + + let missing = SDKGitHubDownloadError.httpStatus(artifact, 404) + #expect(!missing.isRetryable) + #expect(!SDKGitHubDownloadError.emptyBody(artifact).isRetryable) + #expect(SDKGitHubDownloadError.network(artifact, "connection reset").isRetryable) +} + +@Test func sdkGitHubDownloadMapsHTTPError() async throws { + let artifact = SDKGitHubArtifact.openAppleMacros(arch: .x86_64) + let client = ScriptedHTTPClient { _ in + (HTTPResponse(status: .notFound), nil) + } + let downloader = SDKGitHubDownloader(maxAttempts: 3, retryDelay: .zero, sleep: { _ in }) + + await withDependencies { + $0.httpClient = client + } operation: { + await #expect(throws: SDKGitHubDownloadError.httpStatus(artifact, 404)) { + _ = try await downloader.body(for: artifact) + } + } +} + +@Test func sdkGitHubDownloadDoesNotRetryClientErrors() async throws { + let attempts = AttemptBox() + let artifact = SDKGitHubArtifact.toolset(arch: .aarch64) + let client = ScriptedHTTPClient { _ in + attempts.count += 1 + return (HTTPResponse(status: .notFound), nil) + } + let downloader = SDKGitHubDownloader(maxAttempts: 3, retryDelay: .zero, sleep: { _ in }) + + await withDependencies { + $0.httpClient = client + } operation: { + await #expect(throws: SDKGitHubDownloadError.self) { + _ = try await downloader.body(for: artifact) + } + } + #expect(attempts.count == 1) +} + +@Test func sdkGitHubDownloadRetriesServerErrorsThenSucceeds() async throws { + let attempts = AttemptBox() + let sleeps = AttemptBox() + let artifact = SDKGitHubArtifact.toolset(arch: .x86_64) + let payload = Data("toolset".utf8) + let client = ScriptedHTTPClient { _ in + attempts.count += 1 + if attempts.count < 3 { + return (HTTPResponse(status: .init(code: 503, reasonPhrase: "Unavailable")), nil) + } + return (HTTPResponse(status: .ok), HTTPBody(payload)) + } + let downloader = SDKGitHubDownloader( + maxAttempts: 3, + retryDelay: .milliseconds(1), + sleep: { _ in sleeps.count += 1 } + ) + + let body = try await withDependencies { + $0.httpClient = client + } operation: { + try await downloader.body(for: artifact) + } + #expect(attempts.count == 3) + #expect(sleeps.count == 2) + + var collected = Data() + for try await chunk in body { + collected.append(contentsOf: chunk) + } + #expect(collected == payload) +} + +@Test func sdkGitHubDownloadWrapsTransportFailures() async throws { + struct Boom: Error, CustomStringConvertible { + var description: String { "connection timed out" } + } + let artifact = SDKGitHubArtifact.openAppleMacros(arch: .x86_64) + let client = ScriptedHTTPClient { _ in throw Boom() } + let downloader = SDKGitHubDownloader(maxAttempts: 1, retryDelay: .zero, sleep: { _ in }) + + await withDependencies { + $0.httpClient = client + } operation: { + await #expect(throws: SDKGitHubDownloadError.network(artifact, "connection timed out")) { + _ = try await downloader.body(for: artifact) + } + } +} From c65f89ee5f500f2cb7e891f697bbbd23c320e4b2 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:26:11 -0700 Subject: [PATCH 2/2] sdk: read the GitHub artifact body inside the retry boundary A connection drop while SDKBuilder streamed the toolset or OpenAppleMacros body happened after the downloader had returned, so it was neither retried nor reported with the artifact and URL. The downloader now reads each attempt's full body into memory and reports progress while it does. A failed attempt's partial bytes are dropped before the next one, and a body that keeps failing surfaces as the artifact's network error. SDKBuilder pipes the returned data into tar or writes it to the macros binary. Cancellation is rethrown instead of being retried as a network error, and maxAttempts below 1 is rejected with a precondition. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KFTE4BHbGYzgf8Uhzcppdh --- Sources/XToolSupport/SDKBuilder.swift | 58 +++------------ Sources/XToolSupport/SDKGitHubDownload.swift | 37 +++++++++- Tests/XToolTests/SDKGitHubDownloadTests.swift | 74 +++++++++++++++++-- 3 files changed, 113 insertions(+), 56 deletions(-) diff --git a/Sources/XToolSupport/SDKBuilder.swift b/Sources/XToolSupport/SDKBuilder.swift index 90293489..c565f4c7 100644 --- a/Sources/XToolSupport/SDKBuilder.swift +++ b/Sources/XToolSupport/SDKBuilder.swift @@ -243,42 +243,17 @@ struct SDKBuilder { ) let artifact = SDKGitHubArtifact.toolset(arch: arch) - let body: HTTPBody - do { - body = try await SDKGitHubDownloader().body(for: artifact) - } catch let error as SDKGitHubDownloadError { - throw Console.Error(error.description) - } - let length: Int64? = switch body.length { - case .known(let known): known - case .unknown: nil - } + let data = try await download(artifact) defer { print() } try await Subprocess.run( .name("tar"), arguments: ["xzf", "-"], workingDirectory: FilePath(toolsetDir), - input: .inputWriter, + input: .data(data), output: .discarded, error: .discarded, - ) { execution in - var totalWritten: Int64 = 0 - for try await chunk in body { - var remaining = chunk - while !remaining.isEmpty { - let written = try await execution.standardInputWriter.write(Array(remaining)) - remaining = remaining.dropFirst(written) - totalWritten += Int64(written) - - guard let length else { continue } - let progress = Int(Double(totalWritten) / Double(length) * 100) - print("\r[\(artifact.progressLabel)] \(progress)%", terminator: "") - fflush(stdoutSafe) - } - } - try await execution.standardInputWriter.finish() - } + ) .checkSuccess() // swift-build assumes we have an Apple-flavored librarian for Apple platforms @@ -385,19 +360,17 @@ struct SDKBuilder { try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: ld64.path) } - private func installMacros(in output: URL) async throws { - let artifact = SDKGitHubArtifact.openAppleMacros(arch: arch) - let body: HTTPBody + private func download(_ artifact: SDKGitHubArtifact) async throws -> Data { do { - body = try await SDKGitHubDownloader().body(for: artifact) + return try await SDKGitHubDownloader().body(for: artifact) } catch let error as SDKGitHubDownloadError { throw Console.Error(error.description) } + } - let length: Int64? = switch body.length { - case .known(let known): known - case .unknown: nil - } + private func installMacros(in output: URL) async throws { + let artifact = SDKGitHubArtifact.openAppleMacros(arch: arch) + let data = try await download(artifact) let outputPath = output.appendingPathComponent("OpenAppleMacrosServer").path do { @@ -408,18 +381,7 @@ struct SDKBuilder { permissions: [.ownerReadWriteExecute, .groupReadExecute, .otherReadExecute] ) defer { try? handle.close() } - - var written: Int64 = 0 - for try await chunk in body { - try handle.writeAll(chunk) - written += Int64(chunk.count) - - if let length { - let progress = Int(Double(written) / Double(length) * 100) - print("\r[\(artifact.progressLabel)] \(progress)%", terminator: "") - fflush(stdoutSafe) - } - } + try handle.writeAll(data) } print() } diff --git a/Sources/XToolSupport/SDKGitHubDownload.swift b/Sources/XToolSupport/SDKGitHubDownload.swift index 35aad59d..82fb9d09 100644 --- a/Sources/XToolSupport/SDKGitHubDownload.swift +++ b/Sources/XToolSupport/SDKGitHubDownload.swift @@ -102,11 +102,14 @@ struct SDKGitHubDownloader: Sendable { self.sleep = sleep } - func body(for artifact: SDKGitHubArtifact) async throws -> HTTPBody { + func body(for artifact: SDKGitHubArtifact) async throws -> Data { + precondition(maxAttempts >= 1, "maxAttempts must be at least 1") var lastError: SDKGitHubDownloadError? for attempt in 1...maxAttempts { do { return try await fetch(artifact) + } catch let error as CancellationError { + throw error } catch let error as SDKGitHubDownloadError { lastError = error guard error.isRetryable, attempt < maxAttempts else { throw error } @@ -116,12 +119,14 @@ struct SDKGitHubDownloader: Sendable { throw lastError! } - private func fetch(_ artifact: SDKGitHubArtifact) async throws -> HTTPBody { + private func fetch(_ artifact: SDKGitHubArtifact) async throws -> Data { @Dependency(\.httpClient) var httpClient let response: HTTPResponse let body: HTTPBody? do { (response, body) = try await httpClient.send(HTTPRequest(url: artifact.url)) + } catch let error as CancellationError { + throw error } catch { throw SDKGitHubDownloadError.network(artifact, String(describing: error)) } @@ -131,6 +136,32 @@ struct SDKGitHubDownloader: Sendable { guard let body else { throw SDKGitHubDownloadError.emptyBody(artifact) } - return body + return try await consume(body, artifact: artifact) + } + + private func consume(_ body: HTTPBody, artifact: SDKGitHubArtifact) async throws -> Data { + let total: Int64? = switch body.length { + case .known(let known): known + case .unknown: nil + } + var data = Data() + do { + for try await chunk in body { + data.append(contentsOf: chunk) + reportProgress(artifact, written: Int64(data.count), total: total) + } + } catch let error as CancellationError { + throw error + } catch { + throw SDKGitHubDownloadError.network(artifact, String(describing: error)) + } + return data + } + + private func reportProgress(_ artifact: SDKGitHubArtifact, written: Int64, total: Int64?) { + guard let total else { return } + let progress = Int(Double(written) / Double(total) * 100) + print("\r[\(artifact.progressLabel)] \(progress)%", terminator: "") + fflush(stdoutSafe) } } diff --git a/Tests/XToolTests/SDKGitHubDownloadTests.swift b/Tests/XToolTests/SDKGitHubDownloadTests.swift index 2a892dbf..a4aaa999 100644 --- a/Tests/XToolTests/SDKGitHubDownloadTests.swift +++ b/Tests/XToolTests/SDKGitHubDownloadTests.swift @@ -35,6 +35,18 @@ private final class AttemptBox: @unchecked Sendable { var count = 0 } +private struct MidBodyFailure: Error, CustomStringConvertible { + var description: String { "connection reset by peer" } +} + +private func bodyThatFailsMidway(_ prefix: Data) -> HTTPBody { + let stream = AsyncThrowingStream { continuation in + continuation.yield(Array(prefix)[...]) + continuation.finish(throwing: MidBodyFailure()) + } + return HTTPBody(stream, length: .unknown) +} + @Test func sdkGitHubArtifactURLsMatchPublishedReleases() { let toolset = SDKGitHubArtifact.toolset(arch: .x86_64, version: "1.1.0") #expect(toolset.name == "Darwin toolset") @@ -118,19 +130,71 @@ private final class AttemptBox: @unchecked Sendable { sleep: { _ in sleeps.count += 1 } ) - let body = try await withDependencies { + let data = try await withDependencies { $0.httpClient = client } operation: { try await downloader.body(for: artifact) } #expect(attempts.count == 3) #expect(sleeps.count == 2) + #expect(data == payload) +} + +@Test func sdkGitHubDownloadRetriesMidBodyFailureAndDropsPartialBytes() async throws { + let attempts = AttemptBox() + let sleeps = AttemptBox() + let artifact = SDKGitHubArtifact.toolset(arch: .x86_64) + let partial = Data("partial-from-attempt-1".utf8) + let complete = Data("only-attempt-2".utf8) + let client = ScriptedHTTPClient { _ in + attempts.count += 1 + if attempts.count == 1 { + return (HTTPResponse(status: .ok), bodyThatFailsMidway(partial)) + } + return (HTTPResponse(status: .ok), HTTPBody(complete)) + } + let downloader = SDKGitHubDownloader( + maxAttempts: 3, + retryDelay: .milliseconds(1), + sleep: { _ in sleeps.count += 1 } + ) + + let data = try await withDependencies { + $0.httpClient = client + } operation: { + try await downloader.body(for: artifact) + } + #expect(attempts.count == 2) + #expect(sleeps.count == 1) + #expect(data == complete) +} - var collected = Data() - for try await chunk in body { - collected.append(contentsOf: chunk) +@Test func sdkGitHubDownloadFailsWithArtifactErrorWhenBodyAlwaysFails() async throws { + let attempts = AttemptBox() + let sleeps = AttemptBox() + let artifact = SDKGitHubArtifact.openAppleMacros(arch: .aarch64) + let client = ScriptedHTTPClient { _ in + attempts.count += 1 + return (HTTPResponse(status: .ok), bodyThatFailsMidway(Data("partial".utf8))) } - #expect(collected == payload) + let downloader = SDKGitHubDownloader( + maxAttempts: 3, + retryDelay: .zero, + sleep: { _ in sleeps.count += 1 } + ) + + await withDependencies { + $0.httpClient = client + } operation: { + await #expect(throws: SDKGitHubDownloadError.network(artifact, "connection reset by peer")) { + _ = try await downloader.body(for: artifact) + } + } + #expect(attempts.count == 3) + #expect(sleeps.count == 2) + let surfaced = SDKGitHubDownloadError.network(artifact, "connection reset by peer") + #expect(surfaced.description.contains(artifact.name)) + #expect(surfaced.description.contains(artifact.url.absoluteString)) } @Test func sdkGitHubDownloadWrapsTransportFailures() async throws {