diff --git a/Sources/XToolSupport/SDKBuilder.swift b/Sources/XToolSupport/SDKBuilder.swift index 09af2dd7..c565f4c7 100644 --- a/Sources/XToolSupport/SDKBuilder.swift +++ b/Sources/XToolSupport/SDKBuilder.swift @@ -1,5 +1,4 @@ import Foundation -import Dependencies import libunxip import Subprocess import XKit // HTTPClient, stdoutSafe @@ -105,8 +104,9 @@ struct SDKBuilder { withIntermediateDirectories: true ) - // TODO: parallelize these three steps - // we need to synchronize progress reporting though + // Download GitHub artifacts before unxip so a toolchain fetch failure + // is reported immediately and is not mistaken for a corrupt Xcode.xip. + // TODO: parallelize these three steps (need synchronized progress). try await installMacros(in: output) @@ -242,43 +242,18 @@ struct SDKBuilder { withIntermediateDirectories: false ) - @Dependency(\.httpClient) var httpClient - let url = URL(string: """ - https://github.com/xtool-org/darwin-tools-linux-llvm/releases/download/\ - v\(Self.darwinToolsVersion)/toolset-\(arch.rawValue).tar.gz - """)! - let (response, body) = try await httpClient.send(HTTPRequest(url: url)) - guard response.status == 200, let body else { throw Console.Error("Could not fetch toolset") } - let length: Int64? = switch body.length { - case .known(let known): known - case .unknown: nil - } + let artifact = SDKGitHubArtifact.toolset(arch: arch) + let data = try await download(artifact) defer { print() } try await Subprocess.run( .name("tar"), arguments: ["xzf", "-"], workingDirectory: FilePath(toolsetDir), - input: .inputWriter, + input: .data(data), output: .discarded, error: .discarded, - ) { execution in - var totalWritten: Int64 = 0 - for try await chunk in body { - var remaining = chunk - while !remaining.isEmpty { - let written = try await execution.standardInputWriter.write(Array(remaining)) - remaining = remaining.dropFirst(written) - totalWritten += Int64(written) - - guard let length else { continue } - let progress = Int(Double(totalWritten) / Double(length) * 100) - print("\r[Downloading toolset] \(progress)%", terminator: "") - fflush(stdoutSafe) - } - } - try await execution.standardInputWriter.finish() - } + ) .checkSuccess() // swift-build assumes we have an Apple-flavored librarian for Apple platforms @@ -385,20 +360,17 @@ struct SDKBuilder { try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: ld64.path) } - private func installMacros(in output: URL) async throws { - @Dependency(\.httpClient) var httpClient - let url = URL(string: """ - https://github.com/xtool-org/OpenAppleMacros/releases/download/\ - v\(Self.oamVersion)/OpenAppleMacrosServer-\(arch.rawValue) - """)! - - let (response, body) = try await httpClient.send(HTTPRequest(url: url)) - guard response.status == 200, let body else { throw Console.Error("Could not fetch toolset") } - - let length: Int64? = switch body.length { - case .known(let known): known - case .unknown: nil + private func download(_ artifact: SDKGitHubArtifact) async throws -> Data { + do { + return try await SDKGitHubDownloader().body(for: artifact) + } catch let error as SDKGitHubDownloadError { + throw Console.Error(error.description) } + } + + private func installMacros(in output: URL) async throws { + let artifact = SDKGitHubArtifact.openAppleMacros(arch: arch) + let data = try await download(artifact) let outputPath = output.appendingPathComponent("OpenAppleMacrosServer").path do { @@ -409,18 +381,7 @@ struct SDKBuilder { permissions: [.ownerReadWriteExecute, .groupReadExecute, .otherReadExecute] ) defer { try? handle.close() } - - var written: Int64 = 0 - for try await chunk in body { - try handle.writeAll(chunk) - written += Int64(chunk.count) - - if let length { - let progress = Int(Double(written) / Double(length) * 100) - print("\r[Downloading OpenAppleMacros] \(progress)%", terminator: "") - fflush(stdoutSafe) - } - } + try handle.writeAll(data) } print() } diff --git a/Sources/XToolSupport/SDKGitHubDownload.swift b/Sources/XToolSupport/SDKGitHubDownload.swift new file mode 100644 index 00000000..82fb9d09 --- /dev/null +++ b/Sources/XToolSupport/SDKGitHubDownload.swift @@ -0,0 +1,167 @@ +import Foundation +import Dependencies +import HTTPTypesFoundation +import XKit + +struct SDKGitHubArtifact: Equatable, Sendable { + var name: String + var progressLabel: String + var url: URL + + static func toolset( + arch: SDKBuilder.Arch, + version: String = SDKBuilder.darwinToolsVersion + ) -> Self { + Self( + name: "Darwin toolset", + progressLabel: "Downloading toolset", + url: URL(string: """ + https://github.com/xtool-org/darwin-tools-linux-llvm/releases/download/\ + v\(version)/toolset-\(arch.rawValue).tar.gz + """)! + ) + } + + static func openAppleMacros( + arch: SDKBuilder.Arch, + version: String = SDKBuilder.oamVersion + ) -> Self { + Self( + name: "OpenAppleMacros", + progressLabel: "Downloading OpenAppleMacros", + url: URL(string: """ + https://github.com/xtool-org/OpenAppleMacros/releases/download/\ + v\(version)/OpenAppleMacrosServer-\(arch.rawValue) + """)! + ) + } +} + +enum SDKGitHubDownloadError: Error, Equatable, CustomStringConvertible, LocalizedError { + case httpStatus(SDKGitHubArtifact, Int) + case emptyBody(SDKGitHubArtifact) + case network(SDKGitHubArtifact, String) + + var artifact: SDKGitHubArtifact { + switch self { + case .httpStatus(let artifact, _), .emptyBody(let artifact), .network(let artifact, _): + artifact + } + } + + var isRetryable: Bool { + switch self { + case .httpStatus(_, let status): + status == 429 || (500...599).contains(status) + case .emptyBody: + false + case .network: + true + } + } + + var description: String { + let detail: String + switch self { + case .httpStatus(_, let status): + detail = "GitHub responded with HTTP \(status)." + case .emptyBody: + detail = "GitHub returned an empty response body." + case .network(_, let message): + detail = "Network error: \(message)" + } + return """ + Failed to download the \(artifact.name) from GitHub. + + This error is not caused by your Xcode.xip. xtool fetches the Darwin \ + toolchain separately from: + \(artifact.url) + + \(detail) + + Check your network connection and GitHub availability, then retry \ + `xtool sdk install`. You do not need to re-download Xcode. + """ + } + + var errorDescription: String? { description } +} + +struct SDKGitHubDownloader: Sendable { + var maxAttempts: Int + var retryDelay: Duration + var sleep: @Sendable (Duration) async throws -> Void + + init( + maxAttempts: Int = 3, + retryDelay: Duration = .milliseconds(250), + sleep: @escaping @Sendable (Duration) async throws -> Void = { try await Task.sleep(for: $0) } + ) { + self.maxAttempts = maxAttempts + self.retryDelay = retryDelay + self.sleep = sleep + } + + func body(for artifact: SDKGitHubArtifact) async throws -> Data { + precondition(maxAttempts >= 1, "maxAttempts must be at least 1") + var lastError: SDKGitHubDownloadError? + for attempt in 1...maxAttempts { + do { + return try await fetch(artifact) + } catch let error as CancellationError { + throw error + } catch let error as SDKGitHubDownloadError { + lastError = error + guard error.isRetryable, attempt < maxAttempts else { throw error } + try await sleep(retryDelay) + } + } + throw lastError! + } + + private func fetch(_ artifact: SDKGitHubArtifact) async throws -> Data { + @Dependency(\.httpClient) var httpClient + let response: HTTPResponse + let body: HTTPBody? + do { + (response, body) = try await httpClient.send(HTTPRequest(url: artifact.url)) + } catch let error as CancellationError { + throw error + } catch { + throw SDKGitHubDownloadError.network(artifact, String(describing: error)) + } + guard response.status == .ok else { + throw SDKGitHubDownloadError.httpStatus(artifact, response.status.code) + } + guard let body else { + throw SDKGitHubDownloadError.emptyBody(artifact) + } + return try await consume(body, artifact: artifact) + } + + private func consume(_ body: HTTPBody, artifact: SDKGitHubArtifact) async throws -> Data { + let total: Int64? = switch body.length { + case .known(let known): known + case .unknown: nil + } + var data = Data() + do { + for try await chunk in body { + data.append(contentsOf: chunk) + reportProgress(artifact, written: Int64(data.count), total: total) + } + } catch let error as CancellationError { + throw error + } catch { + throw SDKGitHubDownloadError.network(artifact, String(describing: error)) + } + return data + } + + private func reportProgress(_ artifact: SDKGitHubArtifact, written: Int64, total: Int64?) { + guard let total else { return } + let progress = Int(Double(written) / Double(total) * 100) + print("\r[\(artifact.progressLabel)] \(progress)%", terminator: "") + fflush(stdoutSafe) + } +} diff --git a/Tests/XToolTests/SDKGitHubDownloadTests.swift b/Tests/XToolTests/SDKGitHubDownloadTests.swift new file mode 100644 index 00000000..a4aaa999 --- /dev/null +++ b/Tests/XToolTests/SDKGitHubDownloadTests.swift @@ -0,0 +1,215 @@ +import Foundation +import Testing +import Dependencies +import HTTPTypes +import OpenAPIRuntime +import XKit +@testable import XToolSupport + +private struct ScriptedHTTPClient: HTTPClientProtocol, ClientTransport { + var asOpenAPITransport: ClientTransport { self } + var handler: @Sendable (HTTPRequest) async throws -> (HTTPResponse, HTTPBody?) + + func send( + _ request: HTTPRequest, + body: HTTPBody?, + baseURL: URL, + operationID: String + ) async throws -> (HTTPResponse, HTTPBody?) { + try await handler(request) + } + + func withEphemeralClient( + perform: (any HTTPClientProtocol) async throws -> T + ) async throws -> T { + try await perform(self) + } + + func makeWebSocket(url: URL) async throws -> any WebSocketSession { + struct Unused: Error {} + throw Unused() + } +} + +private final class AttemptBox: @unchecked Sendable { + var count = 0 +} + +private struct MidBodyFailure: Error, CustomStringConvertible { + var description: String { "connection reset by peer" } +} + +private func bodyThatFailsMidway(_ prefix: Data) -> HTTPBody { + let stream = AsyncThrowingStream { continuation in + continuation.yield(Array(prefix)[...]) + continuation.finish(throwing: MidBodyFailure()) + } + return HTTPBody(stream, length: .unknown) +} + +@Test func sdkGitHubArtifactURLsMatchPublishedReleases() { + let toolset = SDKGitHubArtifact.toolset(arch: .x86_64, version: "1.1.0") + #expect(toolset.name == "Darwin toolset") + #expect(toolset.url.absoluteString == """ + https://github.com/xtool-org/darwin-tools-linux-llvm/releases/download/v1.1.0/toolset-x86_64.tar.gz + """) + + let macros = SDKGitHubArtifact.openAppleMacros(arch: .aarch64, version: "1.3.0") + #expect(macros.name == "OpenAppleMacros") + #expect(macros.url.absoluteString == """ + https://github.com/xtool-org/OpenAppleMacros/releases/download/v1.3.0/OpenAppleMacrosServer-aarch64 + """) +} + +@Test func sdkGitHubDownloadExplainsGitHubFailureIsNotTheXIP() { + let artifact = SDKGitHubArtifact.toolset(arch: .x86_64) + let error = SDKGitHubDownloadError.httpStatus(artifact, 502) + #expect(error.description.contains("not caused by your Xcode.xip")) + #expect(error.description.contains("Darwin toolset")) + #expect(error.description.contains(artifact.url.absoluteString)) + #expect(error.description.contains("HTTP 502")) + #expect(error.description.contains("You do not need to re-download Xcode")) + #expect(error.isRetryable) + + let missing = SDKGitHubDownloadError.httpStatus(artifact, 404) + #expect(!missing.isRetryable) + #expect(!SDKGitHubDownloadError.emptyBody(artifact).isRetryable) + #expect(SDKGitHubDownloadError.network(artifact, "connection reset").isRetryable) +} + +@Test func sdkGitHubDownloadMapsHTTPError() async throws { + let artifact = SDKGitHubArtifact.openAppleMacros(arch: .x86_64) + let client = ScriptedHTTPClient { _ in + (HTTPResponse(status: .notFound), nil) + } + let downloader = SDKGitHubDownloader(maxAttempts: 3, retryDelay: .zero, sleep: { _ in }) + + await withDependencies { + $0.httpClient = client + } operation: { + await #expect(throws: SDKGitHubDownloadError.httpStatus(artifact, 404)) { + _ = try await downloader.body(for: artifact) + } + } +} + +@Test func sdkGitHubDownloadDoesNotRetryClientErrors() async throws { + let attempts = AttemptBox() + let artifact = SDKGitHubArtifact.toolset(arch: .aarch64) + let client = ScriptedHTTPClient { _ in + attempts.count += 1 + return (HTTPResponse(status: .notFound), nil) + } + let downloader = SDKGitHubDownloader(maxAttempts: 3, retryDelay: .zero, sleep: { _ in }) + + await withDependencies { + $0.httpClient = client + } operation: { + await #expect(throws: SDKGitHubDownloadError.self) { + _ = try await downloader.body(for: artifact) + } + } + #expect(attempts.count == 1) +} + +@Test func sdkGitHubDownloadRetriesServerErrorsThenSucceeds() async throws { + let attempts = AttemptBox() + let sleeps = AttemptBox() + let artifact = SDKGitHubArtifact.toolset(arch: .x86_64) + let payload = Data("toolset".utf8) + let client = ScriptedHTTPClient { _ in + attempts.count += 1 + if attempts.count < 3 { + return (HTTPResponse(status: .init(code: 503, reasonPhrase: "Unavailable")), nil) + } + return (HTTPResponse(status: .ok), HTTPBody(payload)) + } + let downloader = SDKGitHubDownloader( + maxAttempts: 3, + retryDelay: .milliseconds(1), + sleep: { _ in sleeps.count += 1 } + ) + + let data = try await withDependencies { + $0.httpClient = client + } operation: { + try await downloader.body(for: artifact) + } + #expect(attempts.count == 3) + #expect(sleeps.count == 2) + #expect(data == payload) +} + +@Test func sdkGitHubDownloadRetriesMidBodyFailureAndDropsPartialBytes() async throws { + let attempts = AttemptBox() + let sleeps = AttemptBox() + let artifact = SDKGitHubArtifact.toolset(arch: .x86_64) + let partial = Data("partial-from-attempt-1".utf8) + let complete = Data("only-attempt-2".utf8) + let client = ScriptedHTTPClient { _ in + attempts.count += 1 + if attempts.count == 1 { + return (HTTPResponse(status: .ok), bodyThatFailsMidway(partial)) + } + return (HTTPResponse(status: .ok), HTTPBody(complete)) + } + let downloader = SDKGitHubDownloader( + maxAttempts: 3, + retryDelay: .milliseconds(1), + sleep: { _ in sleeps.count += 1 } + ) + + let data = try await withDependencies { + $0.httpClient = client + } operation: { + try await downloader.body(for: artifact) + } + #expect(attempts.count == 2) + #expect(sleeps.count == 1) + #expect(data == complete) +} + +@Test func sdkGitHubDownloadFailsWithArtifactErrorWhenBodyAlwaysFails() async throws { + let attempts = AttemptBox() + let sleeps = AttemptBox() + let artifact = SDKGitHubArtifact.openAppleMacros(arch: .aarch64) + let client = ScriptedHTTPClient { _ in + attempts.count += 1 + return (HTTPResponse(status: .ok), bodyThatFailsMidway(Data("partial".utf8))) + } + let downloader = SDKGitHubDownloader( + maxAttempts: 3, + retryDelay: .zero, + sleep: { _ in sleeps.count += 1 } + ) + + await withDependencies { + $0.httpClient = client + } operation: { + await #expect(throws: SDKGitHubDownloadError.network(artifact, "connection reset by peer")) { + _ = try await downloader.body(for: artifact) + } + } + #expect(attempts.count == 3) + #expect(sleeps.count == 2) + let surfaced = SDKGitHubDownloadError.network(artifact, "connection reset by peer") + #expect(surfaced.description.contains(artifact.name)) + #expect(surfaced.description.contains(artifact.url.absoluteString)) +} + +@Test func sdkGitHubDownloadWrapsTransportFailures() async throws { + struct Boom: Error, CustomStringConvertible { + var description: String { "connection timed out" } + } + let artifact = SDKGitHubArtifact.openAppleMacros(arch: .x86_64) + let client = ScriptedHTTPClient { _ in throw Boom() } + let downloader = SDKGitHubDownloader(maxAttempts: 1, retryDelay: .zero, sleep: { _ in }) + + await withDependencies { + $0.httpClient = client + } operation: { + await #expect(throws: SDKGitHubDownloadError.network(artifact, "connection timed out")) { + _ = try await downloader.body(for: artifact) + } + } +}