sdk: explain GitHub toolchain download failures

Sources/XToolSupport/SDKBuilder.swift

@@ -1,5 +1,4 @@
 import Foundation
-import Dependencies
 import libunxip
 import Subprocess
 import XKit // HTTPClient, stdoutSafe
@@ -105,8 +104,9 @@ struct SDKBuilder {
             withIntermediateDirectories: true
         )
 
-        // TODO: parallelize these three steps
-        // we need to synchronize progress reporting though
+        // Download GitHub artifacts before unxip so a toolchain fetch failure
+        // is reported immediately and is not mistaken for a corrupt Xcode.xip.
+        // TODO: parallelize these three steps (need synchronized progress).
 
         try await installMacros(in: output)
 
@@ -242,43 +242,18 @@ struct SDKBuilder {
             withIntermediateDirectories: false
         )
 
-        @Dependency(\.httpClient) var httpClient
-        let url = URL(string: """
-        https://github.com/xtool-org/darwin-tools-linux-llvm/releases/download/\
-        v\(Self.darwinToolsVersion)/toolset-\(arch.rawValue).tar.gz
-        """)!
-        let (response, body) = try await httpClient.send(HTTPRequest(url: url))
-        guard response.status == 200, let body else { throw Console.Error("Could not fetch toolset") }
-        let length: Int64? = switch body.length {
-        case .known(let known): known
-        case .unknown: nil
-        }
+        let artifact = SDKGitHubArtifact.toolset(arch: arch)
+        let data = try await download(artifact)
 
         defer { print() }
         try await Subprocess.run(
             .name("tar"),
             arguments: ["xzf", "-"],
             workingDirectory: FilePath(toolsetDir),
-            input: .inputWriter,
+            input: .data(data),
             output: .discarded,
             error: .discarded,
-        ) { execution in
-            var totalWritten: Int64 = 0
-            for try await chunk in body {
-                var remaining = chunk
-                while !remaining.isEmpty {
-                    let written = try await execution.standardInputWriter.write(Array(remaining))
-                    remaining = remaining.dropFirst(written)
-                    totalWritten += Int64(written)
-
-                    guard let length else { continue }
-                    let progress = Int(Double(totalWritten) / Double(length) * 100)
-                    print("\r[Downloading toolset] \(progress)%", terminator: "")
-                    fflush(stdoutSafe)
-                }
-            }
-            try await execution.standardInputWriter.finish()
-        }
+        )
         .checkSuccess()
 
         // swift-build assumes we have an Apple-flavored librarian for Apple platforms
@@ -385,20 +360,17 @@ struct SDKBuilder {
         try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: ld64.path)
     }
 
-    private func installMacros(in output: URL) async throws {
-        @Dependency(\.httpClient) var httpClient
-        let url = URL(string: """
-        https://github.com/xtool-org/OpenAppleMacros/releases/download/\
-        v\(Self.oamVersion)/OpenAppleMacrosServer-\(arch.rawValue)
-        """)!
-
-        let (response, body) = try await httpClient.send(HTTPRequest(url: url))
-        guard response.status == 200, let body else { throw Console.Error("Could not fetch toolset") }
-
-        let length: Int64? = switch body.length {
-        case .known(let known): known
-        case .unknown: nil
+    private func download(_ artifact: SDKGitHubArtifact) async throws -> Data {
+        do {
+            return try await SDKGitHubDownloader().body(for: artifact)
+        } catch let error as SDKGitHubDownloadError {
+            throw Console.Error(error.description)
         }
+    }
+
+    private func installMacros(in output: URL) async throws {
+        let artifact = SDKGitHubArtifact.openAppleMacros(arch: arch)
+        let data = try await download(artifact)
 
         let outputPath = output.appendingPathComponent("OpenAppleMacrosServer").path
         do {
@@ -409,18 +381,7 @@ struct SDKBuilder {
                 permissions: [.ownerReadWriteExecute, .groupReadExecute, .otherReadExecute]
             )
             defer { try? handle.close() }
-
-            var written: Int64 = 0
-            for try await chunk in body {
-                try handle.writeAll(chunk)
-                written += Int64(chunk.count)
-
-                if let length {
-                    let progress = Int(Double(written) / Double(length) * 100)
-                    print("\r[Downloading OpenAppleMacros] \(progress)%", terminator: "")
-                    fflush(stdoutSafe)
-                }
-            }
+            try handle.writeAll(data)
         }
         print()
     }

Sources/XToolSupport/SDKGitHubDownload.swift

@@ -0,0 +1,167 @@
+import Foundation
+import Dependencies
+import HTTPTypesFoundation
+import XKit
+
+struct SDKGitHubArtifact: Equatable, Sendable {
+    var name: String
+    var progressLabel: String
+    var url: URL
+
+    static func toolset(
+        arch: SDKBuilder.Arch,
+        version: String = SDKBuilder.darwinToolsVersion
+    ) -> Self {
+        Self(
+            name: "Darwin toolset",
+            progressLabel: "Downloading toolset",
+            url: URL(string: """
+            https://github.com/xtool-org/darwin-tools-linux-llvm/releases/download/\
+            v\(version)/toolset-\(arch.rawValue).tar.gz
+            """)!
+        )
+    }
+
+    static func openAppleMacros(
+        arch: SDKBuilder.Arch,
+        version: String = SDKBuilder.oamVersion
+    ) -> Self {
+        Self(
+            name: "OpenAppleMacros",
+            progressLabel: "Downloading OpenAppleMacros",
+            url: URL(string: """
+            https://github.com/xtool-org/OpenAppleMacros/releases/download/\
+            v\(version)/OpenAppleMacrosServer-\(arch.rawValue)
+            """)!
+        )
+    }
+}
+
+enum SDKGitHubDownloadError: Error, Equatable, CustomStringConvertible, LocalizedError {
+    case httpStatus(SDKGitHubArtifact, Int)
+    case emptyBody(SDKGitHubArtifact)
+    case network(SDKGitHubArtifact, String)
+
+    var artifact: SDKGitHubArtifact {
+        switch self {
+        case .httpStatus(let artifact, _), .emptyBody(let artifact), .network(let artifact, _):
+            artifact
+        }
+    }
+
+    var isRetryable: Bool {
+        switch self {
+        case .httpStatus(_, let status):
+            status == 429 || (500...599).contains(status)
+        case .emptyBody:
+            false
+        case .network:
+            true
+        }
+    }
+
+    var description: String {
+        let detail: String
+        switch self {
+        case .httpStatus(_, let status):
+            detail = "GitHub responded with HTTP \(status)."
+        case .emptyBody:
+            detail = "GitHub returned an empty response body."
+        case .network(_, let message):
+            detail = "Network error: \(message)"
+        }
+        return """
+        Failed to download the \(artifact.name) from GitHub.
+
+        This error is not caused by your Xcode.xip. xtool fetches the Darwin \
+        toolchain separately from:
+          \(artifact.url)
+
+        \(detail)
+
+        Check your network connection and GitHub availability, then retry \
+        `xtool sdk install`. You do not need to re-download Xcode.
+        """
+    }
+
+    var errorDescription: String? { description }
+}
+
+struct SDKGitHubDownloader: Sendable {
+    var maxAttempts: Int
+    var retryDelay: Duration
+    var sleep: @Sendable (Duration) async throws -> Void
+
+    init(
+        maxAttempts: Int = 3,
+        retryDelay: Duration = .milliseconds(250),
+        sleep: @escaping @Sendable (Duration) async throws -> Void = { try await Task.sleep(for: $0) }
+    ) {
+        self.maxAttempts = maxAttempts
+        self.retryDelay = retryDelay
+        self.sleep = sleep
+    }
+
+    func body(for artifact: SDKGitHubArtifact) async throws -> Data {
+        precondition(maxAttempts >= 1, "maxAttempts must be at least 1")
+        var lastError: SDKGitHubDownloadError?
+        for attempt in 1...maxAttempts {
+            do {
+                return try await fetch(artifact)
+            } catch let error as CancellationError {
+                throw error
+            } catch let error as SDKGitHubDownloadError {
+                lastError = error
+                guard error.isRetryable, attempt < maxAttempts else { throw error }
+                try await sleep(retryDelay)
+            }
+        }
+        throw lastError!
+    }
+
+    private func fetch(_ artifact: SDKGitHubArtifact) async throws -> Data {
+        @Dependency(\.httpClient) var httpClient
+        let response: HTTPResponse
+        let body: HTTPBody?
+        do {
+            (response, body) = try await httpClient.send(HTTPRequest(url: artifact.url))
+        } catch let error as CancellationError {
+            throw error
+        } catch {
+            throw SDKGitHubDownloadError.network(artifact, String(describing: error))
+        }
+        guard response.status == .ok else {
+            throw SDKGitHubDownloadError.httpStatus(artifact, response.status.code)
+        }
+        guard let body else {
+            throw SDKGitHubDownloadError.emptyBody(artifact)
+        }
+        return try await consume(body, artifact: artifact)
+    }
+
+    private func consume(_ body: HTTPBody, artifact: SDKGitHubArtifact) async throws -> Data {
+        let total: Int64? = switch body.length {
+        case .known(let known): known
+        case .unknown: nil
+        }
+        var data = Data()
+        do {
+            for try await chunk in body {
+                data.append(contentsOf: chunk)
+                reportProgress(artifact, written: Int64(data.count), total: total)
+            }
+        } catch let error as CancellationError {
+            throw error
+        } catch {
+            throw SDKGitHubDownloadError.network(artifact, String(describing: error))
+        }
+        return data
+    }
+
+    private func reportProgress(_ artifact: SDKGitHubArtifact, written: Int64, total: Int64?) {
+        guard let total else { return }
+        let progress = Int(Double(written) / Double(total) * 100)
+        print("\r[\(artifact.progressLabel)] \(progress)%", terminator: "")
+        fflush(stdoutSafe)
+    }
+}