diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57810c00587..a5426d569b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: - 'crates/**' - 'vendor/**' - 'scripts/diagnostic-build.rs' - - 'scripts/setup.js' + - 'scripts/setup*' - 'Cargo.toml' - 'Cargo.lock' gpui: @@ -273,6 +273,9 @@ jobs: prefix-key: v1-rust-tests save-cache: ${{ github.ref == 'refs/heads/main' }} + - name: Test native dependency setup + run: node --test scripts/setup.test.mjs + - name: Run setup run: node scripts/setup.js diff --git a/scripts/setup.js b/scripts/setup.js index d61d0898c32..c400a297b67 100644 --- a/scripts/setup.js +++ b/scripts/setup.js @@ -71,9 +71,9 @@ async function main() { if (!(await fileExists(nativeDepsTarPath))) { console.log(`Downloading ${nativeDepsTar}`); - const nativeDepsBytes = await fetch(`${NATIVE_DEPS_URL}/${nativeDepsTar}`) - .then((r) => r.blob()) - .then((b) => b.arrayBuffer()); + const nativeDepsBytes = await downloadBytes( + `${NATIVE_DEPS_URL}/${nativeDepsTar}`, + ); await fs.writeFile(nativeDepsTarPath, Buffer.from(nativeDepsBytes)); console.log("Downloaded native deps"); downloadedNativeDeps = true; @@ -148,9 +148,7 @@ async function main() { const ffmpegZip = `ffmpeg-${FFMPEG_VERSION}.zip`; const ffmpegZipPath = path.join(targetDir, ffmpegZip); if (!(await fileExists(ffmpegZipPath))) { - const ffmpegZipBytes = await fetch(FFMPEG_ZIP_URL) - .then((r) => r.blob()) - .then((b) => b.arrayBuffer()); + const ffmpegZipBytes = await downloadBytes(FFMPEG_ZIP_URL); await fs.writeFile(ffmpegZipPath, Buffer.from(ffmpegZipBytes)); console.log(`Downloaded ${ffmpegZip}`); downloadedFfmpeg = true; @@ -273,9 +271,9 @@ async function main() { let downloadedNativeDeps = false; if (!(await fileExists(nativeDepsTarPath))) { console.log(`Downloading ${nativeDepsTar}`); - const bytes = await fetch(`${NATIVE_DEPS_URL}/${nativeDepsTar}`) - .then((r) => r.blob()) - .then((b) => b.arrayBuffer()); + const bytes = await downloadBytes( + `${NATIVE_DEPS_URL}/${nativeDepsTar}`, + ); await fs.writeFile(nativeDepsTarPath, Buffer.from(bytes)); console.log("Downloaded native deps"); downloadedNativeDeps = true; @@ -441,9 +439,7 @@ async function setupMacOSOnnxRuntime() { if (!(await fileExists(archivePath))) { console.log(`Downloading ${asset.name}`); - const bytes = await fetch(url) - .then((r) => r.blob()) - .then((b) => b.arrayBuffer()); + const bytes = await downloadBytes(url); await fs.writeFile(archivePath, Buffer.from(bytes)); console.log(`Downloaded ${asset.name}`); } else console.log(`Using cached ${asset.name}`); @@ -495,9 +491,7 @@ async function setupWindowsOnnxRuntime() { if (!(await fileExists(archivePath))) { console.log(`Downloading ${asset.name}`); - const bytes = await fetch(url) - .then((r) => r.blob()) - .then((b) => b.arrayBuffer()); + const bytes = await downloadBytes(url); await fs.writeFile(archivePath, Buffer.from(bytes)); console.log(`Downloaded ${asset.name}`); } else console.log(`Using cached ${asset.name}`); @@ -706,6 +700,13 @@ async function signMacOSDylib(filePath) { ); } +async function downloadBytes(url) { + const response = await fetch(url); + if (!response.ok) + throw new Error(`Failed to download ${url}: HTTP ${response.status}`); + return response.arrayBuffer(); +} + async function fileExists(path) { return await fs .access(path) diff --git a/scripts/setup.test.mjs b/scripts/setup.test.mjs new file mode 100644 index 00000000000..2f6f979e696 --- /dev/null +++ b/scripts/setup.test.mjs @@ -0,0 +1,180 @@ +import assert from "node:assert/strict"; +import { execFile as execFileCallback } from "node:child_process"; +import { once } from "node:events"; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const execFile = promisify(execFileCallback); +const repositoryRoot = fileURLToPath(new URL("../", import.meta.url)); + +test( + "setup retries failed downloads without poisoning the archive cache", + { + skip: process.platform !== "linux" && "Linux setup fixture", + timeout: 60_000, + }, + async (t) => { + const fixture = mkdtempSync(path.join(tmpdir(), "cap-setup-")); + t.after(() => rmSync(fixture, { recursive: true, force: true })); + for (const file of [ + "package.json", + "scripts/setup.js", + "scripts/linux-bundle-config.mjs", + "apps/desktop/src-tauri/tauri.conf.json", + "apps/desktop-gpui/tauri.conf.json", + ]) { + const destination = path.join(fixture, file); + mkdirSync(path.dirname(destination), { recursive: true }); + copyFileSync(path.join(repositoryRoot, file), destination); + } + + const nativeName = "native-deps-x86_64-linux-gnu.tar.xz"; + const onnxName = "onnxruntime-linux-x64-1.24.3.tgz"; + const nativeUrl = `/spacedriveapp/native-deps/releases/download/v0.26/${nativeName}`; + const onnxUrl = `/microsoft/onnxruntime/releases/download/v1.24.3/${onnxName}`; + const nativeLibrary = "synthetic native library\n"; + const onnxLibrary = "synthetic ONNX library\n"; + const archives = new Map(); + for (const [url, entry, contents] of [ + [nativeUrl, "lib/libavcodec.so.61", nativeLibrary], + [onnxUrl, `${onnxName.slice(0, -4)}/lib/libonnxruntime.so`, onnxLibrary], + ]) { + const archive = path.join(fixture, "assets", path.basename(url)); + const directory = `${archive}.contents`; + mkdirSync(path.dirname(path.join(directory, entry)), { recursive: true }); + writeFileSync(path.join(directory, entry), contents); + await execFile( + "tar", + [url.endsWith(".xz") ? "-cJf" : "-czf", archive, "-C", directory, "."], + { signal: t.signal }, + ); + archives.set(url, readFileSync(archive)); + } + + let failNative = true; + const requests = []; + const server = createServer((request, response) => { + requests.push(request.url); + if (failNative && request.url === nativeUrl) { + response.writeHead(503).end("Service Unavailable\n"); + return; + } + const archive = archives.get(request.url); + response.writeHead(archive ? 200 : 404); + response.end(archive ?? "Unexpected setup download"); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + t.after(async () => { + server.closeAllConnections(); + await promisify(server.close.bind(server))(); + }); + const origin = `http://127.0.0.1:${server.address().port}`; + const preload = path.join(fixture, "http-fixture.mjs"); + writeFileSync( + preload, + `const networkFetch = globalThis.fetch; +globalThis.fetch = (input, options) => { + const url = new URL(input); + if (url.origin !== "https://github.com") + throw new Error(\`Unexpected external request: \${url}\`); + return networkFetch(new URL(url.pathname, ${JSON.stringify(origin)}), options); +}; +`, + ); + const runSetup = () => + execFile(process.execPath, ["--import", preload, "scripts/setup.js"], { + cwd: fixture, + signal: t.signal, + timeout: 20_000, + env: { + ...process.env, + NODE_OPTIONS: "", + RUST_TARGET_TRIPLE: "x86_64-unknown-linux-gnu", + CAP_USE_SCCACHE: "0", + TMPDIR: fixture, + }, + }).then( + ({ stdout, stderr }) => ({ code: 0, stdout, stderr }), + (error) => { + assert.equal(typeof error.code, "number", error.message); + return error; + }, + ); + + const cache = path.join(fixture, "target", `v0.26-${nativeName}`); + const failed = await runSetup(); + assert.equal(failed.code, 1, failed.stderr); + assert.deepEqual(requests, [nativeUrl]); + assert.equal( + existsSync(cache), + false, + "failed downloads must not be cached", + ); + assert.match(failed.stderr, /HTTP 503/); + + failNative = false; + for (const expectedRequests of [[nativeUrl, onnxUrl], []]) { + requests.length = 0; + const result = await runSetup(); + assert.equal(result.code, 0, result.stderr); + assert.deepEqual(requests, expectedRequests); + assert.deepEqual(readFileSync(cache), archives.get(nativeUrl)); + for (const [name, contents] of [ + ["libavcodec.so.61", nativeLibrary], + ["libonnxruntime.so", onnxLibrary], + ["libonnxruntime.so.1", onnxLibrary], + ]) { + for (const directory of [ + "target/debug", + "target/release", + "target/x86_64-unknown-linux-gnu/debug", + "target/x86_64-unknown-linux-gnu/release", + ]) { + assert.equal( + readFileSync(path.join(fixture, directory, name), "utf8"), + contents, + ); + } + for (const directory of [ + "apps/desktop/src-tauri", + "apps/desktop-gpui", + ]) { + const config = JSON.parse( + readFileSync( + path.join(fixture, directory, "tauri.linux.conf.json"), + "utf8", + ), + ); + const source = config.bundle.linux.deb.files[`/usr/lib/cap/${name}`]; + assert.equal( + readFileSync(path.resolve(fixture, directory, source), "utf8"), + contents, + ); + } + } + const cargo = readFileSync( + path.join(fixture, ".cargo/config.toml"), + "utf8", + ); + assert.match(cargo, /\[target\.x86_64-unknown-linux-gnu\]/); + assert.match( + cargo, + /ORT_DYLIB_PATH = .*native-deps\/onnxruntime\/lib\/libonnxruntime\.so/, + ); + } + }, +);