From adba7d0883dc9ccf9e4bd0b461989602deec09a8 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 04:07:44 +0000 Subject: [PATCH] fix(http): preserve binary download and stream output --- CHANGELOG.md | 6 + crates/bashkit/src/builtins/curl.rs | 19 ++- .../integration/http_binary_output_tests.rs | 145 ++++++++++++++++++ crates/bashkit/tests/integration/main.rs | 1 + knowledge/log.md | 7 + knowledge/security/http-transport.md | 11 ++ 6 files changed, 181 insertions(+), 8 deletions(-) create mode 100644 crates/bashkit/tests/integration/http_binary_output_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 713af9ded..af27ac18e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [Unreleased] + +### What's Changed + +- fix(http): preserve binary curl downloads and curl/wget stdout through redirections and pipelines, including compressed responses and byte counts. + ## [0.20.1] - 2026-10-25 ### Highlights diff --git a/crates/bashkit/src/builtins/curl.rs b/crates/bashkit/src/builtins/curl.rs index adf58c63e..55b0a2771 100644 --- a/crates/bashkit/src/builtins/curl.rs +++ b/crates/bashkit/src/builtins/curl.rs @@ -1044,7 +1044,8 @@ async fn execute_curl_request( response.body.clone() }; - // Format output + // Keep response bytes canonical for files, redirects, and + // pipelines. Only text consumers may decode invalid UTF-8. let output = if head_only { // For -I, output headers let mut header_output = format!("HTTP/1.1 {} OK\r\n", response.status); @@ -1052,15 +1053,15 @@ async fn execute_curl_request( header_output.push_str(&format!("{}: {}\r\n", name, value)); } header_output.push_str("\r\n"); - header_output + header_output.into_bytes() } else { - String::from_utf8_lossy(&body_bytes).into_owned() + body_bytes }; // Write to file if -o specified if let Some(file_path) = output_file { let full_path = resolve_path(ctx.cwd, file_path); - if let Err(e) = ctx.fs.write_file(&full_path, output.as_bytes()).await { + if let Err(e) = ctx.fs.write_file(&full_path, &output).await { return Ok(ExecResult::err( format!("curl: failed to write to {}: {}\n", file_path, e), 23, @@ -1075,10 +1076,12 @@ async fn execute_curl_request( } // Append write-out format if specified - let mut final_output = verbose_output; - final_output.push_str(&output); + let mut final_output = verbose_output.into_bytes(); + final_output.extend_from_slice(&output); if let Some(fmt) = write_out { - final_output.push_str(&format_write_out(fmt, &response, output.len())); + final_output.extend_from_slice( + format_write_out(fmt, &response, output.len()).as_bytes(), + ); } return Ok(ExecResult::ok(final_output)); @@ -1491,7 +1494,7 @@ async fn execute_wget_request( let output_path = if let Some(file) = output_file { if file == "-" { // Output to stdout - return Ok(ExecResult::ok(response.body_string())); + return Ok(ExecResult::ok(response.body)); } file.to_string() } else { diff --git a/crates/bashkit/tests/integration/http_binary_output_tests.rs b/crates/bashkit/tests/integration/http_binary_output_tests.rs new file mode 100644 index 000000000..a6d7a1e75 --- /dev/null +++ b/crates/bashkit/tests/integration/http_binary_output_tests.rs @@ -0,0 +1,145 @@ +//! HTTP downloads retain bytes through files, shell streams, and decompression. +#![cfg(feature = "http_client")] + +use async_trait::async_trait; +use base64::Engine; +use bashkit::{ + Bash, FileSystem, HttpResponse, HttpTransport, HttpTransportError, HttpTransportRequest, + InMemoryFs, NetworkAllowlist, +}; +use std::{io::Write, path::Path, sync::Arc}; + +const URL: &str = "http://93.184.216.34/archive"; +const BINARY: &[u8] = &[0x1f, 0x8b, 0x08, 0x00, 0xff, 0x80, b'\n', 0x00]; + +struct FixtureTransport { + body: Vec, + headers: Vec<(String, String)>, + status: u16, +} + +#[async_trait] +impl HttpTransport for FixtureTransport { + async fn execute(&self, _: HttpTransportRequest) -> Result { + Ok(HttpResponse { + status: self.status, + headers: self.headers.clone(), + body: self.body.clone(), + }) + } +} + +fn shell(body: Vec, headers: Vec<(String, String)>, status: u16) -> (Bash, Arc) { + let fs = Arc::new(InMemoryFs::new()); + let bash = Bash::builder() + .fs(fs.clone()) + .network(NetworkAllowlist::allow_all()) + .http_transport(Arc::new(FixtureTransport { + body, + headers, + status, + })) + .build(); + (bash, fs) +} + +#[tokio::test] +async fn downloads_and_redirection_keep_non_utf8_and_nul_bytes() { + for command in [ + format!("curl -s -o /archive {URL}"), + format!("curl -s {URL} > /archive"), + format!("wget -q -O /archive {URL}"), + ] { + let (mut bash, fs) = shell(BINARY.to_vec(), vec![], 200); + let result = bash.exec(&command).await.unwrap(); + assert_eq!(result.exit_code, 0, "{}", result.stderr); + assert_eq!( + fs.read_file(Path::new("/archive")).await.unwrap(), + BINARY, + "{command}" + ); + } +} + +#[tokio::test] +async fn stdout_and_pipelines_keep_binary_response_bytes() { + for command in [format!("curl -s {URL}"), format!("wget -q -O - {URL}")] { + let (mut bash, _) = shell(BINARY.to_vec(), vec![], 200); + let result = bash.exec(&command).await.unwrap(); + assert_eq!(result.exit_code, 0, "{}", result.stderr); + assert_eq!(result.stdout.as_bytes(), BINARY, "{command}"); + let piped = bash.exec(&format!("{command} | base64")).await.unwrap(); + assert_eq!(piped.exit_code, 0, "{}", piped.stderr); + assert_eq!( + piped.stdout.trim(), + base64::engine::general_purpose::STANDARD.encode(BINARY) + ); + } +} + +#[tokio::test] +async fn curl_write_out_counts_original_download_bytes() { + let (mut bash, fs) = shell(BINARY.to_vec(), vec![], 200); + let result = bash + .exec(&format!( + "curl -s -o /archive -w '%{{size_download}}' {URL}" + )) + .await + .unwrap(); + assert_eq!(result.exit_code, 0, "{}", result.stderr); + assert_eq!(result.stdout, BINARY.len().to_string()); + assert_eq!(fs.read_file(Path::new("/archive")).await.unwrap(), BINARY); +} + +#[tokio::test] +async fn compressed_binary_downloads_preserve_decompressed_bytes() { + let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gzip.write_all(BINARY).unwrap(); + let body = gzip.finish().unwrap(); + for suffix in ["-o /archive", "> /archive"] { + let (mut bash, fs) = shell( + body.clone(), + vec![("Content-Encoding".into(), "gzip".into())], + 200, + ); + let result = bash + .exec(&format!("curl -s --compressed {URL} {suffix}")) + .await + .unwrap(); + assert_eq!(result.exit_code, 0, "{}", result.stderr); + assert_eq!(fs.read_file(Path::new("/archive")).await.unwrap(), BINARY); + } +} + +#[tokio::test] +async fn failed_http_download_does_not_overwrite_existing_binary_file() { + let (mut bash, fs) = shell(BINARY.to_vec(), vec![], 404); + fs.write_file(Path::new("/archive"), b"existing") + .await + .unwrap(); + let result = bash + .exec(&format!("curl -s -f -o /archive {URL}")) + .await + .unwrap(); + assert_eq!(result.exit_code, 22); + assert_eq!( + fs.read_file(Path::new("/archive")).await.unwrap(), + b"existing" + ); +} + +#[tokio::test] +async fn downloaded_gzip_archive_can_be_extracted() { + let payload = b"a downloaded skill companion file\n"; + let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gzip.write_all(payload).unwrap(); + let (mut bash, _) = shell(gzip.finish().unwrap(), vec![], 200); + let result = bash + .exec(&format!( + "curl -s -o /package.gz {URL} && gzip -dc /package.gz" + )) + .await + .unwrap(); + assert_eq!(result.exit_code, 0, "{}", result.stderr); + assert_eq!(result.stdout.as_bytes(), payload); +} diff --git a/crates/bashkit/tests/integration/main.rs b/crates/bashkit/tests/integration/main.rs index 1044b5706..4cf3b8c74 100644 --- a/crates/bashkit/tests/integration/main.rs +++ b/crates/bashkit/tests/integration/main.rs @@ -53,6 +53,7 @@ pub mod curl_data_compat_tests; pub mod custom_builtins_tests; pub mod custom_fs_tests; pub mod function_metadata_security_tests; +pub mod http_binary_output_tests; pub mod input_redirect_budget_tests; pub mod interactive_builtins_tests; pub mod make_substitution_tests; diff --git a/knowledge/log.md b/knowledge/log.md index 9fb78a480..df45c6ff6 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -1,5 +1,12 @@ # Bashkit Knowledge Update Log +## 2026-10-26 + +* Preserve binary [HTTP transport](security/http-transport.md) output in curl + files and curl/wget streams. Lossy text conversion belongs at an explicit + consumer boundary; downloads, redirections, pipelines, compressed responses, + and write-out byte counts must retain the original bytes. + ## 2026-10-25 * **npm publish gap**: `tsconfig.json`'s `include` must name every `.ts` source in the native package, including one reached only through an import. tsc compiles a reachable import regardless, so omitting `runtime-functions.ts` broke no build but narrowed what `scripts/tests/test_js_publish_files.py` treats as a tsc output, disabling that guard for the one file it omitted. The `js-stubs` artifact then shipped without `runtime-functions.js` and all fifteen v0.20.0 `Publish JS` test jobs failed on `ERR_MODULE_NOT_FOUND`, leaving both npm packages on 0.19.0 while crates.io and PyPI published. `include` completeness is now guarded; see [Release Process](operations/release-process.md). diff --git a/knowledge/security/http-transport.md b/knowledge/security/http-transport.md index 3d52fd1db..351544c85 100644 --- a/knowledge/security/http-transport.md +++ b/knowledge/security/http-transport.md @@ -43,6 +43,17 @@ Embedding hosts (reference consumer: [everruns](https://github.com/everruns/ever ## Request Pipeline +HTTP response bodies stay byte-native through `curl`/`wget` output files, +stdout, redirections, and pipelines. Invalid UTF-8 and NUL bytes must not be +rewritten before reaching the filesystem or the next command. Text decoding +belongs to an explicit consumer boundary; `curl --compressed` applies that +same rule to the decompressed body. Write-out byte quantities use the original +output bytes rather than a replacement-expanded text representation. + +`tests/integration/http_binary_output_tests.rs` covers raw downloads, streams, +pipelines, compressed responses, gzip extraction, byte counts, and HTTP error +handling without live network dependencies. + ``` 1. Allowlist check ← security gate 2. Private IP / SSRF check ← SSRF protection, produces pinned_addrs