Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 9 additions & 25 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,23 +82,14 @@ jobs:
name: Test (${{ matrix.name }})
runs-on: ${{ matrix.runner }}
timeout-minutes: 60
# manual_only lanes are GitHub-hosted and must not run on push or pull_request.
# Compared as a string: a boolean false in a matrix is easy to misread as the
# truthy string "false" once it crosses an expression boundary.
if: ${{ matrix.manual_only != 'yes' || github.event_name == 'workflow_dispatch' }}
# Select manual lanes before expansion; job-level conditions cannot use matrix.
strategy:
fail-fast: false
matrix:
include:
- name: linux
runner: sm-standard-4
manual_only: "no"
- name: macos-latest, manual
runner: macos-latest
manual_only: "yes"
- name: windows-latest, manual
runner: windows-latest
manual_only: "yes"
include: >-
${{ fromJSON(github.event_name == 'workflow_dispatch' &&
'[{"name":"linux","runner":"sm-standard-4","manual_only":"no"},{"name":"macos-latest, manual","runner":"macos-latest","manual_only":"yes"},{"name":"windows-latest, manual","runner":"windows-latest","manual_only":"yes"}]' ||
'[{"name":"linux","runner":"sm-standard-4","manual_only":"no"}]') }}
steps:
- name: Flag billed GitHub-hosted runner
if: matrix.manual_only == 'yes'
Expand All @@ -121,20 +112,13 @@ jobs:
name: Build (${{ matrix.name }})
runs-on: ${{ matrix.runner }}
timeout-minutes: 90
if: ${{ matrix.manual_only != 'yes' || github.event_name == 'workflow_dispatch' }}
strategy:
fail-fast: false
matrix:
include:
- name: linux
runner: sm-standard-4
manual_only: "no"
- name: macos-latest, manual
runner: macos-latest
manual_only: "yes"
- name: windows-latest, manual
runner: windows-latest
manual_only: "yes"
include: >-
${{ fromJSON(github.event_name == 'workflow_dispatch' &&
'[{"name":"linux","runner":"sm-standard-4","manual_only":"no"},{"name":"macos-latest, manual","runner":"macos-latest","manual_only":"yes"},{"name":"windows-latest, manual","runner":"windows-latest","manual_only":"yes"}]' ||
'[{"name":"linux","runner":"sm-standard-4","manual_only":"no"}]') }}
steps:
- name: Flag billed GitHub-hosted runner
if: matrix.manual_only == 'yes'
Expand Down
8 changes: 7 additions & 1 deletion .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ on:
description: 'Image tag to build and push'
required: true
default: 'latest'
publish_latest:
description: 'Also update latest for a stable version tag'
required: false
type: boolean
default: false

permissions:
contents: read
Expand Down Expand Up @@ -40,6 +45,7 @@ jobs:
env:
EVENT_NAME: ${{ github.event_name }}
MANUAL_TAG: ${{ github.event.inputs.tag }}
PUBLISH_LATEST: ${{ github.event.inputs.publish_latest }}
shell: bash
run: |
set -euo pipefail
Expand All @@ -58,7 +64,7 @@ jobs:
{
echo 'tags<<EOF'
echo "rustfs/rc:${image_tag}"
if [[ "$EVENT_NAME" == "workflow_run" && "$image_tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
if [[ ( "$EVENT_NAME" == "workflow_run" || "$PUBLISH_LATEST" == "true" ) && "$image_tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo 'rustfs/rc:latest'
fi
echo 'EOF'
Expand Down
26 changes: 21 additions & 5 deletions .github/workflows/package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ jobs:
tag: ${{ steps.resolve.outputs.tag }}
head_sha: ${{ steps.resolve.outputs.head_sha }}
steps:
- name: Checkout workflow source
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
persist-credentials: false
- name: Install GitHub CLI
uses: ./.github/actions/setup-self-hosted
with:
Expand Down Expand Up @@ -195,17 +199,18 @@ jobs:
rpm_arch: aarch64
artifact_name: rustfs-cli-linux-arm64
steps:
- name: Install file(1)
uses: ./.github/actions/setup-self-hosted
with:
# Used to prove the packaged binary is statically linked.
profile: file
- name: Checkout exact build source
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
ref: ${{ needs.resolve.outputs.head_sha }}
persist-credentials: false

- name: Install file(1)
uses: ./.github/actions/setup-self-hosted
with:
# Used to prove the packaged binary is statically linked.
profile: file

- name: Download binary artifact from release build
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
with:
Expand Down Expand Up @@ -384,6 +389,11 @@ jobs:
contents: write
actions: read
steps:
- name: Checkout exact build source
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
ref: ${{ needs.resolve.outputs.head_sha }}
persist-credentials: false
- name: Install GitHub CLI
uses: ./.github/actions/setup-self-hosted
with:
Expand Down Expand Up @@ -467,8 +477,14 @@ jobs:
runs-on: sm-standard-2
timeout-minutes: 15
permissions:
contents: read
actions: read
steps:
- name: Checkout exact build source
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
ref: ${{ needs.resolve.outputs.head_sha }}
persist-credentials: false
- name: Install AWS CLI
uses: ./.github/actions/setup-self-hosted
with:
Expand Down
6 changes: 5 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,11 @@ jobs:
permissions:
contents: write
steps:
- name: Checkout release source
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
ref: ${{ needs.build-check.outputs.source_ref }}
persist-credentials: false
- name: Install AWS CLI
uses: ./.github/actions/setup-self-hosted
with:
Expand Down Expand Up @@ -528,7 +533,6 @@ jobs:
package-deb-rpm:
name: Package DEB/RPM
needs: [build-check, upload-release-assets]
timeout-minutes: 90
if: needs.build-check.outputs.is_release == 'true'
uses: ./.github/workflows/package.yml
with:
Expand Down
6 changes: 3 additions & 3 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ resolver = "3"
members = ["crates/cli", "crates/core", "crates/s3"]

[workspace.package]
version = "0.1.24"
version = "0.1.37"
edition = "2024"
rust-version = "1.92"
license = "MIT OR Apache-2.0"
Expand All @@ -16,9 +16,9 @@ categories = ["command-line-utilities", "filesystem"]

[workspace.dependencies]
# RustFS Internal Crates
rc-core = { path = "./crates/core", version = "0.1.24" }
rc-s3 = { path = "./crates/s3", version = "0.1.24" }
rustfs-cli = { path = "./crates/cli", version = "0.1.24" }
rc-core = { path = "./crates/core", version = "0.1.37" }
rc-s3 = { path = "./crates/s3", version = "0.1.37" }
rustfs-cli = { path = "./crates/cli", version = "0.1.37" }

# Async runtime
tokio = { version = "1.49", features = ["full"] }
Expand Down
41 changes: 40 additions & 1 deletion crates/cli/src/launcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
//! Designed for RustFS and other S3-compatible backends.

use clap::Parser;
use tracing_subscriber::{EnvFilter, fmt, prelude::*};
use tracing_subscriber::{EnvFilter, filter::filter_fn, fmt, prelude::*};

use crate::commands::{self, Cli};
use crate::exit_code::ExitCode;
Expand Down Expand Up @@ -34,6 +34,15 @@ fn build_env_filter(debug: bool, env_is_set: bool) -> EnvFilter {
}
}

fn allow_log_metadata(metadata: &tracing::Metadata<'_>) -> bool {
// SDK debug events can include credentials and signed request headers.
*metadata.level() <= tracing::Level::WARN
|| matches!(
metadata.target().split("::").next(),
Some("rustfs_cli" | "rc_core" | "rc_s3")
)
}

/// Run the shared CLI launcher, preserving its worker-thread and exit behavior.
pub fn run() {
run_with_shutdown(|| async {});
Expand All @@ -52,6 +61,7 @@ where

// Initialize tracing subscriber for logging
tracing_subscriber::registry()
.with(filter_fn(allow_log_metadata))
.with(fmt::layer())
.with(env_filter)
.init();
Expand Down Expand Up @@ -158,6 +168,35 @@ mod tests {
assert_eq!(filter.to_string(), "debug");
}

#[test]
fn dependency_debug_logs_are_capped_without_hiding_application_diagnostics() {
let subscriber = tracing_subscriber::registry()
.with(filter_fn(allow_log_metadata))
.with(EnvFilter::new("trace"));
tracing::subscriber::with_default(subscriber, || {
assert!(tracing::enabled!(
target: "rc_s3::client",
tracing::Level::DEBUG
));
assert!(tracing::enabled!(
target: "rustfs_cli::launcher",
tracing::Level::TRACE
));
assert!(tracing::enabled!(
target: "aws_smithy_runtime::client",
tracing::Level::WARN
));
assert!(!tracing::enabled!(
target: "aws_smithy_runtime::client",
tracing::Level::DEBUG
));
assert!(!tracing::enabled!(
target: "reqwest::connect",
tracing::Level::TRACE
));
});
}

#[test]
fn command_launcher_preserves_command_exit_code() {
let exit_code = run_async_command(|| async { ExitCode::UnsupportedFeature }, test_runtime)
Expand Down
46 changes: 40 additions & 6 deletions crates/cli/tests/recursive_remote_copy.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#![cfg(not(windows))]

use std::collections::BTreeMap;
use std::io::{ErrorKind, Read, Write};
use std::io::{BufRead, BufReader, ErrorKind, Read, Write};
use std::net::{TcpListener, TcpStream};
use std::path::PathBuf;
use std::process::{Command, Output, Stdio};
Expand Down Expand Up @@ -104,7 +104,7 @@ impl S3Mock {
.lock()
.expect("record S3 request")
.push(request.clone());
write_response(&mut stream, handler(&request));
write_response(&mut stream, handler(&request), &request.method);
}
});
}
Expand Down Expand Up @@ -193,7 +193,7 @@ fn read_request(stream: &mut TcpStream, pending: &mut Vec<u8>) -> Option<Request
})
}

fn write_response(stream: &mut TcpStream, response: Response) {
fn write_response(stream: &mut TcpStream, response: Response, method: &str) {
let has_content_length = response
.headers
.iter()
Expand All @@ -216,12 +216,46 @@ fn write_response(stream: &mut TcpStream, response: Response) {
stream
.write_all(head.as_bytes())
.expect("write S3 response headers");
stream
.write_all(response.body.as_bytes())
.expect("write S3 response body");
if method != "HEAD" {
stream
.write_all(response.body.as_bytes())
.expect("write S3 response body");
}
stream.flush().expect("flush S3 response");
}

#[test]
fn head_error_response_does_not_corrupt_the_next_keep_alive_response() {
let mock = S3Mock::start(|request| {
if request.method == "HEAD" {
Response::access_denied()
} else {
Response::empty()
}
});
let mut stream = TcpStream::connect(mock.endpoint.strip_prefix("http://").unwrap()).unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(2)))
.unwrap();
let mut reader = BufReader::new(stream.try_clone().unwrap());
stream
.write_all(b"HEAD /bucket/key HTTP/1.1\r\nhost: localhost\r\n\r\n")
.unwrap();
loop {
let mut line = String::new();
assert_ne!(reader.read_line(&mut line).unwrap(), 0);
if line == "\r\n" {
break;
}
}
stream
.write_all(b"GET /bucket/key HTTP/1.1\r\nhost: localhost\r\n\r\n")
.unwrap();
let mut status = String::new();
reader.read_line(&mut status).unwrap();
assert_eq!(status, "HTTP/1.1 200 OK\r\n");
}

fn rc_binary() -> PathBuf {
if let Ok(path) = std::env::var("CARGO_BIN_EXE_rc") {
return PathBuf::from(path);
Expand Down
Loading