diff --git a/.github/workflows/CICD.yml b/.github/workflows/CICD.yml index add9922c933..323a89a21ff 100644 --- a/.github/workflows/CICD.yml +++ b/.github/workflows/CICD.yml @@ -554,15 +554,6 @@ jobs: sudo apt-get -y update sudo apt-get -y install fuse3 libfuse-dev ;; - wasm32-wasi*) - # expr depends on oniguruma, a C library, so cross-compiling needs a WASI sysroot - WASI_SDK=wasi-sdk-25.0-x86_64-linux - curl -sSfL "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/${WASI_SDK}.tar.gz" | tar -xz -C "$HOME" - for abi in wasip1 wasip2; do - echo "CC_wasm32_${abi}=$HOME/${WASI_SDK}/bin/clang" >> $GITHUB_ENV - echo "AR_wasm32_${abi}=$HOME/${WASI_SDK}/bin/llvm-ar" >> $GITHUB_ENV - done - ;; esac case '${{ matrix.job.os }}' in macos-latest) brew install coreutils ;; # needed for testing diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 27cfef76ba8..a00962f07a6 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -119,18 +119,6 @@ jobs: sudo apt-get -y install libselinux1-dev libsystemd-dev ;; esac - - name: Install/setup WASI prerequisites - if: startsWith(matrix.job.target, 'wasm32-wasi') - shell: bash - run: | - ## Install/setup WASI prerequisites - # expr depends on oniguruma, a C library, so cross-compiling needs a WASI sysroot - WASI_SDK=wasi-sdk-25.0-x86_64-linux - curl -sSfL "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/${WASI_SDK}.tar.gz" | tar -xz -C "$HOME" - for abi in wasip1 wasip2; do - echo "CC_wasm32_${abi}=$HOME/${WASI_SDK}/bin/clang" >> $GITHUB_ENV - echo "AR_wasm32_${abi}=$HOME/${WASI_SDK}/bin/llvm-ar" >> $GITHUB_ENV - done - name: "`cargo clippy` lint testing" uses: nick-fields/retry@v4 with: diff --git a/.github/workflows/wasi.yml b/.github/workflows/wasi.yml index faa99e8dfc5..9d48f5daa47 100644 --- a/.github/workflows/wasi.yml +++ b/.github/workflows/wasi.yml @@ -36,15 +36,6 @@ jobs: - uses: Swatinem/rust-cache@v2 with: key: "${{ matrix.job.target }}" - - name: Install WASI sysroot - run: | - # expr depends on oniguruma, a C library, so cross-compiling needs a WASI sysroot - WASI_SDK=wasi-sdk-25.0-x86_64-linux - curl -sSfL "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/${WASI_SDK}.tar.gz" | tar -xz -C "$HOME" - for abi in wasip1 wasip2; do - echo "CC_wasm32_${abi}=$HOME/${WASI_SDK}/bin/clang" >> $GITHUB_ENV - echo "AR_wasm32_${abi}=$HOME/${WASI_SDK}/bin/llvm-ar" >> $GITHUB_ENV - done - name: Install wasmtime run: | curl https://wasmtime.dev/install.sh -sSf | bash diff --git a/Cargo.lock b/Cargo.lock index 09b0147fed3..5a03fe2eea9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -171,6 +171,21 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "1.3.2" @@ -998,6 +1013,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fancy-regex" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "476de73bddf2ef8490aa4ee8f1cf40b430bf1d56c48c22080e5186952cd580e6" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -2132,28 +2158,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "onig" -version = "6.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" -dependencies = [ - "bitflags 2.13.1", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "openssl" version = "0.10.81" @@ -3599,10 +3603,10 @@ version = "0.11.0" dependencies = [ "clap", "codspeed-divan-compat", + "fancy-regex", "fluent", "num-bigint", "num-traits", - "onig", "thiserror 2.0.20", "uucore", ] diff --git a/Cargo.toml b/Cargo.toml index 5ba9dac5e70..33416e5cf3a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,9 +33,8 @@ expensive_tests = [] # Enable with: `cargo build --release --features openssl` # # By default OpenSSL is built from source and statically linked (via the -# `vendored` feature of the `openssl` crate), mirroring how `expr` links -# oniguruma. To link against the system libcrypto/libssl dynamically instead, -# set `OPENSSL_NO_VENDOR=1` at build time. +# `vendored` feature of the `openssl` crate). To link against the system +# libcrypto/libssl dynamically instead, set `OPENSSL_NO_VENDOR=1` at build time. # # The `?/openssl` entries propagate the choice into each standalone checksum # crate only when that crate is itself enabled, so disabling this feature @@ -464,7 +463,10 @@ notify = { version = "8.2.0", features = ["macos_kqueue"] } num-bigint = "0.4.4" num-prime = "0.5.0" num-traits = "0.2.19" -onig = { version = "~6.5.1", default-features = false } +fancy-regex = { version = "0.19.0", default-features = false, features = [ + "std", + "unicode", +] } os_display = "0.1.3" parse_datetime = "0.16.0" phf = "0.14.0" diff --git a/README.md b/README.md index a573e60cd44..d1c05eb0e1f 100644 --- a/README.md +++ b/README.md @@ -142,9 +142,8 @@ the pure-Rust digest crates, enable the `openssl` feature: cargo build --release --features openssl ``` By default OpenSSL is built from source and statically linked into the -binary (mirroring how `expr` links `oniguruma`), so no runtime dependency -on system libcrypto/libssl is added. To link dynamically against the system -libcrypto instead, set `OPENSSL_NO_VENDOR=1` at build time. +binary, so no runtime dependency on system libcrypto/libssl is added. To link +dynamically against the system libcrypto instead, set `OPENSSL_NO_VENDOR=1` at build time. The speedup is largest on CPUs without SHA-NI hardware acceleration. The feature is a no-op on Windows (the pure-Rust implementations are always used diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index f7292d6aecd..2d30136db8b 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -8,6 +8,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "android_system_properties" version = "0.1.6" @@ -114,6 +123,21 @@ dependencies = [ "num-traits", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "1.3.2" @@ -577,6 +601,17 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fancy-regex" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "476de73bddf2ef8490aa4ee8f1cf40b430bf1d56c48c22080e5186952cd580e6" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -1304,28 +1339,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "onig" -version = "6.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" -dependencies = [ - "bitflags 2.13.1", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "ordered-multimap" version = "0.7.3" @@ -1368,12 +1381,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pkg-config" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" - [[package]] name = "portable-atomic" version = "1.14.0" @@ -1490,6 +1497,17 @@ name = "regex-automata" version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "rust-ini" @@ -1920,10 +1938,10 @@ name = "uu_expr" version = "0.11.0" dependencies = [ "clap", + "fancy-regex", "fluent", "num-bigint", "num-traits", - "onig", "thiserror", "uucore", ] diff --git a/src/uu/expr/Cargo.toml b/src/uu/expr/Cargo.toml index 69c07e17880..83fed4854a3 100644 --- a/src/uu/expr/Cargo.toml +++ b/src/uu/expr/Cargo.toml @@ -19,7 +19,7 @@ doctest = false clap = { workspace = true } num-bigint = { workspace = true } num-traits = { workspace = true } -onig = { workspace = true } +fancy-regex = { workspace = true, default-features = false } uucore = { workspace = true, features = ["i18n-collator"] } thiserror = { workspace = true } fluent = { workspace = true } diff --git a/src/uu/expr/src/syntax_tree.rs b/src/uu/expr/src/syntax_tree.rs index 405715057ad..6659d705891 100644 --- a/src/uu/expr/src/syntax_tree.rs +++ b/src/uu/expr/src/syntax_tree.rs @@ -7,9 +7,9 @@ use std::{cell::Cell, collections::BTreeMap}; +use fancy_regex::{Regex, RegexBuilder}; use num_bigint::BigInt; use num_traits::ToPrimitive; -use onig::{MatchParam, Regex, RegexOptions, SearchOptions, Syntax}; use crate::{ ExprError, ExprResult, @@ -208,8 +208,8 @@ where // Check if parsed quantifier is valid let re = Regex::new(r"^([0-9]*,[0-9]*|[0-9]+)$").expect("valid regular expression"); - if let Some(captures) = re.captures(&quantifier) { - let matched = captures.at(0).unwrap_or_default(); + if let Ok(Some(captures)) = re.captures(&quantifier) { + let matched = captures.get(0).map_or("", |m| m.as_str()); match matched.split_once(',') { Some(("", "")) => Ok(()), Some((x, "") | ("", x)) if x.parse::().is_ok() => Ok(()), @@ -230,14 +230,13 @@ where /// Check for errors in a supplied regular expression /// /// GNU coreutils shows messages for invalid regular expressions -/// differently from the oniguruma library used by the regex crate. +/// differently from standard regex engines. /// This method attempts to do these checks manually in one pass /// through the regular expression. /// /// This method is not comprehensively checking all cases in which /// a regular expression could be invalid; any cases not caught will -/// result in a [`ExprError::InvalidRegexExpression`] when passing the -/// regular expression through the Oniguruma bindings. This method is +/// result in a [`ExprError::InvalidRegexExpression`]. This method is /// intended to just identify a few situations for which GNU coreutils /// has specific error messages. fn check_posix_regex_errors(pattern: &str) -> ExprResult<()> { @@ -265,24 +264,16 @@ fn check_posix_regex_errors(pattern: &str) -> ExprResult<()> { } } -/// Build a regex from a pattern string with locale-aware encoding -fn build_regex(pattern_bytes: Vec) -> ExprResult<(Regex, String)> { - use onig::EncodedBytes; - use uucore::i18n::UEncoding; - - let encoding = uucore::i18n::get_locale_encoding(); - - // For pattern processing, we need to handle it based on locale - let pattern_str = String::from_utf8(pattern_bytes.clone()) - .unwrap_or_else(|_| String::from_utf8_lossy(&pattern_bytes).into()); - check_posix_regex_errors(&pattern_str)?; - - // Transpile the input pattern from BRE syntax to `onig` crate's `Syntax::grep` - let mut re_string = String::with_capacity(pattern_str.len() + 1); +/// Transpile the input pattern from BRE syntax to ERE for fancy-regex +fn transpile_bre_to_ere(pattern_str: &str) -> ExprResult { + let mut re_string = String::with_capacity(pattern_str.len() + 8); let mut pattern_chars = pattern_str.chars().peekable(); let mut prev = '\0'; let mut prev_is_escaped = false; + let mut in_bracket = false; + let mut bracket_start_idx = 0; let mut is_start_of_expression = true; + let mut after_anchor_caret = false; // All patterns are anchored so they begin with a caret (^) if pattern_chars.peek() != Some(&'^') { @@ -291,243 +282,244 @@ fn build_regex(pattern_bytes: Vec) -> ExprResult<(Regex, String)> { while let Some(curr) = pattern_chars.next() { let curr_is_escaped = prev == '\\' && !prev_is_escaped; - let is_first_character = prev == '\0'; - - match curr { - // Character class negation "[^a]" - // Explicitly escaped caret "\^" - '^' if !is_start_of_expression && !matches!(prev, '[' | '\\') => { - re_string.push_str(r"\^"); - } - '$' if !curr_is_escaped && !is_end_of_expression(&pattern_chars) => { - re_string.push_str(r"\$"); - } - '\\' if !curr_is_escaped && pattern_chars.peek().is_none() => { - return Err(ExprError::TrailingBackslash); + let mut next_is_start = false; + let mut next_after_anchor = false; + + if in_bracket { + let is_first = re_string.len() == bracket_start_idx + 1 + || (re_string.ends_with("[^") && re_string.len() == bracket_start_idx + 2); + if curr == ']' && !curr_is_escaped && !is_first { + in_bracket = false; } - '{' if curr_is_escaped => { - // Handle '{' literally at the start of an expression - if is_start_of_expression { + re_string.push(curr); + } else { + match curr { + '[' if !curr_is_escaped => { + in_bracket = true; + bracket_start_idx = re_string.len(); + re_string.push('['); + } + // In BRE, '(', ')', '|', '+', '?', '{', '}' are literal by default, + // and become operators only when escaped. ERE has the exact opposite convention. + '(' | ')' | '|' | '+' | '?' | '{' | '}' => { + if curr_is_escaped { + if re_string.ends_with('\\') { + re_string.pop(); + } + match curr { + '(' | '|' => { + re_string.push(curr); + next_is_start = true; + } + '+' | '?' => { + if is_start_of_expression || after_anchor_caret { + re_string.push('\\'); + } + re_string.push(curr); + } + '{' => { + // Handle '{' literally at the start of an expression + if is_start_of_expression || after_anchor_caret { + re_string.push_str(r"\{"); + } else { + // Check if the following section is a valid range quantifier + verify_range_quantifier(&pattern_chars)?; + re_string.push('{'); + // Set the lower bound of range quantifier to 0 if it is missing + if pattern_chars.peek() == Some(&',') { + re_string.push('0'); + } + } + } + _ => re_string.push(curr), // ')' and '}' + } + } else { + // Unescaped metacharacter in BRE -> literal in ERE + re_string.push('\\'); + re_string.push(curr); + } + } + '*' => { + if curr_is_escaped { + re_string.push('*'); + } else if is_start_of_expression || after_anchor_caret { + re_string.push_str(r"\*"); + } else { + re_string.push('*'); + } + } + // Character class negation "[^a]" + // Explicitly escaped caret "\^" + '^' => { + if curr_is_escaped { + re_string.push('^'); + } else if is_start_of_expression { + re_string.push('^'); + next_after_anchor = true; + } else if prev == '[' && !prev_is_escaped { + re_string.push('^'); + } else { + re_string.push_str(r"\^"); + } + } + '$' if !curr_is_escaped && !is_end_of_expression(&pattern_chars) => { + re_string.push_str(r"\$"); + } + '<' if curr_is_escaped => { if re_string.ends_with('\\') { - let _ = re_string.pop(); + re_string.pop(); } - re_string.push(curr); - } else { - // Check if the following section is a valid range quantifier - verify_range_quantifier(&pattern_chars)?; - - re_string.push(curr); - // Set the lower bound of range quantifier to 0 if it is missing - if pattern_chars.peek() == Some(&',') { - re_string.push('0'); + re_string.push_str(r"\b(?=\w)"); + } + '>' if curr_is_escaped => { + if re_string.ends_with('\\') { + re_string.pop(); + } + re_string.push_str(r"\b(?<=\w)"); + } + '\\' if !curr_is_escaped => { + if pattern_chars.peek().is_none() { + return Err(ExprError::TrailingBackslash); + } + // Carry the expression-start / after-anchor state over the + // backslash so the escaped character is still treated as the + // first token of a (sub)expression. + next_is_start = is_start_of_expression; + next_after_anchor = after_anchor_caret; + re_string.push('\\'); + } + _ => { + if curr_is_escaped + && !"123456789.*^$[]\\wWsSbB".contains(curr) + && re_string.ends_with('\\') + { + re_string.pop(); } + re_string.push(curr); } } - _ => re_string.push(curr), } - // Capturing group "\(abc\)" - // Alternative pattern "a\|b" - is_start_of_expression = curr == '\\' && is_first_character - || curr_is_escaped && matches!(curr, '(' | '|') - || curr == '\\' && prev_is_escaped && matches!(prev, '(' | '|'); - + is_start_of_expression = next_is_start; + after_anchor_caret = next_after_anchor; prev_is_escaped = curr_is_escaped; prev = curr; } - // Create regex with proper encoding - let re = match encoding { - UEncoding::Utf8 => { - // For UTF-8 locale, use UTF-8 encoding - Regex::with_options_and_encoding( - &re_string, - RegexOptions::REGEX_OPTION_SINGLELINE | RegexOptions::REGEX_OPTION_MULTILINE, - Syntax::grep(), - ) - } - UEncoding::Ascii => { - // For non-UTF-8 locale, use ASCII encoding - Regex::with_options_and_encoding( - EncodedBytes::ascii(re_string.as_bytes()), - RegexOptions::REGEX_OPTION_SINGLELINE | RegexOptions::REGEX_OPTION_MULTILINE, - Syntax::grep(), - ) - } - } - .map_err(|error| match error.code() { - // "invalid repeat range {lower,upper}" - -123 => ExprError::InvalidBracketContent, - // "too big number for repeat range" - -201 => ExprError::TooBigRangeQuantifierIndex, - _ => ExprError::InvalidRegexExpression, - })?; - - Ok((re, re_string)) + Ok(re_string) } -/// Run a regex search, treating runtime match errors as no match. -fn regex_search( - regex: &Regex, - chars: T, - to: usize, - region: &mut onig::Region, -) -> Option { - regex - .search_with_param( - chars, - 0, - to, - SearchOptions::SEARCH_OPTION_NONE, - Some(region), - MatchParam::default(), - ) - .unwrap_or(None) +/// Build a regex from a pattern string with locale-aware encoding +fn build_regex(pattern_bytes: Vec) -> ExprResult { + use uucore::i18n::UEncoding; + + let encoding = uucore::i18n::get_locale_encoding(); + + // For pattern processing, we need to handle it based on locale + let pattern_str = match encoding { + UEncoding::Utf8 => String::from_utf8(pattern_bytes.clone()) + .unwrap_or_else(|_| String::from_utf8_lossy(&pattern_bytes).into()), + UEncoding::Ascii => pattern_bytes.iter().map(|&b| b as char).collect(), + }; + check_posix_regex_errors(&pattern_str)?; + + let re_string = transpile_bre_to_ere(&pattern_str)?; + + RegexBuilder::new(&format!("(?s){re_string}")) + .oniguruma_mode(true) + .build() + .map_err(|_| ExprError::InvalidRegexExpression) } /// Find matches in the input using the compiled regex -fn find_match(regex: Regex, re_string: String, left_bytes: Vec) -> String { - use onig::EncodedBytes; +fn find_match(regex: Regex, left_bytes: Vec) -> String { use uucore::i18n::UEncoding; let encoding = uucore::i18n::get_locale_encoding(); + let has_captures = regex.captures_len() > 1; // Match against the input using the appropriate encoding - let mut region = onig::Region::new(); match encoding { UEncoding::Utf8 => { // In UTF-8 locale, check if input is valid UTF-8 if let Ok(left_str) = std::str::from_utf8(&left_bytes) { // Valid UTF-8, match as UTF-8 - let pos = regex_search(®ex, left_str, left_str.len(), &mut region); - - if pos.is_some() { - if regex.captures_len() > 0 { + if let Ok(Some(caps)) = regex.captures(left_str) { + return if has_captures { // Get first capture group - region - .pos(1) - .map(|(start, end)| left_str[start..end].to_string()) - .unwrap_or_default() + caps.get(1) + .map_or(String::new(), |m| m.as_str().to_string()) } else { // Count characters in the match - let (start, end) = region.pos(0).unwrap(); - left_str[start..end].chars().count().to_string() - } - } else { - // No match - if regex.captures_len() > 0 { - String::new() - } else { - "0".to_string() - } + caps.get(0).unwrap().as_str().chars().count().to_string() + }; } } else { - // Invalid UTF-8 in UTF-8 locale - // Try to match as bytes using ASCII encoding - let left_encoded = EncodedBytes::ascii(&left_bytes); - // Need to create ASCII version of regex too - let re_ascii = Regex::with_options_and_encoding( - EncodedBytes::ascii(re_string.as_bytes()), - RegexOptions::REGEX_OPTION_SINGLELINE | RegexOptions::REGEX_OPTION_MULTILINE, - Syntax::grep(), - ) - .ok(); - - if let Some(re_ascii) = re_ascii { - let pos = regex_search(&re_ascii, left_encoded, left_bytes.len(), &mut region); - - if pos.is_some() { - if re_ascii.captures_len() > 0 { - // Get first capture group - region - .pos(1) - .map(|(start, end)| { - // Return empty string for invalid UTF-8 capture in UTF-8 locale - if std::str::from_utf8(&left_bytes[start..end]).is_err() { - String::new() - } else { - String::from_utf8_lossy(&left_bytes[start..end]) - .into_owned() - } - }) - .unwrap_or_default() - } else { - // No capture groups - return 0 for invalid UTF-8 in UTF-8 locale - "0".to_string() + // Invalid UTF-8 in UTF-8 locale: match on Latin-1 byte mapping + let left_str: String = left_bytes.iter().map(|&b| b as char).collect(); + if let Ok(Some(caps)) = regex.captures(&left_str) { + if has_captures { + if let Some(m) = caps.get(1) { + let start = m.start(); + let end = m.end(); + // Return empty string for invalid UTF-8 capture in UTF-8 locale + if std::str::from_utf8(&left_bytes[start..end]).is_ok() { + return String::from_utf8_lossy(&left_bytes[start..end]) + .into_owned(); + } } - } else { - // No match - if re_ascii.captures_len() > 0 { - String::new() - } else { - "0".to_string() - } - } - } else { - // Couldn't create ASCII regex - no match - if regex.captures_len() > 0 { - String::new() - } else { - "0".to_string() + return String::new(); } + // No capture groups - return 0 for invalid UTF-8 in UTF-8 locale + return "0".to_string(); } } } UEncoding::Ascii => { - // In ASCII/C locale, work with bytes directly - let left_encoded = EncodedBytes::ascii(&left_bytes); - let pos = regex_search(®ex, left_encoded, left_bytes.len(), &mut region); - - if pos.is_some() { - if regex.captures_len() > 0 { - // Get first capture group - return raw bytes for C locale - if let Some((start, end)) = region.pos(1) { - let capture_bytes = &left_bytes[start..end]; - // Return raw bytes as String for consistency with other cases - return String::from_utf8_lossy(capture_bytes).into_owned(); - } - String::new() + // In ASCII/C locale, work with Latin-1 byte mapping + let left_str: String = left_bytes.iter().map(|&b| b as char).collect(); + if let Ok(Some(caps)) = regex.captures(&left_str) { + return if has_captures { + caps.get(1).map_or_else(String::new, |m| { + let bytes: Vec = m.as_str().chars().map(|c| c as u8).collect(); + String::from_utf8_lossy(&bytes).into_owned() + }) } else { - // Return byte count of match - let (start, end) = region.pos(0).unwrap(); - (end - start).to_string() - } - } else { - // No match - if regex.captures_len() > 0 { - String::new() - } else { - "0".to_string() - } + caps.get(0).unwrap().as_str().chars().count().to_string() + }; } } } + + // No match + if has_captures { + String::new() + } else { + "0".to_string() + } } /// Evaluate a match expression with locale-aware regex matching fn evaluate_match_expression(left_bytes: Vec, right_bytes: Vec) -> ExprResult { use uucore::i18n::UEncoding; - let (regex, re_string) = build_regex(right_bytes)?; + let regex = build_regex(right_bytes)?; // Special case for ASCII locale with capture groups that need to return raw bytes let encoding = uucore::i18n::get_locale_encoding(); - if matches!(encoding, UEncoding::Ascii) && regex.captures_len() > 0 { + if matches!(encoding, UEncoding::Ascii) && regex.captures_len() > 1 { // Try to find the actual capture bytes for ASCII locale - let mut region = onig::Region::new(); - let left_encoded = onig::EncodedBytes::ascii(&left_bytes); - let pos = regex_search(®ex, left_encoded, left_bytes.len(), &mut region); - - if pos.is_some() - && let Some((start, end)) = region.pos(1) + let left_str: String = left_bytes.iter().map(|&b| b as char).collect(); + if let Ok(Some(caps)) = regex.captures(&left_str) + && let Some(m) = caps.get(1) { - let capture_bytes = &left_bytes[start..end]; - return Ok(MaybeNonUtf8String::from(capture_bytes.to_vec()).into()); + let bytes: Vec = m.as_str().chars().map(|c| c as u8).collect(); + return Ok(MaybeNonUtf8String::from(bytes).into()); } } - Ok(find_match(regex, re_string, left_bytes).into()) + Ok(find_match(regex, left_bytes).into()) } /// Precedence for infix binary operators @@ -1600,4 +1592,63 @@ mod test { let result = evaluate_match_expression(b"hello".to_vec(), b"a\\{invalid\\}".to_vec()); assert!(matches!(result, Err(ExprError::InvalidBracketContent))); } + + #[test] + fn test_evaluate_match_expression_multibyte_character_class() { + use super::evaluate_match_expression; + use uucore::i18n::{UEncoding, get_locale_encoding}; + + let result = evaluate_match_expression( + vec![0xce, 0xb1, b'b', b'c', 0xce, 0xb4, b'e', b'f'], + vec![b'[', 0xce, 0xb1, b']'], + ) + .unwrap(); + assert_eq!(result.eval_as_string(), b"1"); + + let result = evaluate_match_expression( + vec![0xce, 0xb1, b'b', b'c', 0xce, 0xb4, b'e', b'f'], + vec![b'\\', b'(', b'[', 0xce, 0xb1, b']', b'\\', b')'], + ) + .unwrap(); + match get_locale_encoding() { + UEncoding::Utf8 => assert_eq!(result.eval_as_string(), &[0xce, 0xb1]), + UEncoding::Ascii => assert_eq!(result.eval_as_string(), &[0xce]), + } + } + + #[test] + fn test_adjacent_quantifiers() { + use super::evaluate_match_expression; + + let result = evaluate_match_expression(b"aaa".to_vec(), br"\(a\)\{2\}*".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"a"); + + let result = evaluate_match_expression(b"aaa".to_vec(), br"a**".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"3"); + } + + #[test] + fn test_gnu_bre_extensions_and_escaped_caret() { + use super::evaluate_match_expression; + + // Word character \w + let result = evaluate_match_expression(b"a1b".to_vec(), br"\w".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"1"); + + // Word boundary \b + let result = evaluate_match_expression(b"abc".to_vec(), br"\ba".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"1"); + + // Beginning of word \< + let result = evaluate_match_expression(b"abc".to_vec(), br"\ + let result = evaluate_match_expression(b"b".to_vec(), br"b\>".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"1"); + + // Escaped caret with quantifier \^* + let result = evaluate_match_expression(b"^".to_vec(), br"\^*".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"1"); + } }