From d0a5fbbe26c04a58bbe28ac1a019cde761c5b981 Mon Sep 17 00:00:00 2001 From: wtcpython <1762993226@qq.com> Date: Sun, 13 Sep 2026 16:04:49 +0800 Subject: [PATCH 1/2] uucore, expr: centralize regex in uucore --- Cargo.lock | 2 +- Cargo.toml | 6 +- fuzz/Cargo.lock | 2 +- src/uu/expr/Cargo.toml | 3 +- src/uu/expr/locales/en-US.ftl | 7 - src/uu/expr/locales/fr-FR.ftl | 7 - src/uu/expr/src/diagnostics.rs | 8 +- src/uu/expr/src/expr.rs | 16 +- src/uu/expr/src/syntax_tree.rs | 371 ++--------------- src/uucore/Cargo.toml | 6 + src/uucore/locales/errors/en-US.ftl | 10 + src/uucore/locales/errors/fr-FR.ftl | 10 + src/uucore/src/lib/features.rs | 2 + src/uucore/src/lib/features/regex.rs | 587 +++++++++++++++++++++++++++ src/uucore/src/lib/lib.rs | 2 + tests/by-util/test_expr.rs | 41 ++ 16 files changed, 697 insertions(+), 383 deletions(-) create mode 100644 src/uucore/src/lib/features/regex.rs diff --git a/Cargo.lock b/Cargo.lock index 8d0655551e2..4d4fd289196 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3460,7 +3460,6 @@ version = "0.13.0" dependencies = [ "clap", "codspeed-divan-compat", - "fancy-regex", "fluent", "num-bigint", "num-traits", @@ -4426,6 +4425,7 @@ dependencies = [ "digest 0.11.3", "dns-lookup", "dunce", + "fancy-regex", "fluent", "fluent-syntax", "glob", diff --git a/Cargo.toml b/Cargo.toml index 9edb70b98ba..266aafca395 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,11 +96,7 @@ num-bigint = "0.4.4" num-integer = "0.1.47" num-prime = "0.7.0" num-traits = "0.2.19" -fancy-regex = { version = "0.19.2", default-features = false, features = [ - "std", - "unicode", - "leftmost_longest", -] } +fancy-regex = { version = "0.19.2", default-features = false } os_display = "0.1.3" parse_datetime = "0.17.0" phf = "0.14.0" diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index b6cae9a026b..745d7c0e692 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -1723,7 +1723,6 @@ name = "uu_expr" version = "0.13.0" dependencies = [ "clap", - "fancy-regex", "fluent", "num-bigint", "num-traits", @@ -1841,6 +1840,7 @@ dependencies = [ "data-encoding-macro", "digest 0.11.3", "dunce", + "fancy-regex", "fluent", "fluent-syntax", "glob", diff --git a/src/uu/expr/Cargo.toml b/src/uu/expr/Cargo.toml index fe4de8d651c..8389cfc36e8 100644 --- a/src/uu/expr/Cargo.toml +++ b/src/uu/expr/Cargo.toml @@ -17,12 +17,11 @@ doctest = false [dependencies] clap = { workspace = true } -fancy-regex = { workspace = true, default-features = false } fluent = { workspace = true } num-bigint = { workspace = true } num-traits = { workspace = true } thiserror = { workspace = true } -uucore = { workspace = true, features = ["i18n-collator"] } +uucore = { workspace = true, features = ["i18n-collator", "regex"] } [dev-dependencies] divan = { workspace = true } diff --git a/src/uu/expr/locales/en-US.ftl b/src/uu/expr/locales/en-US.ftl index 6c600aee0ba..b281e0b26e4 100644 --- a/src/uu/expr/locales/en-US.ftl +++ b/src/uu/expr/locales/en-US.ftl @@ -95,15 +95,8 @@ expr-error-missing-argument = syntax error: missing argument after { $arg } expr-error-non-integer-argument = non-integer argument expr-error-missing-operand = missing operand expr-error-division-by-zero = division by zero -expr-error-invalid-regex-expression = Invalid regex expression expr-error-expected-closing-brace-after = syntax error: expecting ')' after { $arg } expr-error-expected-closing-brace-instead-of = syntax error: expecting ')' instead of { $arg } -expr-error-unmatched-opening-parenthesis = Unmatched ( or \( -expr-error-unmatched-closing-parenthesis = Unmatched ) or \) -expr-error-unmatched-opening-brace = Unmatched {"\\{"} -expr-error-invalid-bracket-content = Invalid content of {"\\{\\}"} -expr-error-trailing-backslash = Trailing backslash -expr-error-too-big-range-quantifier-index = Regular expression too big expr-error-match-utf8 = match does not support invalid UTF-8 encoding in { $arg } # Diagnostic labels, used when errors are rendered with a source snippet diff --git a/src/uu/expr/locales/fr-FR.ftl b/src/uu/expr/locales/fr-FR.ftl index 8a376fa0186..793a2e20e27 100644 --- a/src/uu/expr/locales/fr-FR.ftl +++ b/src/uu/expr/locales/fr-FR.ftl @@ -95,15 +95,8 @@ expr-error-missing-argument = erreur de syntaxe : argument manquant après { $ar expr-error-non-integer-argument = argument non entier expr-error-missing-operand = opérande manquant expr-error-division-by-zero = division par zéro -expr-error-invalid-regex-expression = Expression regex invalide expr-error-expected-closing-brace-after = erreur de syntaxe : ')' attendu après { $arg } expr-error-expected-closing-brace-instead-of = erreur de syntaxe : ')' attendu au lieu de { $arg } -expr-error-unmatched-opening-parenthesis = Parenthèse ouvrante ( ou \( non appariée -expr-error-unmatched-closing-parenthesis = Parenthèse fermante ) ou \) non appariée -expr-error-unmatched-opening-brace = Accolade ouvrante {"\\{"} non appariée -expr-error-invalid-bracket-content = Contenu invalide de {"\\{\\}"} -expr-error-trailing-backslash = Barre oblique inverse en fin -expr-error-too-big-range-quantifier-index = Expression régulière trop grande expr-error-match-utf8 = match ne supporte pas l'encodage UTF-8 invalide dans { $arg } # Étiquettes de diagnostic, utilisées quand les erreurs sont rendues avec un extrait diff --git a/src/uu/expr/src/diagnostics.rs b/src/uu/expr/src/diagnostics.rs index 3a2893c9a8f..4284269920d 100644 --- a/src/uu/expr/src/diagnostics.rs +++ b/src/uu/expr/src/diagnostics.rs @@ -81,13 +81,7 @@ fn locate(snapshot: &Snapshot, err: &ExprError, at: &FailurePoint) -> Option return None, }; diff --git a/src/uu/expr/src/expr.rs b/src/uu/expr/src/expr.rs index 5edc4fed7ff..a4f4fae30c0 100644 --- a/src/uu/expr/src/expr.rs +++ b/src/uu/expr/src/expr.rs @@ -57,26 +57,14 @@ pub enum ExprError { MissingOperand, #[error("{}", translate!("expr-error-division-by-zero"))] DivisionByZero, - #[error("{}", translate!("expr-error-invalid-regex-expression"))] - InvalidRegexExpression, #[error("{}", translate!("expr-error-expected-closing-brace-after", "arg" => _0.quote()))] ExpectedClosingBraceAfter(String), #[error("{}", translate!("expr-error-expected-closing-brace-instead-of", "arg" => _0.quote()))] ExpectedClosingBraceInsteadOf(String), - #[error("{}", translate!("expr-error-unmatched-opening-parenthesis"))] - UnmatchedOpeningParenthesis, - #[error("{}", translate!("expr-error-unmatched-closing-parenthesis"))] - UnmatchedClosingParenthesis, - #[error("{}", translate!("expr-error-unmatched-opening-brace"))] - UnmatchedOpeningBrace, - #[error("{}", translate!("expr-error-invalid-bracket-content"))] - InvalidBracketContent, - #[error("{}", translate!("expr-error-trailing-backslash"))] - TrailingBackslash, - #[error("{}", translate!("expr-error-too-big-range-quantifier-index"))] - TooBigRangeQuantifierIndex, #[error("{}", translate!("expr-error-match-utf8", "arg" => _0.quote()))] UnsupportedNonUtf8Match(String), + #[error(transparent)] + Regex(#[from] uucore::regex::RegexError), } impl UError for ExprError { diff --git a/src/uu/expr/src/syntax_tree.rs b/src/uu/expr/src/syntax_tree.rs index 2121fa34a6c..c5f49656bc7 100644 --- a/src/uu/expr/src/syntax_tree.rs +++ b/src/uu/expr/src/syntax_tree.rs @@ -7,9 +7,9 @@ use std::{cell::Cell, collections::BTreeMap}; -use fancy_regex::{Regex, RegexBuilder}; use num_bigint::BigInt; use num_traits::ToPrimitive; +use uucore::regex::{Regex, RegexBuilder, bre_to_ere}; use crate::{ ExprError, ExprResult, @@ -155,260 +155,6 @@ impl StringOp { } } -/// Check if regex pattern character iterator is at the end of a regex expression or subexpression -fn is_end_of_expression(pattern_chars: &I) -> bool -where - I: Iterator + Clone, -{ - let mut pattern_chars_clone = pattern_chars.clone(); - match pattern_chars_clone.next() { - Some('\\') => matches!(pattern_chars_clone.next(), Some(')' | '|')), - None => true, // No characters left - _ => false, - } -} - -/// Check if regex pattern character iterator is at the start of a valid range quantifier. -/// The iterator's start position is expected to be after the opening brace. -/// Range quantifier ends to closing brace. -/// -/// # Examples of valid range quantifiers -/// -/// - `r"\{3\}"` -/// - `r"\{3,\}"` -/// - `r"\{,6\}"` -/// - `r"\{3,6\}"` -/// - `r"\{,\}"` -fn verify_range_quantifier(pattern_chars: &I) -> Result<(), ExprError> -where - I: Iterator + Clone, -{ - let mut pattern_chars_clone = pattern_chars.clone().peekable(); - if pattern_chars_clone.peek().is_none() { - return Err(ExprError::UnmatchedOpeningBrace); - } - - // Parse the string between braces - let mut quantifier = String::new(); - let mut prev = '\0'; - let mut curr_is_escaped = false; - while let Some(curr) = pattern_chars_clone.next() { - curr_is_escaped = prev == '\\' && !curr_is_escaped; - if curr_is_escaped && curr == '}' { - break; - } - if pattern_chars_clone.peek().is_none() { - return Err(ExprError::UnmatchedOpeningBrace); - } - if prev != '\0' { - quantifier.push(prev); - } - prev = curr; - } - - // Check if parsed quantifier is valid - let re = Regex::new(r"^([0-9]*,[0-9]*|[0-9]+)$").expect("valid regular expression"); - if let Ok(Some(captures)) = re.captures(&quantifier) { - let matched = captures.get(0).map_or("", |m| m.as_str()); - match matched.split_once(',') { - Some(("", "")) => Ok(()), - Some((x, "") | ("", x)) if x.parse::().is_ok() => Ok(()), - Some((_, "") | ("", _)) => Err(ExprError::TooBigRangeQuantifierIndex), - Some((f, l)) => match (f.parse::(), l.parse::()) { - (Ok(f), Ok(l)) if f > l => Err(ExprError::InvalidBracketContent), - (Ok(_), Ok(_)) => Ok(()), - _ => Err(ExprError::TooBigRangeQuantifierIndex), - }, - None if matched.parse::().is_ok() => Ok(()), - None => Err(ExprError::TooBigRangeQuantifierIndex), - } - } else { - Err(ExprError::InvalidBracketContent) - } -} - -/// Check for errors in a supplied regular expression -/// -/// GNU coreutils shows messages for invalid regular expressions -/// differently from standard regex engines. -/// This method attempts to do these checks manually in one pass -/// through the regular expression. -/// -/// This method is not comprehensively checking all cases in which -/// a regular expression could be invalid; any cases not caught will -/// result in a [`ExprError::InvalidRegexExpression`]. This method is -/// intended to just identify a few situations for which GNU coreutils -/// has specific error messages. -fn check_posix_regex_errors(pattern: &str) -> ExprResult<()> { - let mut escaped_parens: u64 = 0; - let mut prev = '\0'; - let mut curr_is_escaped = false; - - for curr in pattern.chars() { - curr_is_escaped = prev == '\\' && !curr_is_escaped; - match (curr_is_escaped, curr) { - (true, '(') => escaped_parens += 1, - (true, ')') => { - escaped_parens = escaped_parens - .checked_sub(1) - .ok_or(ExprError::UnmatchedClosingParenthesis)?; - } - _ => {} - } - prev = curr; - } - - match escaped_parens { - 0 => Ok(()), - _ => Err(ExprError::UnmatchedOpeningParenthesis), - } -} - -/// Transpile the input pattern from BRE syntax to ERE for fancy-regex -fn transpile_bre_to_ere(pattern_str: &str) -> ExprResult { - let mut re_string = String::with_capacity(pattern_str.len() + 8); - let mut pattern_chars = pattern_str.chars().peekable(); - let mut prev = '\0'; - let mut prev_is_escaped = false; - let mut in_bracket = false; - let mut bracket_start_idx = 0; - let mut is_start_of_expression = true; - let mut after_anchor_caret = false; - - // All patterns are anchored so they begin with a caret (^) - if pattern_chars.peek() != Some(&'^') { - re_string.push('^'); - } - - while let Some(curr) = pattern_chars.next() { - let curr_is_escaped = prev == '\\' && !prev_is_escaped; - let mut next_is_start = false; - let mut next_after_anchor = false; - - if in_bracket { - let is_first = re_string.len() == bracket_start_idx + 1 - || (re_string.ends_with("[^") && re_string.len() == bracket_start_idx + 2); - if curr == ']' && !curr_is_escaped && !is_first { - in_bracket = false; - } - re_string.push(curr); - } else { - match curr { - '[' if !curr_is_escaped => { - in_bracket = true; - bracket_start_idx = re_string.len(); - re_string.push('['); - } - // In BRE, '(', ')', '|', '+', '?', '{', '}' are literal by default, - // and become operators only when escaped. ERE has the exact opposite convention. - '(' | ')' | '|' | '+' | '?' | '{' | '}' => { - if curr_is_escaped { - if re_string.ends_with('\\') { - re_string.pop(); - } - match curr { - '(' | '|' => { - re_string.push(curr); - next_is_start = true; - } - '+' | '?' => { - if is_start_of_expression || after_anchor_caret { - re_string.push('\\'); - } - re_string.push(curr); - } - '{' => { - // Handle '{' literally at the start of an expression - if is_start_of_expression || after_anchor_caret { - re_string.push_str(r"\{"); - } else { - // Check if the following section is a valid range quantifier - verify_range_quantifier(&pattern_chars)?; - re_string.push('{'); - // Set the lower bound of range quantifier to 0 if it is missing - if pattern_chars.peek() == Some(&',') { - re_string.push('0'); - } - } - } - _ => re_string.push(curr), // ')' and '}' - } - } else { - // Unescaped metacharacter in BRE -> literal in ERE - re_string.push('\\'); - re_string.push(curr); - } - } - '*' => { - if curr_is_escaped { - re_string.push('*'); - } else if is_start_of_expression || after_anchor_caret { - re_string.push_str(r"\*"); - } else { - re_string.push('*'); - } - } - // Character class negation "[^a]" - // Explicitly escaped caret "\^" - '^' => { - if curr_is_escaped { - re_string.push('^'); - } else if is_start_of_expression { - re_string.push('^'); - next_after_anchor = true; - } else if prev == '[' && !prev_is_escaped { - re_string.push('^'); - } else { - re_string.push_str(r"\^"); - } - } - '$' if !curr_is_escaped && !is_end_of_expression(&pattern_chars) => { - re_string.push_str(r"\$"); - } - '<' if curr_is_escaped => { - if re_string.ends_with('\\') { - re_string.pop(); - } - re_string.push_str(r"\b(?=\w)"); - } - '>' if curr_is_escaped => { - if re_string.ends_with('\\') { - re_string.pop(); - } - re_string.push_str(r"\b(?<=\w)"); - } - '\\' if !curr_is_escaped => { - if pattern_chars.peek().is_none() { - return Err(ExprError::TrailingBackslash); - } - // Carry the expression-start / after-anchor state over the - // backslash so the escaped character is still treated as the - // first token of a (sub)expression. - next_is_start = is_start_of_expression; - next_after_anchor = after_anchor_caret; - re_string.push('\\'); - } - _ => { - if curr_is_escaped - && !"123456789.*^$[]\\wWsSbB".contains(curr) - && re_string.ends_with('\\') - { - re_string.pop(); - } - re_string.push(curr); - } - } - } - - is_start_of_expression = next_is_start; - after_anchor_caret = next_after_anchor; - prev_is_escaped = curr_is_escaped; - prev = curr; - } - - Ok(re_string) -} - /// Build a regex from a pattern string with locale-aware encoding fn build_regex(pattern_bytes: Vec) -> ExprResult { use uucore::i18n::UEncoding; @@ -421,15 +167,17 @@ fn build_regex(pattern_bytes: Vec) -> ExprResult { .unwrap_or_else(|_| String::from_utf8_lossy(&pattern_bytes).into()), UEncoding::Ascii => pattern_bytes.iter().map(|&b| b as char).collect(), }; - check_posix_regex_errors(&pattern_str)?; - let re_string = transpile_bre_to_ere(&pattern_str)?; + let re_string = bre_to_ere(&pattern_str, true)?; - RegexBuilder::new(&format!("(?s){re_string}")) + let regex = RegexBuilder::new(&format!("(?s){re_string}")) .oniguruma_mode(true) .leftmost_longest(true) + .seek(true) .build() - .map_err(|_| ExprError::InvalidRegexExpression) + .map_err(uucore::regex::RegexError::from)?; + + Ok(regex) } /// Find matches in the input using the compiled regex @@ -1115,12 +863,11 @@ pub fn is_truthy(s: &NumOrStr) -> bool { #[cfg(test)] mod test { - use crate::syntax_tree::verify_range_quantifier; use crate::{ExprError, ExprResult}; + use uucore::regex::RegexError; use super::{ - AstNode, AstNodeInner, BinOp, MaybeNonUtf8Str, NumericOp, RelationOp, StringOp, - check_posix_regex_errors, get_next_id, + AstNode, AstNodeInner, BinOp, MaybeNonUtf8Str, NumericOp, RelationOp, StringOp, get_next_id, }; /// Parse an expression, discarding how far the parser got. @@ -1337,78 +1084,6 @@ mod test { assert_eq!(result.eval_as_string(), b"0"); } - #[test] - fn check_regex_valid() { - assert!(check_posix_regex_errors(r"(a+b) \(a* b\)").is_ok()); - } - - #[test] - fn check_regex_simple_repeating_pattern() { - assert!(check_posix_regex_errors(r"\(a+b\)\{4\}").is_ok()); - } - - #[test] - fn check_regex_missing_closing() { - assert_eq!( - check_posix_regex_errors(r"\(abc"), - Err(ExprError::UnmatchedOpeningParenthesis) - ); - } - - #[test] - fn check_regex_missing_opening() { - assert_eq!( - check_posix_regex_errors(r"abc\)"), - Err(ExprError::UnmatchedClosingParenthesis) - ); - } - - #[test] - fn test_is_valid_range_quantifier() { - assert!(verify_range_quantifier(&"3\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&"3,\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&",6\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&"3,6\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&",\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&"32767\\}anything".chars()).is_ok()); - assert_eq!( - verify_range_quantifier(&"\\{3,6\\}".chars()), - Err(ExprError::InvalidBracketContent) - ); - assert_eq!( - verify_range_quantifier(&"\\}".chars()), - Err(ExprError::InvalidBracketContent) - ); - assert_eq!( - verify_range_quantifier(&"".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&"3".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&"3,".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&",6".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&"3,6".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&",".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&"32768\\}".chars()), - Err(ExprError::TooBigRangeQuantifierIndex) - ); - } - #[test] fn test_evaluate_match_expression_basic() { use super::evaluate_match_expression; @@ -1573,23 +1248,29 @@ mod test { let result = evaluate_match_expression(b"hello".to_vec(), b"\\(hello".to_vec()); assert!(matches!( result, - Err(ExprError::UnmatchedOpeningParenthesis) + Err(ExprError::Regex(RegexError::UnmatchedOpeningParenthesis)) )); // Unmatched closing parenthesis let result = evaluate_match_expression(b"hello".to_vec(), b"hello\\)".to_vec()); assert!(matches!( result, - Err(ExprError::UnmatchedClosingParenthesis) + Err(ExprError::Regex(RegexError::UnmatchedClosingParenthesis)) )); // Trailing backslash let result = evaluate_match_expression(b"hello".to_vec(), b"hello\\".to_vec()); - assert!(matches!(result, Err(ExprError::TrailingBackslash))); + assert!(matches!( + result, + Err(ExprError::Regex(RegexError::TrailingBackslash)) + )); // Invalid bracket content let result = evaluate_match_expression(b"hello".to_vec(), b"a\\{invalid\\}".to_vec()); - assert!(matches!(result, Err(ExprError::InvalidBracketContent))); + assert!(matches!( + result, + Err(ExprError::Regex(RegexError::InvalidBracketContent)) + )); } #[test] @@ -1631,10 +1312,14 @@ mod test { use super::evaluate_match_expression; // This test verifies leftmost-longest (POSIX) match semantics. - // Pattern `(a|ab)` against `ab` should capture the longest alternative (`ab`) + // Pattern `\(a\|ab\)` against `ab` should capture the longest alternative (`ab`), // not the first (`a`). let result = evaluate_match_expression(b"ab".to_vec(), br"\(a\|ab\)".to_vec()).unwrap(); assert_eq!(result.eval_as_string(), b"ab"); + + // `aaaaa\|a*` against `aaaaaa` should match all 6 characters + let result = evaluate_match_expression(b"aaaaaa".to_vec(), br"aaaaa\|a*".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"6"); } #[test] @@ -1657,6 +1342,14 @@ mod test { let result = evaluate_match_expression(b"b".to_vec(), br"b\>".to_vec()).unwrap(); assert_eq!(result.eval_as_string(), b"1"); + // Beginning of buffer \` + let result = evaluate_match_expression(b"start".to_vec(), br"\`start".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"5"); + + // End of buffer \' + let result = evaluate_match_expression(b"end".to_vec(), br"end\'".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"3"); + // Escaped caret with quantifier \^* let result = evaluate_match_expression(b"^".to_vec(), br"\^*".to_vec()).unwrap(); assert_eq!(result.eval_as_string(), b"1"); diff --git a/src/uucore/Cargo.toml b/src/uucore/Cargo.toml index b1867dae76e..bc2ca745d24 100644 --- a/src/uucore/Cargo.toml +++ b/src/uucore/Cargo.toml @@ -25,6 +25,11 @@ clap = { workspace = true } data-encoding = { workspace = true, optional = true } data-encoding-macro = { workspace = true, optional = true } dunce = { workspace = true, optional = true } +fancy-regex = { workspace = true, optional = true, features = [ + "std", + "unicode", + "leftmost_longest", +] } glob = { workspace = true, optional = true } itertools = { workspace = true, optional = true } jiff = { workspace = true, optional = true, features = [ @@ -197,6 +202,7 @@ process = ["libc", "nix/user", "rustix/process", "windows-sys"] proc-info = ["tty", "walkdir"] quoting-style = ["i18n-common"] ranges = [] +regex = ["dep:fancy-regex"] ringbuffer = [] safe-copy = ["rustix/fs"] safe-traversal = ["libc", "nix/fs", "nix/dir", "nix/user", "rustix/fs"] diff --git a/src/uucore/locales/errors/en-US.ftl b/src/uucore/locales/errors/en-US.ftl index 5720f768ecd..822cd3ee743 100644 --- a/src/uucore/locales/errors/en-US.ftl +++ b/src/uucore/locales/errors/en-US.ftl @@ -56,3 +56,13 @@ range-diag-label-inverted = this range ends before it starts # Invalid QUOTING_STYLE env var invalid-quoting-style-env-var = ignoring invalid value of environment variable QUOTING_STYLE: { $invalid } + +# Regex parsing messages (expr, grep, sed, ...) +regex-error-unmatched-opening-parenthesis = Unmatched ( or \( +regex-error-unmatched-closing-parenthesis = Unmatched ) or \) +regex-error-trailing-backslash = Trailing backslash +regex-error-unmatched-opening-brace = Unmatched {"\\{"} +regex-error-invalid-bracket-content = Invalid content of {"\\{\\}"} +regex-error-too-big-range-quantifier-index = Regular expression too big +regex-error-invalid-character-class-name = Invalid character class name +regex-error-compilation-failed = Invalid regular expression: { $error } diff --git a/src/uucore/locales/errors/fr-FR.ftl b/src/uucore/locales/errors/fr-FR.ftl index 28642d75ea7..52bf3d8d3ee 100644 --- a/src/uucore/locales/errors/fr-FR.ftl +++ b/src/uucore/locales/errors/fr-FR.ftl @@ -56,3 +56,13 @@ range-diag-label-inverted = cet intervalle se termine avant de commencer # variable d'environnement QUOTING_STYLE invalide invalid-quoting-style-env-var = valeur invalide de la variable d'environnement QUOTING_STYLE ignorée : { $invalid } + +# Messages d'analyse regex (expr, grep, sed, ...) +regex-error-unmatched-opening-parenthesis = Parenthèse ouvrante ( ou \( non appariée +regex-error-unmatched-closing-parenthesis = Parenthèse fermante ) ou \) non appariée +regex-error-trailing-backslash = Barre oblique inverse en fin +regex-error-unmatched-opening-brace = Accolade ouvrante {"\\{"} non appariée +regex-error-invalid-bracket-content = Contenu invalide de {"\\{\\}"} +regex-error-too-big-range-quantifier-index = Expression régulière trop grande +regex-error-invalid-character-class-name = Nom de classe de caractères invalide +regex-error-compilation-failed = Expression régulière invalide : { $error } diff --git a/src/uucore/src/lib/features.rs b/src/uucore/src/lib/features.rs index 4bce7676bd3..f6cbd27a970 100644 --- a/src/uucore/src/lib/features.rs +++ b/src/uucore/src/lib/features.rs @@ -55,6 +55,8 @@ pub mod parser; pub mod quoting_style; #[cfg(feature = "ranges")] pub mod ranges; +#[cfg(feature = "regex")] +pub mod regex; #[cfg(feature = "ringbuffer")] pub mod ringbuffer; #[cfg(feature = "sum")] diff --git a/src/uucore/src/lib/features/regex.rs b/src/uucore/src/lib/features/regex.rs new file mode 100644 index 00000000000..28b405df85b --- /dev/null +++ b/src/uucore/src/lib/features/regex.rs @@ -0,0 +1,587 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Unified regular expression infrastructure for uutils. +//! +//! Provides a clean abstraction layer around [`fancy_regex`] with support for +//! POSIX Basic Regular Expressions (BRE) transpilation, GNU extensions, +//! and POSIX leftmost-longest matching semantics. + +use crate::translate; +pub use fancy_regex::{Regex, RegexBuilder}; + +/// Errors encountered when compiling or transpiling regular expressions. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum RegexError { + #[error("{}", translate!("regex-error-unmatched-opening-parenthesis"))] + UnmatchedOpeningParenthesis, + + #[error("{}", translate!("regex-error-unmatched-closing-parenthesis"))] + UnmatchedClosingParenthesis, + + #[error("{}", translate!("regex-error-trailing-backslash"))] + TrailingBackslash, + + #[error("{}", translate!("regex-error-unmatched-opening-brace"))] + UnmatchedOpeningBrace, + + #[error("{}", translate!("regex-error-invalid-bracket-content"))] + InvalidBracketContent, + + #[error("{}", translate!("regex-error-too-big-range-quantifier-index"))] + TooBigRangeQuantifierIndex, + + #[error("{}", translate!("regex-error-invalid-character-class-name"))] + InvalidCharacterClassName, + + #[error("{}", translate!("regex-error-compilation-failed", "error" => _0))] + CompilationFailed(String), +} + +impl crate::error::UError for RegexError { + fn code(&self) -> i32 { + 2 + } +} + +impl From for RegexError { + fn from(err: fancy_regex::Error) -> Self { + Self::CompilationFailed(err.to_string()) + } +} + +/// Map POSIX character class name (e.g. `"alpha"`, `"digit"`) to its Unicode property equivalent. +pub fn map_posix_class(name: &str) -> Option<&'static str> { + match name { + "alpha" => Some(r"\p{Alphabetic}"), + "lower" => Some(r"\p{Lowercase}"), + "upper" => Some(r"\p{Uppercase}"), + "alnum" => Some(r"\p{Alphabetic}0-9"), + "space" => Some(r"\p{White_Space}"), + "blank" => Some(r"\t\p{Zs}"), + "cntrl" => Some(r"\p{Control}"), + "digit" => Some("0-9"), + "xdigit" => Some("0-9A-Fa-f"), + "punct" => Some(r"\p{Punctuation}"), + "graph" => Some(r"\P{C}&&\P{Z}"), + "print" => Some(r"\P{C}"), + _ => None, + } +} + +/// Check if a regex character iterator is at the start of a valid range quantifier (`\{m,n\}`). +/// +/// The iterator's start position is expected to be immediately after the opening brace. +fn verify_range_quantifier(pattern_chars: &I) -> Result<(), RegexError> +where + I: Iterator + Clone, +{ + let mut pattern_chars_clone = pattern_chars.clone().peekable(); + if pattern_chars_clone.peek().is_none() { + return Err(RegexError::UnmatchedOpeningBrace); + } + + // Parse the string between braces + let mut quantifier = String::new(); + let mut prev = '\0'; + let mut curr_is_escaped = false; + while let Some(curr) = pattern_chars_clone.next() { + curr_is_escaped = prev == '\\' && !curr_is_escaped; + if curr_is_escaped && curr == '}' { + break; + } + if pattern_chars_clone.peek().is_none() { + return Err(RegexError::UnmatchedOpeningBrace); + } + if prev != '\0' { + quantifier.push(prev); + } + prev = curr; + } + + // Check if parsed quantifier is valid + let re = Regex::new(r"^([0-9]*,[0-9]*|[0-9]+)$").expect("valid regular expression"); + if let Ok(Some(captures)) = re.captures(&quantifier) { + let matched = captures.get(0).map_or("", |m| m.as_str()); + match matched.split_once(',') { + Some(("", "")) => Ok(()), + Some((x, "") | ("", x)) if x.parse::().is_ok() => Ok(()), + Some((_, "") | ("", _)) => Err(RegexError::TooBigRangeQuantifierIndex), + Some((f, l)) => match (f.parse::(), l.parse::()) { + (Ok(f), Ok(l)) if f > l => Err(RegexError::InvalidBracketContent), + (Ok(_), Ok(_)) => Ok(()), + _ => Err(RegexError::TooBigRangeQuantifierIndex), + }, + None if matched.parse::().is_ok() => Ok(()), + None => Err(RegexError::TooBigRangeQuantifierIndex), + } + } else { + Err(RegexError::InvalidBracketContent) + } +} + +/// Check for errors in a supplied regular expression +/// +/// GNU coreutils shows messages for invalid regular expressions +/// differently from standard regex engines. +/// This method attempts to do these checks manually in one pass +/// through the regular expression. +/// +/// This method is not comprehensively checking all cases in which +/// a regular expression could be invalid; any cases not caught will +/// fall through to `fancy-regex` compilation. This method is intended to +/// just identify a few situations for which GNU coreutils has specific +/// error messages. +fn check_posix_regex_errors(pattern: &str) -> Result<(), RegexError> { + let mut escaped_parens: u64 = 0; + let mut prev = '\0'; + let mut curr_is_escaped = false; + + for curr in pattern.chars() { + curr_is_escaped = prev == '\\' && !curr_is_escaped; + match (curr_is_escaped, curr) { + (true, '(') => escaped_parens += 1, + (true, ')') => { + escaped_parens = escaped_parens + .checked_sub(1) + .ok_or(RegexError::UnmatchedClosingParenthesis)?; + } + _ => {} + } + prev = curr; + } + + match escaped_parens { + 0 => Ok(()), + _ => Err(RegexError::UnmatchedOpeningParenthesis), + } +} + +/// Check if regex pattern character iterator is at the end of a regex expression or subexpression +fn is_end_of_expression(pattern_chars: &I) -> bool +where + I: Iterator + Clone, +{ + let mut pattern_chars_clone = pattern_chars.clone(); + match pattern_chars_clone.next() { + Some('\\') => matches!(pattern_chars_clone.next(), Some(')' | '|')), + None => true, // No characters left + _ => false, + } +} + +/// Transpile a POSIX Basic Regular Expression (BRE) into Extended Regular Expression (ERE) +/// compatible with `fancy-regex`. +/// +/// If `anchored` is `true`, the pattern is implicitly anchored at the start (`^`), as required by POSIX `expr`. +/// Otherwise, standard unanchored BRE transpilation is performed (e.g. for `grep` and `sed`). +pub fn bre_to_ere(pattern_str: &str, anchored: bool) -> Result { + check_posix_regex_errors(pattern_str)?; + + let mut re_string = String::with_capacity(pattern_str.len() + 8); + let mut pattern_chars = pattern_str.chars().peekable(); + let mut prev = '\0'; + let mut prev_is_escaped = false; + let mut in_bracket = false; + let mut is_start_of_expression = true; + let mut after_anchor_caret = false; + + if anchored && pattern_chars.peek() != Some(&'^') { + re_string.push('^'); + } + + while let Some(curr) = pattern_chars.next() { + let curr_is_escaped = prev == '\\' && !prev_is_escaped; + let mut next_is_start = false; + let mut next_after_anchor = false; + + if in_bracket { + if curr == '[' && pattern_chars.peek() == Some(&':') { + pattern_chars.next(); + let mut name = String::new(); + let mut closed = false; + while let Some(c) = pattern_chars.next() { + if c == ':' && pattern_chars.peek() == Some(&']') { + pattern_chars.next(); + closed = true; + break; + } + name.push(c); + } + if closed { + if let Some(unicode_class) = map_posix_class(&name) { + re_string.push_str(unicode_class); + prev = ']'; + prev_is_escaped = false; + continue; + } + return Err(RegexError::InvalidCharacterClassName); + } + re_string.push_str("[:"); + re_string.push_str(&name); + prev = name.chars().last().unwrap_or(':'); + prev_is_escaped = false; + continue; + } + if curr == ']' && re_string.ends_with(|c| c != '\\' && c != '[' && c != '^') { + in_bracket = false; + } + re_string.push(curr); + prev = curr; + prev_is_escaped = false; + continue; + } + + match curr { + '[' if !curr_is_escaped => { + in_bracket = true; + re_string.push('['); + } + // In BRE, '(', ')', '|', '+', '?', '{', '}' are literal by default, + // and become operators only when escaped. ERE has the exact opposite convention. + '(' | ')' | '|' | '+' | '?' | '{' | '}' => { + if curr_is_escaped { + if re_string.ends_with('\\') { + re_string.pop(); + } + match curr { + '(' | '|' => { + re_string.push(curr); + next_is_start = true; + } + '+' | '?' => { + if is_start_of_expression || after_anchor_caret { + re_string.push('\\'); + } + re_string.push(curr); + } + '{' => { + // Handle '{' literally at the start of an expression + if is_start_of_expression || after_anchor_caret { + re_string.push_str(r"\{"); + } else { + // Check if the following section is a valid range quantifier + verify_range_quantifier(&pattern_chars)?; + re_string.push('{'); + // Set the lower bound of range quantifier to 0 if it is missing + if pattern_chars.peek() == Some(&',') { + re_string.push('0'); + } + } + } + _ => re_string.push(curr), // ')' and '}' + } + } else { + // Unescaped metacharacter in BRE -> literal in ERE + re_string.push('\\'); + re_string.push(curr); + } + } + '*' => { + if curr_is_escaped { + re_string.push('*'); + } else if is_start_of_expression || after_anchor_caret { + re_string.push_str(r"\*"); + } else { + re_string.push('*'); + } + } + // Character class negation "[^a]" + // Explicitly escaped caret "\^" + '^' => { + if curr_is_escaped { + re_string.push('^'); + } else if is_start_of_expression { + re_string.push('^'); + next_after_anchor = true; + } else if prev == '[' && !prev_is_escaped { + re_string.push('^'); + } else { + re_string.push_str(r"\^"); + } + } + '$' if !curr_is_escaped && !is_end_of_expression(&pattern_chars) => { + re_string.push_str(r"\$"); + } + '`' if curr_is_escaped => { + if re_string.ends_with('\\') { + re_string.pop(); + } + re_string.push_str(r"\A"); + } + '\'' if curr_is_escaped => { + if re_string.ends_with('\\') { + re_string.pop(); + } + re_string.push_str(r"\z"); + } + '<' if curr_is_escaped => { + if re_string.ends_with('\\') { + re_string.pop(); + } + re_string.push_str(r"\b(?=\w)"); + } + '>' if curr_is_escaped => { + if re_string.ends_with('\\') { + re_string.pop(); + } + re_string.push_str(r"\b(?<=\w)"); + } + '\\' if !curr_is_escaped => { + if pattern_chars.peek().is_none() { + return Err(RegexError::TrailingBackslash); + } + // Carry the expression-start / after-anchor state over the + // backslash so the escaped character is still treated as the + // first token of a (sub)expression. + next_is_start = is_start_of_expression; + next_after_anchor = after_anchor_caret; + re_string.push('\\'); + } + _ => { + if curr_is_escaped + && !"123456789.*^$[]\\wWsSbB".contains(curr) + && re_string.ends_with('\\') + { + re_string.pop(); + } + re_string.push(curr); + } + } + + is_start_of_expression = next_is_start; + after_anchor_caret = next_after_anchor; + prev_is_escaped = curr_is_escaped; + prev = curr; + } + Ok(re_string) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_bre_groups() { + assert_eq!(bre_to_ere(r"foo\(bar\)baz", false).unwrap(), "foo(bar)baz"); + } + + #[test] + fn test_bre_intervals() { + assert_eq!(bre_to_ere(r"ab\{1,3\}c", false).unwrap(), "ab{1,3}c"); + assert_eq!(bre_to_ere(r"ab\{,3\}c", false).unwrap(), "ab{0,3}c"); + } + + #[test] + fn test_bre_alternation() { + assert_eq!(bre_to_ere(r"foo\|bar", false).unwrap(), "foo|bar"); + } + + #[test] + fn test_bre_literal_specials() { + assert_eq!( + bre_to_ere("a+b?c|d(e)f{g}", false).unwrap(), + r"a\+b\?c\|d\(e\)f\{g\}" + ); + } + + #[test] + fn test_bre_anchors() { + assert_eq!(bre_to_ere("^foo$", false).unwrap(), "^foo$"); + assert_eq!(bre_to_ere("a^b$c", false).unwrap(), r"a\^b\$c"); + } + + #[test] + fn test_bre_word_boundaries() { + assert_eq!( + bre_to_ere(r"\", false).unwrap(), + r"\b(?=\w)word\b(?<=\w)" + ); + } + + #[test] + fn test_bre_buffer_anchors() { + assert_eq!(bre_to_ere(r"\`start", false).unwrap(), r"\Astart"); + assert_eq!(bre_to_ere(r"end\'", false).unwrap(), r"end\z"); + } + + #[test] + fn test_bre_posix_classes() { + assert_eq!( + bre_to_ere(r"[[:alpha:]]", false).unwrap(), + r"[\p{Alphabetic}]" + ); + assert_eq!(bre_to_ere(r"[[:digit:]]", false).unwrap(), r"[0-9]"); + assert_eq!( + bre_to_ere(r"[^[:lower:]]", false).unwrap(), + r"[^\p{Lowercase}]" + ); + assert_eq!( + bre_to_ere(r"[[:alpha:][:digit:]]", false).unwrap(), + r"[\p{Alphabetic}0-9]" + ); + assert_eq!(bre_to_ere(r"[]a]", false).unwrap(), "[]a]"); + assert_eq!(bre_to_ere(r"[^]a]", false).unwrap(), "[^]a]"); + + // Unicode match test: [[:alpha:]] matches 'é' + let transpiled = bre_to_ere(r"[[:alpha:]]", true).unwrap(); + let re = Regex::new(&format!("(?s){transpiled}")).unwrap(); + assert!(re.is_match("é").unwrap()); + } + + #[test] + fn test_bre_invalid_posix_class() { + assert_eq!( + bre_to_ere(r"[[:bogus:]]", false).unwrap_err(), + RegexError::InvalidCharacterClassName + ); + assert_eq!( + bre_to_ere(r"[[:123:]]", false).unwrap_err(), + RegexError::InvalidCharacterClassName + ); + } + + #[test] + fn test_bre_backrefs() { + assert_eq!(bre_to_ere(r"\(foo\)\1", false).unwrap(), r"(foo)\1"); + } + + #[test] + fn test_bre_leading_quantifier() { + assert_eq!(bre_to_ere("*foo", false).unwrap(), r"\*foo"); + assert_eq!(bre_to_ere(r"\(*foo\)", false).unwrap(), r"(\*foo)"); + assert_eq!(bre_to_ere(r"a\|*b", false).unwrap(), r"a|\*b"); + } + + #[test] + fn test_bre_anchored() { + assert_eq!(bre_to_ere("abc", true).unwrap(), "^abc"); + assert_eq!(bre_to_ere("^abc", true).unwrap(), "^abc"); + assert_eq!(bre_to_ere(r"\(foo\)", true).unwrap(), "^(foo)"); + } + + #[test] + fn test_posix_errors() { + assert_eq!( + bre_to_ere(r"\(foo", false).unwrap_err(), + RegexError::UnmatchedOpeningParenthesis + ); + assert_eq!( + bre_to_ere(r"foo\)", false).unwrap_err(), + RegexError::UnmatchedClosingParenthesis + ); + assert_eq!( + bre_to_ere(r"foo\", false).unwrap_err(), + RegexError::TrailingBackslash + ); + assert_eq!( + bre_to_ere(r"foo\{1", false).unwrap_err(), + RegexError::UnmatchedOpeningBrace + ); + assert_eq!( + bre_to_ere(r"foo\{5,2\}", false).unwrap_err(), + RegexError::InvalidBracketContent + ); + } + + #[test] + fn test_leftmost_longest_semantics() { + // POSIX leftmost-longest requires (a|ab) against "ab" to match "ab" (longest), not "a". + let transpiled = bre_to_ere(r"\(a\|ab\)", false).unwrap(); + let re = RegexBuilder::new(&transpiled) + .oniguruma_mode(true) + .leftmost_longest(true) + .seek(true) + .build() + .unwrap(); + + let caps = re.captures("ab").unwrap().expect("should match"); + assert_eq!(caps.get(1).unwrap().as_str(), "ab"); + + // "aaaaa|a*" against "aaaaaa" should match all 6 "a"s + let transpiled2 = bre_to_ere(r"aaaaa\|a*", false).unwrap(); + let re2 = RegexBuilder::new(&transpiled2) + .oniguruma_mode(true) + .leftmost_longest(true) + .seek(true) + .build() + .unwrap(); + + let m = re2.find("aaaaaa").unwrap().expect("should match"); + assert_eq!(m.as_str(), "aaaaaa"); + } + + #[test] + fn check_regex_valid() { + assert!(check_posix_regex_errors(r"(a+b) \(a* b\)").is_ok()); + } + + #[test] + fn check_regex_simple_repeating_pattern() { + assert!(check_posix_regex_errors(r"\(a+b\)\{4\}").is_ok()); + } + + #[test] + fn check_regex_missing_closing() { + assert_eq!( + check_posix_regex_errors(r"\(abc"), + Err(RegexError::UnmatchedOpeningParenthesis) + ); + } + + #[test] + fn check_regex_missing_opening() { + assert_eq!( + check_posix_regex_errors(r"abc\)"), + Err(RegexError::UnmatchedClosingParenthesis) + ); + } + + #[test] + fn test_is_valid_range_quantifier() { + assert!(verify_range_quantifier(&"3\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&"3,\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&",6\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&"3,6\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&",\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&"32767\\}anything".chars()).is_ok()); + assert_eq!( + verify_range_quantifier(&"\\{3,6\\}".chars()), + Err(RegexError::InvalidBracketContent) + ); + assert_eq!( + verify_range_quantifier(&"\\}".chars()), + Err(RegexError::InvalidBracketContent) + ); + assert_eq!( + verify_range_quantifier(&"".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&"3".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&"3,".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&",6".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&"3,6".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&",".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&"32768\\}".chars()), + Err(RegexError::TooBigRangeQuantifierIndex) + ); + } +} diff --git a/src/uucore/src/lib/lib.rs b/src/uucore/src/lib/lib.rs index 7f58e384db1..b9451bf8229 100644 --- a/src/uucore/src/lib/lib.rs +++ b/src/uucore/src/lib/lib.rs @@ -90,6 +90,8 @@ pub use crate::features::parser; pub use crate::features::quoting_style; #[cfg(feature = "ranges")] pub use crate::features::ranges; +#[cfg(feature = "regex")] +pub use crate::features::regex; #[cfg(feature = "ringbuffer")] pub use crate::features::ringbuffer; #[cfg(feature = "sum")] diff --git a/tests/by-util/test_expr.rs b/tests/by-util/test_expr.rs index 9cb2a118d9c..481284fa7b2 100644 --- a/tests/by-util/test_expr.rs +++ b/tests/by-util/test_expr.rs @@ -505,6 +505,47 @@ fn test_regex_leftmost_longest_match_semantics() { .args(&["ab", ":", "a\\|ab"]) .succeeds() .stdout_only("2\n"); + + new_ucmd!() + .args(&["aaaaaa", ":", "aaaaa\\|a*"]) + .succeeds() + .stdout_only("6\n"); + + new_ucmd!() + .args(&["ab", ":", "\\(a\\|ab\\)"]) + .succeeds() + .stdout_only("ab\n"); + + new_ucmd!() + .args(&["aaaaaa", ":", "\\(aaaaa\\|a*\\)"]) + .succeeds() + .stdout_only("aaaaaa\n"); +} + +#[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data, every locale is C")] +fn test_regex_posix_character_classes() { + new_ucmd!() + .args(&["é", ":", "[[:alpha:]]"]) + .succeeds() + .stdout_only("1\n"); + + new_ucmd!() + .env("LC_ALL", "en_US.UTF-8") + .args(&["é", ":", "[[:lower:]]"]) + .succeeds() + .stdout_only("1\n"); + + new_ucmd!() + .env("LC_ALL", "en_US.UTF-8") + .args(&["É", ":", "[[:upper:]]"]) + .succeeds() + .stdout_only("1\n"); + + new_ucmd!() + .args(&["a", ":", "[[:bogus:]]"]) + .fails_with_code(2) + .stderr_contains("Invalid character class name"); } #[test] From 7a8eb708c2b0e950e75659cd510caa4ee05a9764 Mon Sep 17 00:00:00 2001 From: CodSpeed Bot Date: Wed, 16 Sep 2026 14:23:06 +0000 Subject: [PATCH 2/2] uucore: reach the error-only strings only for the ids they define The error-only resource is parsed on the first lookup that misses every ordinary bundle. That is not only an error path: a binary that cannot resolve its own strings misses on every id it asks for, and a bench binary calling `uumain` directly is exactly that. Every benchmark in the tree therefore parsed the whole resource -- 106k of the 319k instructions hostname_basic measures, and 16 allocations it never frees. Growing the resource consequently charged every benchmark, whether or not the utility can reach the new strings. The eight regex messages added here moved hostname_basic from 318,823 to 335,034 instructions (+5.1%) and every utility's peak memory by 4.2 KB, in utilities that never compile a regex. Generate the ids the resource defines at build time and check that before building the bundle, so a lookup that cannot be in there costs a match instead of a parse: hostname_basic instructions main (94811cc) 318823 this branch 335034 this branch, this commit 212632 Measured with `codspeed run --mode simulation -- cargo codspeed run -p uu_hostname`. The expr benchmarks are unchanged (1,448,685 against 1,448,761 on main), and expr still reports its regex diagnostics, in English and in French. --- src/uucore/build.rs | 85 +++++++++++++++++++++++++++++++ src/uucore/src/lib/mods/locale.rs | 9 ++++ 2 files changed, 94 insertions(+) diff --git a/src/uucore/build.rs b/src/uucore/build.rs index 5d968159706..ae2c036349a 100644 --- a/src/uucore/build.rs +++ b/src/uucore/build.rs @@ -74,10 +74,76 @@ pub fn main() -> Result<(), Box> { writeln!(embedded_file, " }}")?; writeln!(embedded_file, "}}")?; + generate_error_locale_ids(&mut embedded_file)?; + embedded_file.flush()?; Ok(()) } +/// Generate the lookup telling whether an id belongs to the error-only strings. +/// +/// Reaching those strings costs a parse of the whole resource, so it is only +/// worth doing for an id the resource actually defines. Which ids those are is +/// known at compile time, so the check itself costs nothing. +/// +/// # Errors +/// +/// Returns an error if `CARGO_MANIFEST_DIR` is not set, if the resource exists +/// but cannot be read, or if writing to `embedded_file` fails. +fn generate_error_locale_ids(embedded_file: &mut File) -> Result<(), Box> { + use std::fs; + + let path = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).join("locales/errors/en-US.ftl"); + let content = if path.is_file() { + println!("cargo:rerun-if-changed={}", path.display()); + fs::read_to_string(&path)? + } else { + String::new() + }; + let ids = message_ids(&content); + + writeln!(embedded_file)?; + writeln!( + embedded_file, + "/// Whether the error-only strings define `id`." + )?; + writeln!(embedded_file, "fn is_error_locale_id(id: &str) -> bool {{")?; + if ids.is_empty() { + writeln!(embedded_file, " let _ = id;")?; + writeln!(embedded_file, " false")?; + } else { + writeln!(embedded_file, " matches!(")?; + writeln!(embedded_file, " id,")?; + for (i, id) in ids.iter().enumerate() { + let separator = if i == 0 { ' ' } else { '|' }; + writeln!(embedded_file, " {separator} \"{id}\"")?; + } + writeln!(embedded_file, " )")?; + } + writeln!(embedded_file, "}}")?; + + Ok(()) +} + +/// The ids of the messages `content` defines, in the order they appear. +fn message_ids(content: &str) -> Vec<&str> { + content + .lines() + .filter_map(|line| { + let id = line.split_once('=')?.0.trim_end(); + // A message starts in the first column: an indented line continues + // the message before it, and a leading '-' makes it a term, which + // no lookup ever asks for by name. + id.starts_with(|c: char| c.is_ascii_alphanumeric()) + .then_some(id) + .filter(|id| { + id.chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') + }) + }) + .collect() +} + /// Get the project root directory /// /// # Errors @@ -480,6 +546,25 @@ where mod tests { use super::*; + #[test] + fn message_ids_are_the_ones_starting_a_line() { + let content = concat!( + "# a comment = not a message\n", + "\n", + "first-id = a value\n", + "second-id = a value with an = in it\n", + " a continuation = still the second value\n", + "-a-term = not asked for by name\n", + "third_id = another value\n", + ); + + assert_eq!( + message_ids(content), + vec!["first-id", "second-id", "third_id"] + ); + assert!(message_ids("").is_empty()); + } + #[test] fn consumer_lock_file_found_above_target_dir() { let tmp = std::env::temp_dir().join(format!("uucore-build-{}", std::process::id())); diff --git a/src/uucore/src/lib/mods/locale.rs b/src/uucore/src/lib/mods/locale.rs index dac7bff0ddd..be6667b81be 100644 --- a/src/uucore/src/lib/mods/locale.rs +++ b/src/uucore/src/lib/mods/locale.rs @@ -125,6 +125,15 @@ fn errors_message( id: &str, args: Option<&FluentArgs>, ) -> Option { + // Only an id this resource defines is worth that parse. Lookups reach here + // whenever they miss every ordinary bundle, which is not only error paths: + // a binary that cannot find its own strings misses on all of them, and a + // bench binary calling `uumain` directly is exactly that. The ids come + // from the English resource, the one every locale is a translation of. + if !is_error_locale_id(id) { + return None; + } + ERRORS_BUNDLE.with(|cell| { let bundle = cell.get_or_init(|| build_errors_bundle(locales)).as_ref()?; let message = bundle.get_message(id)?.value()?;