diff --git a/aw-query/src/datatype.rs b/aw-query/src/datatype.rs index 847129bc0..dcbc73afb 100644 --- a/aw-query/src/datatype.rs +++ b/aw-query/src/datatype.rs @@ -427,6 +427,26 @@ impl TryFrom<&DataType> for Rule { if rtype == "none" { Ok(Self::None) } else if rtype == "regex" { + let ignore_case_val = match obj.get("ignore_case") { + Some(case_val) => case_val, + None => &DataType::Bool(false), + }; + let ignore_case = match ignore_case_val { + DataType::Bool(b) => b, + _ => { + return Err(QueryError::InvalidFunctionParameters( + "the ignore_case field of the regex rule is not a bool".to_string(), + )) + } + }; + // Option B for aw-webui#939: per-field patterns on the existing + // `regex` type instead of a new rule type. A `fields` rule carries + // no `regex` of its own, so a server that predates it either rejects + // the rule or (with an empty `regex`) matches nothing for it, rather + // than misclassifying events. + if obj.contains_key("fields") { + return parse_fields_rule(obj, *ignore_case); + } let regex_val = match obj.get("regex") { Some(regex_val) => regex_val, None => { @@ -443,18 +463,6 @@ impl TryFrom<&DataType> for Rule { )) } }; - let ignore_case_val = match obj.get("ignore_case") { - Some(case_val) => case_val, - None => &DataType::Bool(false), - }; - let ignore_case = match ignore_case_val { - DataType::Bool(b) => b, - _ => { - return Err(QueryError::InvalidFunctionParameters( - "the ignore_case field of the regex rule is not a bool".to_string(), - )) - } - }; let select_keys = match obj.get("select_keys") { Some(DataType::List(keys)) => { let mut select_keys = Vec::with_capacity(keys.len()); @@ -492,3 +500,95 @@ impl TryFrom<&DataType> for Rule { } } } + +/// Parse a `regex` rule that carries a per-field `fields` map (Option B for +/// aw-webui#939), for example: +/// +/// ```text +/// { "type": "regex", "fields": { "app": "mstsc", "title": "office" } } +/// ``` +/// +/// Every named field must be present, a string, and match its own pattern +/// (unanchored, like a plain `regex`). `regex` and `select_keys` are rejected +/// when set: silently ignoring them would turn a rule that older servers read +/// as a single-pattern rule into something else. An empty `regex`/`select_keys` +/// counts as unset, matching aw-core's Python `Rule` and keeping such a rule +/// fail-closed on servers that predate the field map. +fn parse_fields_rule( + obj: &HashMap, + ignore_case: bool, +) -> Result { + match obj.get("regex") { + None => {} + Some(DataType::String(s)) if s.is_empty() => {} + Some(DataType::String(_)) => { + return Err(QueryError::InvalidFunctionParameters( + "a regex rule with 'fields' must not also carry a 'regex' pattern \ + (an empty string counts as unset)" + .to_string(), + )) + } + Some(_) => { + return Err(QueryError::InvalidFunctionParameters( + "the regex field of the regex rule is not a string".to_string(), + )) + } + } + match obj.get("select_keys") { + None => {} + Some(DataType::List(keys)) if keys.is_empty() => {} + Some(DataType::List(_)) => { + return Err(QueryError::InvalidFunctionParameters( + "a regex rule with 'fields' must not also carry 'select_keys' \ + (an empty list counts as unset)" + .to_string(), + )) + } + Some(_) => { + return Err(QueryError::InvalidFunctionParameters( + "the select_keys field of the regex rule is not a list".to_string(), + )) + } + } + let fields = match obj.get("fields") { + Some(DataType::Dict(fields)) => fields, + Some(_) => { + return Err(QueryError::InvalidFunctionParameters( + "the fields of the regex rule is not a dict".to_string(), + )) + } + None => { + return Err(QueryError::InvalidFunctionParameters( + "a regex rule with 'fields' is missing the 'fields' map".to_string(), + )) + } + }; + if fields.is_empty() { + return Err(QueryError::InvalidFunctionParameters( + "the fields of the regex rule must not be empty".to_string(), + )); + } + let mut patterns = HashMap::with_capacity(fields.len()); + for (field, pattern) in fields { + let pattern = match pattern { + DataType::String(pattern) if !pattern.is_empty() => pattern.clone(), + DataType::String(_) => { + return Err(QueryError::InvalidFunctionParameters(format!( + "the pattern for field '{field}' in the regex rule is empty" + ))) + } + _ => { + return Err(QueryError::InvalidFunctionParameters(format!( + "the pattern for field '{field}' in the regex rule is not a string" + ))) + } + }; + patterns.insert(field.clone(), pattern); + } + match RegexRule::from_fields(patterns, ignore_case) { + Ok(rule) => Ok(Rule::Regex(rule)), + Err(err) => Err(QueryError::RegexCompileError(format!( + "Failed to compile a pattern in the 'fields' map of the regex rule: {err:?}" + ))), + } +} diff --git a/aw-query/tests/query.rs b/aw-query/tests/query.rs index dc7e6d3d4..b66443ff1 100644 --- a/aw-query/tests/query.rs +++ b/aw-query/tests/query.rs @@ -657,6 +657,126 @@ mod query_tests { assert_eq!(tags, &serde_json::json!(vec!["title-match"])); } + #[test] + fn test_tag_fields_map() { + // Option B for ActivityWatch/aw-webui#939: per-field patterns on the + // existing `regex` rule, ANDed. mstsc and winbox show the same host in + // their titles, so only app AND title distinguish them. + let ds = setup_datastore_with_bucket(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + let event = |app: &str, title: &str| Event { + id: None, + timestamp: chrono::Utc::now(), + duration: Duration::seconds(1), + data: json_map! { "app": json!(app), "title": json!(title) }, + }; + ds.insert_events( + BUCKET_ID, + &[ + event("mstsc.exe", "office.example.com - Remote Desktop"), + event("winbox64.exe", "admin@office.example.com"), + event("firefox", "office.example.com"), + ], + ) + .unwrap(); + + let code = format!( + r#" + events = query_bucket("{}"); + events = tag(events, [["rdp", {{ "type": "regex", "fields": {{ "app": "mstsc.exe", "title": "office.example" }} }}], ["winbox", {{ "type": "regex", "fields": {{ "app": "winbox", "title": "office.example" }} }}]]); + return events;"#, + BUCKET_ID + ); + let result: DataType = aw_query::query(&code, &interval, &ds).unwrap(); + let events: Vec = Vec::try_from(&result).unwrap(); + assert_eq!(events.len(), 3); + + // `query_bucket` returns events newest-first, so key by app instead of + // relying on insertion order. + let tags_by_app: std::collections::HashMap = events + .iter() + .map(|e| { + ( + e.data.get("app").unwrap().as_str().unwrap().to_string(), + e.data.get("$tags").cloned().unwrap(), + ) + }) + .collect(); + assert_eq!(tags_by_app["mstsc.exe"], serde_json::json!(vec!["rdp"])); + assert_eq!( + tags_by_app["winbox64.exe"], + serde_json::json!(vec!["winbox"]) + ); + assert_eq!( + tags_by_app["firefox"], + serde_json::json!(Vec::::new()) + ); + + // Empty `regex`/`select_keys` count as unset, so a config that serializes + // them next to `fields` still parses (and stays fail-closed on servers + // that predate the field map, which read the empty regex as no match). + let code = format!( + r#" + events = query_bucket("{}"); + events = tag(events, [["rdp", {{ "type": "regex", "regex": "", "select_keys": [], "fields": {{ "app": "mstsc" }} }}]]); + return events;"#, + BUCKET_ID + ); + let result: DataType = aw_query::query(&code, &interval, &ds).unwrap(); + let events: Vec = Vec::try_from(&result).unwrap(); + let rdp_tags: Vec<_> = events + .iter() + .filter(|e| e.data.get("app").unwrap().as_str() == Some("mstsc.exe")) + .map(|e| e.data.get("$tags").cloned().unwrap()) + .collect(); + assert_eq!(rdp_tags, vec![serde_json::json!(vec!["rdp"])]); + } + + #[test] + fn test_rule_parsing_fields_map() { + let ds = setup_datastore_populated(); + let interval = TimeInterval::new_from_string(TIME_INTERVAL).unwrap(); + + // A fields rule may not smuggle in a single pattern: older servers read + // only `regex`, so silently ignoring it would change what the rule means. + for member in [r#""regex": "value""#, r#""select_keys": ["app"]"#] { + let code = format!( + r#" + events = []; + events = tag(events, [["t", {{ "type": "regex", {member}, "fields": {{ "app": "value" }} }}]]); + return events;"# + ); + let res = aw_query::query(&code, &interval, &ds); + assert_err_type!(res, QueryError::InvalidFunctionParameters(_)); + } + + // Malformed or empty field maps are rejected rather than matching all. + for fields in [ + r#""fields": []"#, + r#""fields": {}"#, + r#""fields": { "app": 5 }"#, + r#""fields": { "app": "" }"#, + ] { + let code = format!( + r#" + events = []; + events = tag(events, [["t", {{ "type": "regex", {fields} }}]]); + return events;"# + ); + let res = aw_query::query(&code, &interval, &ds); + assert_err_type!(res, QueryError::InvalidFunctionParameters(_)); + } + + // An uncompilable per-field pattern is reported as a regex error. + let code = r#" + events = []; + events = tag(events, [["t", { "type": "regex", "fields": { "app": "!#¤%&/(=" } }]]); + return events;"#; + let res = aw_query::query(code, &interval, &ds); + assert_err_type!(res, QueryError::RegexCompileError(_)); + } + #[test] fn test_rule_parsing() { let ds = setup_datastore_populated(); diff --git a/aw-transform/src/classify.rs b/aw-transform/src/classify.rs index df42275da..38bd07f02 100644 --- a/aw-transform/src/classify.rs +++ b/aw-transform/src/classify.rs @@ -36,7 +36,10 @@ impl RuleTrait for Rule { fn needs_values(&self) -> bool { match self { Rule::None => false, - Rule::Regex(regex_rule) => regex_rule.select_keys.is_none(), + // Delegate so a `fields` rule is classified from the same place the + // matching does; duplicating the check here would silently keep + // extracting the shared value set for field rules. + Rule::Regex(regex_rule) => regex_rule.needs_values(), } } } @@ -61,8 +64,13 @@ impl<'a> MatchValues<'a> { } pub struct RegexRule { - regex: Arc, + /// Pattern matched against `select_keys` (or every string value of the + /// event). `None` for a `fields` rule, which carries one pattern per field + /// instead, see [`RegexRule::from_fields`]. + regex: Option>, select_keys: Option>, + /// Per-field patterns, every one of which must match (logical AND). + fields: Option>>, // An empty pattern compiles to a match-all regex, which would silently // categorize every event. aw-core's Python `Rule` deliberately never // matches on an empty regex (see ActivityWatch/aw-webui#1026); mirror @@ -88,34 +96,39 @@ impl RegexRule { } } - let never_matches = regex_str.is_empty(); - - // can't use `RegexBuilder::case_insensitive` because it's not supported by fancy_regex, - // so we need to prefix with `(?i)` to make it case insensitive. - let full_regex_str = if ignore_case { - format!("(?i){regex_str}") - } else { - regex_str.to_string() - }; - - let cache = REGEX_CACHE.get_or_init(|| { - Mutex::new(LruCache::new( - NonZeroUsize::new(REGEX_CACHE_CAPACITY).unwrap(), - )) - }); - let mut cache = cache.lock().unwrap(); - - let regex = if let Some(re) = cache.get(&full_regex_str) { - re.clone() - } else { - let re = Arc::new(Regex::new(&full_regex_str)?); - cache.put(full_regex_str.clone(), re.clone()); - re - }; - Ok(RegexRule { - regex, + regex: Some(compile_pattern(regex_str, ignore_case)?), select_keys, + fields: None, + never_matches: regex_str.is_empty(), + }) + } + + /// A rule that tests each named field against its own pattern and requires + /// all of them to match (logical AND) — `{"app": …, "title": …}`. + /// + /// Each per-field pattern keeps the unanchored `search` semantics of a plain + /// `regex` rule (`^…$` is how an exact match is asked for), so a single + /// field matches exactly what the equivalent `select_keys` rule would. + /// Field values must be strings: a missing or non-string value fails the + /// whole rule. + /// + /// An empty map or an empty pattern would match every event, so such a rule + /// is built in the never-matching state instead. + pub fn from_fields( + fields: HashMap, + ignore_case: bool, + ) -> Result { + let mut never_matches = fields.is_empty(); + let mut compiled = HashMap::with_capacity(fields.len()); + for (field, pattern) in &fields { + never_matches |= pattern.is_empty(); + compiled.insert(field.clone(), compile_pattern(pattern, ignore_case)?); + } + Ok(RegexRule { + regex: None, + select_keys: None, + fields: Some(compiled), never_matches, }) } @@ -124,11 +137,60 @@ impl RegexRule { if self.never_matches { return false; } + // A `fields` rule never reaches the select_keys/all-values path. + let Some(regex) = self.regex.as_ref() else { + return false; + }; match value.as_str() { - Some(value) => self.regex.is_match(value).unwrap_or(false), + Some(value) => regex.is_match(value).unwrap_or(false), None => false, } } + + /// Whether every field named by a `fields` rule is present, is a string, + /// and matches its pattern. + fn fields_match(&self, event: &Event) -> bool { + let Some(fields) = self.fields.as_ref() else { + return false; + }; + if self.never_matches || fields.is_empty() { + return false; + } + fields.iter().all(|(field, regex)| { + event + .data + .get(field) + .and_then(|value| value.as_str()) + .is_some_and(|value| regex.is_match(value).unwrap_or(false)) + }) + } +} + +/// Compile `pattern`, reusing the shared cache. +/// +/// Can't use `RegexBuilder::case_insensitive` because it's not supported by +/// fancy_regex, so `ignore_case` is expressed by prefixing `(?i)`. The cache key +/// is the prepared pattern, so both cases get one entry each. +fn compile_pattern(pattern: &str, ignore_case: bool) -> Result, fancy_regex::Error> { + let full_regex_str = if ignore_case { + format!("(?i){pattern}") + } else { + pattern.to_string() + }; + + let cache = REGEX_CACHE.get_or_init(|| { + Mutex::new(LruCache::new( + NonZeroUsize::new(REGEX_CACHE_CAPACITY).unwrap(), + )) + }); + let mut cache = cache.lock().unwrap(); + + if let Some(re) = cache.get(&full_regex_str) { + return Ok(re.clone()); + } + let re = Arc::new(Regex::new(&full_regex_str)?); + cache.put(full_regex_str, re.clone()); + Ok(re) } /// This struct defines the rules for classification. @@ -141,6 +203,9 @@ impl RuleTrait for RegexRule { if self.never_matches { return false; } + if self.fields.is_some() { + return self.fields_match(event); + } match &self.select_keys { Some(select_keys) => select_keys .iter() @@ -148,15 +213,20 @@ impl RuleTrait for RegexRule { .any(|val| self.value_matches(val)), // `values` holds the same strings (in the same order) that the // previous `event.data.values()` iteration produced. - None => values - .0 - .iter() - .any(|value| self.regex.is_match(value).unwrap_or(false)), + None => match self.regex.as_ref() { + Some(regex) => values + .0 + .iter() + .any(|value| regex.is_match(value).unwrap_or(false)), + // A `fields` rule has no single pattern and is handled above. + None => false, + }, } } fn needs_values(&self) -> bool { - self.select_keys.is_none() + // A `fields` rule reads its named keys, not the shared value set. + self.select_keys.is_none() && self.fields.is_none() } } @@ -167,8 +237,9 @@ impl From for Rule { // cannot silently match every event. let never_matches = re.as_str().is_empty(); Rule::Regex(RegexRule { - regex: Arc::new(re), + regex: Some(Arc::new(re)), select_keys: None, + fields: None, never_matches, }) } @@ -825,9 +896,28 @@ fn naive_rule_matches(rule: &Rule, event: &Event) -> bool { match rule { Rule::None => false, Rule::Regex(r) => { + if r.never_matches { + return false; + } + if let Some(fields) = &r.fields { + if fields.is_empty() { + return false; + } + return fields.iter().all(|(key, regex)| { + event + .data + .get(key) + .and_then(|v| v.as_str()) + .map(|s| regex.is_match(s).unwrap_or(false)) + .unwrap_or(false) + }); + } + let Some(regex) = r.regex.as_ref() else { + return false; + }; let matches_value = |v: &serde_json::Value| { v.as_str() - .map(|s| r.regex.is_match(s).unwrap_or(false)) + .map(|s| regex.is_match(s).unwrap_or(false)) .unwrap_or(false) }; match &r.select_keys { @@ -885,12 +975,25 @@ fn test_categorize_matches_naive_best_of_all() { for _ in 0..rule_count { let pattern = PATTERNS[rng.below(PATTERNS.len() as u64) as usize]; let ignore_case = rng.below(2) == 0; - let select_keys = if rng.below(2) == 0 { - None + let rule = if rng.below(4) == 0 { + // Per-field (AND) rules too; `pid` is numeric, so a field rule + // naming it must never match — the oracle covers that case. + let field_count = 1 + rng.below(2) as usize; + let mut fields = HashMap::new(); + for _ in 0..field_count { + let key = KEYS[rng.below(KEYS.len() as u64) as usize]; + let pattern = PATTERNS[rng.below(PATTERNS.len() as u64) as usize]; + fields.insert(key.to_string(), pattern.to_string()); + } + Rule::Regex(RegexRule::from_fields(fields, ignore_case).unwrap()) } else { - Some(vec![KEYS[rng.below(KEYS.len() as u64) as usize].to_string()]) + let select_keys = if rng.below(2) == 0 { + None + } else { + Some(vec![KEYS[rng.below(KEYS.len() as u64) as usize].to_string()]) + }; + Rule::Regex(RegexRule::new(pattern, ignore_case, select_keys).unwrap()) }; - let rule = Rule::Regex(RegexRule::new(pattern, ignore_case, select_keys).unwrap()); let depth = 1 + rng.below(3) as usize; let category: Vec = (0..depth).map(|i| format!("Cat{case}_{i}")).collect(); let mut cr = CategoryRule::new(category, rule); @@ -939,3 +1042,189 @@ fn test_categorize_matches_naive_best_of_all() { ); } } + +#[cfg(test)] +fn event_with_app_title(app: &str, title: &str) -> Event { + let mut e = Event::default(); + e.data.insert("app".into(), serde_json::json!(app)); + e.data.insert("title".into(), serde_json::json!(title)); + e +} + +#[cfg(test)] +fn fields_rule(fields: &[(&str, &str)], ignore_case: bool) -> Rule { + let fields = fields + .iter() + .map(|(key, pattern)| (key.to_string(), pattern.to_string())) + .collect::>(); + Rule::Regex(RegexRule::from_fields(fields, ignore_case).unwrap()) +} + +#[test] +fn test_rule_fields_split_same_title_by_app() { + // Reported case (ActivityWatch/aw-webui#939): mstsc and winbox can show the + // same host in their window titles, so only app AND title tell them apart. + let events = vec![ + event_with_app_title("mstsc.exe", "office.example.com - Remote Desktop"), + event_with_app_title("winbox64.exe", "admin@office.example.com"), + event_with_app_title("firefox", "office.example.com"), + ]; + let out = categorize( + events.clone(), + &[ + CategoryRule::new( + vec!["Remote".into(), "RDP".into()], + fields_rule( + &[("app", r"mstsc\.exe"), ("title", r"office\.example")], + false, + ), + ), + CategoryRule::new( + vec!["Remote".into(), "Winbox".into()], + fields_rule(&[("app", "winbox"), ("title", r"office\.example")], false), + ), + ], + ); + assert_eq!(category_of(&out), &serde_json::json!(vec!["Remote", "RDP"])); + assert_eq!( + category_of(&out[1..]), + &serde_json::json!(vec!["Remote", "Winbox"]) + ); + assert_eq!( + category_of(&out[2..]), + &serde_json::json!(vec!["Uncategorized"]) + ); + + // `Rule::matches` agrees with the cached/ranked `categorize` path. + let rdp_rule = fields_rule( + &[("app", r"mstsc\.exe"), ("title", r"office\.example")], + false, + ); + let matched: Vec = events.iter().map(|e| rule_matches(&rdp_rule, e)).collect(); + assert_eq!(matched, vec![true, false, false]); +} + +#[test] +fn test_rule_fields_require_present_string_values() { + let rule = fields_rule(&[("app", "mstsc"), ("title", "office")], false); + assert!(rule_matches( + &rule, + &event_with_app_title("mstsc.exe", "office") + )); + assert!(!rule_matches( + &rule, + &event_with_app_title("mstsc.exe", "home") + )); + + // A missing field fails the whole rule. + let mut missing = event_with_app_title("mstsc.exe", "office"); + missing.data.remove("title"); + assert!(!rule_matches(&rule, &missing)); + + // A non-string value fails it too (watchers do store numbers). + let mut numeric = event_with_app_title("mstsc.exe", "office"); + numeric.data.insert("title".into(), serde_json::json!(123)); + assert!(!rule_matches(&rule, &numeric)); +} + +#[test] +fn test_rule_fields_search_semantics_and_ignore_case() { + let event = event_with_app_title("mstsc.exe", "x"); + // Unanchored like a plain `regex` rule; `^...$` is how exact match is asked for. + assert!(rule_matches( + &fields_rule(&[("app", "^mstsc")], false), + &event + )); + assert!(!rule_matches( + &fields_rule(&[("app", "^mstsc$")], false), + &event + )); + assert!(!rule_matches( + &fields_rule(&[("app", "MSTSC")], false), + &event + )); + assert!(rule_matches( + &fields_rule(&[("app", "MSTSC")], true), + &event + )); +} + +#[test] +fn test_rule_fields_single_field_matches_select_keys() { + // A one-field rule with the same pattern must agree with the equivalent + // `select_keys` rule, so migrating an editor config cannot change results. + let events = vec![ + event_with_app_title("mstsc.exe", "office"), + event_with_app_title("firefox", "office"), + event_with_app_title("office", "mstsc"), + ]; + let by_fields = fields_rule(&[("app", "office")], false); + let by_select_keys = + Rule::Regex(RegexRule::new("office", false, Some(vec!["app".into()])).unwrap()); + for event in &events { + assert_eq!( + rule_matches(&by_fields, event), + rule_matches(&by_select_keys, event), + "fields and select_keys disagree on {event:?}" + ); + } +} + +#[test] +fn test_rule_fields_empty_map_or_pattern_never_matches() { + // An empty map is vacuously true and an empty pattern compiles to a + // match-all regex, so neither may match every event. + let event = event_with_app_title("mstsc.exe", "office"); + for rule in [ + fields_rule(&[], false), + fields_rule(&[("app", "")], false), + fields_rule(&[("app", "mstsc"), ("title", "")], false), + ] { + assert!(!rule_matches(&rule, &event)); + } + + let out = categorize( + vec![event], + &[CategoryRule::new( + vec!["Everything".into()], + fields_rule(&[], false), + )], + ); + assert_eq!(category_of(&out), &serde_json::json!(vec!["Uncategorized"])); +} + +#[test] +fn test_rule_fields_work_for_tags() { + let rules = vec![( + "rdp".to_string(), + fields_rule(&[("app", r"mstsc\.exe"), ("title", "office")], false), + )]; + let out = tag( + vec![ + event_with_app_title("mstsc.exe", "office"), + event_with_app_title("winbox64.exe", "office"), + ], + &rules, + ); + assert_eq!(out[0].data.get("$tags"), Some(&serde_json::json!(["rdp"]))); + assert_eq!(out[1].data.get("$tags"), Some(&serde_json::json!([]))); +} + +#[test] +fn test_rule_fields_do_not_need_shared_values() { + // `needs_values` decides whether per-event JSON is flattened into the + // shared value set. A field rule reads named keys directly, so a rule set + // of only field rules must skip that extraction. Regression guard: the + // `Rule` impl used to duplicate the `select_keys` check instead of + // delegating, so adding `fields` changed matching but not this predicate. + let fields = fields_rule(&[("app", "mstsc")], false); + assert!(!fields.needs_values()); + + let plain = Rule::Regex(RegexRule::new("test", false, None).unwrap()); + assert!(plain.needs_values()); + + let selected = Rule::Regex(RegexRule::new("test", false, Some(vec!["app".into()])).unwrap()); + assert!(!selected.needs_values()); + + assert!(!Rule::None.needs_values()); +}