Skip to content

Commit 3191f27

Browse files
committed
Merge remote-tracking branch 'origin/wip/redaction-gaps' into feat/ui-node-errors-job-detail-redaction
2 parents 2ecd551 + bdacec2 commit 3191f27

4 files changed

Lines changed: 127 additions & 16 deletions

File tree

‎crates/sqlbuild-core/src/redaction/_helpers/patterns.rs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ use std::sync::LazyLock;
55
use regex::{Captures, Regex, RegexBuilder};
66

77
use crate::redaction::constants::{
8-
AUTHORIZATION_PATTERN, BARE_USERINFO_PATTERN, BEARER_PATTERN, PATTERN_SIZE_LIMIT,
9-
SECRET_VALUE_PATTERN, URI_CREDENTIAL_PATTERN,
8+
AUTHORIZATION_PATTERN, BARE_USERINFO_PATTERN, BEARER_PATTERN, DDL_PASSWORD_PATTERN,
9+
PATTERN_SIZE_LIMIT, SECRET_VALUE_PATTERN, URI_CREDENTIAL_PATTERN,
1010
};
1111

1212
pub(crate) static AUTHORIZATION: LazyLock<Result<Regex, regex::Error>> =
@@ -15,6 +15,8 @@ pub(crate) static BEARER: LazyLock<Result<Regex, regex::Error>> =
1515
LazyLock::new(|| compiled(BEARER_PATTERN));
1616
pub(crate) static SECRET_VALUE: LazyLock<Result<Regex, regex::Error>> =
1717
LazyLock::new(|| compiled(SECRET_VALUE_PATTERN));
18+
pub(crate) static DDL_PASSWORD: LazyLock<Result<Regex, regex::Error>> =
19+
LazyLock::new(|| compiled(DDL_PASSWORD_PATTERN));
1820
pub(crate) static URI_CREDENTIAL: LazyLock<Result<Regex, regex::Error>> =
1921
LazyLock::new(|| compiled(URI_CREDENTIAL_PATTERN));
2022
static BARE_USERINFO: LazyLock<Result<Regex, regex::Error>> =
@@ -38,15 +40,15 @@ pub(crate) fn substituted(
3840
}
3941
}
4042

41-
/// A cut URI's password redacted; an all-digit value stays, since it may be a port.
43+
/// A port is all digits and the authority ends after it (`/`, `?`, `#`); anything else may be a password.
4244
pub(crate) fn bare_userinfo_redacted(text: &str) -> String {
4345
match BARE_USERINFO.as_ref() {
4446
Ok(regex) => regex
4547
.replace_all(text, |found: &Captures<'_>| {
46-
if found[2].bytes().all(|byte| byte.is_ascii_digit()) {
48+
if !found[3].is_empty() && found[2].bytes().all(|byte| byte.is_ascii_digit()) {
4749
found[0].to_owned()
4850
} else {
49-
format!("{}[REDACTED]", &found[1])
51+
format!("{}[REDACTED]{}", &found[1], &found[3])
5052
}
5153
})
5254
.into_owned(),

‎crates/sqlbuild-core/src/redaction/constants.rs‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,26 +13,36 @@ pub(crate) const SECRET_FIELD_PARTS: [&str; 11] = [
1313
"secret",
1414
"token",
1515
];
16+
/// A key may be glued to a prefix, as in `db_password` or `PGPASSWORD`; `pwd` stands alone so `OLDPWD` and `cwd` keep their paths.
1617
pub(crate) const SECRET_VALUE_PATTERN: &str = concat!(
17-
r"(?i)\b(api[-_]?key|access[-_]?key|private[-_]?key|authorization|",
18+
r"(?i)\b([a-z0-9_.-]*?(?:api[-_]?key|access[-_]?key|private[-_]?key|authorization|",
1819
r"client[-_]?secret|refresh[-_]?token|",
19-
r"password|passwd|pwd|secret|token|dsn|connection_string)",
20+
r"password|passwd|secret|token|dsn|connection_string)|pwd)",
2021
r#"(\s*[=:]\s*)("[^"\r\n]{0,512}"|'[^'\r\n]{0,512}'|[^\s,;]{1,512})"#,
2122
);
2223
pub(crate) const AUTHORIZATION_PATTERN: &str = concat!(
2324
r"(?i)\b(authorization)(\s*[=:]\s*)(?:basic|bearer)\s+",
2425
r#"("[^"\r\n]{1,512}"|'[^'\r\n]{1,512}'|[^\s,;]{1,512})"#,
2526
);
2627
pub(crate) const BEARER_PATTERN: &str = r"(?i)\bbearer\s+[a-z0-9._~+/=-]{1,512}";
27-
pub(crate) const URI_CREDENTIAL_PATTERN: &str =
28-
r"(?i)\b([a-z][a-z0-9+.-]*://[^\s:/@]+:)([^\s@]+)(@)";
28+
/// The password runs to the last `@` before the host, so an unencoded `@` inside it is covered.
29+
pub(crate) const URI_CREDENTIAL_PATTERN: &str = concat!(
30+
r"(?i)\b([a-z][a-z0-9+.-]*://[^\s:/@]+:)([^\s]+?)",
31+
r#"(@[^\s@/?#,;'"]*)([\s/?#,;'"]|$)"#,
32+
);
33+
/// SQL that sets a password: `PASSWORD '…'` (quoted only, so prose such as "password authentication" stays) and `IDENTIFIED [WITH plugin] BY …`.
34+
pub(crate) const DDL_PASSWORD_PATTERN: &str = concat!(
35+
r#"(?i)\b(password)(\s+)("[^"\r\n]{0,512}"|'[^'\r\n]{0,512}')|"#,
36+
r#"\b(identified\s+(?:with\s+\w+\s+)?by)(\s+)("[^"\r\n]{0,512}"|'[^'\r\n]{0,512}'|[^\s,;]{1,512})"#,
37+
);
38+
pub(crate) const DDL_REPLACEMENT: &str = "${1}${2}${4}${5}[REDACTED]";
2939
pub(crate) const PAIR_REPLACEMENT: &str = "${1}${2}[REDACTED]";
3040
pub(crate) const BEARER_REPLACEMENT: &str = "Bearer [REDACTED]";
31-
pub(crate) const URI_REPLACEMENT: &str = "${1}[REDACTED]${3}";
41+
pub(crate) const URI_REPLACEMENT: &str = "${1}[REDACTED]${3}${4}";
3242
/// Compiled size allowance for the bounded Unicode repetitions above.
3343
pub(crate) const PATTERN_SIZE_LIMIT: usize = 1 << 28;
34-
/// `scheme://user:secret` with no `@` after it, as when a warehouse cuts a quoted connection string short.
44+
/// `scheme://name:value` with no `@` after it, as in a cut connection string; group 3 ends the authority.
3545
pub(crate) const BARE_USERINFO_PATTERN: &str =
36-
r#"(?i)\b([a-z][a-z0-9+.-]*://[^\s:/@'"]+:)([^\s/@'"]+)"#;
46+
r#"(?i)\b([a-z][a-z0-9+.-]*://[^\s:/@'"]+:)([^\s/?#@'"]+)([/?#]?)"#;
3747
/// The most characters of error text kept after redaction.
3848
pub(crate) const ERROR_TEXT_CHARS: usize = 2000;
Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
11
//! `redact_text`: obvious named secrets and URI passwords removed from diagnostic text.
22
33
use crate::redaction::_helpers::patterns::{
4-
AUTHORIZATION, BEARER, SECRET_VALUE, URI_CREDENTIAL, substituted,
4+
AUTHORIZATION, BEARER, DDL_PASSWORD, SECRET_VALUE, URI_CREDENTIAL, substituted,
5+
};
6+
use crate::redaction::constants::{
7+
BEARER_REPLACEMENT, DDL_REPLACEMENT, PAIR_REPLACEMENT, URI_REPLACEMENT,
58
};
6-
use crate::redaction::constants::{BEARER_REPLACEMENT, PAIR_REPLACEMENT, URI_REPLACEMENT};
79

8-
/// Authorization headers, bearer tokens, `name=value` secrets, then URI passwords redacted.
10+
/// Authorization headers, bearer tokens, `name=value` secrets, SQL passwords, then URI passwords redacted.
911
#[must_use]
1012
pub fn redact_text(value: &str) -> String {
1113
let authorization: String = substituted(&AUTHORIZATION, value, PAIR_REPLACEMENT);
1214
let bearer: String = substituted(&BEARER, &authorization, BEARER_REPLACEMENT);
1315
let named: String = substituted(&SECRET_VALUE, &bearer, PAIR_REPLACEMENT);
14-
substituted(&URI_CREDENTIAL, &named, URI_REPLACEMENT)
16+
let statement: String = substituted(&DDL_PASSWORD, &named, DDL_REPLACEMENT);
17+
substituted(&URI_CREDENTIAL, &statement, URI_REPLACEMENT)
1518
}

‎crates/sqlbuild-core/src/redaction/tests/test_redact_text.rs‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
use crate::redaction::main::redact_error_text::redact_error_text;
12
use crate::redaction::main::redact_text::redact_text;
23
use crate::redaction::tests::test_types::RedactTestCase;
34

@@ -19,6 +20,61 @@ fn given_diagnostic_text_when_redacting_then_matches_python_substitutions() {
1920
text: "nothing to see",
2021
expected_text: "nothing to see",
2122
},
23+
RedactTestCase {
24+
description: "a key glued to a prefix with a separator",
25+
text: "db_password=orders-pass host=warehouse",
26+
expected_text: "db_password=[REDACTED] host=warehouse",
27+
},
28+
RedactTestCase {
29+
description: "a key glued to a prefix without a separator",
30+
text: "PGPASSWORD=orders-pass psql",
31+
expected_text: "PGPASSWORD=[REDACTED] psql",
32+
},
33+
RedactTestCase {
34+
description: "near miss: a key that only starts with a secret word keeps its value",
35+
text: "password_policy=strict max_tokens=5 tokenizer: words passwords: 3 tries",
36+
expected_text: "password_policy=strict max_tokens=5 tokenizer: words passwords: 3 tries",
37+
},
38+
RedactTestCase {
39+
description: "near miss: pwd stays a whole word, so working directories are kept",
40+
text: "OLDPWD=/srv/orders cwd=/srv/orders pwd=orders-pass",
41+
expected_text: "OLDPWD=/srv/orders cwd=/srv/orders pwd=[REDACTED]",
42+
},
43+
RedactTestCase {
44+
description: "a quoted PASSWORD in DDL",
45+
text: "CREATE ROLE orders_app WITH LOGIN PASSWORD 'orders-pass' VALID UNTIL 'infinity'",
46+
expected_text: "CREATE ROLE orders_app WITH LOGIN PASSWORD [REDACTED] VALID UNTIL 'infinity'",
47+
},
48+
RedactTestCase {
49+
description: "IDENTIFIED BY with a quoted and a bare password",
50+
text: "CREATE USER orders IDENTIFIED BY 'orders-pass'; ALTER USER stock IDENTIFIED BY stockpass",
51+
expected_text: "CREATE USER orders IDENTIFIED BY [REDACTED]; ALTER USER stock IDENTIFIED BY [REDACTED]",
52+
},
53+
RedactTestCase {
54+
description: "IDENTIFIED WITH a plugin BY a password, as ClickHouse writes it",
55+
text: "CREATE USER orders IDENTIFIED WITH sha256_password BY 'orders-pass' HOST ANY",
56+
expected_text: "CREATE USER orders IDENTIFIED WITH sha256_password BY [REDACTED] HOST ANY",
57+
},
58+
RedactTestCase {
59+
description: "near miss: prose about passwords and IDENTIFIED WITH keep their words",
60+
text: "password authentication failed for user \"orders\"; IDENTIFIED WITH caching_sha2_password",
61+
expected_text: "password authentication failed for user \"orders\"; IDENTIFIED WITH caching_sha2_password",
62+
},
63+
RedactTestCase {
64+
description: "an unencoded @ inside a URI password",
65+
text: "url=postgres://orders:p@ss@warehouse.example.test/orders",
66+
expected_text: "url=postgres://orders:[REDACTED]@warehouse.example.test/orders",
67+
},
68+
RedactTestCase {
69+
description: "near miss: an @ after the host belongs to the path or query, not the password",
70+
text: "postgres://orders:pw@warehouse.example.test/orders?contact=support@example.com, https://api.example.test/v1",
71+
expected_text: "postgres://orders:[REDACTED]@warehouse.example.test/orders?contact=support@example.com, https://api.example.test/v1",
72+
},
73+
RedactTestCase {
74+
description: "two URIs in a list each lose their password",
75+
text: "postgres://orders:one@h1.example.test,postgres://stock:two@h2.example.test",
76+
expected_text: "postgres://orders:[REDACTED]@h1.example.test,postgres://stock:[REDACTED]@h2.example.test",
77+
},
2278
];
2379

2480
for test_case in test_cases {
@@ -30,3 +86,43 @@ fn given_diagnostic_text_when_redacting_then_matches_python_substitutions() {
3086
);
3187
}
3288
}
89+
90+
#[test]
91+
fn given_error_text_with_cut_uris_when_redacting_then_position_tells_a_port_from_a_password() {
92+
let test_cases = [
93+
RedactTestCase {
94+
description: "an all-digit password in a URI cut before its host",
95+
text: "could not parse 'postgres://orders:1234",
96+
expected_text: "could not parse 'postgres://orders:[REDACTED]",
97+
},
98+
RedactTestCase {
99+
description: "a password cut at a closing quote",
100+
text: "dsn 'postgres://orders:orders-pass' rejected",
101+
expected_text: "dsn 'postgres://orders:[REDACTED]' rejected",
102+
},
103+
RedactTestCase {
104+
description: "a cut password containing / is not a port",
105+
text: "cut 'postgres://orders:pa/ss",
106+
expected_text: "cut 'postgres://orders:[REDACTED]/ss",
107+
},
108+
RedactTestCase {
109+
description: "near miss: a port the authority ends after stays",
110+
text: "GET http://warehouse.example.test:8080/health failed; retry http://warehouse.example.test:8443?attempt=2",
111+
expected_text: "GET http://warehouse.example.test:8080/health failed; retry http://warehouse.example.test:8443?attempt=2",
112+
},
113+
RedactTestCase {
114+
description: "a complete URI is redacted once and a second pass changes nothing",
115+
text: "postgres://orders:[REDACTED]@warehouse.example.test:5432/orders",
116+
expected_text: "postgres://orders:[REDACTED]@warehouse.example.test:5432/orders",
117+
},
118+
];
119+
120+
for test_case in test_cases {
121+
assert_eq!(
122+
redact_error_text(test_case.text),
123+
test_case.expected_text,
124+
"{}",
125+
test_case.description
126+
);
127+
}
128+
}

0 commit comments

Comments
 (0)