Repository navigation
Expand file tree
/
Copy pathsecrets.py
More file actions
416 lines (349 loc) · 16.2 KB
/
Copy pathsecrets.py
File metadata and controls
416 lines (349 loc) · 16.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
"""Secret detection base (architecture §25 plugins/secrets).
Secrets are string literals of the PyHIR. ``literals`` walks every string literal with
the name it is bound to (assignment target, keyword argument, dictionary key) and the
enclosing function; ``SecretDetector`` reports at most one finding per literal: a
provider pattern first (``hardcoded-secret``), then a credential-like name with a real
value (``hardcoded-credential``), then a high-entropy token on its own
(``high-entropy-string``). Messages and metadata carry a redacted preview, never the
secret. Only Python sources are scanned; configuration files are not.
"""
from __future__ import annotations
import json
import math
import re
import tomllib
from collections import Counter
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import ClassVar
from coretrace_python.findings import Confidence, Finding, Severity
from coretrace_python.hir import nodes
from coretrace_python.hir.visitors import Node, children
from coretrace_python.interprocedural import discover_files
from coretrace_python.plugins.api import Plugin, PluginContext
from coretrace_python.source import SourceId, SourceSpan, decode_text
Literal = tuple[str, str | None, SourceSpan, str | None]
_HEX = re.compile(r"^[0-9a-fA-F]+$")
_TOKEN = re.compile(r"^[A-Za-z0-9+/=_-]+$")
_PLACEHOLDER = re.compile(r"^(<.*>|\$\{.*\}|\{\{.*\}\}|%.*%|\.{3,}|(.)\2*)$")
_PLACEHOLDER_WORDS = frozenset(
{"", "changeme", "change_me", "password", "passwd", "secret", "token", "example", "xxx", "todo", "none", "null"}
)
_NOT_CREDENTIAL_SUFFIXES = (
"_name", "_field", "_file", "_path", "_url", "_id", "_env", "_var", "_header", "_param",
# ``token_type = "bearer"``: the kind of a credential, not one.
"_type", "_kind", "_scheme", "_format", "_algorithm", "_method", "_mode",
)
# MD5, SHA-1, SHA-256 and SHA-512 digests, as hex.
_DIGEST_LENGTHS = frozenset({32, 40, 64, 128})
_DIGEST_WORDS = ("hash", "digest", "sha", "md5", "checksum", "commit", "etag", "fingerprint", "revision", "version")
# Subresource integrity, as in lock files: ``sha512-<base64>``.
_INTEGRITY = re.compile(r"^(md5|sha1|sha256|sha384|sha512)-[A-Za-z0-9+/=]+$", re.IGNORECASE)
_ALPHABET_WORDS = ("alphabet", "charset", "characters", "letters", "digits")
# A password hash names its algorithm, then its parameters, salt and digest, each in a
# hash alphabet: modular crypt and PHC strings (``$6$salt$digest``, ``$argon2id$v=19$...``;
# bcrypt has a fixed shape), Werkzeug's ``method:parameters$salt$hex digest`` and
# Django's ``algorithm$...$digest``. An algorithm identifier alone is an incomplete hash.
_HASH_FIELD = r"[A-Za-z0-9./+=,-]"
_CRYPT_ALGORITHMS = r"1|5|6|7|y|apr1|argon2(?:i|d|id)|pbkdf2(?:-sha(?:1|256|512))?|scrypt"
_PASSWORD_HASHES = tuple(
re.compile(regex)
for regex in (
r"^\$2[abxy]\$\d\d\$[./A-Za-z0-9]{53}$",
rf"^\$(?:{_CRYPT_ALGORITHMS}|2[abxy])\$$",
rf"^\$(?:{_CRYPT_ALGORITHMS})(?:\${_HASH_FIELD}+){{2,}}$",
r"^(?:scrypt|pbkdf2):[a-z0-9:]+\$[^$\s]+\$[0-9a-f]+$",
rf"^(?:pbkdf2_sha(?:1|256)|argon2|bcrypt(?:_sha256)?|scrypt|md5|sha1)(?:\${_HASH_FIELD}*){{2,}}$",
)
)
def is_password_hash(value: str) -> bool:
"""A password hash of a known format, which an application stores instead of the
password and which does not reveal it: the algorithm identifier followed by fields
of that format's shape, or the identifier alone. An identifier followed by anything
else (``$2y$Summer2024!``) is a plaintext password."""
return any(pattern.match(value) is not None for pattern in _PASSWORD_HASHES)
def looks_like_digest(value: str, name: str | None) -> bool:
"""A hex value of digest length, a hex value whose name names a hash, or a
subresource-integrity hash: a commit id, a checksum, a content hash, never a secret
unless a credential name says so."""
if _INTEGRITY.match(value) is not None or (name is not None and name.lower() == "integrity"):
return True
if _HEX.match(value) is None:
return False
if len(value) in _DIGEST_LENGTHS:
return True
return name is not None and any(word in name.lower() for word in _DIGEST_WORDS)
_TEST_DIRECTORIES = frozenset({"test", "tests", "testing", "fixtures", "__tests__"})
_TEST_FILE = re.compile(r"^(test_.*|.*_test|conftest)\.py$")
_EXAMPLE_SUFFIXES = (".example", ".sample", ".dist", ".tpl")
def credential_context(path: str) -> str | None:
"""``"test"`` for test code and fixtures, ``"example"`` for template configuration
files (``.env.example``), ``None`` for anything else."""
parts = Path(path).parts
if any(part in _TEST_DIRECTORIES for part in parts[:-1]) or _TEST_FILE.match(parts[-1]):
return "test"
if parts[-1].endswith(_EXAMPLE_SUFFIXES):
return "example"
return None
def is_alphabet(value: str, name: str | None) -> bool:
"""A constant that spells out a character set is high-entropy by construction: its
name says so, or every character in it is distinct, which no random token is."""
if name is not None and any(word in name.lower() for word in _ALPHABET_WORDS):
return True
return len(value) >= 16 and len(set(value)) == len(value)
def shannon_entropy(text: str) -> float:
"""Bits of information per character of ``text``."""
if not text:
return 0.0
counts = Counter(text)
length = len(text)
return -sum(count / length * math.log2(count / length) for count in counts.values())
def literals(module: nodes.Module) -> Iterator[Literal]:
"""Every string literal of the module as ``(value, bound name, span, function)``."""
for statement in module.body:
yield from _walk(statement, None, None)
def _walk(node: Node, name: str | None, function: str | None) -> Iterator[Literal]:
if isinstance(node, nodes.Constant):
if isinstance(node.value, str):
yield node.value, name, node.span, function
return
if isinstance(node, nodes.Function):
for child in children(node):
yield from _walk(child, None, node.name)
return
if isinstance(node, nodes.Class):
for child in children(node):
yield from _walk(child, None, None)
return
if isinstance(node, nodes.Assign):
yield from _walk(node.value, _bound_name(node.target), function)
return
if isinstance(node, nodes.Keyword):
yield from _walk(node.value, node.name, function)
return
if isinstance(node, nodes.Parameter):
if node.default is not None:
yield from _walk(node.default, node.name, function)
return
if isinstance(node, nodes.Call) and _is_environment_lookup(node):
# ``os.getenv("APP_TOKEN", "fallback")``: the fallback is bound to the variable name.
variable, fallback = node.arguments[0], node.arguments[1]
assert isinstance(variable, nodes.Constant) and isinstance(variable.value, str)
yield from _walk(variable, None, function)
yield from _walk(fallback, variable.value, function)
for keyword in node.keywords:
yield from _walk(keyword, None, function)
return
if isinstance(node, nodes.Dict):
for key, value in node.items:
# A constant key names the value; it is not a value itself.
if isinstance(key, nodes.Constant):
bound = key.value if isinstance(key.value, str) else None
else:
bound = None
if key is not None:
yield from _walk(key, None, function)
yield from _walk(value, bound, function)
return
for child in children(node):
yield from _walk(child, None, function)
def _is_environment_lookup(call: nodes.Call) -> bool:
callee = call.callee
return (
isinstance(callee, nodes.Attribute)
and callee.name in ("getenv", "get")
and len(call.arguments) >= 2
and isinstance(call.arguments[0], nodes.Constant)
and isinstance(call.arguments[0].value, str)
)
def _bound_name(target: nodes.Target) -> str | None:
"""The name an assignment binds: ``name``, ``obj.attr`` or ``mapping['key']``."""
if isinstance(target, nodes.Name):
return target.identifier
if isinstance(target, nodes.Attribute):
return target.name
if isinstance(target, nodes.Subscript) and isinstance(target.key, nodes.Constant):
return target.key.value if isinstance(target.key.value, str) else None
return None
@dataclass(frozen=True)
class SecretPattern:
"""A provider-specific secret format."""
provider: str
regex: str
def matches(self, text: str) -> bool:
return re.search(self.regex, text) is not None
DEFAULT_PATTERNS: tuple[SecretPattern, ...] = (
SecretPattern("aws", r"\b(AKIA|ASIA)[0-9A-Z]{16}\b"),
SecretPattern("github", r"\bgh[pousr]_[A-Za-z0-9]{36,}\b"),
SecretPattern("github", r"\bgithub_pat_[A-Za-z0-9]{22}_[A-Za-z0-9]{59}\b"),
SecretPattern("slack", r"\bxox[abpr]-[0-9]{10,}-[0-9A-Za-z-]{10,}"),
SecretPattern("stripe", r"\b[sr]k_(live|test)_[0-9A-Za-z]{24,}\b"),
SecretPattern("google", r"\bAIza[0-9A-Za-z_-]{35}\b"),
SecretPattern("private-key", r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
SecretPattern("jwt", r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"),
SecretPattern("sendgrid", r"\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b"),
SecretPattern("twilio", r"\bSK[0-9a-fA-F]{32}\b"),
SecretPattern(
"url",
r"://[^/\s:@'\"]+:[^/\s@'\"]+@"
r"|[?&](?:password|passwd|pwd|token|api_key|apikey|secret|access_key)=[^&\s'\"]{3,}",
),
)
DEFAULT_CREDENTIAL_NAMES: tuple[str, ...] = (
"password",
"passwd",
"pwd",
"secret",
"token",
"api_key",
"apikey",
"api-key",
"private_key",
"access_key",
"credential",
)
CONFIG_SUFFIXES = frozenset({".env", ".yaml", ".yml", ".toml", ".json", ".ini", ".cfg", ".properties", ".conf"})
_PAIR = re.compile(r"^\s*(?:-\s+)?([A-Za-z_][\w.-]*)\s*[:=]\s*(.*?)\s*$")
_MAX_CONFIG_BYTES = 8_000_000
def config_literals(root: Path) -> Iterator[Literal]:
"""Every string value of the configuration files under ``root`` with the key it is
bound to: ``.env``, YAML, TOML, JSON, INI and properties files, decoded by their
byte order mark. Python files are left to ``literals``."""
for path in discover_files(root):
if not (path.suffix in CONFIG_SUFFIXES or path.name.startswith(".env")):
continue
try:
data = path.read_bytes()
if len(data) > _MAX_CONFIG_BYTES:
continue
text = decode_text(data)
except (OSError, UnicodeDecodeError):
continue
source = SourceId(str(path))
if path.suffix == ".json":
yield from _structured(source, text, _load_json(text))
elif path.suffix == ".toml":
yield from _structured(source, text, _load_toml(text))
else:
yield from _pairs(source, text)
def _load_json(text: str) -> object:
try:
return json.loads(text)
except ValueError:
return None
def _load_toml(text: str) -> object:
try:
return tomllib.loads(text)
except tomllib.TOMLDecodeError:
return None
def _structured(source: SourceId, text: str, data: object) -> Iterator[Literal]:
lines = text.splitlines()
def line_of(key: str) -> int:
for number, line in enumerate(lines, start=1):
if re.match(rf'^\s*"?{re.escape(key)}"?\s*[:=]', line) or f'"{key}"' in line:
return number
return 1
def walk(node: object, key: str | None) -> Iterator[Literal]:
if isinstance(node, dict):
for name, value in node.items():
yield from walk(value, str(name))
elif isinstance(node, list):
for item in node:
yield from walk(item, key)
elif isinstance(node, str) and key is not None:
yield node, key, SourceSpan(source, line_of(key), 1), None
yield from walk(data, None)
def _pairs(source: SourceId, text: str) -> Iterator[Literal]:
for number, line in enumerate(text.splitlines(), start=1):
stripped = line.strip()
if not stripped or stripped[0] in "#;[":
continue
match = _PAIR.match(line)
if match is None:
continue
key, value = match.group(1), match.group(2)
if value.startswith("#"):
continue
if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
value = value[1:-1]
if value:
yield value, key, SourceSpan(source, number, 1), None
def redacted(value: str) -> str:
preview = value[:4] if len(value) > 8 else value[:1]
return f"{preview}… ({len(value)} characters)"
def is_placeholder(value: str) -> bool:
lowered = value.strip().lower()
return len(lowered) < 4 or lowered in _PLACEHOLDER_WORDS or _PLACEHOLDER.match(lowered) is not None
class SecretDetector(Plugin):
"""Report hardcoded secrets among the module's string literals."""
patterns: ClassVar[tuple[SecretPattern, ...]] = ()
credential_names: ClassVar[tuple[str, ...]] = ()
entropy_threshold: ClassVar[float] = 4.5
hex_entropy_threshold: ClassVar[float] = 3.5
minimum_length: ClassVar[int] = 32
def analyze(self, ctx: PluginContext) -> Sequence[Finding]:
findings: list[Finding] = []
for value, name, span, function in literals(ctx.module):
finding = self.judge(value, name, span, function)
if finding is not None:
findings.append(finding)
return findings
def judge(self, value: str, name: str | None, span: SourceSpan, function: str | None) -> Finding | None:
where = f"in {name}" if name is not None else "in a string literal"
for pattern in self.patterns:
if pattern.matches(value):
return Finding(
"hardcoded-secret",
f"Hardcoded {pattern.provider} secret {where}: {redacted(value)}",
Severity.HIGH,
Confidence.HIGH,
span,
function,
{"provider": pattern.provider, "name": name or "", "length": str(len(value))},
)
if (
name is not None
and self.is_credential_name(name)
and not is_placeholder(value)
and not is_password_hash(value)
):
# A password in a test fixture or a template file is rarely a leak; a name
# is a hint, so the finding stays, at low confidence.
context = credential_context(str(span.source_id))
metadata = {"name": name, "length": str(len(value))}
if context is not None:
metadata["context"] = context
return Finding(
"hardcoded-credential",
f"Hardcoded credential {where}: {redacted(value)}",
Severity.HIGH,
Confidence.LOW if context is not None else Confidence.MEDIUM,
span,
function,
metadata,
)
entropy = self.entropy_of(value)
if entropy is not None and not looks_like_digest(value, name) and not is_alphabet(value, name):
return Finding(
"high-entropy-string",
f"High-entropy string {where}: {redacted(value)}",
Severity.MEDIUM,
Confidence.LOW,
span,
function,
{"name": name or "", "length": str(len(value)), "entropy": f"{entropy:.2f}"},
)
return None
def is_credential_name(self, name: str) -> bool:
lowered = name.lower()
if lowered.endswith(_NOT_CREDENTIAL_SUFFIXES):
return False
return any(word in lowered for word in self.credential_names)
def entropy_of(self, value: str) -> float | None:
"""The entropy of ``value`` when it looks like an opaque token, else ``None``."""
if len(value) < self.minimum_length or _TOKEN.match(value) is None:
return None
entropy = shannon_entropy(value)
threshold = self.hex_entropy_threshold if _HEX.match(value) else self.entropy_threshold
return entropy if entropy >= threshold else None