Skip to content

Commit fda02aa

Browse files
committed
test: define numeric, validator and authorization evidence
1 parent d7acb28 commit fda02aa

1 file changed

Lines changed: 337 additions & 0 deletions

File tree

‎tests/test_refutation_evidence.py‎

Lines changed: 337 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,337 @@
1+
"""Acceptance tests for the remaining evidence of the proof engine (``docs/architecture.md``
2+
§24, §25 plugins/proof; roadmap issue #32).
3+
4+
- Numeric ranges: ``abstract.ranges`` keeps an interval for every value proven numeric
5+
(numeric constants, arithmetic on numbers, ``int()``, ``len()`` and friends), refined
6+
on the branches of comparisons and widened over loops. A tainted origin whose every
7+
dependence path to the sink argument goes through a numeric value cannot inject.
8+
- Validators: a ``Validator`` model names a callable whose truth proves one of its
9+
arguments safe, so ``re.fullmatch`` and framework validators refute like ``isdigit()``.
10+
- Authorization: an ``AuthorizationGuard`` model names a decorator or a condition that
11+
restricts who reaches the sink; a flow behind one is a hotspot, not a vulnerability.
12+
13+
Plugins contribute both models through ``Plugin.models``, the same way they contribute
14+
sources and sinks: that is the plugin extension point for refutation evidence.
15+
16+
Expected to remain red until ``abstract.ranges``, ``Validator`` and ``AuthorizationGuard``
17+
exist and the refutation engine consumes them.
18+
"""
19+
20+
from __future__ import annotations
21+
22+
from pathlib import Path
23+
24+
import pytest
25+
26+
from coretrace_python import engine
27+
from coretrace_python.analysis import AnalysisManager
28+
from coretrace_python.findings import Confidence
29+
from coretrace_python.findings.refutation import RefutationAnalysis, Status, Verdict, Verdicts
30+
from coretrace_python.frontend import build_hir
31+
from coretrace_python.hir import nodes
32+
from coretrace_python.ir.model import Call, Symbol
33+
from coretrace_python.ir.ssa import SSAAnalysis
34+
from coretrace_python.plugins import ModelPlugin
35+
from coretrace_python.semantic.symbols import SymbolId
36+
from coretrace_python.source import SourceManager
37+
from coretrace_python.taint import (
38+
SecurityModelAnalysis,
39+
SecurityModelRegistry,
40+
Sink,
41+
Source,
42+
TaintKind,
43+
)
44+
45+
try:
46+
from coretrace_python.abstract import Interval, RangeAnalysis
47+
from coretrace_python.taint import AuthorizationGuard, Validator
48+
except ImportError as error: # pragma: no cover - red until the evidence lands
49+
MISSING = error
50+
else:
51+
MISSING = None
52+
53+
54+
@pytest.fixture(autouse=True)
55+
def require_evidence() -> None:
56+
if MISSING is not None:
57+
pytest.fail(f"refutation evidence is not implemented yet: {MISSING}")
58+
59+
60+
REPO = Path(__file__).resolve().parent.parent
61+
PLUGINS = REPO / "plugins"
62+
63+
OS_SYSTEM = Sink(SymbolId("python.os.system"), TaintKind.COMMAND)
64+
STDIN = Source(SymbolId("python.builtins.input"), "stdin")
65+
LOGIN = SymbolId("python.flask_login.current_user.is_authenticated")
66+
LOGIN_REQUIRED = SymbolId("python.flask_login.login_required")
67+
68+
PRELUDE = "import os\nimport re\nfrom flask_login import current_user, login_required\n\n"
69+
70+
71+
def manager_for(body: str, *models: object) -> AnalysisManager:
72+
module = build_hir(SourceManager().add_source("proof.py", PRELUDE + body))
73+
manager = AnalysisManager(module)
74+
manager.register(*engine.ALL_ANALYSES)
75+
registry = SecurityModelRegistry()
76+
registry.register(STDIN, OS_SYSTEM, *models) # type: ignore[arg-type]
77+
manager.provide(SecurityModelAnalysis, registry.freeze())
78+
return manager
79+
80+
81+
def first_function(manager: AnalysisManager) -> nodes.Function:
82+
return next(s for s in manager.module.body if isinstance(s, nodes.Function))
83+
84+
85+
def verdict(body: str, *models: object) -> Verdict:
86+
manager = manager_for(body, *models)
87+
result: Verdicts = manager.get(RefutationAnalysis, first_function(manager))
88+
(found,) = result.all()
89+
return found
90+
91+
92+
def ranges_at_sink(body: str): # type: ignore[no-untyped-def]
93+
"""The intervals in the block calling ``os.system``, keyed by argument position."""
94+
95+
manager = manager_for(body)
96+
function = first_function(manager)
97+
ssa = manager.get(SSAAnalysis, function)
98+
facts = manager.get(RangeAnalysis, function)
99+
defs = {i.result: i for block in ssa.blocks for i in block.instructions if i.result}
100+
for block in ssa.blocks:
101+
for instruction in block.instructions:
102+
callee = defs.get(instruction.callee) if isinstance(instruction, Call) else None
103+
if isinstance(callee, Symbol) and callee.symbol_id == OS_SYSTEM.symbol:
104+
at = facts.at(block.id)
105+
return [at.get(argument) for argument in instruction.arguments]
106+
raise AssertionError("no call to os.system")
107+
108+
109+
# --------------------------------------------------------------------------- ranges
110+
111+
112+
def test_intervals_join_and_widen() -> None:
113+
assert Interval(0, 5).join(Interval(3, 9)) == Interval(0, 9)
114+
assert Interval(0, 5).widen(Interval(0, 6)) == Interval(0, float("inf"))
115+
assert Interval(0, 5).widen(Interval(-1, 5)) == Interval(float("-inf"), 5)
116+
assert Interval(0, 5).widen(Interval(1, 4)) == Interval(0, 5)
117+
assert str(Interval(0, float("inf"))) == "[0, inf]"
118+
119+
120+
def test_numeric_constants_and_arithmetic_have_ranges() -> None:
121+
assert ranges_at_sink("def f():\n n = 2 + 3 * 4\n os.system(n, n - 20, -n)\n") == [
122+
Interval(14, 14),
123+
Interval(-6, -6),
124+
Interval(-14, -14),
125+
]
126+
127+
128+
def test_conversions_and_lengths_are_numeric() -> None:
129+
low, length, raw = ranges_at_sink(
130+
"def f(x):\n os.system(int(x), len(x), x)\n"
131+
)
132+
assert low == Interval(float("-inf"), float("inf"))
133+
assert length == Interval(0, float("inf"))
134+
assert raw is None
135+
136+
137+
def test_branches_refine_ranges_on_each_side() -> None:
138+
then_side = ranges_at_sink("def f(x):\n n = int(x)\n if n < 10:\n os.system(n)\n")
139+
else_side = ranges_at_sink("def f(x):\n n = int(x)\n if n < 10:\n pass\n else:\n os.system(n)\n")
140+
chained = ranges_at_sink("def f(x):\n n = int(x)\n if 0 < n <= 100:\n os.system(n)\n")
141+
142+
assert then_side == [Interval(float("-inf"), 10)]
143+
assert else_side == [Interval(10, float("inf"))]
144+
assert chained == [Interval(0, 100)]
145+
146+
147+
def test_loops_terminate_by_widening() -> None:
148+
(counter,) = ranges_at_sink(
149+
"def f(x):\n i = 0\n while i < len(x):\n i = i + 1\n os.system(i)\n"
150+
)
151+
assert counter == Interval(0, float("inf"))
152+
153+
154+
def test_range_analysis_is_a_function_analysis_over_ssa() -> None:
155+
from coretrace_python.cfg import CFGAnalysis
156+
157+
assert RangeAnalysis.name == "abstract.ranges"
158+
assert {SSAAnalysis, CFGAnalysis} <= RangeAnalysis.requires
159+
assert RangeAnalysis in engine.ALL_ANALYSES
160+
161+
162+
# --------------------------------------------------------------------------- numeric evidence
163+
164+
165+
def test_numeric_values_cannot_inject() -> None:
166+
found = verdict("def f():\n n = int(input())\n os.system('sleep ' + str(n))\n")
167+
168+
assert found.status is Status.REFUTED
169+
assert "numeric" in found.evidence
170+
171+
172+
def test_ranges_show_in_the_evidence() -> None:
173+
found = verdict(
174+
"def f():\n n = int(input())\n if 0 <= n < 60:\n os.system('sleep ' + str(n))\n"
175+
)
176+
assert found.status is Status.REFUTED
177+
assert "[0, 60]" in found.evidence
178+
179+
180+
def test_a_numeric_path_does_not_cover_a_raw_one() -> None:
181+
found = verdict("def f():\n x = input()\n os.system(x + str(int(x)))\n")
182+
assert found.status is Status.VULNERABILITY
183+
184+
185+
def test_a_bounded_length_proves_nothing_about_the_string() -> None:
186+
found = verdict("def f():\n x = input()\n if len(x) < 5:\n os.system(x)\n")
187+
assert found.status is Status.HOTSPOT
188+
189+
190+
# --------------------------------------------------------------------------- validators
191+
192+
193+
FULLMATCH = Validator(SymbolId("python.re.fullmatch"), argument=1) if MISSING is None else None
194+
195+
196+
def test_validators_and_authorization_guards_are_indexed_in_the_model_table() -> None:
197+
guard = AuthorizationGuard(LOGIN, "login")
198+
registry = SecurityModelRegistry()
199+
registry.register(FULLMATCH, guard, Sink(LOGIN, TaintKind.HTML))
200+
table = registry.freeze()
201+
202+
assert table.validator(FULLMATCH.symbol) == FULLMATCH
203+
assert table.validator(LOGIN) is None
204+
assert table.authorization(LOGIN) == guard
205+
assert table.authorization(FULLMATCH.symbol) is None
206+
assert table.validators == (FULLMATCH,)
207+
assert table.authorizations == (guard,)
208+
assert Validator(LOGIN).argument == 0
209+
extended = table.extended(Sink(SymbolId("python.x"), TaintKind.ADVISORY))
210+
assert extended.validators == (FULLMATCH,) and extended.authorizations == (guard,)
211+
212+
213+
def test_a_true_validator_refutes_the_argument_it_validates() -> None:
214+
found = verdict(
215+
"def f():\n x = input()\n if re.fullmatch('[a-z]+', x):\n os.system(x)\n",
216+
FULLMATCH,
217+
)
218+
assert found.status is Status.REFUTED
219+
assert found.evidence == "validated by python.re.fullmatch"
220+
221+
222+
def test_a_validator_only_proves_its_declared_argument() -> None:
223+
found = verdict(
224+
"def f():\n x = input()\n if re.fullmatch(x, 'abc'):\n os.system(x)\n",
225+
FULLMATCH,
226+
)
227+
assert found.status is Status.HOTSPOT
228+
229+
230+
def test_validators_on_the_wrong_branch_prove_nothing() -> None:
231+
found = verdict(
232+
"def f():\n x = input()\n if not re.fullmatch('[a-z]+', x):\n os.system(x)\n",
233+
FULLMATCH,
234+
)
235+
assert found.status is Status.VULNERABILITY
236+
237+
238+
def test_compiled_pattern_validators_resolve_through_derived_symbols() -> None:
239+
found = verdict(
240+
"def f():\n x = input()\n pattern = re.compile('[a-z]+')\n"
241+
" if pattern.fullmatch(x):\n os.system(x)\n",
242+
Validator(SymbolId("python.re.compile.fullmatch")),
243+
)
244+
assert found.status is Status.REFUTED
245+
246+
247+
# --------------------------------------------------------------------------- authorization
248+
249+
250+
def test_a_dominating_authorization_condition_makes_a_hotspot() -> None:
251+
found = verdict(
252+
"def f():\n if current_user.is_authenticated:\n os.system(input())\n",
253+
AuthorizationGuard(LOGIN, "login"),
254+
)
255+
assert found.status is Status.HOTSPOT
256+
assert found.evidence == "behind authorization (login) at line 6"
257+
258+
259+
def test_an_early_return_on_missing_authorization_counts() -> None:
260+
found = verdict(
261+
"def f():\n if not current_user.is_authenticated:\n return\n os.system(input())\n",
262+
AuthorizationGuard(LOGIN, "login"),
263+
)
264+
assert found.status is Status.HOTSPOT
265+
266+
267+
def test_an_authorization_decorator_counts() -> None:
268+
found = verdict(
269+
"@login_required\ndef f():\n os.system(input())\n",
270+
AuthorizationGuard(LOGIN_REQUIRED, "login"),
271+
)
272+
assert found.status is Status.HOTSPOT
273+
assert found.evidence == "behind authorization (login) by decorator"
274+
275+
276+
def test_without_the_model_the_same_code_is_a_vulnerability() -> None:
277+
found = verdict("def f():\n if current_user.is_authenticated:\n os.system(input())\n")
278+
assert found.status is Status.VULNERABILITY
279+
280+
281+
def test_authorization_does_not_override_a_refutation() -> None:
282+
found = verdict(
283+
"@login_required\ndef f():\n x = input()\n if x.isdigit():\n os.system(x)\n",
284+
AuthorizationGuard(LOGIN_REQUIRED, "login"),
285+
)
286+
assert found.status is Status.REFUTED
287+
288+
289+
# --------------------------------------------------------------------------- plugins
290+
291+
292+
def test_any_plugin_contributes_refutation_models() -> None:
293+
class Evidence(ModelPlugin):
294+
name = "evidence"
295+
models = (FULLMATCH, AuthorizationGuard(LOGIN, "login"))
296+
297+
table = engine.plugin_models([Evidence()])
298+
299+
assert table.validators == (FULLMATCH,)
300+
assert table.authorizations == (AuthorizationGuard(LOGIN, "login"),)
301+
302+
303+
def test_shipped_models_refute_fullmatch_and_flag_login_required_routes() -> None:
304+
findings = engine.check(
305+
SourceManager().add_source(
306+
"app.py",
307+
"import os\nimport re\nfrom flask import Flask, request\nfrom flask_login import login_required\n\n"
308+
"app = Flask(__name__)\n\n"
309+
"@app.route('/ping')\n"
310+
"def ping():\n"
311+
" host = request.args['host']\n"
312+
" if re.fullmatch(r'[a-z.]+', host):\n"
313+
" os.system('ping ' + host)\n\n"
314+
"@app.route('/admin')\n"
315+
"@login_required\n"
316+
"def admin():\n"
317+
" os.system(request.args['cmd'])\n",
318+
),
319+
[PLUGINS],
320+
)
321+
322+
assert [(f.function, f.confidence) for f in findings] == [("admin", Confidence.MEDIUM)]
323+
assert findings[0].metadata["evidence"] == "behind authorization (login) by decorator"
324+
325+
326+
def test_shipped_django_login_required_view_is_a_hotspot() -> None:
327+
findings = engine.check(
328+
SourceManager().add_source(
329+
"views.py",
330+
"import os\nfrom django.contrib.auth.decorators import login_required\n\n"
331+
"@login_required\n"
332+
"def run(request):\n"
333+
" os.system(request.GET['cmd'])\n",
334+
),
335+
[PLUGINS],
336+
)
337+
assert [(f.rule_id, f.confidence) for f in findings] == [("command-injection", Confidence.MEDIUM)]

0 commit comments

Comments
 (0)