|
| 1 | +"""Acceptance tests for the remaining evidence of the proof engine (``docs/architecture.md`` |
| 2 | +§24, §25 plugins/proof; roadmap issue #32). |
| 3 | +
|
| 4 | +- Numeric ranges: ``abstract.ranges`` keeps an interval for every value proven numeric |
| 5 | + (numeric constants, arithmetic on numbers, ``int()``, ``len()`` and friends), refined |
| 6 | + on the branches of comparisons and widened over loops. A tainted origin whose every |
| 7 | + dependence path to the sink argument goes through a numeric value cannot inject. |
| 8 | +- Validators: a ``Validator`` model names a callable whose truth proves one of its |
| 9 | + arguments safe, so ``re.fullmatch`` and framework validators refute like ``isdigit()``. |
| 10 | +- Authorization: an ``AuthorizationGuard`` model names a decorator or a condition that |
| 11 | + restricts who reaches the sink; a flow behind one is a hotspot, not a vulnerability. |
| 12 | +
|
| 13 | +Plugins contribute both models through ``Plugin.models``, the same way they contribute |
| 14 | +sources and sinks: that is the plugin extension point for refutation evidence. |
| 15 | +
|
| 16 | +Expected to remain red until ``abstract.ranges``, ``Validator`` and ``AuthorizationGuard`` |
| 17 | +exist and the refutation engine consumes them. |
| 18 | +""" |
| 19 | + |
| 20 | +from __future__ import annotations |
| 21 | + |
| 22 | +from pathlib import Path |
| 23 | + |
| 24 | +import pytest |
| 25 | + |
| 26 | +from coretrace_python import engine |
| 27 | +from coretrace_python.analysis import AnalysisManager |
| 28 | +from coretrace_python.findings import Confidence |
| 29 | +from coretrace_python.findings.refutation import RefutationAnalysis, Status, Verdict, Verdicts |
| 30 | +from coretrace_python.frontend import build_hir |
| 31 | +from coretrace_python.hir import nodes |
| 32 | +from coretrace_python.ir.model import Call, Symbol |
| 33 | +from coretrace_python.ir.ssa import SSAAnalysis |
| 34 | +from coretrace_python.plugins import ModelPlugin |
| 35 | +from coretrace_python.semantic.symbols import SymbolId |
| 36 | +from coretrace_python.source import SourceManager |
| 37 | +from coretrace_python.taint import ( |
| 38 | + SecurityModelAnalysis, |
| 39 | + SecurityModelRegistry, |
| 40 | + Sink, |
| 41 | + Source, |
| 42 | + TaintKind, |
| 43 | +) |
| 44 | + |
| 45 | +try: |
| 46 | + from coretrace_python.abstract import Interval, RangeAnalysis |
| 47 | + from coretrace_python.taint import AuthorizationGuard, Validator |
| 48 | +except ImportError as error: # pragma: no cover - red until the evidence lands |
| 49 | + MISSING = error |
| 50 | +else: |
| 51 | + MISSING = None |
| 52 | + |
| 53 | + |
| 54 | +@pytest.fixture(autouse=True) |
| 55 | +def require_evidence() -> None: |
| 56 | + if MISSING is not None: |
| 57 | + pytest.fail(f"refutation evidence is not implemented yet: {MISSING}") |
| 58 | + |
| 59 | + |
| 60 | +REPO = Path(__file__).resolve().parent.parent |
| 61 | +PLUGINS = REPO / "plugins" |
| 62 | + |
| 63 | +OS_SYSTEM = Sink(SymbolId("python.os.system"), TaintKind.COMMAND) |
| 64 | +STDIN = Source(SymbolId("python.builtins.input"), "stdin") |
| 65 | +LOGIN = SymbolId("python.flask_login.current_user.is_authenticated") |
| 66 | +LOGIN_REQUIRED = SymbolId("python.flask_login.login_required") |
| 67 | + |
| 68 | +PRELUDE = "import os\nimport re\nfrom flask_login import current_user, login_required\n\n" |
| 69 | + |
| 70 | + |
| 71 | +def manager_for(body: str, *models: object) -> AnalysisManager: |
| 72 | + module = build_hir(SourceManager().add_source("proof.py", PRELUDE + body)) |
| 73 | + manager = AnalysisManager(module) |
| 74 | + manager.register(*engine.ALL_ANALYSES) |
| 75 | + registry = SecurityModelRegistry() |
| 76 | + registry.register(STDIN, OS_SYSTEM, *models) # type: ignore[arg-type] |
| 77 | + manager.provide(SecurityModelAnalysis, registry.freeze()) |
| 78 | + return manager |
| 79 | + |
| 80 | + |
| 81 | +def first_function(manager: AnalysisManager) -> nodes.Function: |
| 82 | + return next(s for s in manager.module.body if isinstance(s, nodes.Function)) |
| 83 | + |
| 84 | + |
| 85 | +def verdict(body: str, *models: object) -> Verdict: |
| 86 | + manager = manager_for(body, *models) |
| 87 | + result: Verdicts = manager.get(RefutationAnalysis, first_function(manager)) |
| 88 | + (found,) = result.all() |
| 89 | + return found |
| 90 | + |
| 91 | + |
| 92 | +def ranges_at_sink(body: str): # type: ignore[no-untyped-def] |
| 93 | + """The intervals in the block calling ``os.system``, keyed by argument position.""" |
| 94 | + |
| 95 | + manager = manager_for(body) |
| 96 | + function = first_function(manager) |
| 97 | + ssa = manager.get(SSAAnalysis, function) |
| 98 | + facts = manager.get(RangeAnalysis, function) |
| 99 | + defs = {i.result: i for block in ssa.blocks for i in block.instructions if i.result} |
| 100 | + for block in ssa.blocks: |
| 101 | + for instruction in block.instructions: |
| 102 | + callee = defs.get(instruction.callee) if isinstance(instruction, Call) else None |
| 103 | + if isinstance(callee, Symbol) and callee.symbol_id == OS_SYSTEM.symbol: |
| 104 | + at = facts.at(block.id) |
| 105 | + return [at.get(argument) for argument in instruction.arguments] |
| 106 | + raise AssertionError("no call to os.system") |
| 107 | + |
| 108 | + |
| 109 | +# --------------------------------------------------------------------------- ranges |
| 110 | + |
| 111 | + |
| 112 | +def test_intervals_join_and_widen() -> None: |
| 113 | + assert Interval(0, 5).join(Interval(3, 9)) == Interval(0, 9) |
| 114 | + assert Interval(0, 5).widen(Interval(0, 6)) == Interval(0, float("inf")) |
| 115 | + assert Interval(0, 5).widen(Interval(-1, 5)) == Interval(float("-inf"), 5) |
| 116 | + assert Interval(0, 5).widen(Interval(1, 4)) == Interval(0, 5) |
| 117 | + assert str(Interval(0, float("inf"))) == "[0, inf]" |
| 118 | + |
| 119 | + |
| 120 | +def test_numeric_constants_and_arithmetic_have_ranges() -> None: |
| 121 | + assert ranges_at_sink("def f():\n n = 2 + 3 * 4\n os.system(n, n - 20, -n)\n") == [ |
| 122 | + Interval(14, 14), |
| 123 | + Interval(-6, -6), |
| 124 | + Interval(-14, -14), |
| 125 | + ] |
| 126 | + |
| 127 | + |
| 128 | +def test_conversions_and_lengths_are_numeric() -> None: |
| 129 | + low, length, raw = ranges_at_sink( |
| 130 | + "def f(x):\n os.system(int(x), len(x), x)\n" |
| 131 | + ) |
| 132 | + assert low == Interval(float("-inf"), float("inf")) |
| 133 | + assert length == Interval(0, float("inf")) |
| 134 | + assert raw is None |
| 135 | + |
| 136 | + |
| 137 | +def test_branches_refine_ranges_on_each_side() -> None: |
| 138 | + then_side = ranges_at_sink("def f(x):\n n = int(x)\n if n < 10:\n os.system(n)\n") |
| 139 | + else_side = ranges_at_sink("def f(x):\n n = int(x)\n if n < 10:\n pass\n else:\n os.system(n)\n") |
| 140 | + chained = ranges_at_sink("def f(x):\n n = int(x)\n if 0 < n <= 100:\n os.system(n)\n") |
| 141 | + |
| 142 | + assert then_side == [Interval(float("-inf"), 10)] |
| 143 | + assert else_side == [Interval(10, float("inf"))] |
| 144 | + assert chained == [Interval(0, 100)] |
| 145 | + |
| 146 | + |
| 147 | +def test_loops_terminate_by_widening() -> None: |
| 148 | + (counter,) = ranges_at_sink( |
| 149 | + "def f(x):\n i = 0\n while i < len(x):\n i = i + 1\n os.system(i)\n" |
| 150 | + ) |
| 151 | + assert counter == Interval(0, float("inf")) |
| 152 | + |
| 153 | + |
| 154 | +def test_range_analysis_is_a_function_analysis_over_ssa() -> None: |
| 155 | + from coretrace_python.cfg import CFGAnalysis |
| 156 | + |
| 157 | + assert RangeAnalysis.name == "abstract.ranges" |
| 158 | + assert {SSAAnalysis, CFGAnalysis} <= RangeAnalysis.requires |
| 159 | + assert RangeAnalysis in engine.ALL_ANALYSES |
| 160 | + |
| 161 | + |
| 162 | +# --------------------------------------------------------------------------- numeric evidence |
| 163 | + |
| 164 | + |
| 165 | +def test_numeric_values_cannot_inject() -> None: |
| 166 | + found = verdict("def f():\n n = int(input())\n os.system('sleep ' + str(n))\n") |
| 167 | + |
| 168 | + assert found.status is Status.REFUTED |
| 169 | + assert "numeric" in found.evidence |
| 170 | + |
| 171 | + |
| 172 | +def test_ranges_show_in_the_evidence() -> None: |
| 173 | + found = verdict( |
| 174 | + "def f():\n n = int(input())\n if 0 <= n < 60:\n os.system('sleep ' + str(n))\n" |
| 175 | + ) |
| 176 | + assert found.status is Status.REFUTED |
| 177 | + assert "[0, 60]" in found.evidence |
| 178 | + |
| 179 | + |
| 180 | +def test_a_numeric_path_does_not_cover_a_raw_one() -> None: |
| 181 | + found = verdict("def f():\n x = input()\n os.system(x + str(int(x)))\n") |
| 182 | + assert found.status is Status.VULNERABILITY |
| 183 | + |
| 184 | + |
| 185 | +def test_a_bounded_length_proves_nothing_about_the_string() -> None: |
| 186 | + found = verdict("def f():\n x = input()\n if len(x) < 5:\n os.system(x)\n") |
| 187 | + assert found.status is Status.HOTSPOT |
| 188 | + |
| 189 | + |
| 190 | +# --------------------------------------------------------------------------- validators |
| 191 | + |
| 192 | + |
| 193 | +FULLMATCH = Validator(SymbolId("python.re.fullmatch"), argument=1) if MISSING is None else None |
| 194 | + |
| 195 | + |
| 196 | +def test_validators_and_authorization_guards_are_indexed_in_the_model_table() -> None: |
| 197 | + guard = AuthorizationGuard(LOGIN, "login") |
| 198 | + registry = SecurityModelRegistry() |
| 199 | + registry.register(FULLMATCH, guard, Sink(LOGIN, TaintKind.HTML)) |
| 200 | + table = registry.freeze() |
| 201 | + |
| 202 | + assert table.validator(FULLMATCH.symbol) == FULLMATCH |
| 203 | + assert table.validator(LOGIN) is None |
| 204 | + assert table.authorization(LOGIN) == guard |
| 205 | + assert table.authorization(FULLMATCH.symbol) is None |
| 206 | + assert table.validators == (FULLMATCH,) |
| 207 | + assert table.authorizations == (guard,) |
| 208 | + assert Validator(LOGIN).argument == 0 |
| 209 | + extended = table.extended(Sink(SymbolId("python.x"), TaintKind.ADVISORY)) |
| 210 | + assert extended.validators == (FULLMATCH,) and extended.authorizations == (guard,) |
| 211 | + |
| 212 | + |
| 213 | +def test_a_true_validator_refutes_the_argument_it_validates() -> None: |
| 214 | + found = verdict( |
| 215 | + "def f():\n x = input()\n if re.fullmatch('[a-z]+', x):\n os.system(x)\n", |
| 216 | + FULLMATCH, |
| 217 | + ) |
| 218 | + assert found.status is Status.REFUTED |
| 219 | + assert found.evidence == "validated by python.re.fullmatch" |
| 220 | + |
| 221 | + |
| 222 | +def test_a_validator_only_proves_its_declared_argument() -> None: |
| 223 | + found = verdict( |
| 224 | + "def f():\n x = input()\n if re.fullmatch(x, 'abc'):\n os.system(x)\n", |
| 225 | + FULLMATCH, |
| 226 | + ) |
| 227 | + assert found.status is Status.HOTSPOT |
| 228 | + |
| 229 | + |
| 230 | +def test_validators_on_the_wrong_branch_prove_nothing() -> None: |
| 231 | + found = verdict( |
| 232 | + "def f():\n x = input()\n if not re.fullmatch('[a-z]+', x):\n os.system(x)\n", |
| 233 | + FULLMATCH, |
| 234 | + ) |
| 235 | + assert found.status is Status.VULNERABILITY |
| 236 | + |
| 237 | + |
| 238 | +def test_compiled_pattern_validators_resolve_through_derived_symbols() -> None: |
| 239 | + found = verdict( |
| 240 | + "def f():\n x = input()\n pattern = re.compile('[a-z]+')\n" |
| 241 | + " if pattern.fullmatch(x):\n os.system(x)\n", |
| 242 | + Validator(SymbolId("python.re.compile.fullmatch")), |
| 243 | + ) |
| 244 | + assert found.status is Status.REFUTED |
| 245 | + |
| 246 | + |
| 247 | +# --------------------------------------------------------------------------- authorization |
| 248 | + |
| 249 | + |
| 250 | +def test_a_dominating_authorization_condition_makes_a_hotspot() -> None: |
| 251 | + found = verdict( |
| 252 | + "def f():\n if current_user.is_authenticated:\n os.system(input())\n", |
| 253 | + AuthorizationGuard(LOGIN, "login"), |
| 254 | + ) |
| 255 | + assert found.status is Status.HOTSPOT |
| 256 | + assert found.evidence == "behind authorization (login) at line 6" |
| 257 | + |
| 258 | + |
| 259 | +def test_an_early_return_on_missing_authorization_counts() -> None: |
| 260 | + found = verdict( |
| 261 | + "def f():\n if not current_user.is_authenticated:\n return\n os.system(input())\n", |
| 262 | + AuthorizationGuard(LOGIN, "login"), |
| 263 | + ) |
| 264 | + assert found.status is Status.HOTSPOT |
| 265 | + |
| 266 | + |
| 267 | +def test_an_authorization_decorator_counts() -> None: |
| 268 | + found = verdict( |
| 269 | + "@login_required\ndef f():\n os.system(input())\n", |
| 270 | + AuthorizationGuard(LOGIN_REQUIRED, "login"), |
| 271 | + ) |
| 272 | + assert found.status is Status.HOTSPOT |
| 273 | + assert found.evidence == "behind authorization (login) by decorator" |
| 274 | + |
| 275 | + |
| 276 | +def test_without_the_model_the_same_code_is_a_vulnerability() -> None: |
| 277 | + found = verdict("def f():\n if current_user.is_authenticated:\n os.system(input())\n") |
| 278 | + assert found.status is Status.VULNERABILITY |
| 279 | + |
| 280 | + |
| 281 | +def test_authorization_does_not_override_a_refutation() -> None: |
| 282 | + found = verdict( |
| 283 | + "@login_required\ndef f():\n x = input()\n if x.isdigit():\n os.system(x)\n", |
| 284 | + AuthorizationGuard(LOGIN_REQUIRED, "login"), |
| 285 | + ) |
| 286 | + assert found.status is Status.REFUTED |
| 287 | + |
| 288 | + |
| 289 | +# --------------------------------------------------------------------------- plugins |
| 290 | + |
| 291 | + |
| 292 | +def test_any_plugin_contributes_refutation_models() -> None: |
| 293 | + class Evidence(ModelPlugin): |
| 294 | + name = "evidence" |
| 295 | + models = (FULLMATCH, AuthorizationGuard(LOGIN, "login")) |
| 296 | + |
| 297 | + table = engine.plugin_models([Evidence()]) |
| 298 | + |
| 299 | + assert table.validators == (FULLMATCH,) |
| 300 | + assert table.authorizations == (AuthorizationGuard(LOGIN, "login"),) |
| 301 | + |
| 302 | + |
| 303 | +def test_shipped_models_refute_fullmatch_and_flag_login_required_routes() -> None: |
| 304 | + findings = engine.check( |
| 305 | + SourceManager().add_source( |
| 306 | + "app.py", |
| 307 | + "import os\nimport re\nfrom flask import Flask, request\nfrom flask_login import login_required\n\n" |
| 308 | + "app = Flask(__name__)\n\n" |
| 309 | + "@app.route('/ping')\n" |
| 310 | + "def ping():\n" |
| 311 | + " host = request.args['host']\n" |
| 312 | + " if re.fullmatch(r'[a-z.]+', host):\n" |
| 313 | + " os.system('ping ' + host)\n\n" |
| 314 | + "@app.route('/admin')\n" |
| 315 | + "@login_required\n" |
| 316 | + "def admin():\n" |
| 317 | + " os.system(request.args['cmd'])\n", |
| 318 | + ), |
| 319 | + [PLUGINS], |
| 320 | + ) |
| 321 | + |
| 322 | + assert [(f.function, f.confidence) for f in findings] == [("admin", Confidence.MEDIUM)] |
| 323 | + assert findings[0].metadata["evidence"] == "behind authorization (login) by decorator" |
| 324 | + |
| 325 | + |
| 326 | +def test_shipped_django_login_required_view_is_a_hotspot() -> None: |
| 327 | + findings = engine.check( |
| 328 | + SourceManager().add_source( |
| 329 | + "views.py", |
| 330 | + "import os\nfrom django.contrib.auth.decorators import login_required\n\n" |
| 331 | + "@login_required\n" |
| 332 | + "def run(request):\n" |
| 333 | + " os.system(request.GET['cmd'])\n", |
| 334 | + ), |
| 335 | + [PLUGINS], |
| 336 | + ) |
| 337 | + assert [(f.rule_id, f.confidence) for f in findings] == [("command-injection", Confidence.MEDIUM)] |
0 commit comments