Skip to content

Commit 930d90d

Browse files
authored
Merge pull request #275 from CoreTrace/fix/templates-unclosed-constructs
fix(templates): read what a template leaves open, and comments, as a browser does
2 parents ff49a2c + 0ec8c4c commit 930d90d

3 files changed

Lines changed: 370 additions & 54 deletions

File tree

‎docs/usage.md‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -197,10 +197,17 @@ It establishes escaping only when all of the following hold:
197197
or an attribute name.
198198
- No Python file of the project sets `autoescape` to `False`.
199199

200-
Start tags and their attributes are read as a browser's HTML tokenizer reads them, in time
201-
linear in the size of the template. A tag, or a quoted attribute value, a template leaves
202-
open at its end is checked too: once the template is included, it runs into the markup
203-
that follows, the value staying quoted until its quote closes.
200+
Templates are read as a browser's HTML tokenizer reads them, in time linear in the size
201+
of the template. What a template leaves open at its end is checked too, since once the
202+
template is included it runs into the markup that follows: a tag's attributes are
203+
checked, a quoted value stays quoted until its quote closes, and a variable in a
204+
`<script>` or `<style>` block or an end tag left open is refused, even where the page
205+
ends there and the browser would never run it. A template including one that ends inside
206+
such a block, a tag or an end tag is not established either: the markup after the
207+
include continues it. A template extending another ends where that one ends. A comment
208+
hides what it holds only up to its first template tag, since an include there may close
209+
it and a condition around its `<!--` may leave it out. A `{% block %}` or
210+
`{% include %}` placed inside a parent's `<script>` is not covered yet (#273).
204211

205212
The analyzer never assumes escaping where it cannot read the template. That covers a
206213
template file it cannot read, a template named by an expression, a template found

‎src/coretrace_python/taint/templates.py‎

Lines changed: 120 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,15 @@
1010
- no variable where HTML escaping does not protect it: in a ``<script>`` or ``<style>``
1111
block, an event handler, a ``style`` or ``srcdoc`` attribute, a URL attribute whose
1212
value does not start with a fixed relative or ``http(s)`` prefix, an unquoted
13-
attribute, an attribute or a tag name;
13+
attribute, an attribute or a tag name, including in such a block or tag the template
14+
leaves open, which once included runs into the markup that follows;
15+
- no include of a template that ends inside a ``<script>`` or ``<style>`` block, a tag
16+
or an end tag, which the markup after the include then continues;
1417
- no Python file of the project turning autoescaping off.
1518
19+
A comment hides what it holds only up to its first template tag: an include there may
20+
close it, and a condition around its ``<!--`` may leave it out.
21+
1622
Templates are found under the project's ``templates`` directories, by their path below
1723
one. A template the engine cannot find — named by an expression, under a ``DIRS`` entry
1824
of another name, shipped by an installed package — or cannot read is never assumed to
@@ -44,7 +50,7 @@
4450
from dataclasses import dataclass, field
4551
from pathlib import Path
4652
from types import MappingProxyType
47-
from typing import ClassVar
53+
from typing import ClassVar, NamedTuple
4854

4955
from coretrace_python.analysis import Analysis, AnalysisContext
5056
from coretrace_python.frontend import HIRBuildError, ParseError, build_hir
@@ -153,10 +159,17 @@
153159
_ATTRIBUTE_NAME = r"(?:=[^\s/>=]*+|[^\s/>=]++)"
154160
_ATTRIBUTE_VALUE = r"(?:\"[^\"]*+(?:\"|\Z)|'[^']*+(?:'|\Z)|[^\s>]*+)"
155161
_ATTRIBUTE = re.compile(rf"({_ATTRIBUTE_NAME})(?:\s*+=\s*+({_ATTRIBUTE_VALUE}))?")
156-
_START_TAG = re.compile(rf"<([A-Za-z][^\s/>]*+)((?:[\s/]++|{_ATTRIBUTE_NAME}(?:\s*+=\s*+{_ATTRIBUTE_VALUE})?)*+)>?")
157-
_RAW_TEXT = re.compile(r"<(script|style)\b[^>]*>(.*?)</\1\s*>", re.DOTALL | re.IGNORECASE)
158-
_COMMENT = re.compile(r"<!--.*?-->", re.DOTALL)
159-
_END_TAG = re.compile(r"</[A-Za-z][^>]*>")
162+
_START_TAG = re.compile(rf"<([A-Za-z][^\s/>]*+)((?:[\s/]++|{_ATTRIBUTE_NAME}(?:\s*+=\s*+{_ATTRIBUTE_VALUE})?)*+)(>)?")
163+
# A script or style block and an end tag the text leaves open run to its end as well, for
164+
# the same reason (#272); their last group, like the start tag's, is then empty or None.
165+
# Only ``script`` and ``style`` themselves: a tag name runs to whitespace, ``/`` or ``>``,
166+
# so ``<style-switcher>`` is another element, whose content is markup.
167+
_RAW_TEXT = re.compile(r"<(script|style)(?=[\s/>])[^>]*+>(.*?)(</\1\s*>|\Z)", re.DOTALL | re.IGNORECASE)
168+
_END_TAG = re.compile(r"</[A-Za-z][^>]*+(>)?")
169+
# ``-->``, ``--!>``, or the ``>`` that ends ``<!-->`` and ``<!--->`` at once.
170+
_COMMENT_END = re.compile(r"--!?>")
171+
# ``<{{ name }}`` or ``</ {{ name }}``: the variable would name a tag.
172+
_NAMED_BY_OUTPUT = re.compile(rf"<[/ \t\n]*{_OUTPUT}")
160173
# Django's filter expressions: a constant or a variable, then filters, each with an
161174
# optional argument, the way ``django.template.base.FilterExpression`` reads them.
162175
_STRING = r"\"[^\"\\]*(?:\\.[^\"\\]*)*\"|'[^'\\]*(?:\\.[^'\\]*)*'"
@@ -179,7 +192,10 @@
179192

180193
def escaped_templates(root: Path) -> frozenset[str]:
181194
"""The names of the templates under ``root`` that escape every variable they render,
182-
as ``render_to_string`` names them (their path below a ``templates`` directory)."""
195+
as ``render_to_string`` names them (their path below a ``templates`` directory). A
196+
template including one that ends inside a script or style block, a tag or an end tag
197+
is not among them: the markup after its include continues that construct. A
198+
template extending another renders it, and ends where it ends."""
183199

184200
found: dict[str, list[Path]] = {}
185201
for path in discover_files(root):
@@ -189,11 +205,20 @@ def escaped_templates(root: Path) -> frozenset[str]:
189205
found.setdefault(name, []).append(path)
190206
local: dict[str, bool] = {}
191207
requires: dict[str, set[str]] = {}
208+
extends: dict[str, set[str]] = {}
209+
includes: dict[str, set[str]] = {}
210+
open_ended: set[str] = set()
192211
for name, paths in found.items():
193212
verdicts = [_verdict(path) for path in paths]
194-
local[name] = all(safe for safe, _ in verdicts)
195-
requires[name] = {needed for _, needs in verdicts for needed in needs}
196-
escaped = {name for name, safe in local.items() if safe}
213+
local[name] = all(verdict.escapes for verdict in verdicts)
214+
requires[name] = {needed for verdict in verdicts for needed in verdict.extends | verdict.includes}
215+
extends[name] = {extended for verdict in verdicts for extended in verdict.extends}
216+
includes[name] = {included for verdict in verdicts for included in verdict.includes}
217+
if any(verdict.open_end for verdict in verdicts):
218+
open_ended.add(name)
219+
while grown := {name for name, extended in extends.items() if extended & open_ended} - open_ended:
220+
open_ended |= grown
221+
escaped = {name for name, safe in local.items() if safe and not includes[name] & open_ended}
197222
changed = True
198223
while changed:
199224
kept = {name for name in escaped if requires[name] <= escaped}
@@ -212,12 +237,26 @@ def _read(path: Path) -> str:
212237
return _text(path) or ""
213238

214239

215-
def _verdict(path: Path) -> tuple[bool, set[str]]:
216-
"""Whether one template file escapes all it prints, and what it needs: a file the
217-
engine cannot read is not established to escape, like a template it cannot find."""
240+
class _Verdict(NamedTuple):
241+
"""Whether a template escapes all it prints itself; the templates it extends and
242+
those it includes, which must escape too; and whether its text ends inside a
243+
construct, which the markup after its include continues."""
244+
245+
escapes: bool
246+
extends: frozenset[str]
247+
includes: frozenset[str]
248+
open_end: bool
249+
250+
251+
_REFUSED = _Verdict(escapes=False, extends=frozenset(), includes=frozenset(), open_end=False)
252+
253+
254+
def _verdict(path: Path) -> _Verdict:
255+
"""What one template file establishes: a file the engine cannot read is not
256+
established to escape, like a template it cannot find."""
218257

219258
text = _text(path)
220-
return (False, set()) if text is None else _inspect(text)
259+
return _REFUSED if text is None else _inspect(text)
221260

222261

223262
def _text(path: Path) -> str | None:
@@ -229,11 +268,11 @@ def _text(path: Path) -> str | None:
229268
return None
230269

231270

232-
def _inspect(text: str) -> tuple[bool, set[str]]:
233-
"""Whether the template escapes all it prints itself, and the templates it extends
234-
or includes, which must escape too."""
271+
def _inspect(text: str) -> _Verdict:
272+
"""What the template of ``text`` establishes (``_Verdict``)."""
235273

236-
requires: set[str] = set()
274+
extends: set[str] = set()
275+
includes: set[str] = set()
237276
html: list[str] = []
238277
skipping: str | None = None
239278
position = 0
@@ -251,7 +290,7 @@ def _inspect(text: str) -> tuple[bool, set[str]]:
251290
position = match.end()
252291
if variable is not None:
253292
if not _escaping_filters(variable):
254-
return False, requires
293+
return _REFUSED
255294
html.append(_OUTPUT)
256295
elif words and words[0] in ("comment", "verbatim"):
257296
skipping = f"end{words[0]}"
@@ -260,23 +299,24 @@ def _inspect(text: str) -> tuple[bool, set[str]]:
260299
elif words:
261300
name = words[0]
262301
if name not in _TAGS:
263-
return False, requires
302+
return _REFUSED
264303
if name == "load" and not set(_loaded(words[1:])) <= _LIBRARIES:
265-
return False, requires
304+
return _REFUSED
266305
if name == "autoescape" and words[1:] != ["on"]:
267-
return False, requires
306+
return _REFUSED
268307
if name == "filter" and not _escaping_filters(tag.split(None, 1)[1] if len(words) > 1 else ""):
269-
return False, requires
308+
return _REFUSED
270309
if name in ("extends", "include"):
271310
needed = _constant(words[1] if len(words) > 1 else "")
272311
if needed is None:
273-
return False, requires
274-
requires.add(needed)
312+
return _REFUSED
313+
(extends if name == "extends" else includes).add(needed)
275314
html.append(_OUTPUT if name in _PRINTING else _URL if name in ("url", "static") else _TAG)
276315
if skipping is not None:
277-
return False, requires
316+
return _REFUSED
278317
html.append(text[position:])
279-
return _escapes_where_printed("".join(html)), requires
318+
reading = _read_markup("".join(html))
319+
return _Verdict(reading.escapes, frozenset(extends), frozenset(includes), reading.open_end)
280320

281321

282322
def _loaded(words: list[str]) -> list[str]:
@@ -294,36 +334,66 @@ def _constant(word: str) -> str | None:
294334
return word[1:-1] if len(word) >= 2 and word[0] == word[-1] and word[0] in "\"'" else None
295335

296336

297-
def _escapes_where_printed(html: str) -> bool:
298-
"""Whether every output marker of ``html`` stands where HTML escaping protects it."""
337+
class _Reading(NamedTuple):
338+
"""What one pass over a template's markup establishes: whether every output marker
339+
stands where HTML escaping protects it, and whether the text ends inside a script or
340+
style block, a tag or an end tag, which the markup after its include continues."""
341+
342+
escapes: bool
343+
open_end: bool
344+
345+
346+
def _read_markup(html: str) -> _Reading:
347+
"""Read ``html`` once, as the HTML tokenizer reads the page it is part of. Each
348+
construct is matched where it starts and one the text leaves open runs to its end,
349+
so every character is read once, where copying the rest of the text at each ``<``
350+
and rescanning what never closed took quadratic time (#272).
351+
352+
A comment ends at its close, as a browser closes it, or at the first template tag
353+
inside it, whichever comes first, and what follows the tag is read as markup: an
354+
include there may close the comment, a condition around its ``<!--`` leave it out.
355+
A variable cannot close it, escaped. A comment left open, with no tag after it,
356+
hides the rest of the text, and hands nothing on: what follows its include stays
357+
inside it."""
299358

300359
position = 0
360+
open_end = False
361+
# The first comment close after the comment being read, searched once for all the
362+
# comments that open before it; and the position after which none closes.
363+
close: re.Match[str] | None = None
364+
unclosed_from = len(html) + 1
301365
while (start := html.find("<", position)) != -1:
302-
rest = html[start:]
303-
comment = _COMMENT.match(rest)
304-
raw = _RAW_TEXT.match(rest)
305-
tag = _START_TAG.match(rest)
306-
end = _END_TAG.match(rest)
307-
if comment:
308-
position = start + comment.end()
309-
elif raw:
310-
if _OUTPUT in raw.group(2) or not _attributes_escape(raw.group(0)[: raw.start(2)]):
311-
return False
312-
position = start + raw.end()
313-
elif tag:
366+
if html.startswith("<!--", start):
367+
if start >= unclosed_from:
368+
close = None
369+
elif close is None or close.start() < start + 2:
370+
close = _COMMENT_END.search(html, start + 2)
371+
if close is None:
372+
unclosed_from = start
373+
first_tag = html.find(_TAG, start + 4, len(html) if close is None else close.start())
374+
if first_tag != -1:
375+
position = first_tag + 1
376+
elif close is not None:
377+
position = close.end()
378+
else:
379+
break
380+
elif raw := _RAW_TEXT.match(html, start):
381+
if _OUTPUT in raw.group(2) or not _attributes_escape(html[start : raw.start(2)]):
382+
return _Reading(escapes=False, open_end=False)
383+
position, open_end = raw.end(), not raw.group(3)
384+
elif tag := _START_TAG.match(html, start):
314385
if _OUTPUT in tag.group(1) or not _attributes_escape(tag.group(2)):
315-
return False
316-
position = start + tag.end()
317-
elif end:
386+
return _Reading(escapes=False, open_end=False)
387+
position, open_end = tag.end(), tag.group(3) is None
388+
elif end := _END_TAG.match(html, start):
318389
if _OUTPUT in end.group(0):
319-
return False
320-
position = start + end.end()
390+
return _Reading(escapes=False, open_end=False)
391+
position, open_end = end.end(), end.group(1) is None
392+
elif _NAMED_BY_OUTPUT.match(html, start):
393+
return _Reading(escapes=False, open_end=False)
321394
else:
322-
# ``<{{ name }}``: the variable would name a tag, escaping cannot stop it.
323-
if rest[1:].lstrip("/ \t\n").startswith(_OUTPUT):
324-
return False
325395
position = start + 1
326-
return True
396+
return _Reading(escapes=True, open_end=open_end)
327397

328398

329399
def _attributes_escape(attributes: str) -> bool:

0 commit comments

Comments
 (0)