1010- no variable where HTML escaping does not protect it: in a ``<script>`` or ``<style>``
1111 block, an event handler, a ``style`` or ``srcdoc`` attribute, a URL attribute whose
1212 value does not start with a fixed relative or ``http(s)`` prefix, an unquoted
13- attribute, an attribute or a tag name;
13+ attribute, an attribute or a tag name, including in such a block or tag the template
14+ leaves open, which once included runs into the markup that follows;
15+ - no include of a template that ends inside a ``<script>`` or ``<style>`` block, a tag
16+ or an end tag, which the markup after the include then continues;
1417- no Python file of the project turning autoescaping off.
1518
19+ A comment hides what it holds only up to its first template tag: an include there may
20+ close it, and a condition around its ``<!--`` may leave it out.
21+
1622Templates are found under the project's ``templates`` directories, by their path below
1723one. A template the engine cannot find — named by an expression, under a ``DIRS`` entry
1824of another name, shipped by an installed package — or cannot read is never assumed to
4450from dataclasses import dataclass , field
4551from pathlib import Path
4652from types import MappingProxyType
47- from typing import ClassVar
53+ from typing import ClassVar , NamedTuple
4854
4955from coretrace_python .analysis import Analysis , AnalysisContext
5056from coretrace_python .frontend import HIRBuildError , ParseError , build_hir
153159_ATTRIBUTE_NAME = r"(?:=[^\s/>=]*+|[^\s/>=]++)"
154160_ATTRIBUTE_VALUE = r"(?:\"[^\"]*+(?:\"|\Z)|'[^']*+(?:'|\Z)|[^\s>]*+)"
155161_ATTRIBUTE = re .compile (rf"({ _ATTRIBUTE_NAME } )(?:\s*+=\s*+({ _ATTRIBUTE_VALUE } ))?" )
156- _START_TAG = re .compile (rf"<([A-Za-z][^\s/>]*+)((?:[\s/]++|{ _ATTRIBUTE_NAME } (?:\s*+=\s*+{ _ATTRIBUTE_VALUE } )?)*+)>?" )
157- _RAW_TEXT = re .compile (r"<(script|style)\b[^>]*>(.*?)</\1\s*>" , re .DOTALL | re .IGNORECASE )
158- _COMMENT = re .compile (r"<!--.*?-->" , re .DOTALL )
159- _END_TAG = re .compile (r"</[A-Za-z][^>]*>" )
162+ _START_TAG = re .compile (rf"<([A-Za-z][^\s/>]*+)((?:[\s/]++|{ _ATTRIBUTE_NAME } (?:\s*+=\s*+{ _ATTRIBUTE_VALUE } )?)*+)(>)?" )
163+ # A script or style block and an end tag the text leaves open run to its end as well, for
164+ # the same reason (#272); their last group, like the start tag's, is then empty or None.
165+ # Only ``script`` and ``style`` themselves: a tag name runs to whitespace, ``/`` or ``>``,
166+ # so ``<style-switcher>`` is another element, whose content is markup.
167+ _RAW_TEXT = re .compile (r"<(script|style)(?=[\s/>])[^>]*+>(.*?)(</\1\s*>|\Z)" , re .DOTALL | re .IGNORECASE )
168+ _END_TAG = re .compile (r"</[A-Za-z][^>]*+(>)?" )
169+ # ``-->``, ``--!>``, or the ``>`` that ends ``<!-->`` and ``<!--->`` at once.
170+ _COMMENT_END = re .compile (r"--!?>" )
171+ # ``<{{ name }}`` or ``</ {{ name }}``: the variable would name a tag.
172+ _NAMED_BY_OUTPUT = re .compile (rf"<[/ \t\n]*{ _OUTPUT } " )
160173# Django's filter expressions: a constant or a variable, then filters, each with an
161174# optional argument, the way ``django.template.base.FilterExpression`` reads them.
162175_STRING = r"\"[^\"\\]*(?:\\.[^\"\\]*)*\"|'[^'\\]*(?:\\.[^'\\]*)*'"
179192
180193def escaped_templates (root : Path ) -> frozenset [str ]:
181194 """The names of the templates under ``root`` that escape every variable they render,
182- as ``render_to_string`` names them (their path below a ``templates`` directory)."""
195+ as ``render_to_string`` names them (their path below a ``templates`` directory). A
196+ template including one that ends inside a script or style block, a tag or an end tag
197+ is not among them: the markup after its include continues that construct. A
198+ template extending another renders it, and ends where it ends."""
183199
184200 found : dict [str , list [Path ]] = {}
185201 for path in discover_files (root ):
@@ -189,11 +205,20 @@ def escaped_templates(root: Path) -> frozenset[str]:
189205 found .setdefault (name , []).append (path )
190206 local : dict [str , bool ] = {}
191207 requires : dict [str , set [str ]] = {}
208+ extends : dict [str , set [str ]] = {}
209+ includes : dict [str , set [str ]] = {}
210+ open_ended : set [str ] = set ()
192211 for name , paths in found .items ():
193212 verdicts = [_verdict (path ) for path in paths ]
194- local [name ] = all (safe for safe , _ in verdicts )
195- requires [name ] = {needed for _ , needs in verdicts for needed in needs }
196- escaped = {name for name , safe in local .items () if safe }
213+ local [name ] = all (verdict .escapes for verdict in verdicts )
214+ requires [name ] = {needed for verdict in verdicts for needed in verdict .extends | verdict .includes }
215+ extends [name ] = {extended for verdict in verdicts for extended in verdict .extends }
216+ includes [name ] = {included for verdict in verdicts for included in verdict .includes }
217+ if any (verdict .open_end for verdict in verdicts ):
218+ open_ended .add (name )
219+ while grown := {name for name , extended in extends .items () if extended & open_ended } - open_ended :
220+ open_ended |= grown
221+ escaped = {name for name , safe in local .items () if safe and not includes [name ] & open_ended }
197222 changed = True
198223 while changed :
199224 kept = {name for name in escaped if requires [name ] <= escaped }
@@ -212,12 +237,26 @@ def _read(path: Path) -> str:
212237 return _text (path ) or ""
213238
214239
215- def _verdict (path : Path ) -> tuple [bool , set [str ]]:
216- """Whether one template file escapes all it prints, and what it needs: a file the
217- engine cannot read is not established to escape, like a template it cannot find."""
240+ class _Verdict (NamedTuple ):
241+ """Whether a template escapes all it prints itself; the templates it extends and
242+ those it includes, which must escape too; and whether its text ends inside a
243+ construct, which the markup after its include continues."""
244+
245+ escapes : bool
246+ extends : frozenset [str ]
247+ includes : frozenset [str ]
248+ open_end : bool
249+
250+
251+ _REFUSED = _Verdict (escapes = False , extends = frozenset (), includes = frozenset (), open_end = False )
252+
253+
254+ def _verdict (path : Path ) -> _Verdict :
255+ """What one template file establishes: a file the engine cannot read is not
256+ established to escape, like a template it cannot find."""
218257
219258 text = _text (path )
220- return ( False , set ()) if text is None else _inspect (text )
259+ return _REFUSED if text is None else _inspect (text )
221260
222261
223262def _text (path : Path ) -> str | None :
@@ -229,11 +268,11 @@ def _text(path: Path) -> str | None:
229268 return None
230269
231270
232- def _inspect (text : str ) -> tuple [bool , set [str ]]:
233- """Whether the template escapes all it prints itself, and the templates it extends
234- or includes, which must escape too."""
271+ def _inspect (text : str ) -> _Verdict :
272+ """What the template of ``text`` establishes (``_Verdict``)."""
235273
236- requires : set [str ] = set ()
274+ extends : set [str ] = set ()
275+ includes : set [str ] = set ()
237276 html : list [str ] = []
238277 skipping : str | None = None
239278 position = 0
@@ -251,7 +290,7 @@ def _inspect(text: str) -> tuple[bool, set[str]]:
251290 position = match .end ()
252291 if variable is not None :
253292 if not _escaping_filters (variable ):
254- return False , requires
293+ return _REFUSED
255294 html .append (_OUTPUT )
256295 elif words and words [0 ] in ("comment" , "verbatim" ):
257296 skipping = f"end{ words [0 ]} "
@@ -260,23 +299,24 @@ def _inspect(text: str) -> tuple[bool, set[str]]:
260299 elif words :
261300 name = words [0 ]
262301 if name not in _TAGS :
263- return False , requires
302+ return _REFUSED
264303 if name == "load" and not set (_loaded (words [1 :])) <= _LIBRARIES :
265- return False , requires
304+ return _REFUSED
266305 if name == "autoescape" and words [1 :] != ["on" ]:
267- return False , requires
306+ return _REFUSED
268307 if name == "filter" and not _escaping_filters (tag .split (None , 1 )[1 ] if len (words ) > 1 else "" ):
269- return False , requires
308+ return _REFUSED
270309 if name in ("extends" , "include" ):
271310 needed = _constant (words [1 ] if len (words ) > 1 else "" )
272311 if needed is None :
273- return False , requires
274- requires .add (needed )
312+ return _REFUSED
313+ ( extends if name == "extends" else includes ) .add (needed )
275314 html .append (_OUTPUT if name in _PRINTING else _URL if name in ("url" , "static" ) else _TAG )
276315 if skipping is not None :
277- return False , requires
316+ return _REFUSED
278317 html .append (text [position :])
279- return _escapes_where_printed ("" .join (html )), requires
318+ reading = _read_markup ("" .join (html ))
319+ return _Verdict (reading .escapes , frozenset (extends ), frozenset (includes ), reading .open_end )
280320
281321
282322def _loaded (words : list [str ]) -> list [str ]:
@@ -294,36 +334,66 @@ def _constant(word: str) -> str | None:
294334 return word [1 :- 1 ] if len (word ) >= 2 and word [0 ] == word [- 1 ] and word [0 ] in "\" '" else None
295335
296336
297- def _escapes_where_printed (html : str ) -> bool :
298- """Whether every output marker of ``html`` stands where HTML escaping protects it."""
337+ class _Reading (NamedTuple ):
338+ """What one pass over a template's markup establishes: whether every output marker
339+ stands where HTML escaping protects it, and whether the text ends inside a script or
340+ style block, a tag or an end tag, which the markup after its include continues."""
341+
342+ escapes : bool
343+ open_end : bool
344+
345+
346+ def _read_markup (html : str ) -> _Reading :
347+ """Read ``html`` once, as the HTML tokenizer reads the page it is part of. Each
348+ construct is matched where it starts and one the text leaves open runs to its end,
349+ so every character is read once, where copying the rest of the text at each ``<``
350+ and rescanning what never closed took quadratic time (#272).
351+
352+ A comment ends at its close, as a browser closes it, or at the first template tag
353+ inside it, whichever comes first, and what follows the tag is read as markup: an
354+ include there may close the comment, a condition around its ``<!--`` leave it out.
355+ A variable cannot close it, escaped. A comment left open, with no tag after it,
356+ hides the rest of the text, and hands nothing on: what follows its include stays
357+ inside it."""
299358
300359 position = 0
360+ open_end = False
361+ # The first comment close after the comment being read, searched once for all the
362+ # comments that open before it; and the position after which none closes.
363+ close : re .Match [str ] | None = None
364+ unclosed_from = len (html ) + 1
301365 while (start := html .find ("<" , position )) != - 1 :
302- rest = html [start :]
303- comment = _COMMENT .match (rest )
304- raw = _RAW_TEXT .match (rest )
305- tag = _START_TAG .match (rest )
306- end = _END_TAG .match (rest )
307- if comment :
308- position = start + comment .end ()
309- elif raw :
310- if _OUTPUT in raw .group (2 ) or not _attributes_escape (raw .group (0 )[: raw .start (2 )]):
311- return False
312- position = start + raw .end ()
313- elif tag :
366+ if html .startswith ("<!--" , start ):
367+ if start >= unclosed_from :
368+ close = None
369+ elif close is None or close .start () < start + 2 :
370+ close = _COMMENT_END .search (html , start + 2 )
371+ if close is None :
372+ unclosed_from = start
373+ first_tag = html .find (_TAG , start + 4 , len (html ) if close is None else close .start ())
374+ if first_tag != - 1 :
375+ position = first_tag + 1
376+ elif close is not None :
377+ position = close .end ()
378+ else :
379+ break
380+ elif raw := _RAW_TEXT .match (html , start ):
381+ if _OUTPUT in raw .group (2 ) or not _attributes_escape (html [start : raw .start (2 )]):
382+ return _Reading (escapes = False , open_end = False )
383+ position , open_end = raw .end (), not raw .group (3 )
384+ elif tag := _START_TAG .match (html , start ):
314385 if _OUTPUT in tag .group (1 ) or not _attributes_escape (tag .group (2 )):
315- return False
316- position = start + tag .end ()
317- elif end :
386+ return _Reading ( escapes = False , open_end = False )
387+ position , open_end = tag .end (), tag . group ( 3 ) is None
388+ elif end := _END_TAG . match ( html , start ) :
318389 if _OUTPUT in end .group (0 ):
319- return False
320- position = start + end .end ()
390+ return _Reading (escapes = False , open_end = False )
391+ position , open_end = end .end (), end .group (1 ) is None
392+ elif _NAMED_BY_OUTPUT .match (html , start ):
393+ return _Reading (escapes = False , open_end = False )
321394 else :
322- # ``<{{ name }}``: the variable would name a tag, escaping cannot stop it.
323- if rest [1 :].lstrip ("/ \t \n " ).startswith (_OUTPUT ):
324- return False
325395 position = start + 1
326- return True
396+ return _Reading ( escapes = True , open_end = open_end )
327397
328398
329399def _attributes_escape (attributes : str ) -> bool :
0 commit comments