Skip to content

Commit 408e8a2

Browse files
committed
fix(source): count every line number by one rule, and place pyproject requirements
Review of the baseline line fix: line numbers had no single rule. The baseline now counted line feeds while the .env, YAML, INI and requirements readers used splitlines and the parser also breaks on a lone carriage return, so a fingerprint or an inline suppression could refer to another line and a rotated secret could be absorbed. lines_of, in the source layer, counts lines as Python does (\n, \r\n, a lone \r; never U+2028 or a form feed), and every producer and consumer of a line number now uses it: the baseline, inline suppressions, the configuration and requirements readers, and the JSON and TOML positions. pyproject.toml requirements were placed at the first line containing their text, or line 1 by default; the position component now finds the element of an array a key holds, and a requirement it cannot place gets a file location. Refs #207, #210
1 parent 1e794c8 commit 408e8a2

10 files changed

Lines changed: 316 additions & 47 deletions

File tree

‎docs/usage.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,9 @@ the directory of the checked file; a path outside it is printed as it is.
277277
}
278278
```
279279

280-
A value of a JSON or TOML file, or a package of a lock file, is located at its own line,
280+
Lines are counted as Python counts them, by `\n`, `\r\n` or a lone `\r` (never U+2028
281+
or a form feed), in every report, suppression and baseline. A value of a JSON or TOML
282+
file, a package of a lock file, or a requirement of `pyproject.toml`, is located at its own line,
281283
which a separate component establishes from the value's structural path and checks
282284
against the value. When it cannot (a dotted TOML key, an inline table, a multi-line
283285
value), the finding gets a file location instead of a guessed line: `line`, `column`,

‎src/coretrace_python/dependency/graph.py‎

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,16 @@
1313
import re
1414
import tomllib
1515
from collections.abc import Iterable, Iterator, Mapping
16-
from dataclasses import dataclass
16+
from dataclasses import dataclass, replace
1717
from pathlib import PurePath, PurePosixPath
1818
from types import MappingProxyType
1919
from typing import Any, ClassVar
2020

2121
from coretrace_python.analysis import Analysis, AnalysisContext
2222
from coretrace_python.findings import Severity
2323
from coretrace_python.semantic.symbols import SymbolId
24-
from coretrace_python.source import Location, SourceFile, SourceId, SourceSpan
25-
from coretrace_python.source.positions import TomlPositions, file_location
24+
from coretrace_python.source import Location, SourceFile, SourceId, SourceSpan, lines_of
25+
from coretrace_python.source.positions import Pointer, TomlPositions, file_location
2626

2727
_REQUIREMENT = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)\s*(\[[^\]]*\])?\s*(.*)$")
2828
_CLAUSE = re.compile(r"^(===|==|!=|<=|>=|~=|<|>|\^)\s*([0-9][0-9A-Za-z.*+!-]*)$")
@@ -349,7 +349,7 @@ def parse_dependencies(source: SourceFile) -> DependencyGraph:
349349

350350
def _parse_requirements_txt(source: SourceFile) -> DependencyGraph:
351351
found: dict[str, Requirement] = {}
352-
for number, line in enumerate(source.text.splitlines(), start=1):
352+
for number, line in enumerate(lines_of(source.text), start=1):
353353
stripped = line.strip()
354354
if not stripped or stripped.startswith(("#", "-")):
355355
continue
@@ -370,25 +370,32 @@ def _parse_toml(source: SourceFile, extract: Any, dependents: Any = None) -> Dep
370370
return DependencyGraph(found, dependents=None if dependents is None else dependents(data))
371371

372372

373-
def _line_of(source: SourceFile, key: str, default: int = 1) -> int:
374-
for number, line in enumerate(source.text.splitlines(), start=1):
375-
if key in line:
376-
return number
377-
return default
373+
def _pyproject_requirements(data: Mapping[str, Any], source: SourceFile) -> list[Requirement]:
374+
"""The requirements ``pyproject.toml`` declares (PEP 621 and Poetry), each located at
375+
its own array element or key, which the position component checks, or given a file
376+
location when it cannot be placed."""
378377

378+
positions = TomlPositions(source.text)
379379

380-
def _pyproject_requirements(data: Mapping[str, Any], source: SourceFile) -> list[Requirement]:
381-
found: list[Requirement] = []
380+
def declared(text: str, path: Pointer, value: object, optional: bool = False) -> Requirement | None:
381+
requirement = Requirement.parse(text, source.source_id, 1, optional)
382+
if requirement is None:
383+
return None
384+
place = positions.locate(path, value)
385+
location: Location = (
386+
SourceSpan(source.source_id, *place)
387+
if place is not None
388+
else file_location(source.source_id, path, text)
389+
)
390+
return replace(requirement, span=location)
391+
392+
found: list[Requirement | None] = []
382393
project = data.get("project", {})
383-
for text in project.get("dependencies", []) or []:
384-
requirement = Requirement.parse(text, source.source_id, _line_of(source, text))
385-
if requirement is not None:
386-
found.append(requirement)
387-
for group in (project.get("optional-dependencies", {}) or {}).values():
388-
for text in group or []:
389-
requirement = Requirement.parse(text, source.source_id, _line_of(source, text), True)
390-
if requirement is not None:
391-
found.append(requirement)
394+
for index, text in enumerate(project.get("dependencies", []) or []):
395+
found.append(declared(text, ("project", "dependencies", index), text))
396+
for group, texts in (project.get("optional-dependencies", {}) or {}).items():
397+
for index, text in enumerate(texts or []):
398+
found.append(declared(text, ("project", "optional-dependencies", group, index), text, True))
392399
poetry = data.get("tool", {}).get("poetry", {})
393400
for section, optional in (("dependencies", False), ("dev-dependencies", True)):
394401
for name, spec in (poetry.get(section, {}) or {}).items():
@@ -397,12 +404,8 @@ def _pyproject_requirements(data: Mapping[str, Any], source: SourceFile) -> list
397404
specifier = spec.get("version", "") if isinstance(spec, dict) else str(spec)
398405
if specifier == "*":
399406
specifier = ""
400-
requirement = Requirement.parse(
401-
f"{name}{specifier}", source.source_id, _line_of(source, name), optional
402-
)
403-
if requirement is not None:
404-
found.append(requirement)
405-
return found
407+
found.append(declared(f"{name}{specifier}", ("tool", "poetry", section, name), spec, optional))
408+
return [requirement for requirement in found if requirement is not None]
406409

407410

408411
def _lock_requirements(data: Mapping[str, Any], source: SourceFile) -> list[Requirement]:

‎src/coretrace_python/findings/baseline.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
from pathlib import Path
2525

2626
from coretrace_python.findings.model import Finding
27-
from coretrace_python.source import SourceSpan, decode_text
27+
from coretrace_python.source import SourceSpan, decode_text, lines_of
2828

2929
BASELINE_SCHEMA = 2
3030
_LEGACY_SCHEMA = 1
@@ -53,11 +53,11 @@ def fingerprint(finding: Finding, root: Path) -> Fingerprint:
5353

5454

5555
def _line_text(path: Path, line: int) -> str:
56-
"""The text of ``line``, lines counted by line feeds as the parser and the position
57-
component count them (``splitlines`` also breaks on U+2028 and other separators)."""
56+
"""The text of ``line``, lines counted as every line number of the engine counts
57+
them (``lines_of``)."""
5858

5959
try:
60-
lines = [text.removesuffix("\r") for text in decode_text(path.read_bytes()).split("\n")]
60+
lines = lines_of(decode_text(path.read_bytes()))
6161
except (OSError, UnicodeDecodeError):
6262
return str(line)
6363
if not 1 <= line <= len(lines):

‎src/coretrace_python/findings/suppressions.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from collections.abc import Callable, Iterable, Mapping
1515

1616
from coretrace_python.findings.model import Finding
17-
from coretrace_python.source import SourceId, SourceSpan
17+
from coretrace_python.source import SourceId, SourceSpan, lines_of
1818

1919
MARKER = re.compile(r"#\s*coretrace:\s*ignore(?:\[([^\]]*)\])?")
2020

@@ -38,10 +38,14 @@ def suppressions_in(text: str) -> Suppressions:
3838

3939

4040
def _comments(text: str) -> Iterable[tuple[int, str]]:
41+
"""Every comment of ``text`` with its line, lines counted as every line number of the
42+
engine counts them (``lines_of``)."""
43+
44+
lines = lines_of(text)
4145
try:
42-
tokens = list(tokenize.generate_tokens(io.StringIO(text).readline))
46+
tokens = list(tokenize.generate_tokens(io.StringIO("".join(f"{line}\n" for line in lines)).readline))
4347
except (tokenize.TokenError, SyntaxError):
44-
for number, line in enumerate(text.splitlines(), 1):
48+
for number, line in enumerate(lines, 1):
4549
if "#" in line:
4650
yield number, line[line.index("#") :]
4751
return

‎src/coretrace_python/plugins/secrets.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
from coretrace_python.hir.visitors import Node, children
2929
from coretrace_python.interprocedural import discover_files
3030
from coretrace_python.plugins.api import Plugin, PluginContext
31-
from coretrace_python.source import Location, SourceId, SourceSpan, decode_text
31+
from coretrace_python.source import Location, SourceId, SourceSpan, decode_text, lines_of
3232
from coretrace_python.source.positions import JsonPositions, Pointer, TomlPositions, file_location
3333

3434
# A value, the name it is bound to, where it is, and its enclosing function. A value of
@@ -347,7 +347,7 @@ def walk(node: object, path: Pointer, key: str | None, named: bool = True) -> It
347347

348348

349349
def _pairs(source: SourceId, text: str) -> Iterator[Literal]:
350-
for number, line in enumerate(text.splitlines(), start=1):
350+
for number, line in enumerate(lines_of(text), start=1):
351351
stripped = line.strip()
352352
if not stripped or stripped[0] in "#;[":
353353
continue
Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
11
"""Source storage and location primitives."""
22

3-
from coretrace_python.source.manager import SourceManager, decode_text
3+
from coretrace_python.source.manager import LINE_BREAK, SourceManager, decode_text, lines_of
44
from coretrace_python.source.model import FileLocation, Location, SourceFile, SourceId, SourceSpan
55

66
__all__ = [
7+
"LINE_BREAK",
78
"FileLocation",
89
"Location",
910
"SourceFile",
1011
"SourceId",
1112
"SourceManager",
1213
"SourceSpan",
1314
"decode_text",
15+
"lines_of",
1416
]
1517

‎src/coretrace_python/source/manager.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
from __future__ import annotations
44

5+
import re
56
from pathlib import Path
67

78
from coretrace_python.source.model import SourceFile, SourceId
@@ -30,6 +31,21 @@ def decode_text(data: bytes) -> str:
3031
return data.decode("utf-8")
3132

3233

34+
# A line break as Python reads one.
35+
LINE_BREAK = re.compile(r"\r\n|\r|\n")
36+
37+
38+
def lines_of(text: str) -> list[str]:
39+
"""The lines of ``text`` as Python counts them: broken by ``\\n``, ``\\r\\n`` or a lone
40+
``\\r``, never by U+2028, U+0085, a form feed or the other separators
41+
``str.splitlines`` also breaks on. Every line number of the engine counts by it."""
42+
43+
lines = LINE_BREAK.split(text)
44+
if lines[-1] == "":
45+
lines.pop()
46+
return lines
47+
48+
3349
class SourceManager:
3450
"""Own source text and return one canonical object for each source ID."""
3551

‎src/coretrace_python/source/positions.py‎

Lines changed: 99 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import tomllib
1919
from bisect import bisect_right
2020

21+
from coretrace_python.source.manager import LINE_BREAK, lines_of
2122
from coretrace_python.source.model import FileLocation, SourceId
2223

2324
Pointer = tuple[str | int, ...]
@@ -48,7 +49,7 @@ def pointer_text(path: Pointer) -> str:
4849

4950
class _Lines:
5051
def __init__(self, text: str) -> None:
51-
self._starts = [0, *(match.end() for match in re.finditer("\n", text))]
52+
self._starts = [0, *(match.end() for match in LINE_BREAK.finditer(text))]
5253

5354
def position(self, offset: int) -> tuple[int, int]:
5455
"""The one-based line and column of ``offset``."""
@@ -158,7 +159,7 @@ def _space(self, position: int) -> int:
158159
class TomlPositions:
159160
"""The key lines of each section of a TOML document, by the structural path of its
160161
table: the root, ``[a.b]``, or the element ``[[a]]`` opens, numbered in order. Lines
161-
are counted by line feeds, as the rest of the engine counts them; a line that starts
162+
are counted as the rest of the engine counts them (``lines_of``); a line that starts
162163
inside a multi-line string or an open array or inline table is a continuation, never
163164
a header or a key."""
164165

@@ -167,9 +168,9 @@ def __init__(self, text: str) -> None:
167168
self._keys: dict[Pointer, dict[str, list[tuple[int, int, str]]]] = {(): {}}
168169
arrays: dict[Pointer, int] = {}
169170
current: Pointer = ()
171+
self._lines = lines_of(text)
170172
state = _LexState()
171-
for number, line in enumerate(text.split("\n"), start=1):
172-
line = line.removesuffix("\r")
173+
for number, line in enumerate(self._lines, start=1):
173174
continued = state.inside()
174175
state.read(line)
175176
if continued:
@@ -205,17 +206,36 @@ def locate(self, path: Pointer, value: object) -> tuple[int, int] | None:
205206
"""The line and column ``value`` starts at under ``path``, or None when the
206207
section its path names has no line setting that key to that value."""
207208

208-
if not path or isinstance(path[-1], int):
209+
if not path:
209210
return None
211+
if isinstance(path[-1], int):
212+
return self._element(path, value)
210213
for number, column, written in self._keys.get(path[:-1], {}).get(path[-1], ()):
211-
try:
212-
found = tomllib.loads(f"v = {written}")["v"]
213-
except tomllib.TOMLDecodeError:
214-
continue
215-
if _same(found, value):
214+
if _same(_value(written), value):
216215
return number, column
217216
return None
218217

218+
def _element(self, path: Pointer, value: object) -> tuple[int, int] | None:
219+
"""An element of an array a key holds, written over one or several lines:
220+
``dependencies = [\\n "a",\\n "b",\\n]``."""
221+
222+
if len(path) < 2 or isinstance(path[-2], int):
223+
return None
224+
index = path[-1]
225+
assert isinstance(index, int)
226+
for number, column, _ in self._keys.get(path[:-2], {}).get(path[-2], ()):
227+
text = "\n".join(self._lines[number - 1 :])[column - 1 :]
228+
elements = _array_elements(text)
229+
if index >= len(elements):
230+
continue
231+
start, end = elements[index]
232+
if not _same(_value(text[start:end]), value):
233+
continue
234+
before = text[:start]
235+
line = number + before.count("\n")
236+
return line, (start - before.rfind("\n") if "\n" in before else column + start)
237+
return None
238+
219239

220240
class _LexState:
221241
"""Where a TOML line leaves the reader: inside a multi-line string (its delimiter),
@@ -266,6 +286,75 @@ def read(self, line: str) -> None:
266286
position += 1
267287

268288

289+
def _value(written: str) -> object:
290+
"""The value TOML text denotes, or None when it is not one value."""
291+
292+
try:
293+
return tomllib.loads(f"v = {written}")["v"]
294+
except tomllib.TOMLDecodeError:
295+
return None
296+
297+
298+
def _array_elements(text: str) -> list[tuple[int, int]]:
299+
"""The start and end offsets of the elements of the array ``text`` opens, skipping
300+
strings and comments; empty when ``text`` does not open an array."""
301+
302+
if not text.startswith("["):
303+
return []
304+
elements: list[tuple[int, int]] = []
305+
depth, start, position = 0, -1, 0
306+
while position < len(text):
307+
character = text[position]
308+
if character in "\"'":
309+
if depth == 1 and start < 0:
310+
start = position
311+
position = _string_end(text, position)
312+
continue
313+
if character == "#":
314+
newline = text.find("\n", position)
315+
position = len(text) if newline < 0 else newline
316+
continue
317+
if character in "[{":
318+
if depth == 1 and start < 0:
319+
start = position
320+
depth += 1
321+
elif character in "]}":
322+
depth -= 1
323+
if depth == 0:
324+
if start >= 0:
325+
elements.append((start, position))
326+
return elements
327+
elif character == "," and depth == 1:
328+
if start >= 0:
329+
elements.append((start, position))
330+
start = -1
331+
elif depth == 1 and start < 0 and not character.isspace():
332+
start = position
333+
position += 1
334+
return []
335+
336+
337+
def _string_end(text: str, start: int) -> int:
338+
"""The offset just past the TOML string that starts at ``start``."""
339+
340+
quote = text[start]
341+
if text.startswith(quote * 3, start):
342+
position = start + 3
343+
while position < len(text):
344+
if quote == '"' and text[position] == "\\":
345+
position += 2
346+
elif text.startswith(quote * 3, position):
347+
run = len(text[position:]) - len(text[position:].lstrip(quote))
348+
return position + min(run, 5)
349+
else:
350+
position += 1
351+
return len(text)
352+
position = start + 1
353+
while position < len(text) and text[position] not in (quote, "\n"):
354+
position += 2 if quote == '"' and text[position] == "\\" else 1
355+
return position + 1
356+
357+
269358
def _resolve(keys: tuple[str, ...], arrays: dict[Pointer, int]) -> Pointer:
270359
"""The structural path of a table named by ``keys``: a key that names an array of
271360
tables stands for its last element."""

0 commit comments

Comments
 (0)