Repository navigation
Expand file tree
/
Copy pathissue_patterns.py
More file actions
293 lines (241 loc) · 10.5 KB
/
Copy pathissue_patterns.py
File metadata and controls
293 lines (241 loc) · 10.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
"""Issue-ID patterns: which tokens in a turn count as ticket references.
The dashed pattern (Plane, Linear, Jira) is always on. A config file adds
presets, ``regex:`` patterns and ``deny:`` prefixes; see
``issue_ids.conf.example``. Matching ignores case, and tokens are stored in
upper case.
"""
from __future__ import annotations
import logging
import os
import re
import threading
from dataclasses import dataclass, field
from typing import Dict, List, Optional, Pattern, Tuple
logger = logging.getLogger("sessionflow.issue_patterns")
ENV_VAR = "SESSIONFLOW_ISSUE_ID_PATTERNS"
DEFAULT_PATH = os.path.join(os.path.expanduser("~"), ".sessionflow", "issue_ids.conf")
PRESETS: Dict[str, str] = {
"dashed": r"\b[A-Z][A-Z0-9]+-\d+\b",
"servicenow": r"\b(?:INC|CHG|REQ|RITM|CTASK|PRB|SCTASK|TASK)\d{7}\b",
}
# Technical-standard prefixes that match the dashed pattern but never name a ticket.
BUILTIN_DENYLIST = frozenset(
{"UTF", "SHA", "HTTP", "HTTPS", "ISO", "RFC", "IPV", "MD", "BASE"}
)
# A stored token sits in a comma-wrapped field and in Milvus LIKE and FTS MATCH
# filters, so it holds only letters, digits and inner dashes.
_SAFE_TOKEN = re.compile(r"^[A-Z0-9]+(?:-[A-Z0-9]+)*$")
_MAX_TOKEN_CHARS = 64
_DENY_PREFIX = re.compile(r"^[A-Z0-9]+$")
_LEADING_LETTERS = re.compile(r"^[A-Z]+")
# A lookaround match depends on text outside the token, so ``matches`` could not
# validate what extraction stores.
_LOOKAROUND = re.compile(r"\(\?<?[=!]")
@dataclass(frozen=True)
class IssuePatterns:
"""The compiled issue-ID patterns and prefix denylist in force.
Attributes:
labels: One label per pattern, such as ``preset:dashed`` or ``regex:<pattern>``.
regexes: The compiled patterns, in ``labels`` order.
deny: Prefixes added by the config file, on top of ``BUILTIN_DENYLIST``.
source: The config file named, or None for the built-in default.
skipped: Config lines that were not understood.
error: Why ``source`` could not be read, or None.
"""
labels: Tuple[str, ...]
regexes: Tuple[Pattern[str], ...]
deny: frozenset = field(default_factory=frozenset)
source: Optional[str] = None
skipped: int = 0
error: Optional[str] = None
def _denied(self, token: str) -> bool:
if "-" in token:
prefix = token.split("-", 1)[0]
else:
match = _LEADING_LETTERS.match(token)
prefix = match.group(0) if match else ""
return prefix in BUILTIN_DENYLIST or prefix in self.deny
def _keep(self, token: str) -> bool:
"""Whether an upper-case token is safe, not denied, and matched in full by a pattern.
Extraction and ``matches`` both use it, so every stored token passes an
``issue_id`` filter.
"""
return (len(token) <= _MAX_TOKEN_CHARS and _SAFE_TOKEN.fullmatch(token) is not None
and not self._denied(token)
and any(regex.fullmatch(token) for regex in self.regexes))
def extract(self, text: str) -> List[str]:
"""Return the issue tokens in ``text``, upper case, once each, in text order.
Args:
text: Turn or memory text to scan.
Returns:
Tokens matched by any pattern, minus denied prefixes, tokens with
characters other than letters, digits and inner dashes, and tokens
no pattern matches on their own.
"""
if not text or not isinstance(text, str):
return []
hits = []
for regex in self.regexes:
for match in regex.finditer(text):
hits.append((match.start(), match.group(0).upper()))
hits.sort(key=lambda hit: hit[0])
seen: List[str] = []
seen_set: set = set()
for _, token in hits:
if token in seen_set or not self._keep(token):
continue
seen_set.add(token)
seen.append(token)
return seen
def matches(self, raw: object) -> bool:
"""Whether ``raw`` is one whole issue token that extraction would store.
Args:
raw: A caller-supplied filter value; whitespace and case are ignored.
Returns:
False for non-strings, unsafe characters, denied prefixes and tokens
no pattern matches in full.
"""
if not isinstance(raw, str):
return False
return self._keep(raw.strip().upper())
def describe(self) -> Dict[str, object]:
"""Return the config file, pattern labels, extra denied prefixes, skipped lines and read error."""
return {
"source": self.source,
"patterns": list(self.labels),
"deny": sorted(self.deny),
"skipped": self.skipped,
"error": self.error,
}
def format_lines(info: Dict[str, object]) -> List[str]:
"""Return bullet lines for an ``IssuePatterns.describe`` dict.
Args:
info: The dict from ``describe``, as ``get_stats`` reports it.
Returns:
Lines naming the patterns, any extra denied prefixes and the config file.
"""
lines = [f"- Patterns: {', '.join(info.get('patterns') or [])}"]
if info.get("deny"):
lines.append(f"- Denied prefixes: {', '.join(info['deny'])}")
source = str(info.get("source") or "none (built-in default)")
skipped = int(info.get("skipped") or 0)
if info.get("error"):
source += f" (not read: {info['error']}; only the dashed preset applies)"
if skipped:
source += f" ({skipped} line{'' if skipped == 1 else 's'} skipped)"
lines.append(f"- Config: {source}")
return lines
def _compile(pattern: str) -> Pattern[str]:
return re.compile(pattern, re.IGNORECASE)
def _default() -> IssuePatterns:
return IssuePatterns(labels=("preset:dashed",), regexes=(_compile(PRESETS["dashed"]),))
def _strip_comment(value: str) -> str:
"""Drop a `` # comment`` from a preset or deny value; regex values keep every character."""
return re.split(r"\s+#", value, maxsplit=1)[0].strip()
def parse(lines: List[str], source: Optional[str] = None) -> IssuePatterns:
"""Build issue patterns from config lines.
Args:
lines: Config file lines. Blank lines and ``#`` comments are ignored.
source: The file the lines came from, for warnings and ``describe``.
Returns:
The dashed preset plus every valid line; a bad line logs a WARN with
its line number and counts in ``skipped``.
"""
labels: List[str] = ["preset:dashed"]
regexes: List[Pattern[str]] = [_compile(PRESETS["dashed"])]
deny: set = set()
skipped = 0
where = source or "issue ID config"
def bad(line_no: int, reason: str) -> None:
nonlocal skipped
skipped += 1
logger.warning("Skipping %s line %d: %s", where, line_no, reason)
for line_no, line in enumerate(lines, start=1):
stripped = line.strip()
if not stripped or stripped.startswith("#"):
continue
kind, sep, value = stripped.partition(":")
kind = kind.strip().lower()
if not sep:
bad(line_no, "expected preset:, regex: or deny:")
elif kind == "preset":
name = _strip_comment(value).lower()
if name not in PRESETS:
bad(line_no, f"unknown preset {name!r}")
elif f"preset:{name}" not in labels:
labels.append(f"preset:{name}")
regexes.append(_compile(PRESETS[name]))
elif kind == "regex":
pattern = value.strip()
try:
compiled = _compile(pattern)
except re.error as exc:
bad(line_no, f"invalid regex ({exc})")
continue
if not pattern:
bad(line_no, "empty regex")
elif _LOOKAROUND.search(pattern):
bad(line_no, "lookaround is not supported; a token must match on its own")
elif f"regex:{pattern}" not in labels:
labels.append(f"regex:{pattern}")
regexes.append(compiled)
elif kind == "deny":
prefix = _strip_comment(value).upper()
if not _DENY_PREFIX.match(prefix):
bad(line_no, "a deny prefix holds only letters and digits")
else:
deny.add(prefix)
else:
bad(line_no, f"unknown entry {kind!r}")
return IssuePatterns(labels=tuple(labels), regexes=tuple(regexes),
deny=frozenset(deny), source=source, skipped=skipped)
def load(path: Optional[str]) -> IssuePatterns:
"""Read issue patterns from ``path``.
Args:
path: Config file path; None returns the built-in default.
Returns:
The parsed patterns, or the default with a WARN when the file cannot be read.
"""
if not path:
return _default()
try:
with open(path, "r", encoding="utf-8") as handle:
lines = handle.readlines()
except (OSError, UnicodeDecodeError) as exc:
logger.warning("Could not read issue ID config %s: %s; using the dashed preset",
path, exc)
default = _default()
reason = exc.strerror if isinstance(exc, OSError) else None
return IssuePatterns(labels=default.labels, regexes=default.regexes, source=path,
error=reason or ("not UTF-8" if isinstance(exc, UnicodeDecodeError)
else type(exc).__name__))
return parse(lines, source=path)
_cache: Dict[str, object] = {}
_cache_lock = threading.Lock()
def _resolve_path() -> Tuple[Optional[str], Optional[float]]:
"""Return the config path in force and its mtime; a missing default file means none."""
configured = os.getenv(ENV_VAR)
path = configured or DEFAULT_PATH
try:
return path, os.path.getmtime(path)
except OSError:
return (path, None) if configured else (None, None)
def active() -> IssuePatterns:
"""Return the patterns in force, reloading the config file when its mtime changes.
The file is ``SESSIONFLOW_ISSUE_ID_PATTERNS``, else ``~/.sessionflow/issue_ids.conf``
when it exists. A configured path that is missing warns once and uses the default.
"""
path, mtime = _resolve_path()
key = (path, mtime)
with _cache_lock:
if _cache.get("key") == key:
return _cache["patterns"] # type: ignore[return-value]
patterns = load(path)
with _cache_lock:
_cache["key"] = key
_cache["patterns"] = patterns
return patterns
def reset() -> None:
"""Forget the cached patterns, so the next ``active`` call reads the config again."""
with _cache_lock:
_cache.clear()