From 9f6d1ef53391ffaf7c8639e0055f573691241821 Mon Sep 17 00:00:00 2001 From: reacher-z <68239571+reacher-z@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:38:15 -0700 Subject: [PATCH 1/2] verifier: strip comments for code redundancy in linear time strip_comments ran one regular expression whose block-comment alternative (/\*.*?\*/ for C-style and SQL sources, for HTML) searches to the end of the text from every opener that is never closed. A file with many unclosed openers took quadratic time: 4 s for 64 KiB, about 20 minutes extrapolated to the 1 MiB per-file cap, all before the time budget is next checked. An opener can only close when the last closer in the text starts after it, so the scan now uses the full scanner before that point and the same scanner without its block-comment alternative after it, and tries a match only at characters where one can start. The output is unchanged (checked against the old implementation on 360,000 generated inputs and on the verifier's own sources); 1 MiB of unclosed openers now takes 0.1 s. All 70 copies of redundancy.py stay byte-identical. The change alters the tasks' content hashes, so it ships with the next task release. --- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- .../tests/websitebench/harbor/redundancy.py | 81 ++++++++++++++++--- tests/test_redundancy.py | 62 ++++++++++++++ 71 files changed, 5032 insertions(+), 700 deletions(-) create mode 100644 tests/test_redundancy.py diff --git a/tasks/1password-web/tests/websitebench/harbor/redundancy.py b/tasks/1password-web/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/1password-web/tests/websitebench/harbor/redundancy.py +++ b/tasks/1password-web/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/33/tests/websitebench/harbor/redundancy.py b/tasks/33/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/33/tests/websitebench/harbor/redundancy.py +++ b/tasks/33/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/ace-hardware/tests/websitebench/harbor/redundancy.py b/tasks/ace-hardware/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/ace-hardware/tests/websitebench/harbor/redundancy.py +++ b/tasks/ace-hardware/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/airtable/tests/websitebench/harbor/redundancy.py b/tasks/airtable/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/airtable/tests/websitebench/harbor/redundancy.py +++ b/tasks/airtable/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/amazon-home-services/tests/websitebench/harbor/redundancy.py b/tasks/amazon-home-services/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/amazon-home-services/tests/websitebench/harbor/redundancy.py +++ b/tasks/amazon-home-services/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/amazon/tests/websitebench/harbor/redundancy.py b/tasks/amazon/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/amazon/tests/websitebench/harbor/redundancy.py +++ b/tasks/amazon/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/amc-theatres/tests/websitebench/harbor/redundancy.py b/tasks/amc-theatres/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/amc-theatres/tests/websitebench/harbor/redundancy.py +++ b/tasks/amc-theatres/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/angi/tests/websitebench/harbor/redundancy.py b/tasks/angi/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/angi/tests/websitebench/harbor/redundancy.py +++ b/tasks/angi/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/apple/tests/websitebench/harbor/redundancy.py b/tasks/apple/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/apple/tests/websitebench/harbor/redundancy.py +++ b/tasks/apple/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/article/tests/websitebench/harbor/redundancy.py b/tasks/article/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/article/tests/websitebench/harbor/redundancy.py +++ b/tasks/article/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/asana/tests/websitebench/harbor/redundancy.py b/tasks/asana/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/asana/tests/websitebench/harbor/redundancy.py +++ b/tasks/asana/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/aspca-pet-insurance/tests/websitebench/harbor/redundancy.py b/tasks/aspca-pet-insurance/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/aspca-pet-insurance/tests/websitebench/harbor/redundancy.py +++ b/tasks/aspca-pet-insurance/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/autoslash/tests/websitebench/harbor/redundancy.py b/tasks/autoslash/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/autoslash/tests/websitebench/harbor/redundancy.py +++ b/tasks/autoslash/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/autotrader/tests/websitebench/harbor/redundancy.py b/tasks/autotrader/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/autotrader/tests/websitebench/harbor/redundancy.py +++ b/tasks/autotrader/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/backcountry/tests/websitebench/harbor/redundancy.py b/tasks/backcountry/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/backcountry/tests/websitebench/harbor/redundancy.py +++ b/tasks/backcountry/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/bean-box/tests/websitebench/harbor/redundancy.py b/tasks/bean-box/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/bean-box/tests/websitebench/harbor/redundancy.py +++ b/tasks/bean-box/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/beeradvocate/tests/websitebench/harbor/redundancy.py b/tasks/beeradvocate/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/beeradvocate/tests/websitebench/harbor/redundancy.py +++ b/tasks/beeradvocate/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/blinkist/tests/websitebench/harbor/redundancy.py b/tasks/blinkist/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/blinkist/tests/websitebench/harbor/redundancy.py +++ b/tasks/blinkist/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/blueapron/tests/websitebench/harbor/redundancy.py b/tasks/blueapron/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/blueapron/tests/websitebench/harbor/redundancy.py +++ b/tasks/blueapron/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/bluemercury/tests/websitebench/harbor/redundancy.py b/tasks/bluemercury/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/bluemercury/tests/websitebench/harbor/redundancy.py +++ b/tasks/bluemercury/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/boardgamegeek/tests/websitebench/harbor/redundancy.py b/tasks/boardgamegeek/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/boardgamegeek/tests/websitebench/harbor/redundancy.py +++ b/tasks/boardgamegeek/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/bookshop/tests/websitebench/harbor/redundancy.py b/tasks/bookshop/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/bookshop/tests/websitebench/harbor/redundancy.py +++ b/tasks/bookshop/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/breakoutgames-2/tests/websitebench/harbor/redundancy.py b/tasks/breakoutgames-2/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/breakoutgames-2/tests/websitebench/harbor/redundancy.py +++ b/tasks/breakoutgames-2/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/brooklinen/tests/websitebench/harbor/redundancy.py b/tasks/brooklinen/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/brooklinen/tests/websitebench/harbor/redundancy.py +++ b/tasks/brooklinen/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/capterra/tests/websitebench/harbor/redundancy.py b/tasks/capterra/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/capterra/tests/websitebench/harbor/redundancy.py +++ b/tasks/capterra/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/change/tests/websitebench/harbor/redundancy.py b/tasks/change/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/change/tests/websitebench/harbor/redundancy.py +++ b/tasks/change/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/columbia/tests/websitebench/harbor/redundancy.py b/tasks/columbia/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/columbia/tests/websitebench/harbor/redundancy.py +++ b/tasks/columbia/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/crumbl-cookies/tests/websitebench/harbor/redundancy.py b/tasks/crumbl-cookies/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/crumbl-cookies/tests/websitebench/harbor/redundancy.py +++ b/tasks/crumbl-cookies/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/crunchyroll/tests/websitebench/harbor/redundancy.py b/tasks/crunchyroll/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/crunchyroll/tests/websitebench/harbor/redundancy.py +++ b/tasks/crunchyroll/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/curology/tests/websitebench/harbor/redundancy.py b/tasks/curology/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/curology/tests/websitebench/harbor/redundancy.py +++ b/tasks/curology/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/dooney-bourke/tests/websitebench/harbor/redundancy.py b/tasks/dooney-bourke/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/dooney-bourke/tests/websitebench/harbor/redundancy.py +++ b/tasks/dooney-bourke/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/eatthismuch/tests/websitebench/harbor/redundancy.py b/tasks/eatthismuch/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/eatthismuch/tests/websitebench/harbor/redundancy.py +++ b/tasks/eatthismuch/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/etsy/tests/websitebench/harbor/redundancy.py b/tasks/etsy/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/etsy/tests/websitebench/harbor/redundancy.py +++ b/tasks/etsy/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/eventbrite/tests/websitebench/harbor/redundancy.py b/tasks/eventbrite/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/eventbrite/tests/websitebench/harbor/redundancy.py +++ b/tasks/eventbrite/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/everydollar/tests/websitebench/harbor/redundancy.py b/tasks/everydollar/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/everydollar/tests/websitebench/harbor/redundancy.py +++ b/tasks/everydollar/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/everyplate/tests/websitebench/harbor/redundancy.py b/tasks/everyplate/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/everyplate/tests/websitebench/harbor/redundancy.py +++ b/tasks/everyplate/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/fandango/tests/websitebench/harbor/redundancy.py b/tasks/fandango/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/fandango/tests/websitebench/harbor/redundancy.py +++ b/tasks/fandango/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/freshdirect/tests/websitebench/harbor/redundancy.py b/tasks/freshdirect/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/freshdirect/tests/websitebench/harbor/redundancy.py +++ b/tasks/freshdirect/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/hipcamp/tests/websitebench/harbor/redundancy.py b/tasks/hipcamp/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/hipcamp/tests/websitebench/harbor/redundancy.py +++ b/tasks/hipcamp/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/hmart/tests/websitebench/harbor/redundancy.py b/tasks/hmart/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/hmart/tests/websitebench/harbor/redundancy.py +++ b/tasks/hmart/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/imdb/tests/websitebench/harbor/redundancy.py b/tasks/imdb/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/imdb/tests/websitebench/harbor/redundancy.py +++ b/tasks/imdb/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/insureon/tests/websitebench/harbor/redundancy.py b/tasks/insureon/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/insureon/tests/websitebench/harbor/redundancy.py +++ b/tasks/insureon/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/insurify/tests/websitebench/harbor/redundancy.py b/tasks/insurify/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/insurify/tests/websitebench/harbor/redundancy.py +++ b/tasks/insurify/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/ioby/tests/websitebench/harbor/redundancy.py b/tasks/ioby/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/ioby/tests/websitebench/harbor/redundancy.py +++ b/tasks/ioby/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/kanetix-ca/tests/websitebench/harbor/redundancy.py b/tasks/kanetix-ca/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/kanetix-ca/tests/websitebench/harbor/redundancy.py +++ b/tasks/kanetix-ca/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/leetcode/tests/websitebench/harbor/redundancy.py b/tasks/leetcode/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/leetcode/tests/websitebench/harbor/redundancy.py +++ b/tasks/leetcode/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/legalnature/tests/websitebench/harbor/redundancy.py b/tasks/legalnature/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/legalnature/tests/websitebench/harbor/redundancy.py +++ b/tasks/legalnature/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/linkedin/tests/websitebench/harbor/redundancy.py b/tasks/linkedin/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/linkedin/tests/websitebench/harbor/redundancy.py +++ b/tasks/linkedin/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/llbean/tests/websitebench/harbor/redundancy.py b/tasks/llbean/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/llbean/tests/websitebench/harbor/redundancy.py +++ b/tasks/llbean/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/lowes/tests/websitebench/harbor/redundancy.py b/tasks/lowes/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/lowes/tests/websitebench/harbor/redundancy.py +++ b/tasks/lowes/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/medium/tests/websitebench/harbor/redundancy.py b/tasks/medium/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/medium/tests/websitebench/harbor/redundancy.py +++ b/tasks/medium/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/meetup/tests/websitebench/harbor/redundancy.py b/tasks/meetup/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/meetup/tests/websitebench/harbor/redundancy.py +++ b/tasks/meetup/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/notion/tests/websitebench/harbor/redundancy.py b/tasks/notion/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/notion/tests/websitebench/harbor/redundancy.py +++ b/tasks/notion/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/ollie/tests/websitebench/harbor/redundancy.py b/tasks/ollie/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/ollie/tests/websitebench/harbor/redundancy.py +++ b/tasks/ollie/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/ontario-parks/tests/websitebench/harbor/redundancy.py b/tasks/ontario-parks/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/ontario-parks/tests/websitebench/harbor/redundancy.py +++ b/tasks/ontario-parks/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/opentable/tests/websitebench/harbor/redundancy.py b/tasks/opentable/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/opentable/tests/websitebench/harbor/redundancy.py +++ b/tasks/opentable/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/petco/tests/websitebench/harbor/redundancy.py b/tasks/petco/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/petco/tests/websitebench/harbor/redundancy.py +++ b/tasks/petco/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/petfinder/tests/websitebench/harbor/redundancy.py b/tasks/petfinder/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/petfinder/tests/websitebench/harbor/redundancy.py +++ b/tasks/petfinder/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/petsmart/tests/websitebench/harbor/redundancy.py b/tasks/petsmart/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/petsmart/tests/websitebench/harbor/redundancy.py +++ b/tasks/petsmart/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/ratemyprofessors/tests/websitebench/harbor/redundancy.py b/tasks/ratemyprofessors/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/ratemyprofessors/tests/websitebench/harbor/redundancy.py +++ b/tasks/ratemyprofessors/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/resy/tests/websitebench/harbor/redundancy.py b/tasks/resy/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/resy/tests/websitebench/harbor/redundancy.py +++ b/tasks/resy/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/simplify-jobs/tests/websitebench/harbor/redundancy.py b/tasks/simplify-jobs/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/simplify-jobs/tests/websitebench/harbor/redundancy.py +++ b/tasks/simplify-jobs/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/target/tests/websitebench/harbor/redundancy.py b/tasks/target/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/target/tests/websitebench/harbor/redundancy.py +++ b/tasks/target/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/taskrabbit/tests/websitebench/harbor/redundancy.py b/tasks/taskrabbit/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/taskrabbit/tests/websitebench/harbor/redundancy.py +++ b/tasks/taskrabbit/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/tripadvisor/tests/websitebench/harbor/redundancy.py b/tasks/tripadvisor/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/tripadvisor/tests/websitebench/harbor/redundancy.py +++ b/tasks/tripadvisor/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/tripit/tests/websitebench/harbor/redundancy.py b/tasks/tripit/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/tripit/tests/websitebench/harbor/redundancy.py +++ b/tasks/tripit/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/ubereats/tests/websitebench/harbor/redundancy.py b/tasks/ubereats/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/ubereats/tests/websitebench/harbor/redundancy.py +++ b/tasks/ubereats/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/warbyparker/tests/websitebench/harbor/redundancy.py b/tasks/warbyparker/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/warbyparker/tests/websitebench/harbor/redundancy.py +++ b/tasks/warbyparker/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/wayfair/tests/websitebench/harbor/redundancy.py b/tasks/wayfair/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/wayfair/tests/websitebench/harbor/redundancy.py +++ b/tasks/wayfair/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/zennioptical/tests/websitebench/harbor/redundancy.py b/tasks/zennioptical/tests/websitebench/harbor/redundancy.py index 34b97007..acc92e69 100644 --- a/tasks/zennioptical/tests/websitebench/harbor/redundancy.py +++ b/tasks/zennioptical/tests/websitebench/harbor/redundancy.py @@ -198,11 +198,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +272,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tests/test_redundancy.py b/tests/test_redundancy.py new file mode 100644 index 00000000..64ef1ab1 --- /dev/null +++ b/tests/test_redundancy.py @@ -0,0 +1,62 @@ +"""Comment stripping in the shared code-redundancy module.""" + +import importlib.util +import sys +import time +import unittest +from pathlib import Path + +MODULE = ( + Path(__file__).resolve().parents[1] + / "tasks/medium/tests/websitebench/harbor/redundancy.py" +) + + +def load_redundancy(): + spec = importlib.util.spec_from_file_location("task_redundancy", MODULE) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +redundancy = load_redundancy() + + +class StripComments(unittest.TestCase): + def test_strips_c_style_comments_and_keeps_strings(self): + text = 'a = "/* kept */"; // gone\nb = 1; /* gone\nstill gone */ c = 2;\n' + self.assertEqual( + redundancy.strip_comments(text, "javascript"), + 'a = "/* kept */"; \nb = 1; \n c = 2;\n', + ) + + def test_unclosed_opener_is_left_as_code(self): + text = "x = 1; /* closed */ y = a /* b\nz = 2;\n" + self.assertEqual( + redundancy.strip_comments(text, "javascript"), + "x = 1; y = a /* b\nz = 2;\n", + ) + + def test_unclosed_html_and_sql_openers_are_left_as_code(self): + self.assertEqual( + redundancy.strip_comments("