diff --git a/tasks/1password-web/tests/websitebench/harbor/redundancy.py b/tasks/1password-web/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/1password-web/tests/websitebench/harbor/redundancy.py +++ b/tasks/1password-web/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/33/tests/websitebench/harbor/redundancy.py b/tasks/33/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/33/tests/websitebench/harbor/redundancy.py +++ b/tasks/33/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/ace-hardware/tests/websitebench/harbor/redundancy.py b/tasks/ace-hardware/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/ace-hardware/tests/websitebench/harbor/redundancy.py +++ b/tasks/ace-hardware/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/airtable/tests/websitebench/harbor/redundancy.py b/tasks/airtable/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/airtable/tests/websitebench/harbor/redundancy.py +++ b/tasks/airtable/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/amazon-home-services/tests/websitebench/harbor/redundancy.py b/tasks/amazon-home-services/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/amazon-home-services/tests/websitebench/harbor/redundancy.py +++ b/tasks/amazon-home-services/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/amazon/tests/websitebench/harbor/redundancy.py b/tasks/amazon/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/amazon/tests/websitebench/harbor/redundancy.py +++ b/tasks/amazon/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/amc-theatres/tests/websitebench/harbor/redundancy.py b/tasks/amc-theatres/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/amc-theatres/tests/websitebench/harbor/redundancy.py +++ b/tasks/amc-theatres/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/angi/tests/websitebench/harbor/redundancy.py b/tasks/angi/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/angi/tests/websitebench/harbor/redundancy.py +++ b/tasks/angi/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/apple/tests/websitebench/harbor/redundancy.py b/tasks/apple/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/apple/tests/websitebench/harbor/redundancy.py +++ b/tasks/apple/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/article/tests/websitebench/harbor/redundancy.py b/tasks/article/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/article/tests/websitebench/harbor/redundancy.py +++ b/tasks/article/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/asana/tests/websitebench/harbor/redundancy.py b/tasks/asana/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/asana/tests/websitebench/harbor/redundancy.py +++ b/tasks/asana/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/aspca-pet-insurance/tests/websitebench/harbor/redundancy.py b/tasks/aspca-pet-insurance/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/aspca-pet-insurance/tests/websitebench/harbor/redundancy.py +++ b/tasks/aspca-pet-insurance/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/autoslash/tests/websitebench/harbor/redundancy.py b/tasks/autoslash/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/autoslash/tests/websitebench/harbor/redundancy.py +++ b/tasks/autoslash/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/autotrader/tests/websitebench/harbor/redundancy.py b/tasks/autotrader/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/autotrader/tests/websitebench/harbor/redundancy.py +++ b/tasks/autotrader/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/backcountry/tests/websitebench/harbor/redundancy.py b/tasks/backcountry/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/backcountry/tests/websitebench/harbor/redundancy.py +++ b/tasks/backcountry/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/bean-box/tests/websitebench/harbor/redundancy.py b/tasks/bean-box/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/bean-box/tests/websitebench/harbor/redundancy.py +++ b/tasks/bean-box/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/beeradvocate/tests/websitebench/harbor/redundancy.py b/tasks/beeradvocate/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/beeradvocate/tests/websitebench/harbor/redundancy.py +++ b/tasks/beeradvocate/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/blinkist/tests/websitebench/harbor/redundancy.py b/tasks/blinkist/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/blinkist/tests/websitebench/harbor/redundancy.py +++ b/tasks/blinkist/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/blueapron/tests/websitebench/harbor/redundancy.py b/tasks/blueapron/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/blueapron/tests/websitebench/harbor/redundancy.py +++ b/tasks/blueapron/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/bluemercury/tests/websitebench/harbor/redundancy.py b/tasks/bluemercury/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/bluemercury/tests/websitebench/harbor/redundancy.py +++ b/tasks/bluemercury/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/boardgamegeek/tests/websitebench/harbor/redundancy.py b/tasks/boardgamegeek/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/boardgamegeek/tests/websitebench/harbor/redundancy.py +++ b/tasks/boardgamegeek/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/bookshop/tests/websitebench/harbor/redundancy.py b/tasks/bookshop/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/bookshop/tests/websitebench/harbor/redundancy.py +++ b/tasks/bookshop/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/breakoutgames-2/tests/websitebench/harbor/redundancy.py b/tasks/breakoutgames-2/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/breakoutgames-2/tests/websitebench/harbor/redundancy.py +++ b/tasks/breakoutgames-2/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/brooklinen/tests/websitebench/harbor/redundancy.py b/tasks/brooklinen/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/brooklinen/tests/websitebench/harbor/redundancy.py +++ b/tasks/brooklinen/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/capterra/tests/websitebench/harbor/redundancy.py b/tasks/capterra/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/capterra/tests/websitebench/harbor/redundancy.py +++ b/tasks/capterra/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/change/tests/websitebench/harbor/redundancy.py b/tasks/change/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/change/tests/websitebench/harbor/redundancy.py +++ b/tasks/change/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/columbia/tests/websitebench/harbor/redundancy.py b/tasks/columbia/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/columbia/tests/websitebench/harbor/redundancy.py +++ b/tasks/columbia/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/crumbl-cookies/tests/websitebench/harbor/redundancy.py b/tasks/crumbl-cookies/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/crumbl-cookies/tests/websitebench/harbor/redundancy.py +++ b/tasks/crumbl-cookies/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/crunchyroll/tests/websitebench/harbor/redundancy.py b/tasks/crunchyroll/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/crunchyroll/tests/websitebench/harbor/redundancy.py +++ b/tasks/crunchyroll/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/curology/tests/websitebench/harbor/redundancy.py b/tasks/curology/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/curology/tests/websitebench/harbor/redundancy.py +++ b/tasks/curology/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/dooney-bourke/tests/websitebench/harbor/redundancy.py b/tasks/dooney-bourke/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/dooney-bourke/tests/websitebench/harbor/redundancy.py +++ b/tasks/dooney-bourke/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/eatthismuch/tests/websitebench/harbor/redundancy.py b/tasks/eatthismuch/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/eatthismuch/tests/websitebench/harbor/redundancy.py +++ b/tasks/eatthismuch/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/etsy/tests/websitebench/harbor/redundancy.py b/tasks/etsy/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/etsy/tests/websitebench/harbor/redundancy.py +++ b/tasks/etsy/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/eventbrite/tests/websitebench/harbor/redundancy.py b/tasks/eventbrite/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/eventbrite/tests/websitebench/harbor/redundancy.py +++ b/tasks/eventbrite/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/everydollar/tests/websitebench/harbor/redundancy.py b/tasks/everydollar/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/everydollar/tests/websitebench/harbor/redundancy.py +++ b/tasks/everydollar/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/everyplate/tests/websitebench/harbor/redundancy.py b/tasks/everyplate/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/everyplate/tests/websitebench/harbor/redundancy.py +++ b/tasks/everyplate/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/fandango/tests/websitebench/harbor/redundancy.py b/tasks/fandango/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/fandango/tests/websitebench/harbor/redundancy.py +++ b/tasks/fandango/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/freshdirect/tests/websitebench/harbor/redundancy.py b/tasks/freshdirect/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/freshdirect/tests/websitebench/harbor/redundancy.py +++ b/tasks/freshdirect/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/hipcamp/tests/websitebench/harbor/redundancy.py b/tasks/hipcamp/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/hipcamp/tests/websitebench/harbor/redundancy.py +++ b/tasks/hipcamp/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/hmart/tests/websitebench/harbor/redundancy.py b/tasks/hmart/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/hmart/tests/websitebench/harbor/redundancy.py +++ b/tasks/hmart/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/imdb/tests/websitebench/harbor/redundancy.py b/tasks/imdb/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/imdb/tests/websitebench/harbor/redundancy.py +++ b/tasks/imdb/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/insureon/tests/websitebench/harbor/redundancy.py b/tasks/insureon/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/insureon/tests/websitebench/harbor/redundancy.py +++ b/tasks/insureon/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/insurify/tests/websitebench/harbor/redundancy.py b/tasks/insurify/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/insurify/tests/websitebench/harbor/redundancy.py +++ b/tasks/insurify/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/ioby/tests/websitebench/harbor/redundancy.py b/tasks/ioby/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/ioby/tests/websitebench/harbor/redundancy.py +++ b/tasks/ioby/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/kanetix-ca/tests/websitebench/harbor/redundancy.py b/tasks/kanetix-ca/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/kanetix-ca/tests/websitebench/harbor/redundancy.py +++ b/tasks/kanetix-ca/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/leetcode/tests/websitebench/harbor/redundancy.py b/tasks/leetcode/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/leetcode/tests/websitebench/harbor/redundancy.py +++ b/tasks/leetcode/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/legalnature/tests/websitebench/harbor/redundancy.py b/tasks/legalnature/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/legalnature/tests/websitebench/harbor/redundancy.py +++ b/tasks/legalnature/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/linkedin/tests/websitebench/harbor/redundancy.py b/tasks/linkedin/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/linkedin/tests/websitebench/harbor/redundancy.py +++ b/tasks/linkedin/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/llbean/tests/websitebench/harbor/redundancy.py b/tasks/llbean/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/llbean/tests/websitebench/harbor/redundancy.py +++ b/tasks/llbean/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/lowes/tests/websitebench/harbor/redundancy.py b/tasks/lowes/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/lowes/tests/websitebench/harbor/redundancy.py +++ b/tasks/lowes/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/medium/tests/websitebench/harbor/redundancy.py b/tasks/medium/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/medium/tests/websitebench/harbor/redundancy.py +++ b/tasks/medium/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/meetup/tests/websitebench/harbor/redundancy.py b/tasks/meetup/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/meetup/tests/websitebench/harbor/redundancy.py +++ b/tasks/meetup/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/notion/tests/websitebench/harbor/redundancy.py b/tasks/notion/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/notion/tests/websitebench/harbor/redundancy.py +++ b/tasks/notion/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/ollie/tests/websitebench/harbor/redundancy.py b/tasks/ollie/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/ollie/tests/websitebench/harbor/redundancy.py +++ b/tasks/ollie/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/ontario-parks/tests/websitebench/harbor/redundancy.py b/tasks/ontario-parks/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/ontario-parks/tests/websitebench/harbor/redundancy.py +++ b/tasks/ontario-parks/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/opentable/tests/websitebench/harbor/redundancy.py b/tasks/opentable/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/opentable/tests/websitebench/harbor/redundancy.py +++ b/tasks/opentable/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/petco/tests/websitebench/harbor/redundancy.py b/tasks/petco/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/petco/tests/websitebench/harbor/redundancy.py +++ b/tasks/petco/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/petfinder/tests/websitebench/harbor/redundancy.py b/tasks/petfinder/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/petfinder/tests/websitebench/harbor/redundancy.py +++ b/tasks/petfinder/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/petsmart/tests/websitebench/harbor/redundancy.py b/tasks/petsmart/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/petsmart/tests/websitebench/harbor/redundancy.py +++ b/tasks/petsmart/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/ratemyprofessors/tests/websitebench/harbor/redundancy.py b/tasks/ratemyprofessors/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/ratemyprofessors/tests/websitebench/harbor/redundancy.py +++ b/tasks/ratemyprofessors/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/resy/tests/websitebench/harbor/redundancy.py b/tasks/resy/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/resy/tests/websitebench/harbor/redundancy.py +++ b/tasks/resy/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/simplify-jobs/tests/websitebench/harbor/redundancy.py b/tasks/simplify-jobs/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/simplify-jobs/tests/websitebench/harbor/redundancy.py +++ b/tasks/simplify-jobs/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/target/tests/websitebench/harbor/redundancy.py b/tasks/target/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/target/tests/websitebench/harbor/redundancy.py +++ b/tasks/target/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/taskrabbit/tests/websitebench/harbor/redundancy.py b/tasks/taskrabbit/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/taskrabbit/tests/websitebench/harbor/redundancy.py +++ b/tasks/taskrabbit/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/tripadvisor/tests/websitebench/harbor/redundancy.py b/tasks/tripadvisor/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/tripadvisor/tests/websitebench/harbor/redundancy.py +++ b/tasks/tripadvisor/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/tripit/tests/websitebench/harbor/redundancy.py b/tasks/tripit/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/tripit/tests/websitebench/harbor/redundancy.py +++ b/tasks/tripit/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/ubereats/tests/websitebench/harbor/redundancy.py b/tasks/ubereats/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/ubereats/tests/websitebench/harbor/redundancy.py +++ b/tasks/ubereats/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/warbyparker/tests/websitebench/harbor/redundancy.py b/tasks/warbyparker/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/warbyparker/tests/websitebench/harbor/redundancy.py +++ b/tasks/warbyparker/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/wayfair/tests/websitebench/harbor/redundancy.py b/tasks/wayfair/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/wayfair/tests/websitebench/harbor/redundancy.py +++ b/tasks/wayfair/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tasks/zennioptical/tests/websitebench/harbor/redundancy.py b/tasks/zennioptical/tests/websitebench/harbor/redundancy.py index 34b97007..37c8a40b 100644 --- a/tasks/zennioptical/tests/websitebench/harbor/redundancy.py +++ b/tasks/zennioptical/tests/websitebench/harbor/redundancy.py @@ -107,6 +107,7 @@ ".cjs": "javascript", ".cpp": "c", ".cs": "csharp", + ".css": "css", ".cts": "typescript", ".ejs": "template", ".go": "go", @@ -198,11 +199,51 @@ ) _HTML_SCANNER = re.compile(r"(?P)", re.DOTALL) +# The same scanners without their block-comment alternative. A block comment +# can only match where its closer still occurs later in the text, so past that +# point these give the same result without searching to the end of the text +# from every unclosed opener, which made stripping quadratic. +_C_SCANNER_OPEN = re.compile( + r""" + (?P "(?:\\.|[^"\\\n])*" + | '(?:\\.|[^'\\\n])*' + | `(?:\\.|[^`\\])*` ) + | (?P //[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) +_SQL_SCANNER_OPEN = re.compile( + r""" + (?P '(?:''|[^'\n])*' ) + | (?P --[^\n]* ) + """, + re.VERBOSE | re.DOTALL, +) + + +class _Scanner: + """A comment scanner, its block-comment delimiters and match start characters.""" + + def __init__( + self, + pattern: re.Pattern[str], + without_block: re.Pattern[str] | None, + opener: str, + closer: str, + starts: str, + ) -> None: + self.pattern = pattern + self.without_block = without_block + self.opener = opener + self.closer = closer + self.starts = re.compile(f"[{re.escape(starts)}]") + + _SCANNERS = { - "c": _C_SCANNER, - "hash": _HASH_SCANNER, - "sql": _SQL_SCANNER, - "html": _HTML_SCANNER, + "c": _Scanner(_C_SCANNER, _C_SCANNER_OPEN, "/*", "*/", "\"'`/"), + "hash": _Scanner(_HASH_SCANNER, None, "", "", "\"'#"), + "sql": _Scanner(_SQL_SCANNER, _SQL_SCANNER_OPEN, "/*", "*/", "'-/"), + "html": _Scanner(_HTML_SCANNER, None, "", "<"), } @@ -232,13 +273,34 @@ def strip_comments(text: str, language: str) -> str: scanner = _SCANNERS.get(COMMENT_STYLES.get(language, "")) if scanner is None: return text - - def replace(match: re.Match[str]) -> str: + # An opener at ``position`` can close only if the last closer starts at or + # after ``position + len(opener)``; from ``boundary`` on, none can. + boundary = len(text) + if scanner.opener: + boundary = max(0, text.rfind(scanner.closer) - len(scanner.opener) + 1) + parts: list[str] = [] + copied = 0 + position = 0 + while True: + start = scanner.starts.search(text, position) + if start is None: + break + position = start.start() + pattern = scanner.pattern if position < boundary else scanner.without_block + if pattern is None: + break + match = pattern.match(text, position) + if match is None: + position += 1 + continue + parts.append(text[copied:position]) if match.lastgroup == "keep": - return match.group() - return "\n" * match.group().count("\n") - - return scanner.sub(replace, text) + parts.append(match.group()) + else: + parts.append("\n" * match.group().count("\n")) + copied = position = match.end() + parts.append(text[copied:]) + return "".join(parts) def normalize_lines(text: str, language: str) -> list[tuple[int, str]]: diff --git a/tests/test_redundancy.py b/tests/test_redundancy.py new file mode 100644 index 00000000..4ac1b327 --- /dev/null +++ b/tests/test_redundancy.py @@ -0,0 +1,79 @@ +"""Comment stripping in the shared code-redundancy module.""" + +import importlib.util +import sys +import tempfile +import time +import unittest +from pathlib import Path + +MODULE = ( + Path(__file__).resolve().parents[1] + / "tasks/medium/tests/websitebench/harbor/redundancy.py" +) + + +def load_redundancy(): + spec = importlib.util.spec_from_file_location("task_redundancy", MODULE) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +redundancy = load_redundancy() + + +class StripComments(unittest.TestCase): + def test_strips_c_style_comments_and_keeps_strings(self): + text = 'a = "/* kept */"; // gone\nb = 1; /* gone\nstill gone */ c = 2;\n' + self.assertEqual( + redundancy.strip_comments(text, "javascript"), + 'a = "/* kept */"; \nb = 1; \n c = 2;\n', + ) + + def test_unclosed_opener_is_left_as_code(self): + text = "x = 1; /* closed */ y = a /* b\nz = 2;\n" + self.assertEqual( + redundancy.strip_comments(text, "javascript"), + "x = 1; y = a /* b\nz = 2;\n", + ) + + def test_unclosed_html_and_sql_openers_are_left_as_code(self): + self.assertEqual( + redundancy.strip_comments("