diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py index dd6e5d15..c69e9a8b 100644 --- a/sqlparse/keywords.py +++ b/sqlparse/keywords.py @@ -200,7 +200,11 @@ def find_delimited_spans(text): # JSON operators (r'(\->>?|#>>?|@>|<@|\?\|?|\?&|\-|#\-)', tokens.Operator), (r'[<>=~!]+', tokens.Operator.Comparison), - (r'[+/@#%^&|^-]+', tokens.Operator), + # Negative lookahead keeps a `--` (or `# `) comment marker from being + # swallowed into a preceding operator run, e.g. `||--comment` must + # tokenize as `||` followed by a comment, not a single operator. + # See issue #722. + (r'(?:(?!--|# )[+/@#%^&|^-])+', tokens.Operator), ] KEYWORDS = { diff --git a/tests/test_format.py b/tests/test_format.py index 93495067..f314b46e 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -68,6 +68,12 @@ def test_strip_comments_single(self): res = '' assert res == sqlparse.format(sql, strip_comments=True) + def test_strip_comments_single_no_space_before_operator(self): + # see issue722 + sql = 'foo || bar ||--comment\nbaz' + res = sqlparse.format(sql, strip_comments=True) + assert res == 'foo || bar ||\nbaz' + def test_strip_comments_invalid_option(self): sql = 'select-- foo\nfrom -- bar\nwhere' with pytest.raises(SQLParseError): diff --git a/tests/test_split.py b/tests/test_split.py index 92c3fefe..9dfaba88 100644 --- a/tests/test_split.py +++ b/tests/test_split.py @@ -90,6 +90,19 @@ def test_split_comment_end_of_line(): assert str(stmts[0]) == 'select * from foo; -- foo\n' +def test_split_comment_directly_after_operator(): + # see issue722: a `--` comment marker glued directly onto a preceding + # operator run (e.g. `||--`) must still be recognized as a comment, not + # merged into the operator token. The `;` right after `--` is part of + # the comment text (not a statement terminator), so this is genuinely + # a single statement; previously the bug caused it to be split in two. + sql = ('myval := oneval || otherval ||--;\n' + 'continuedvals;\n') + stmts = sqlparse.split(sql) + assert len(stmts) == 1 + assert stmts[0] == 'myval := oneval || otherval ||--;\ncontinuedvals;' + + def test_split_casewhen(): sql = ("SELECT case when val = 1 then 2 else null end as foo;\n" "comment on table actor is 'The actor table.';")