Fix comment marker swallowed by preceding operator run

sqlparse/keywords.py

@@ -200,7 +200,11 @@ def find_delimited_spans(text):
     # JSON operators
     (r'(\->>?|#>>?|@>|<@|\?\|?|\?&|\-|#\-)', tokens.Operator),
     (r'[<>=~!]+', tokens.Operator.Comparison),
-    (r'[+/@#%^&|^-]+', tokens.Operator),
+    # Negative lookahead keeps a `--` (or `# `) comment marker from being
+    # swallowed into a preceding operator run, e.g. `||--comment` must
+    # tokenize as `||` followed by a comment, not a single operator.
+    # See issue #722.
+    (r'(?:(?!--|# )[+/@#%^&|^-])+', tokens.Operator),
 ]
 
 KEYWORDS = {

tests/test_format.py

@@ -68,6 +68,12 @@ def test_strip_comments_single(self):
         res = ''
         assert res == sqlparse.format(sql, strip_comments=True)
 
+    def test_strip_comments_single_no_space_before_operator(self):
+        # see issue722
+        sql = 'foo || bar ||--comment\nbaz'
+        res = sqlparse.format(sql, strip_comments=True)
+        assert res == 'foo || bar ||\nbaz'
+
     def test_strip_comments_invalid_option(self):
         sql = 'select-- foo\nfrom -- bar\nwhere'
         with pytest.raises(SQLParseError):

tests/test_split.py

@@ -90,6 +90,19 @@ def test_split_comment_end_of_line():
     assert str(stmts[0]) == 'select * from foo; -- foo\n'
 
 
+def test_split_comment_directly_after_operator():
+    # see issue722: a `--` comment marker glued directly onto a preceding
+    # operator run (e.g. `||--`) must still be recognized as a comment, not
+    # merged into the operator token. The `;` right after `--` is part of
+    # the comment text (not a statement terminator), so this is genuinely
+    # a single statement; previously the bug caused it to be split in two.
+    sql = ('myval := oneval || otherval ||--;\n'
+           'continuedvals;\n')
+    stmts = sqlparse.split(sql)
+    assert len(stmts) == 1
+    assert stmts[0] == 'myval := oneval || otherval ||--;\ncontinuedvals;'
+
+
 def test_split_casewhen():
     sql = ("SELECT case when val = 1 then 2 else null end as foo;\n"
            "comment on table actor is 'The actor table.';")