From b0c0208703636aa8b9a3c70f2c5f19d044c96882 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sun, 12 Jul 2026 03:12:30 -0700 Subject: [PATCH 1/2] Allow comments in refactoring test files Skip semicolon comment lines in the test language tokenizer's initial lexer, the same way whitespace is skipped, so they never reach the grammar. Comments are only recognized between statements, not inside code blocks, where a semicolon stays part of the code. Fixes #350 Co-Authored-By: Claude Opus 4.8 --- test/private/tokenizer.rkt | 15 +++++++++++++++ test/testing-lang-test.rkt | 1 + 2 files changed, 16 insertions(+) diff --git a/test/private/tokenizer.rkt b/test/private/tokenizer.rkt index 375f586a..03f0468d 100644 --- a/test/private/tokenizer.rkt +++ b/test/private/tokenizer.rkt @@ -27,6 +27,8 @@ (define-lex-abbrev rest-of-line (concatenation (complement (concatenation any-string #\newline any-string)) #\newline)) +(define-lex-abbrev comment-line (concatenation ";" rest-of-line)) + (define-lex-abbrev dash-line (concatenation (repetition 3 +inf.0 #\-) #\newline)) (define-lex-abbrev equals-line (concatenation (repetition 3 +inf.0 #\=) #\newline)) (define-lex-abbrev pipe-dash-line (concatenation "|" dash-line)) @@ -83,6 +85,7 @@ (define initial-lexer (lexer-src-pos [whitespace (return-without-pos (initial-lexer input-port))] + [comment-line (return-without-pos (initial-lexer input-port))] [":" (token-COLON)] ["@" (token-AT-SIGN)] [".." (token-DOUBLE-DOT)] @@ -166,6 +169,18 @@ (position-token (token-CODE-LINE "#lang racket\n") (position 11 2 2) (position 24 3 0)))) (check-equal? (tokenize-until-eof tokenizer) expected-tokens)) + (test-case "comments between statements" + (define input (open-input-string "header:\n; a comment\n- #lang racket\n")) + (port-count-lines! input) + (define tokenizer (make-refactoring-test-tokenizer input)) + (define expected-tokens + (list + (position-token (token-IDENTIFIER 'header) (position 1 1 0) (position 7 1 6)) + (position-token (token-COLON) (position 7 1 6) (position 8 1 7)) + (position-token (token-SINGLE-DASH) (position 21 3 0) (position 23 3 2)) + (position-token (token-CODE-LINE "#lang racket\n") (position 23 3 2) (position 36 4 0)))) + (check-equal? (tokenize-until-eof tokenizer) expected-tokens)) + (test-case "code blocks" (define input (open-input-string "---\n#lang racket/base\n(void)\n---\n")) (port-count-lines! input) diff --git a/test/testing-lang-test.rkt b/test/testing-lang-test.rkt index ba588a1a..8565105a 100644 --- a/test/testing-lang-test.rkt +++ b/test/testing-lang-test.rkt @@ -4,6 +4,7 @@ header: - #lang resyntax/test +; Comments are allowed between statements (see issue #350). test: "unnecessary multi-line code blocks in tests refactorable to single-line code blocks" |------------------- | test: "foo" From a3ff24d21b5679e74dfe8959307f15baae27df4c Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sun, 12 Jul 2026 19:10:24 -0700 Subject: [PATCH 2/2] Use // instead of ; for comments in refactoring tests // reads as a comment to people unfamiliar with Racket, who are a common audience for these test files. Co-Authored-By: Claude Opus 4.8 --- test/private/tokenizer.rkt | 8 ++++---- test/testing-lang-test.rkt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/private/tokenizer.rkt b/test/private/tokenizer.rkt index 03f0468d..28d6ea89 100644 --- a/test/private/tokenizer.rkt +++ b/test/private/tokenizer.rkt @@ -27,7 +27,7 @@ (define-lex-abbrev rest-of-line (concatenation (complement (concatenation any-string #\newline any-string)) #\newline)) -(define-lex-abbrev comment-line (concatenation ";" rest-of-line)) +(define-lex-abbrev comment-line (concatenation "//" rest-of-line)) (define-lex-abbrev dash-line (concatenation (repetition 3 +inf.0 #\-) #\newline)) (define-lex-abbrev equals-line (concatenation (repetition 3 +inf.0 #\=) #\newline)) @@ -170,15 +170,15 @@ (check-equal? (tokenize-until-eof tokenizer) expected-tokens)) (test-case "comments between statements" - (define input (open-input-string "header:\n; a comment\n- #lang racket\n")) + (define input (open-input-string "header:\n// a comment\n- #lang racket\n")) (port-count-lines! input) (define tokenizer (make-refactoring-test-tokenizer input)) (define expected-tokens (list (position-token (token-IDENTIFIER 'header) (position 1 1 0) (position 7 1 6)) (position-token (token-COLON) (position 7 1 6) (position 8 1 7)) - (position-token (token-SINGLE-DASH) (position 21 3 0) (position 23 3 2)) - (position-token (token-CODE-LINE "#lang racket\n") (position 23 3 2) (position 36 4 0)))) + (position-token (token-SINGLE-DASH) (position 22 3 0) (position 24 3 2)) + (position-token (token-CODE-LINE "#lang racket\n") (position 24 3 2) (position 37 4 0)))) (check-equal? (tokenize-until-eof tokenizer) expected-tokens)) (test-case "code blocks" diff --git a/test/testing-lang-test.rkt b/test/testing-lang-test.rkt index 8565105a..aa8098ef 100644 --- a/test/testing-lang-test.rkt +++ b/test/testing-lang-test.rkt @@ -4,7 +4,7 @@ header: - #lang resyntax/test -; Comments are allowed between statements (see issue #350). +// Comments are allowed between statements (see issue #350). test: "unnecessary multi-line code blocks in tests refactorable to single-line code blocks" |------------------- | test: "foo"