From ad09844f20740235b69f36b324bcb68a8631e37d Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:08:54 -0700 Subject: [PATCH 01/17] Move linemap module into resyntax/grimoire Co-Authored-By: Claude Fable 5 --- {private => grimoire}/linemap.rkt | 0 private/analysis.rkt | 2 +- private/line-replacement.rkt | 2 +- private/refactoring-result.rkt | 2 +- private/syntax-replacement.rkt | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename {private => grimoire}/linemap.rkt (100%) diff --git a/private/linemap.rkt b/grimoire/linemap.rkt similarity index 100% rename from private/linemap.rkt rename to grimoire/linemap.rkt diff --git a/private/analysis.rkt b/private/analysis.rkt index e723a5be..9e214090 100644 --- a/private/analysis.rkt +++ b/private/analysis.rkt @@ -42,7 +42,7 @@ resyntax/default-recommendations/analyzers/ignored-result-values resyntax/default-recommendations/analyzers/variable-mutability resyntax/private/analyzer - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/private/logger resyntax/grimoire/source resyntax/private/string-indent diff --git a/private/line-replacement.rkt b/private/line-replacement.rkt index 639f74f2..5e52a294 100644 --- a/private/line-replacement.rkt +++ b/private/line-replacement.rkt @@ -27,7 +27,7 @@ rebellion/streaming/reducer rebellion/streaming/transducer rebellion/type/record - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/grimoire/string-replacement) diff --git a/private/refactoring-result.rkt b/private/refactoring-result.rkt index 23c17bb6..4d03977f 100644 --- a/private/refactoring-result.rkt +++ b/private/refactoring-result.rkt @@ -50,7 +50,7 @@ resyntax/private/code-snippet resyntax/private/commit resyntax/private/line-replacement - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/private/logger resyntax/grimoire/source resyntax/grimoire/string-replacement diff --git a/private/syntax-replacement.rkt b/private/syntax-replacement.rkt index 0789a78c..8dd87c12 100644 --- a/private/syntax-replacement.rkt +++ b/private/syntax-replacement.rkt @@ -42,7 +42,7 @@ rebellion/collection/range-set rebellion/private/static-name rebellion/type/record - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/private/logger resyntax/grimoire/source resyntax/private/string-indent From 3210cc2c0cdacc2458929e566df3eab38387bcee Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:08:54 -0700 Subject: [PATCH 02/17] Make string-linemap produce immutable line strings The linemap-lines contract has always promised immutable strings, but the implementation built lines with substring, which returns mutable ones. Any use of linemap-lines through the contract boundary raised a broke-its-own- contract error. It has no callers outside the module, so nothing noticed. Co-Authored-By: Claude Fable 5 --- grimoire/linemap.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grimoire/linemap.rkt b/grimoire/linemap.rkt index f499ce64..536c43e6 100644 --- a/grimoire/linemap.rkt +++ b/grimoire/linemap.rkt @@ -45,12 +45,12 @@ (let loop ([line-number 1] [line-start-index 0] [index 0]) (cond [(= index char-count) - (define last-line (substring str line-start-index index)) + (define last-line (string->immutable-string (substring str line-start-index index))) (vector-builder-add lines last-line) (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number) (vector-builder-add start-positions-by-line-number (add1 line-start-index))] [(equal? (string-ref str index) #\newline) - (define next-line (substring str line-start-index index)) + (define next-line (string->immutable-string (substring str line-start-index index))) (vector-builder-add lines next-line) (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number) (vector-builder-add start-positions-by-line-number (add1 line-start-index)) From ec2016dbfe449612d79b22b8ce3eb1ba6922048e Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:08:54 -0700 Subject: [PATCH 03/17] Draft API reference for linemaps Co-Authored-By: Claude Fable 5 --- grimoire.scrbl | 1 + grimoire/linemap.scrbl | 96 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 grimoire/linemap.scrbl diff --git a/grimoire.scrbl b/grimoire.scrbl index 02bf46d0..5f058ee9 100644 --- a/grimoire.scrbl +++ b/grimoire.scrbl @@ -18,3 +18,4 @@ programmatically on anything found here. @include-section[(lib "resyntax/grimoire/syntax-property-bundle.scrbl")] @include-section[(lib "resyntax/grimoire/expansion-analyzers.scrbl")] @include-section[(lib "resyntax/grimoire/string-replacement.scrbl")] +@include-section[(lib "resyntax/grimoire/linemap.scrbl")] diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl new file mode 100644 index 00000000..6c64fade --- /dev/null +++ b/grimoire/linemap.scrbl @@ -0,0 +1,96 @@ +#lang scribble/manual + + +@(require (for-label racket/base + racket/contract/base + rebellion/base/comparator + rebellion/base/range + resyntax/grimoire/linemap)) + + +@title[#:tag "linemap"]{Linemaps} +@defmodule[resyntax/grimoire/linemap] + +A @deftech{linemap} is a precomputed index of a string's line structure that supports converting +between character positions and line numbers. Resyntax straddles two views of source code: the +line-oriented view, used by @tech{source groups} and the command-line interface (see +@secref["cli"]) to select which code to analyze, and the character-oriented view, used by +@tech{string replacements} and syntax object source locations. Linemaps are the bridge between them --- Resyntax uses linemaps to compute which +lines a refactoring suggestion modifies, to render string replacements as line-based diffs, and to +restrict analysis to the requested lines. + +@bold{All positions and line numbers in a linemap are one-based}, matching the conventions of +@racket[syntax-position] and @racket[syntax-line]: the first character of a string is at position +@racket[1], on line @racket[1]. Beware that this is the @emph{opposite} of the convention used by +@tech{string replacements}, which address characters with @emph{zero-based} indices. Converting +between the two worlds requires adding or subtracting one, as discussed in +@secref["string-replacement"]. + +The lines of a string are the segments separated by newline characters. The terminating newline is +not part of a line's contents, but positions of newline characters belong to the lines they +terminate. A string that ends with a newline has a final empty line after it, and the empty string +consists of a single empty line. + + +@defproc[(linemap? [v any/c]) boolean?]{ + A predicate that recognizes @tech{linemaps}.} + + +@defproc[(string-linemap [str string?]) linemap?]{ + Constructs a @tech{linemap} of the lines in @racket[str]. Only @racket[#\newline] characters are + treated as line separators.} + + +@defproc[(linemap-lines [map linemap?]) + (vectorof (and/c string? immutable?) #:immutable #true)]{ + Returns a vector of the lines of the string that @racket[map] was built from, without their + terminating newlines. Note that because line numbers are one-based and vector indices are + zero-based, line @racket[_n] is at index @racket[_n] minus one in the returned vector.} + + +@defproc[(linemap-position-to-line [map linemap?] [position exact-positive-integer?]) + exact-positive-integer?]{ + Returns the line number of the line containing @racket[position]. The position of a newline + character is considered contained by the line that the newline terminates. Positions beyond the + end of the string do not raise an error; they are all treated as belonging to the last line.} + + +@defproc[(linemap-line-start-position [map linemap?] [line exact-positive-integer?]) + exact-positive-integer?]{ + Returns the position of the first character of @racket[line]. If the string ends with a newline, + the start position of its final, empty line is one past the end of the string. Raises an error if + @racket[line] is greater than the number of lines in the string.} + + +@defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-positive-integer?]) + exact-positive-integer?]{ + Returns the position of the first character of the line containing @racket[position]. Equivalent + to composing @racket[linemap-position-to-line] with @racket[linemap-line-start-position].} + + +@defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-positive-integer?]) + exact-positive-integer?]{ + Returns the position just past the last character of the contents of the line containing + @racket[position] --- that is, the position of the line's terminating newline, or one past the + end of the string if the line is the last one.} + + +@defproc[(syntax-start-line-position [stx syntax?] [#:linemap map linemap?]) + exact-positive-integer?]{ + Returns the position of the start of the line on which @racket[stx] begins. The + @tech[#:doc '(lib "scribblings/reference/reference.scrbl")]{source location} of @racket[stx] must + refer to positions within the string that @racket[map] was built from.} + + +@defproc[(syntax-end-line-position [stx syntax?] [#:linemap map linemap?]) + exact-positive-integer?]{ + Returns the position of the end of the line on which @racket[stx] ends, in the same sense as + @racket[linemap-position-to-end-of-line]. The source location of @racket[stx] must refer to + positions within the string that @racket[map] was built from.} + + +@defproc[(syntax-line-range [stx syntax?] [#:linemap map linemap?]) range?]{ + Returns a closed range (with @racket[natural<=>] as its comparator) containing the line numbers + of every line that @racket[stx] spans, from the line on which it begins to the line on which it + ends. The source location of @racket[stx] must refer to positions within the string that + @racket[map] was built from.} From a22b8915425842456484b4fe548f621489b56dce Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:18:19 -0700 Subject: [PATCH 04/17] Delete unused linemap bindings Co-Authored-By: Claude Fable 5 --- grimoire/linemap.rkt | 12 ------------ grimoire/linemap.scrbl | 37 +++++-------------------------------- 2 files changed, 5 insertions(+), 44 deletions(-) diff --git a/grimoire/linemap.rkt b/grimoire/linemap.rkt index 536c43e6..4b052437 100644 --- a/grimoire/linemap.rkt +++ b/grimoire/linemap.rkt @@ -8,13 +8,9 @@ (contract-out [string-linemap (-> string? linemap?)] [linemap? (-> any/c boolean?)] - [linemap-lines (-> linemap? (vectorof (and/c string? immutable?) #:immutable #true #:flat? #true))] [linemap-position-to-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [linemap-line-start-position (-> linemap? exact-positive-integer? exact-positive-integer?)] [linemap-position-to-start-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)] [linemap-position-to-end-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [syntax-start-line-position (-> syntax? #:linemap linemap? exact-positive-integer?)] - [syntax-end-line-position (-> syntax? #:linemap linemap? exact-positive-integer?)] [syntax-line-range (-> syntax? #:linemap linemap? range?)])) @@ -88,14 +84,6 @@ (linemap-line-end-position map (linemap-position-to-line map position))) -(define (syntax-start-line-position stx #:linemap map) - (linemap-position-to-start-of-line map (syntax-position stx))) - - -(define (syntax-end-line-position stx #:linemap map) - (linemap-position-to-end-of-line map (+ (syntax-position stx) (syntax-span stx)))) - - (define (syntax-line-range stx #:linemap map) (define first-line (syntax-line stx)) (define last-line (linemap-position-to-line map (+ (syntax-position stx) (syntax-span stx)))) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index 6c64fade..de76b97c 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -41,13 +41,6 @@ consists of a single empty line. treated as line separators.} -@defproc[(linemap-lines [map linemap?]) - (vectorof (and/c string? immutable?) #:immutable #true)]{ - Returns a vector of the lines of the string that @racket[map] was built from, without their - terminating newlines. Note that because line numbers are one-based and vector indices are - zero-based, line @racket[_n] is at index @racket[_n] minus one in the returned vector.} - - @defproc[(linemap-position-to-line [map linemap?] [position exact-positive-integer?]) exact-positive-integer?]{ Returns the line number of the line containing @racket[position]. The position of a newline @@ -55,17 +48,11 @@ consists of a single empty line. end of the string do not raise an error; they are all treated as belonging to the last line.} -@defproc[(linemap-line-start-position [map linemap?] [line exact-positive-integer?]) - exact-positive-integer?]{ - Returns the position of the first character of @racket[line]. If the string ends with a newline, - the start position of its final, empty line is one past the end of the string. Raises an error if - @racket[line] is greater than the number of lines in the string.} - - @defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-positive-integer?]) exact-positive-integer?]{ - Returns the position of the first character of the line containing @racket[position]. Equivalent - to composing @racket[linemap-position-to-line] with @racket[linemap-line-start-position].} + Returns the position of the first character of the line containing @racket[position]. If the + string ends with a newline and @racket[position] is on the final, empty line after it, that + line's start position is one past the end of the string.} @defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-positive-integer?]) @@ -75,22 +62,8 @@ consists of a single empty line. end of the string if the line is the last one.} -@defproc[(syntax-start-line-position [stx syntax?] [#:linemap map linemap?]) - exact-positive-integer?]{ - Returns the position of the start of the line on which @racket[stx] begins. The - @tech[#:doc '(lib "scribblings/reference/reference.scrbl")]{source location} of @racket[stx] must - refer to positions within the string that @racket[map] was built from.} - - -@defproc[(syntax-end-line-position [stx syntax?] [#:linemap map linemap?]) - exact-positive-integer?]{ - Returns the position of the end of the line on which @racket[stx] ends, in the same sense as - @racket[linemap-position-to-end-of-line]. The source location of @racket[stx] must refer to - positions within the string that @racket[map] was built from.} - - @defproc[(syntax-line-range [stx syntax?] [#:linemap map linemap?]) range?]{ Returns a closed range (with @racket[natural<=>] as its comparator) containing the line numbers of every line that @racket[stx] spans, from the line on which it begins to the line on which it - ends. The source location of @racket[stx] must refer to positions within the string that - @racket[map] was built from.} + ends. The @tech[#:doc '(lib "scribblings/reference/reference.scrbl")]{source location} of + @racket[stx] must refer to positions within the string that @racket[map] was built from.} From fc553cc4117ce57c895170c13aeb0f24be62102b Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:35:35 -0700 Subject: [PATCH 05/17] Make linemap positions zero-indexed Linemaps are built from strings, and Racket string APIs (and Resyntax's string replacements) index characters from zero. One-indexed positions are a file port and IDE convention that doesn't belong here. Line numbers stay one-indexed, since they're primarily useful in UI contexts. Every caller was converting with add1/sub1 at the boundary; those shims are now gone, and syntax-line-range converts from one-indexed syntax positions itself. Co-Authored-By: Claude Fable 5 --- grimoire/linemap.rkt | 139 +++++++++++++++++---------------- grimoire/linemap.scrbl | 31 ++++---- private/line-replacement.rkt | 11 +-- private/refactoring-result.rkt | 8 +- private/syntax-replacement.rkt | 7 +- 5 files changed, 100 insertions(+), 96 deletions(-) diff --git a/grimoire/linemap.rkt b/grimoire/linemap.rkt index 4b052437..c88634bc 100644 --- a/grimoire/linemap.rkt +++ b/grimoire/linemap.rkt @@ -8,9 +8,11 @@ (contract-out [string-linemap (-> string? linemap?)] [linemap? (-> any/c boolean?)] - [linemap-position-to-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [linemap-position-to-start-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [linemap-position-to-end-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)] + [linemap-position-to-line (-> linemap? exact-nonnegative-integer? exact-positive-integer?)] + [linemap-position-to-start-of-line + (-> linemap? exact-nonnegative-integer? exact-nonnegative-integer?)] + [linemap-position-to-end-of-line + (-> linemap? exact-nonnegative-integer? exact-nonnegative-integer?)] [syntax-line-range (-> syntax? #:linemap linemap? range?)])) @@ -43,13 +45,13 @@ [(= index char-count) (define last-line (string->immutable-string (substring str line-start-index index))) (vector-builder-add lines last-line) - (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number) - (vector-builder-add start-positions-by-line-number (add1 line-start-index))] + (sorted-map-builder-put line-numbers-by-start-position line-start-index line-number) + (vector-builder-add start-positions-by-line-number line-start-index)] [(equal? (string-ref str index) #\newline) (define next-line (string->immutable-string (substring str line-start-index index))) (vector-builder-add lines next-line) - (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number) - (vector-builder-add start-positions-by-line-number (add1 line-start-index)) + (sorted-map-builder-put line-numbers-by-start-position line-start-index line-number) + (vector-builder-add start-positions-by-line-number line-start-index) (define next-index (add1 index)) (loop (add1 line-number) next-index next-index)] [else (loop line-number line-start-index (add1 index))])) @@ -86,7 +88,9 @@ (define (syntax-line-range stx #:linemap map) (define first-line (syntax-line stx)) - (define last-line (linemap-position-to-line map (+ (syntax-position stx) (syntax-span stx)))) + ;; Syntax object positions are one-indexed, unlike linemap positions. + (define last-line + (linemap-position-to-line map (+ (sub1 (syntax-position stx)) (syntax-span stx)))) (unless (<= first-line last-line) (raise-arguments-error 'syntax-line-range "syntax object's last line number is before its first line number" @@ -105,118 +109,119 @@ (define (nat-map . args) (apply sorted-map #:key-comparator natural<=> args)) - (check-equal? (string-linemap "") (linemap #("") (nat-map 1 1) #(1))) - (check-equal? (string-linemap "a") (linemap #("a") (nat-map 1 1) #(1))) - (check-equal? (string-linemap "λ") (linemap #("λ") (nat-map 1 1) #(1))) - (check-equal? (string-linemap "a\n") (linemap #("a" "") (nat-map 1 1 3 2) #(1 3))) - (check-equal? (string-linemap "λ\n") (linemap #("λ" "") (nat-map 1 1 3 2) #(1 3))) - (check-equal? (string-linemap "aaa\n") (linemap #("aaa" "") (nat-map 1 1 5 2) #(1 5))) - (check-equal? (string-linemap "aaa\nbbb") (linemap #("aaa" "bbb") (nat-map 1 1 5 2) #(1 5))) + (check-equal? (string-linemap "") (linemap #("") (nat-map 0 1) #(0))) + (check-equal? (string-linemap "a") (linemap #("a") (nat-map 0 1) #(0))) + (check-equal? (string-linemap "λ") (linemap #("λ") (nat-map 0 1) #(0))) + (check-equal? (string-linemap "a\n") (linemap #("a" "") (nat-map 0 1 2 2) #(0 2))) + (check-equal? (string-linemap "λ\n") (linemap #("λ" "") (nat-map 0 1 2 2) #(0 2))) + (check-equal? (string-linemap "aaa\n") (linemap #("aaa" "") (nat-map 0 1 4 2) #(0 4))) + (check-equal? (string-linemap "aaa\nbbb") (linemap #("aaa" "bbb") (nat-map 0 1 4 2) #(0 4))) (check-equal? (string-linemap "aaa\nbbb\n") - (linemap #("aaa" "bbb" "") (nat-map 1 1 5 2 9 3) #(1 5 9))) + (linemap #("aaa" "bbb" "") (nat-map 0 1 4 2 8 3) #(0 4 8))) (check-equal? (string-linemap "a\n\n\nb") - (linemap #("a" "" "" "b") (nat-map 1 1 3 2 4 3 5 4) #(1 3 4 5)))) + (linemap #("a" "" "" "b") (nat-map 0 1 2 2 3 3 4 4) #(0 2 3 4)))) (test-case (name-string linemap-position-to-line) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) + (check-equal? (linemap-position-to-line map 0) 1) (check-equal? (linemap-position-to-line map 1) 1) (check-equal? (linemap-position-to-line map 2) 1) (check-equal? (linemap-position-to-line map 3) 1) (check-equal? (linemap-position-to-line map 4) 1) (check-equal? (linemap-position-to-line map 5) 1) - (check-equal? (linemap-position-to-line map 6) 1) + (check-equal? (linemap-position-to-line map 6) 2) (check-equal? (linemap-position-to-line map 7) 2) (check-equal? (linemap-position-to-line map 8) 2) (check-equal? (linemap-position-to-line map 9) 2) (check-equal? (linemap-position-to-line map 10) 2) (check-equal? (linemap-position-to-line map 11) 2) - (check-equal? (linemap-position-to-line map 12) 2) - (check-equal? (linemap-position-to-line map 13) 3)) + (check-equal? (linemap-position-to-line map 12) 3)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) + (check-equal? (linemap-position-to-line map 0) 1) (check-equal? (linemap-position-to-line map 1) 1) - (check-equal? (linemap-position-to-line map 2) 1) - (check-equal? (linemap-position-to-line map 3) 2) - (check-equal? (linemap-position-to-line map 4) 3) - (check-equal? (linemap-position-to-line map 5) 3))) + (check-equal? (linemap-position-to-line map 2) 2) + (check-equal? (linemap-position-to-line map 3) 3) + (check-equal? (linemap-position-to-line map 4) 3))) (test-case (name-string linemap-line-start-position) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-line-start-position map 1) 1) - (check-equal? (linemap-line-start-position map 2) 7) - (check-equal? (linemap-line-start-position map 3) 13)) + (check-equal? (linemap-line-start-position map 1) 0) + (check-equal? (linemap-line-start-position map 2) 6) + (check-equal? (linemap-line-start-position map 3) 12)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-line-start-position map 1) 1) - (check-equal? (linemap-line-start-position map 2) 3) - (check-equal? (linemap-line-start-position map 3) 4))) + (check-equal? (linemap-line-start-position map 1) 0) + (check-equal? (linemap-line-start-position map 2) 2) + (check-equal? (linemap-line-start-position map 3) 3))) (test-case (name-string linemap-line-end-position) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-line-end-position map 1) 6) - (check-equal? (linemap-line-end-position map 2) 12)) + (check-equal? (linemap-line-end-position map 1) 5) + (check-equal? (linemap-line-end-position map 2) 11)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-line-end-position map 1) 2) - (check-equal? (linemap-line-end-position map 2) 3) - (check-equal? (linemap-line-end-position map 3) 5))) + (check-equal? (linemap-line-end-position map 1) 1) + (check-equal? (linemap-line-end-position map 2) 2) + (check-equal? (linemap-line-end-position map 3) 4))) (test-case (name-string linemap-position-to-start-of-line) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-position-to-start-of-line map 1) 1) - (check-equal? (linemap-position-to-start-of-line map 2) 1) - (check-equal? (linemap-position-to-start-of-line map 3) 1) - (check-equal? (linemap-position-to-start-of-line map 4) 1) - (check-equal? (linemap-position-to-start-of-line map 5) 1) - (check-equal? (linemap-position-to-start-of-line map 6) 1) - (check-equal? (linemap-position-to-start-of-line map 7) 7) - (check-equal? (linemap-position-to-start-of-line map 8) 7) - (check-equal? (linemap-position-to-start-of-line map 9) 7) - (check-equal? (linemap-position-to-start-of-line map 10) 7) - (check-equal? (linemap-position-to-start-of-line map 11) 7) - (check-equal? (linemap-position-to-start-of-line map 12) 7) - (check-equal? (linemap-position-to-start-of-line map 13) 13)) + (check-equal? (linemap-position-to-start-of-line map 0) 0) + (check-equal? (linemap-position-to-start-of-line map 1) 0) + (check-equal? (linemap-position-to-start-of-line map 2) 0) + (check-equal? (linemap-position-to-start-of-line map 3) 0) + (check-equal? (linemap-position-to-start-of-line map 4) 0) + (check-equal? (linemap-position-to-start-of-line map 5) 0) + (check-equal? (linemap-position-to-start-of-line map 6) 6) + (check-equal? (linemap-position-to-start-of-line map 7) 6) + (check-equal? (linemap-position-to-start-of-line map 8) 6) + (check-equal? (linemap-position-to-start-of-line map 9) 6) + (check-equal? (linemap-position-to-start-of-line map 10) 6) + (check-equal? (linemap-position-to-start-of-line map 11) 6) + (check-equal? (linemap-position-to-start-of-line map 12) 12)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-position-to-start-of-line map 1) 1) - (check-equal? (linemap-position-to-start-of-line map 2) 1) + (check-equal? (linemap-position-to-start-of-line map 0) 0) + (check-equal? (linemap-position-to-start-of-line map 1) 0) + (check-equal? (linemap-position-to-start-of-line map 2) 2) (check-equal? (linemap-position-to-start-of-line map 3) 3) - (check-equal? (linemap-position-to-start-of-line map 4) 4) - (check-equal? (linemap-position-to-start-of-line map 5) 4))) + (check-equal? (linemap-position-to-start-of-line map 4) 3))) (test-case (name-string linemap-position-to-end-of-line) (test-case "two-line string with ending newline" (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-position-to-end-of-line map 1) 6) - (check-equal? (linemap-position-to-end-of-line map 2) 6) - (check-equal? (linemap-position-to-end-of-line map 3) 6) - (check-equal? (linemap-position-to-end-of-line map 4) 6) - (check-equal? (linemap-position-to-end-of-line map 5) 6) - (check-equal? (linemap-position-to-end-of-line map 6) 6) - (check-equal? (linemap-position-to-end-of-line map 7) 12) - (check-equal? (linemap-position-to-end-of-line map 8) 12) - (check-equal? (linemap-position-to-end-of-line map 9) 12) - (check-equal? (linemap-position-to-end-of-line map 10) 12) - (check-equal? (linemap-position-to-end-of-line map 11) 12) + (check-equal? (linemap-position-to-end-of-line map 0) 5) + (check-equal? (linemap-position-to-end-of-line map 1) 5) + (check-equal? (linemap-position-to-end-of-line map 2) 5) + (check-equal? (linemap-position-to-end-of-line map 3) 5) + (check-equal? (linemap-position-to-end-of-line map 4) 5) + (check-equal? (linemap-position-to-end-of-line map 5) 5) + (check-equal? (linemap-position-to-end-of-line map 6) 11) + (check-equal? (linemap-position-to-end-of-line map 7) 11) + (check-equal? (linemap-position-to-end-of-line map 8) 11) + (check-equal? (linemap-position-to-end-of-line map 9) 11) + (check-equal? (linemap-position-to-end-of-line map 10) 11) + (check-equal? (linemap-position-to-end-of-line map 11) 11) (check-equal? (linemap-position-to-end-of-line map 12) 12)) (test-case "multiple blank lines" (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-position-to-end-of-line map 1) 2) + (check-equal? (linemap-position-to-end-of-line map 0) 1) + (check-equal? (linemap-position-to-end-of-line map 1) 1) (check-equal? (linemap-position-to-end-of-line map 2) 2) - (check-equal? (linemap-position-to-end-of-line map 3) 3) - (check-equal? (linemap-position-to-end-of-line map 4) 5) - (check-equal? (linemap-position-to-end-of-line map 5) 5)))) + (check-equal? (linemap-position-to-end-of-line map 3) 4) + (check-equal? (linemap-position-to-end-of-line map 4) 4)))) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index de76b97c..0b6cc910 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -19,12 +19,15 @@ line-oriented view, used by @tech{source groups} and the command-line interface lines a refactoring suggestion modifies, to render string replacements as line-based diffs, and to restrict analysis to the requested lines. -@bold{All positions and line numbers in a linemap are one-based}, matching the conventions of -@racket[syntax-position] and @racket[syntax-line]: the first character of a string is at position -@racket[1], on line @racket[1]. Beware that this is the @emph{opposite} of the convention used by -@tech{string replacements}, which address characters with @emph{zero-based} indices. Converting -between the two worlds requires adding or subtracting one, as discussed in -@secref["string-replacement"]. +@bold{Positions in a linemap are zero-based, but line numbers are one-based.} Positions are +character indices into the string, following the same convention as Racket's string operations and +as @tech{string replacements}: the first character of a string is at position @racket[0]. Line +numbers instead begin at line @racket[1], matching @racket[syntax-line] and the conventions of +code editors --- line numbers are almost exclusively useful in user interfaces, where one-based +numbering is expected. Beware that @racket[syntax-position] and file port positions are +@emph{one-based}, unlike linemap positions. The @racket[syntax-line-range] operation performs that +conversion itself, but positions obtained from syntax objects by other means must be converted +before use with a linemap. The lines of a string are the segments separated by newline characters. The terminating newline is not part of a line's contents, but positions of newline characters belong to the lines they @@ -41,25 +44,25 @@ consists of a single empty line. treated as line separators.} -@defproc[(linemap-position-to-line [map linemap?] [position exact-positive-integer?]) +@defproc[(linemap-position-to-line [map linemap?] [position exact-nonnegative-integer?]) exact-positive-integer?]{ Returns the line number of the line containing @racket[position]. The position of a newline character is considered contained by the line that the newline terminates. Positions beyond the end of the string do not raise an error; they are all treated as belonging to the last line.} -@defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-positive-integer?]) - exact-positive-integer?]{ +@defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-nonnegative-integer?]) + exact-nonnegative-integer?]{ Returns the position of the first character of the line containing @racket[position]. If the string ends with a newline and @racket[position] is on the final, empty line after it, that - line's start position is one past the end of the string.} + line's start position is equal to the length of the string.} -@defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-positive-integer?]) - exact-positive-integer?]{ +@defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-nonnegative-integer?]) + exact-nonnegative-integer?]{ Returns the position just past the last character of the contents of the line containing - @racket[position] --- that is, the position of the line's terminating newline, or one past the - end of the string if the line is the last one.} + @racket[position] --- that is, the position of the line's terminating newline, or the length of + the string if the line is the last one.} @defproc[(syntax-line-range [stx syntax?] [#:linemap map linemap?]) range?]{ diff --git a/private/line-replacement.rkt b/private/line-replacement.rkt index 5e52a294..37d8df23 100644 --- a/private/line-replacement.rkt +++ b/private/line-replacement.rkt @@ -77,16 +77,13 @@ (define new-lmap (string-linemap new-string)) (define start-line - (linemap-position-to-line orig-lmap (add1 (string-replacement-start replacement)))) + (linemap-position-to-line orig-lmap (string-replacement-start replacement))) (define start-pos - (sub1 - (linemap-position-to-start-of-line orig-lmap (add1 (string-replacement-start replacement))))) + (linemap-position-to-start-of-line orig-lmap (string-replacement-start replacement))) (define original-end-pos - (sub1 - (linemap-position-to-end-of-line orig-lmap - (add1 (string-replacement-original-end replacement))))) + (linemap-position-to-end-of-line orig-lmap (string-replacement-original-end replacement))) (define new-end-pos - (sub1 (linemap-position-to-end-of-line new-lmap (add1 (string-replacement-new-end replacement))))) + (linemap-position-to-end-of-line new-lmap (string-replacement-new-end replacement))) (define original-substr (substring original-string start-pos original-end-pos)) (define new-substr (substring new-string start-pos new-end-pos)) diff --git a/private/refactoring-result.rkt b/private/refactoring-result.rkt index 4d03977f..733d98cf 100644 --- a/private/refactoring-result.rkt +++ b/private/refactoring-result.rkt @@ -192,9 +192,9 @@ (define lmap (string-linemap full-orig-code)) (define start (string-replacement-start replacement)) (define end (string-replacement-original-end replacement)) - (define start-column (- (add1 start) (linemap-position-to-start-of-line lmap (add1 start)))) + (define start-column (- start (linemap-position-to-start-of-line lmap start))) (define raw-text (string->immutable-string (substring full-orig-code start end))) - (code-snippet raw-text start-column (linemap-position-to-line lmap (add1 start)))) + (code-snippet raw-text start-column (linemap-position-to-line lmap start))) (define (refactoring-result-new-code result) @@ -203,8 +203,8 @@ (source->string (syntax-replacement-source (refactoring-result-syntax-replacement result)))) (define lmap (string-linemap full-orig-code)) (define start (string-replacement-start replacement)) - (define original-line (linemap-position-to-line lmap (add1 start))) - (define original-column (- (add1 start) (linemap-position-to-start-of-line lmap (add1 start)))) + (define original-line (linemap-position-to-line lmap start)) + (define original-column (- start (linemap-position-to-start-of-line lmap start))) (define refactored-source-code (string-replacement-apply replacement full-orig-code)) (define new-code-string (substring refactored-source-code diff --git a/private/syntax-replacement.rkt b/private/syntax-replacement.rkt index 8dd87c12..3e130aea 100644 --- a/private/syntax-replacement.rkt +++ b/private/syntax-replacement.rkt @@ -267,10 +267,9 @@ (define trailing-text-length (let ([linemap (string-linemap original)] [orig-end (string-replacement-original-end replacement)]) - ;; Convert from 0-based string indices to 1-based positions for linemap - (if (= (linemap-position-to-line linemap (add1 start)) - (linemap-position-to-line linemap (add1 orig-end))) - (- (linemap-position-to-end-of-line linemap (add1 orig-end)) (add1 orig-end)) + (if (= (linemap-position-to-line linemap start) + (linemap-position-to-line linemap orig-end)) + (- (linemap-position-to-end-of-line linemap orig-end) orig-end) 0))) (define allowed-width (- base-allowed-width trailing-text-length)) From dc06edfe6d155128ade505f33f780b74e434f7de Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:40:29 -0700 Subject: [PATCH 06/17] Document newline normalization in source reading The fix for #272 (Windows \r\n newlines breaking analysis, PR #274) lives in with-input-from-source, which reencodes every source port with newline conversion. That behavior and the invariant it protects were undocumented. Co-Authored-By: Claude Fable 5 --- grimoire/linemap.scrbl | 8 ++++++-- grimoire/source.scrbl | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index 0b6cc910..ae605971 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -5,7 +5,8 @@ racket/contract/base rebellion/base/comparator rebellion/base/range - resyntax/grimoire/linemap)) + resyntax/grimoire/linemap + resyntax/grimoire/source)) @title[#:tag "linemap"]{Linemaps} @@ -41,7 +42,10 @@ consists of a single empty line. @defproc[(string-linemap [str string?]) linemap?]{ Constructs a @tech{linemap} of the lines in @racket[str]. Only @racket[#\newline] characters are - treated as line separators.} + treated as line separators. In particular, Windows-style @racket["\r\n"] line endings are not + understood. This never arises in practice, because Resyntax normalizes all newlines to + @racket[#\newline] when reading @tech{source code} --- see @racket[with-input-from-source] for + details on that normalization and why it matters.} @defproc[(linemap-position-to-line [map linemap?] [position exact-nonnegative-integer?]) diff --git a/grimoire/source.scrbl b/grimoire/source.scrbl index 6984a381..9682ce10 100644 --- a/grimoire/source.scrbl +++ b/grimoire/source.scrbl @@ -4,6 +4,7 @@ @(require (for-label racket/base racket/contract/base racket/path + racket/port rebellion/base/immutable-string rebellion/collection/range-set resyntax/grimoire/source @@ -104,13 +105,26 @@ stack of dependent changes to commit in series without actually mutating the fil @defproc[(source->string [code source?]) immutable-string?]{ Returns the full text of @racket[code], reading it from the filesystem if necessary. For @racket[modified-source?] values, this returns the new, updated text rather than the original - unmodified text.} + unmodified text. The returned text has its newlines normalized, as described in + @racket[with-input-from-source].} @defproc[(with-input-from-source [code source?] [proc (-> any)]) any]{ Calls @racket[proc] with @racket[current-input-port] set to a freshly opened input port reading the contents of @racket[code]. For unmodified file sources, this opens a file port. For modified - sources and string sources, this opens a string port without interacting with the filesystem.} + sources and string sources, this opens a string port without interacting with the filesystem. + + The opened port decodes the source as UTF-8 and @bold{normalizes newlines}, as in + @racket[reencode-input-port]: Windows-style @racket["\r\n"] sequences (and other newline + conventions) are converted into single @racket[#\newline] characters. Every operation that reads + a source goes through this normalization, including @racket[source->string] and + @racket[source-read-syntax]. Normalizing consistently is load-bearing: when line counting is + enabled on a port, Racket counts a @racket["\r\n"] sequence as a @emph{single} position, so + without normalization the source locations of syntax objects read from a source would disagree + with the character indices of that source's text. Resyntax relies on the assumption that a syntax + object's position and span identify exactly the range of characters it was read from. Analyzing + code with Windows-style newlines used to violate that assumption and break Resyntax in + hard-to-diagnose ways.} @section{Parsing, Expanding, and Compiling Sources} From 2f52d88d8df3cda203b3ef14966cf686e726df70 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 00:56:36 -0700 Subject: [PATCH 07/17] Normalize newlines when constructing string and modified sources Previously only the port-reading path normalized newlines, so two string sources could be non-equal? yet denote the same text via source->string. Normalizing eagerly in the struct guards removes that corner case as early in the data flow as possible. The guard conversion matches the full set of sequences that reencode-input-port converts, verified empirically: \r\n, \r NEL, \r, NEL, and LS all become \n. Co-Authored-By: Claude Fable 5 --- grimoire/source.rkt | 32 ++++++++++++++++++++++++++++++-- grimoire/source.scrbl | 13 ++++++++++--- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/grimoire/source.rkt b/grimoire/source.rkt index 8350f50a..112cbe46 100644 --- a/grimoire/source.rkt +++ b/grimoire/source.rkt @@ -53,6 +53,19 @@ ;@---------------------------------------------------------------------------------------------------- +;; Matches every newline sequence that reencode-input-port's newline conversion recognizes. +(define newline-sequence-pattern + (let ([cr (string #\return)] + [lf (string #\newline)] + [nel (string (integer->char #x85))] + [ls (string (integer->char #x2028))]) + (regexp (string-append cr lf "|" cr nel "|" cr "|" nel "|" ls)))) + + +(define (string-normalize-newlines str) + (regexp-replace* newline-sequence-pattern str (string #\newline))) + + (struct source () #:transparent) @@ -66,12 +79,13 @@ (struct string-source unmodified-source (contents) #:transparent - #:guard (λ (contents _) (string->immutable-string contents))) + #:guard (λ (contents _) (string->immutable-string (string-normalize-newlines contents)))) (struct modified-source source (original contents) #:transparent - #:guard (λ (original contents _) (values original (string->immutable-string contents)))) + #:guard (λ (original contents _) + (values original (string->immutable-string (string-normalize-newlines contents))))) (define (source-name src) @@ -137,6 +151,20 @@ (module+ test + (test-case "string and modified sources normalize newlines upon construction" + (define cr (string #\return)) + (define lf (string #\newline)) + (define nel (string (integer->char #x85))) + (define ls (string (integer->char #x2028))) + (define normalized (string-source (string-append "a" lf "b" lf "c"))) + (check-equal? (string-source (string-append "a" cr lf "b" cr lf "c")) normalized) + (check-equal? (string-source (string-append "a" cr "b" cr "c")) normalized) + (check-equal? (string-source (string-append "a" nel "b" cr nel "c")) normalized) + (check-equal? (string-source (string-append "a" ls "b" ls "c")) normalized) + (define base (string-source "base")) + (check-equal? (modified-source base (string-append "a" cr lf "b")) + (modified-source base (string-append "a" lf "b")))) + (test-case "source-read-language" (check-equal? (source-read-language (string-source "#lang racket")) 'racket) (check-equal? (source-read-language (string-source "#lang at-exp racket")) 'at-exp) diff --git a/grimoire/source.scrbl b/grimoire/source.scrbl index 9682ce10..30db79c2 100644 --- a/grimoire/source.scrbl +++ b/grimoire/source.scrbl @@ -64,7 +64,10 @@ stack of dependent changes to commit in series without actually mutating the fil @defproc[(string-source [contents string?]) string-source?]{ - Constructs a source string containing @racket[contents] directly.} + Constructs a source string containing @racket[contents] directly. The newlines of + @racket[contents] are normalized at construction time, in the same manner described in + @racket[with-input-from-source]. Normalizing eagerly ensures that two string sources denoting the + same text are @racket[equal?] even if they were constructed with different newline conventions.} @defproc[(modified-source? [v any/c]) boolean?]{ @@ -77,7 +80,8 @@ stack of dependent changes to commit in series without actually mutating the fil Constructs a modified source that replaces the contents of @racket[original] with @racket[new-contents]. This represents a whole-file replacement --- the @emph{complete} contents of @racket[original] are @emph{entirely} swapped out with @racket[new-contents]. Modified sources cannot - represent partial edits on their own.} + represent partial edits on their own. Like @racket[string-source], the newlines of + @racket[new-contents] are normalized at construction time.} @defproc[(source-name [code source?]) (or/c path? symbol?)]{ @@ -124,7 +128,10 @@ stack of dependent changes to commit in series without actually mutating the fil with the character indices of that source's text. Resyntax relies on the assumption that a syntax object's position and span identify exactly the range of characters it was read from. Analyzing code with Windows-style newlines used to violate that assumption and break Resyntax in - hard-to-diagnose ways.} + hard-to-diagnose ways. + + String sources and modified sources also apply this normalization eagerly, when the source value + is constructed, so the port-level conversion only has a visible effect for file sources.} @section{Parsing, Expanding, and Compiling Sources} From 81f276b8e02794b45142224d34f56bd35cf84187 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 01:02:38 -0700 Subject: [PATCH 08/17] Normalize newlines through reencode-input-port itself Rather than a regexp that reproduces the port's conversion behavior, round- trip the string through an actual newline-converting reencoded port. Slightly more copying, but it can never fall out of sync with what reading a source does. Co-Authored-By: Claude Fable 5 --- grimoire/source.rkt | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/grimoire/source.rkt b/grimoire/source.rkt index 112cbe46..9cb4f2d7 100644 --- a/grimoire/source.rkt +++ b/grimoire/source.rkt @@ -53,17 +53,13 @@ ;@---------------------------------------------------------------------------------------------------- -;; Matches every newline sequence that reencode-input-port's newline conversion recognizes. -(define newline-sequence-pattern - (let ([cr (string #\return)] - [lf (string #\newline)] - [nel (string (integer->char #x85))] - [ls (string (integer->char #x2028))]) - (regexp (string-append cr lf "|" cr nel "|" cr "|" nel "|" ls)))) - - +;; Reads str back out through a newline-converting reencoded port, guaranteeing that this +;; normalization can never disagree with the one performed by with-input-from-source. (define (string-normalize-newlines str) - (regexp-replace* newline-sequence-pattern str (string #\newline))) + (define reencoded-in + (reencode-input-port + (open-input-string str) "UTF-8" #false #false 'string-normalize-newlines #true)) + (port->string reencoded-in)) (struct source () #:transparent) From ce825dc3b8c3fd2580b20ad1e8dfd42dddd5cab7 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 01:29:09 -0700 Subject: [PATCH 09/17] Share one reencode-input-port wrapper across all source reading Co-Authored-By: Claude Fable 5 --- grimoire/source.rkt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/grimoire/source.rkt b/grimoire/source.rkt index 9cb4f2d7..a7bc1502 100644 --- a/grimoire/source.rkt +++ b/grimoire/source.rkt @@ -53,13 +53,14 @@ ;@---------------------------------------------------------------------------------------------------- -;; Reads str back out through a newline-converting reencoded port, guaranteeing that this -;; normalization can never disagree with the one performed by with-input-from-source. +;; All source text flows through this port wrapper, which decodes it as UTF-8 and converts +;; newline sequences like \r\n into single \n characters. +(define (reencoded-source-input-port in) + (reencode-input-port in "UTF-8" #false #false (object-name in) #true)) + + (define (string-normalize-newlines str) - (define reencoded-in - (reencode-input-port - (open-input-string str) "UTF-8" #false #false 'string-normalize-newlines #true)) - (port->string reencoded-in)) + (port->string (reencoded-source-input-port (open-input-string str)))) (struct source () #:transparent) @@ -93,8 +94,7 @@ (define (with-input-from-source code proc) (define (call-proc-with-reencoded-input in) - (define reencoded-in (reencode-input-port in "UTF-8" #false #false (object-name in) #true)) - (parameterize ([current-input-port reencoded-in]) + (parameterize ([current-input-port (reencoded-source-input-port in)]) (proc))) (match code From 74178306cefca2a11153b72c86d3865ea04e2ee9 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 01:51:57 -0700 Subject: [PATCH 10/17] Link line-counting docs from newline normalization prose Co-Authored-By: Claude Fable 5 --- grimoire/source.scrbl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/grimoire/source.scrbl b/grimoire/source.scrbl index 30db79c2..0f413516 100644 --- a/grimoire/source.scrbl +++ b/grimoire/source.scrbl @@ -122,13 +122,15 @@ stack of dependent changes to commit in series without actually mutating the fil @racket[reencode-input-port]: Windows-style @racket["\r\n"] sequences (and other newline conventions) are converted into single @racket[#\newline] characters. Every operation that reads a source goes through this normalization, including @racket[source->string] and - @racket[source-read-syntax]. Normalizing consistently is load-bearing: when line counting is - enabled on a port, Racket counts a @racket["\r\n"] sequence as a @emph{single} position, so - without normalization the source locations of syntax objects read from a source would disagree - with the character indices of that source's text. Resyntax relies on the assumption that a syntax - object's position and span identify exactly the range of characters it was read from. Analyzing - code with Windows-style newlines used to violate that assumption and break Resyntax in - hard-to-diagnose ways. + @racket[source-read-syntax]. Normalizing consistently is load-bearing: as described in + @secref["linecol" #:doc '(lib "scribblings/reference/reference.scrbl")], Racket performs this + same conversion whenever it counts line and column numbers, treating a @racket["\r\n"] sequence + as a @emph{single} position --- and line counting must be enabled for syntax objects to receive + line numbers in their source locations. Without normalization, then, the source locations of + syntax objects read from a source would disagree with the character indices of that source's + text. Resyntax relies on the assumption that a syntax object's position and span identify exactly + the range of characters it was read from. Analyzing code with Windows-style newlines used to + violate that assumption and break Resyntax in hard-to-diagnose ways. String sources and modified sources also apply this normalization eagerly, when the source value is constructed, so the port-level conversion only has a visible effect for file sources.} From 135c14eac698d2f560f1c7546940a5e2128b552f Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 14:42:37 -0700 Subject: [PATCH 11/17] Edit linemap overview docs --- grimoire/linemap.scrbl | 47 +++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index ae605971..f59b13c1 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -13,27 +13,40 @@ @defmodule[resyntax/grimoire/linemap] A @deftech{linemap} is a precomputed index of a string's line structure that supports converting -between character positions and line numbers. Resyntax straddles two views of source code: the -line-oriented view, used by @tech{source groups} and the command-line interface (see -@secref["cli"]) to select which code to analyze, and the character-oriented view, used by -@tech{string replacements} and syntax object source locations. Linemaps are the bridge between them --- Resyntax uses linemaps to compute which -lines a refactoring suggestion modifies, to render string replacements as line-based diffs, and to -restrict analysis to the requested lines. - -@bold{Positions in a linemap are zero-based, but line numbers are one-based.} Positions are -character indices into the string, following the same convention as Racket's string operations and -as @tech{string replacements}: the first character of a string is at position @racket[0]. Line -numbers instead begin at line @racket[1], matching @racket[syntax-line] and the conventions of -code editors --- line numbers are almost exclusively useful in user interfaces, where one-based -numbering is expected. Beware that @racket[syntax-position] and file port positions are -@emph{one-based}, unlike linemap positions. The @racket[syntax-line-range] operation performs that -conversion itself, but positions obtained from syntax objects by other means must be converted -before use with a linemap. +between character positions and line numbers. Source code is frequently viewed from one of two +perspectives: + +@itemlist[ + @item{A @emph{human} perspective, either reader or writer, who looks at code as a 2D grid composed of + lines and columns.} + + @item{A @emph{machine} perspective, that looks at code as one line sequence of characters (or perhaps + even just plain bytes).}] + +Tools that serve as a human-machine interface for code often have to juggle these two perspectives. +Linemaps are Resyntax's tool for doing so. They are used in various places where human concerns +related to viewing and describing source code and source edits come up, such as displaying or +consuming line-based diffs in the command-line interface. See @secref["cli"] for further details on +that matter. + +@bold{Positions in a linemap are zero-based, but line numbers are one-based.} Positions +are character indices into the string. This follows the same convention as Racket's string operations +such as @racket[string-ref], as well as Resyntax's conventions for @tech{string replacements}. Line +numbers, however, follow the conventions outlined in +@secref["linecol" #:doc '(lib "scribblings/reference/reference.scrbl")]: source files begin at line +@racket[1]. This matches @racket[syntax-line] and the conventions of code editors --- line numbers +are almost exclusively useful in user interfaces, where one-based numbering is expected. + +@bold{Beware that @racket[syntax-position] and file port positions are @emph{one-based}, unlike + linemap positions.} The @racket[syntax-line-range] operation performs that conversion itself, but +positions obtained from syntax objects by other means must be converted before use with a linemap. The lines of a string are the segments separated by newline characters. The terminating newline is not part of a line's contents, but positions of newline characters belong to the lines they terminate. A string that ends with a newline has a final empty line after it, and the empty string -consists of a single empty line. +consists of a single empty line. Note that there are multiple distinct byte sequences that Racket +treats as a newline when reading source code --- for further details on how Resyntax handles these +cases, see the notes in @racket[with-input-from-source]. @defproc[(linemap? [v any/c]) boolean?]{ From 642a1a398cb5e278065dc3a04975b219e52f1f8e Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 14:46:07 -0700 Subject: [PATCH 12/17] Fix typo in linemap docs Co-Authored-By: Claude Fable 5 --- grimoire/linemap.scrbl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index f59b13c1..42fc607c 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -20,8 +20,8 @@ perspectives: @item{A @emph{human} perspective, either reader or writer, who looks at code as a 2D grid composed of lines and columns.} - @item{A @emph{machine} perspective, that looks at code as one line sequence of characters (or perhaps - even just plain bytes).}] + @item{A @emph{machine} perspective, that looks at code as one linear sequence of characters (or + perhaps even just plain bytes).}] Tools that serve as a human-machine interface for code often have to juggle these two perspectives. Linemaps are Resyntax's tool for doing so. They are used in various places where human concerns From 3c631f2d39bb9198d77cae1332f1e79c33a8988a Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Sat, 11 Jul 2026 14:53:52 -0700 Subject: [PATCH 13/17] Fix 1 occurrence of `define-let-to-multi-define` Applies Resyntax's own suggestion from the PR analyzer bot, which the GitHub UI couldn't apply because the suggestion range overlapped deleted lines. Co-Authored-By: Claude Fable 5 --- private/syntax-replacement.rkt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/private/syntax-replacement.rkt b/private/syntax-replacement.rkt index 3e130aea..6a05271c 100644 --- a/private/syntax-replacement.rkt +++ b/private/syntax-replacement.rkt @@ -264,13 +264,12 @@ ;; For single-line replacements, we need to account for trailing text after the replacement ;; to ensure the formatted code doesn't exceed the line length limit. + (define linemap (string-linemap original)) + (define orig-end (string-replacement-original-end replacement)) (define trailing-text-length - (let ([linemap (string-linemap original)] - [orig-end (string-replacement-original-end replacement)]) - (if (= (linemap-position-to-line linemap start) - (linemap-position-to-line linemap orig-end)) - (- (linemap-position-to-end-of-line linemap orig-end) orig-end) - 0))) + (if (= (linemap-position-to-line linemap start) (linemap-position-to-line linemap orig-end)) + (- (linemap-position-to-end-of-line linemap orig-end) orig-end) + 0)) (define allowed-width (- base-allowed-width trailing-text-length)) (define formatted-code-substring From 7003c47a9d76a8678aa807b7100b8c750392eeae Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Thu, 16 Jul 2026 00:33:35 -0700 Subject: [PATCH 14/17] Add @examples to the linemap grimoire docs Co-Authored-By: Claude Fable 5 --- grimoire/linemap.scrbl | 59 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index 42fc607c..141f463a 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -6,7 +6,15 @@ rebellion/base/comparator rebellion/base/range resyntax/grimoire/linemap - resyntax/grimoire/source)) + resyntax/grimoire/source) + scribble/example + (submod resyntax/private/scribble-evaluator-factory doc)) + + +@(define make-evaluator + (make-module-sharing-evaluator-factory + #:public (list 'resyntax/grimoire/linemap) + #:private (list 'racket/base))) @title[#:tag "linemap"]{Linemaps} @@ -58,32 +66,71 @@ cases, see the notes in @racket[with-input-from-source]. treated as line separators. In particular, Windows-style @racket["\r\n"] line endings are not understood. This never arises in practice, because Resyntax normalizes all newlines to @racket[#\newline] when reading @tech{source code} --- see @racket[with-input-from-source] for - details on that normalization and why it matters.} + details on that normalization and why it matters. + + @(examples + #:eval (make-evaluator) #:once + (eval:no-prompt (define lines (string-linemap "hello\nworld\n"))) + (linemap-position-to-line lines 0) + (code:comment "The trailing newline creates an empty third line.") + (linemap-position-to-line lines 12))} @defproc[(linemap-position-to-line [map linemap?] [position exact-nonnegative-integer?]) exact-positive-integer?]{ Returns the line number of the line containing @racket[position]. The position of a newline character is considered contained by the line that the newline terminates. Positions beyond the - end of the string do not raise an error; they are all treated as belonging to the last line.} + end of the string do not raise an error; they are all treated as belonging to the last line. + + @(examples + #:eval (make-evaluator) #:once + (eval:no-prompt (define lines (string-linemap "hello\nworld\n"))) + (code:comment "Position 5 is line 1's terminating newline, so it belongs to line 1.") + (linemap-position-to-line lines 5) + (linemap-position-to-line lines 6) + (code:comment "Positions past the end of the string belong to the last line.") + (linemap-position-to-line lines 100))} @defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-nonnegative-integer?]) exact-nonnegative-integer?]{ Returns the position of the first character of the line containing @racket[position]. If the string ends with a newline and @racket[position] is on the final, empty line after it, that - line's start position is equal to the length of the string.} + line's start position is equal to the length of the string. + + @(examples + #:eval (make-evaluator) #:once + (eval:no-prompt (define lines (string-linemap "hello\nworld\n"))) + (linemap-position-to-start-of-line lines 8) + (code:comment "The final empty line starts at the very end of the string.") + (linemap-position-to-start-of-line lines 12))} @defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-nonnegative-integer?]) exact-nonnegative-integer?]{ Returns the position just past the last character of the contents of the line containing @racket[position] --- that is, the position of the line's terminating newline, or the length of - the string if the line is the last one.} + the string if the line is the last one. + + @(examples + #:eval (make-evaluator) #:once + (eval:no-prompt (define lines (string-linemap "hello\nworld\n"))) + (code:comment "Line 1's contents end just before its newline at position 5.") + (linemap-position-to-end-of-line lines 2) + (linemap-position-to-end-of-line lines 8))} @defproc[(syntax-line-range [stx syntax?] [#:linemap map linemap?]) range?]{ Returns a closed range (with @racket[natural<=>] as its comparator) containing the line numbers of every line that @racket[stx] spans, from the line on which it begins to the line on which it ends. The @tech[#:doc '(lib "scribblings/reference/reference.scrbl")]{source location} of - @racket[stx] must refer to positions within the string that @racket[map] was built from.} + @racket[stx] must refer to positions within the string that @racket[map] was built from. + + @(examples + #:eval (make-evaluator) #:once + (eval:no-prompt + (define src "(define (f x)\n (* x 2))\n") + (define in (open-input-string src)) + (port-count-lines! in) + (define stx (read-syntax 'example in))) + (syntax-line-range stx #:linemap (string-linemap src)))} From a81ed51778e0500cd568c7795118c1119987dd5a Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Thu, 16 Jul 2026 22:12:45 -0700 Subject: [PATCH 15/17] Reject out-of-bounds positions in linemap operations Positions past the end of the string now raise contract errors instead of silently mapping to the last line, and syntax-line-range checks that the syntax object's source location fits within the linemap's string. The position equal to the string's length remains valid: it's the exclusive end position that callers legitimately pass, and it belongs to the last line. Co-Authored-By: Claude Fable 5 --- grimoire/linemap.rkt | 47 ++++++++++++++++++++++++++++++++++++++++-- grimoire/linemap.scrbl | 25 +++++++++++++++------- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/grimoire/linemap.rkt b/grimoire/linemap.rkt index c88634bc..25207fb9 100644 --- a/grimoire/linemap.rkt +++ b/grimoire/linemap.rkt @@ -65,6 +65,7 @@ (define (linemap-position-to-line map position) + (check-position-within-string 'linemap-position-to-line map position) (define line-numbers (linemap-line-numbers-by-start-position map)) (entry-value (present-value (sorted-map-entry-at-most line-numbers position)))) @@ -78,19 +79,43 @@ (string-length (linemap-line map line)))) +(define (linemap-string-length map) + (linemap-line-end-position map (vector-length (linemap-lines map)))) + + +;; The position equal to the string's length is within bounds: it's the exclusive end position of +;; the string, and it refers to the last line. +(define (check-position-within-string who map position) + (define length (linemap-string-length map)) + (when (> position length) + (raise-arguments-error who + "position is past the end of the string" + "position" position + "string length" length))) + + (define (linemap-position-to-start-of-line map position) + (check-position-within-string 'linemap-position-to-start-of-line map position) (linemap-line-start-position map (linemap-position-to-line map position))) (define (linemap-position-to-end-of-line map position) + (check-position-within-string 'linemap-position-to-end-of-line map position) (linemap-line-end-position map (linemap-position-to-line map position))) (define (syntax-line-range stx #:linemap map) (define first-line (syntax-line stx)) ;; Syntax object positions are one-indexed, unlike linemap positions. - (define last-line - (linemap-position-to-line map (+ (sub1 (syntax-position stx)) (syntax-span stx)))) + (define end-position (+ (sub1 (syntax-position stx)) (syntax-span stx))) + (when (> end-position (linemap-string-length map)) + (raise-arguments-error 'syntax-line-range + "syntax object's source location is out of bounds for the linemap" + "syntax" stx + "syntax position" (syntax-position stx) + "syntax span" (syntax-span stx) + "string length" (linemap-string-length map))) + (define last-line (linemap-position-to-line map end-position)) (unless (<= first-line last-line) (raise-arguments-error 'syntax-line-range "syntax object's last line number is before its first line number" @@ -147,6 +172,24 @@ (check-equal? (linemap-position-to-line map 3) 3) (check-equal? (linemap-position-to-line map 4) 3))) + (test-case "positions past the end of the string raise errors" + (define map (string-linemap "a\n\nb")) + (check-exn #rx"position is past the end" (λ () (linemap-position-to-line map 5))) + (check-exn #rx"position is past the end" (λ () (linemap-position-to-start-of-line map 5))) + (check-exn #rx"position is past the end" (λ () (linemap-position-to-end-of-line map 5)))) + + (test-case "syntax-line-range" + (define src "(define (f x)\n (* x 2))\n") + (define in (open-input-string src)) + (port-count-lines! in) + (define stx (read-syntax 'example in)) + (check-equal? (syntax-line-range stx #:linemap (string-linemap src)) + (closed-range 1 2 #:comparator natural<=>)) + + (test-case "out-of-bounds syntax" + (check-exn #rx"out of bounds" + (λ () (syntax-line-range stx #:linemap (string-linemap "short")))))) + (test-case (name-string linemap-line-start-position) (test-case "two-line string with ending newline" diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl index 141f463a..72171ec0 100644 --- a/grimoire/linemap.scrbl +++ b/grimoire/linemap.scrbl @@ -79,8 +79,9 @@ cases, see the notes in @racket[with-input-from-source]. @defproc[(linemap-position-to-line [map linemap?] [position exact-nonnegative-integer?]) exact-positive-integer?]{ Returns the line number of the line containing @racket[position]. The position of a newline - character is considered contained by the line that the newline terminates. Positions beyond the - end of the string do not raise an error; they are all treated as belonging to the last line. + character is considered contained by the line that the newline terminates. The position equal to + the length of the string --- the string's exclusive end position --- is allowed, and belongs to + the last line. Positions greater than that raise a contract error. @(examples #:eval (make-evaluator) #:once @@ -88,15 +89,17 @@ cases, see the notes in @racket[with-input-from-source]. (code:comment "Position 5 is line 1's terminating newline, so it belongs to line 1.") (linemap-position-to-line lines 5) (linemap-position-to-line lines 6) - (code:comment "Positions past the end of the string belong to the last line.") - (linemap-position-to-line lines 100))} + (code:comment "Positions past the end of the string are out of bounds.") + (eval:error (linemap-position-to-line lines 100)))} @defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-nonnegative-integer?]) exact-nonnegative-integer?]{ Returns the position of the first character of the line containing @racket[position]. If the string ends with a newline and @racket[position] is on the final, empty line after it, that - line's start position is equal to the length of the string. + line's start position is equal to the length of the string. Like + @racket[linemap-position-to-line], positions greater than the length of the string raise a + contract error. @(examples #:eval (make-evaluator) #:once @@ -110,7 +113,8 @@ cases, see the notes in @racket[with-input-from-source]. exact-nonnegative-integer?]{ Returns the position just past the last character of the contents of the line containing @racket[position] --- that is, the position of the line's terminating newline, or the length of - the string if the line is the last one. + the string if the line is the last one. Like @racket[linemap-position-to-line], positions greater + than the length of the string raise a contract error. @(examples #:eval (make-evaluator) #:once @@ -124,7 +128,10 @@ cases, see the notes in @racket[with-input-from-source]. Returns a closed range (with @racket[natural<=>] as its comparator) containing the line numbers of every line that @racket[stx] spans, from the line on which it begins to the line on which it ends. The @tech[#:doc '(lib "scribblings/reference/reference.scrbl")]{source location} of - @racket[stx] must refer to positions within the string that @racket[map] was built from. + @racket[stx] must refer to positions within the string that @racket[map] was built from. If the + source location extends past the end of that string, a contract error is raised. This is only + partial protection against accidentally pairing a syntax object with a linemap built from some + other, unrelated string, but it's better than nothing. @(examples #:eval (make-evaluator) #:once @@ -133,4 +140,6 @@ cases, see the notes in @racket[with-input-from-source]. (define in (open-input-string src)) (port-count-lines! in) (define stx (read-syntax 'example in))) - (syntax-line-range stx #:linemap (string-linemap src)))} + (syntax-line-range stx #:linemap (string-linemap src)) + (code:comment "Pairing a syntax object with the wrong linemap is caught (sometimes).") + (eval:error (syntax-line-range stx #:linemap (string-linemap "some other string"))))} From 54ae6ba9c8ec5650d8eddfa5d9888f557b4f3282 Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Thu, 16 Jul 2026 22:40:42 -0700 Subject: [PATCH 16/17] Editing pass over newline normalization docs --- grimoire/source.scrbl | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/grimoire/source.scrbl b/grimoire/source.scrbl index 0f413516..4598aeab 100644 --- a/grimoire/source.scrbl +++ b/grimoire/source.scrbl @@ -122,18 +122,19 @@ stack of dependent changes to commit in series without actually mutating the fil @racket[reencode-input-port]: Windows-style @racket["\r\n"] sequences (and other newline conventions) are converted into single @racket[#\newline] characters. Every operation that reads a source goes through this normalization, including @racket[source->string] and - @racket[source-read-syntax]. Normalizing consistently is load-bearing: as described in - @secref["linecol" #:doc '(lib "scribblings/reference/reference.scrbl")], Racket performs this - same conversion whenever it counts line and column numbers, treating a @racket["\r\n"] sequence - as a @emph{single} position --- and line counting must be enabled for syntax objects to receive - line numbers in their source locations. Without normalization, then, the source locations of - syntax objects read from a source would disagree with the character indices of that source's - text. Resyntax relies on the assumption that a syntax object's position and span identify exactly - the range of characters it was read from. Analyzing code with Windows-style newlines used to - violate that assumption and break Resyntax in hard-to-diagnose ways. - - String sources and modified sources also apply this normalization eagerly, when the source value - is constructed, so the port-level conversion only has a visible effect for file sources.} + @racket[source-read-syntax]. This normalization mirrors the normalization Racket performs when + reading from ports with line counting enabled, as described in + @secref["linecol" #:doc '(lib "scribblings/reference/reference.scrbl")]. + + Newline normalization avoids various problems in Resyntax. Primarily, it ensures that the character + positions of source text strings match with the position numbers in syntax object source locations. + Without normalization, replacements could be misapplied in files if they contain Windows-style + newlines --- an especially difficult problem to diagnose given that newline characters are invisible + and their conventions platform-specific. See @hyperlink["TODO(claude)"]{this motivating bug report} + for an example. + + Note that string sources and modified sources apply this normalization eagerly, when the source value + is constructed, so the port-level conversion only has a visible effect for unmodified file sources.} @section{Parsing, Expanding, and Compiling Sources} From 1e05a6f2c52eb37b54e22e0cba1d5b6ffa4c013e Mon Sep 17 00:00:00 2001 From: Jack Firth Date: Thu, 16 Jul 2026 22:42:16 -0700 Subject: [PATCH 17/17] Link issue #272 from newline normalization docs Co-Authored-By: Claude Fable 5 --- grimoire/source.scrbl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grimoire/source.scrbl b/grimoire/source.scrbl index 4598aeab..e9fbb644 100644 --- a/grimoire/source.scrbl +++ b/grimoire/source.scrbl @@ -130,7 +130,7 @@ stack of dependent changes to commit in series without actually mutating the fil positions of source text strings match with the position numbers in syntax object source locations. Without normalization, replacements could be misapplied in files if they contain Windows-style newlines --- an especially difficult problem to diagnose given that newline characters are invisible - and their conventions platform-specific. See @hyperlink["TODO(claude)"]{this motivating bug report} + and their conventions platform-specific. See @hyperlink["https://github.com/jackfirth/resyntax/issues/272"]{this motivating bug report} for an example. Note that string sources and modified sources apply this normalization eagerly, when the source value