diff --git a/grimoire.scrbl b/grimoire.scrbl index 02bf46d0..5f058ee9 100644 --- a/grimoire.scrbl +++ b/grimoire.scrbl @@ -18,3 +18,4 @@ programmatically on anything found here. @include-section[(lib "resyntax/grimoire/syntax-property-bundle.scrbl")] @include-section[(lib "resyntax/grimoire/expansion-analyzers.scrbl")] @include-section[(lib "resyntax/grimoire/string-replacement.scrbl")] +@include-section[(lib "resyntax/grimoire/linemap.scrbl")] diff --git a/grimoire/linemap.rkt b/grimoire/linemap.rkt new file mode 100644 index 00000000..25207fb9 --- /dev/null +++ b/grimoire/linemap.rkt @@ -0,0 +1,270 @@ +#lang racket/base + + +(require racket/contract/base) + + +(provide + (contract-out + [string-linemap (-> string? linemap?)] + [linemap? (-> any/c boolean?)] + [linemap-position-to-line (-> linemap? exact-nonnegative-integer? exact-positive-integer?)] + [linemap-position-to-start-of-line + (-> linemap? exact-nonnegative-integer? exact-nonnegative-integer?)] + [linemap-position-to-end-of-line + (-> linemap? exact-nonnegative-integer? exact-nonnegative-integer?)] + [syntax-line-range (-> syntax? #:linemap linemap? range?)])) + + +(require rebellion/base/comparator + rebellion/base/option + rebellion/base/range + rebellion/collection/entry + rebellion/collection/sorted-map + rebellion/collection/vector/builder) + + +(module+ test + (require rackunit + rebellion/private/static-name)) + + +;@---------------------------------------------------------------------------------------------------- + + +(struct linemap (lines line-numbers-by-start-position start-positions-by-line-number) #:transparent) + + +(define (string-linemap str) + (define lines (make-vector-builder)) + (define line-numbers-by-start-position (make-sorted-map-builder natural<=>)) + (define start-positions-by-line-number (make-vector-builder)) + (define char-count (string-length str)) + (let loop ([line-number 1] [line-start-index 0] [index 0]) + (cond + [(= index char-count) + (define last-line (string->immutable-string (substring str line-start-index index))) + (vector-builder-add lines last-line) + (sorted-map-builder-put line-numbers-by-start-position line-start-index line-number) + (vector-builder-add start-positions-by-line-number line-start-index)] + [(equal? (string-ref str index) #\newline) + (define next-line (string->immutable-string (substring str line-start-index index))) + (vector-builder-add lines next-line) + (sorted-map-builder-put line-numbers-by-start-position line-start-index line-number) + (vector-builder-add start-positions-by-line-number line-start-index) + (define next-index (add1 index)) + (loop (add1 line-number) next-index next-index)] + [else (loop line-number line-start-index (add1 index))])) + (linemap (build-vector lines) + (build-sorted-map line-numbers-by-start-position) + (build-vector start-positions-by-line-number))) + + +(define (linemap-line map line) + (vector-ref (linemap-lines map) (sub1 line))) + + +(define (linemap-position-to-line map position) + (check-position-within-string 'linemap-position-to-line map position) + (define line-numbers (linemap-line-numbers-by-start-position map)) + (entry-value (present-value (sorted-map-entry-at-most line-numbers position)))) + + +(define (linemap-line-start-position map line) + (vector-ref (linemap-start-positions-by-line-number map) (sub1 line))) + + +(define (linemap-line-end-position map line) + (+ (linemap-line-start-position map line) + (string-length (linemap-line map line)))) + + +(define (linemap-string-length map) + (linemap-line-end-position map (vector-length (linemap-lines map)))) + + +;; The position equal to the string's length is within bounds: it's the exclusive end position of +;; the string, and it refers to the last line. +(define (check-position-within-string who map position) + (define length (linemap-string-length map)) + (when (> position length) + (raise-arguments-error who + "position is past the end of the string" + "position" position + "string length" length))) + + +(define (linemap-position-to-start-of-line map position) + (check-position-within-string 'linemap-position-to-start-of-line map position) + (linemap-line-start-position map (linemap-position-to-line map position))) + + +(define (linemap-position-to-end-of-line map position) + (check-position-within-string 'linemap-position-to-end-of-line map position) + (linemap-line-end-position map (linemap-position-to-line map position))) + + +(define (syntax-line-range stx #:linemap map) + (define first-line (syntax-line stx)) + ;; Syntax object positions are one-indexed, unlike linemap positions. + (define end-position (+ (sub1 (syntax-position stx)) (syntax-span stx))) + (when (> end-position (linemap-string-length map)) + (raise-arguments-error 'syntax-line-range + "syntax object's source location is out of bounds for the linemap" + "syntax" stx + "syntax position" (syntax-position stx) + "syntax span" (syntax-span stx) + "string length" (linemap-string-length map))) + (define last-line (linemap-position-to-line map end-position)) + (unless (<= first-line last-line) + (raise-arguments-error 'syntax-line-range + "syntax object's last line number is before its first line number" + "syntax" stx + "first line" first-line + "last line" last-line + "position" (syntax-position stx) + "span" (syntax-span stx))) + (closed-range first-line last-line #:comparator natural<=>)) + + +(module+ test + + (test-case (name-string string-linemap) + + (define (nat-map . args) + (apply sorted-map #:key-comparator natural<=> args)) + + (check-equal? (string-linemap "") (linemap #("") (nat-map 0 1) #(0))) + (check-equal? (string-linemap "a") (linemap #("a") (nat-map 0 1) #(0))) + (check-equal? (string-linemap "λ") (linemap #("λ") (nat-map 0 1) #(0))) + (check-equal? (string-linemap "a\n") (linemap #("a" "") (nat-map 0 1 2 2) #(0 2))) + (check-equal? (string-linemap "λ\n") (linemap #("λ" "") (nat-map 0 1 2 2) #(0 2))) + (check-equal? (string-linemap "aaa\n") (linemap #("aaa" "") (nat-map 0 1 4 2) #(0 4))) + (check-equal? (string-linemap "aaa\nbbb") (linemap #("aaa" "bbb") (nat-map 0 1 4 2) #(0 4))) + (check-equal? (string-linemap "aaa\nbbb\n") + (linemap #("aaa" "bbb" "") (nat-map 0 1 4 2 8 3) #(0 4 8))) + (check-equal? (string-linemap "a\n\n\nb") + (linemap #("a" "" "" "b") (nat-map 0 1 2 2 3 3 4 4) #(0 2 3 4)))) + + (test-case (name-string linemap-position-to-line) + + (test-case "two-line string with ending newline" + (define map (string-linemap "hello\nworld\n")) + (check-equal? (linemap-position-to-line map 0) 1) + (check-equal? (linemap-position-to-line map 1) 1) + (check-equal? (linemap-position-to-line map 2) 1) + (check-equal? (linemap-position-to-line map 3) 1) + (check-equal? (linemap-position-to-line map 4) 1) + (check-equal? (linemap-position-to-line map 5) 1) + (check-equal? (linemap-position-to-line map 6) 2) + (check-equal? (linemap-position-to-line map 7) 2) + (check-equal? (linemap-position-to-line map 8) 2) + (check-equal? (linemap-position-to-line map 9) 2) + (check-equal? (linemap-position-to-line map 10) 2) + (check-equal? (linemap-position-to-line map 11) 2) + (check-equal? (linemap-position-to-line map 12) 3)) + + (test-case "multiple blank lines" + (define map (string-linemap "a\n\nb")) + (check-equal? (linemap-position-to-line map 0) 1) + (check-equal? (linemap-position-to-line map 1) 1) + (check-equal? (linemap-position-to-line map 2) 2) + (check-equal? (linemap-position-to-line map 3) 3) + (check-equal? (linemap-position-to-line map 4) 3))) + + (test-case "positions past the end of the string raise errors" + (define map (string-linemap "a\n\nb")) + (check-exn #rx"position is past the end" (λ () (linemap-position-to-line map 5))) + (check-exn #rx"position is past the end" (λ () (linemap-position-to-start-of-line map 5))) + (check-exn #rx"position is past the end" (λ () (linemap-position-to-end-of-line map 5)))) + + (test-case "syntax-line-range" + (define src "(define (f x)\n (* x 2))\n") + (define in (open-input-string src)) + (port-count-lines! in) + (define stx (read-syntax 'example in)) + (check-equal? (syntax-line-range stx #:linemap (string-linemap src)) + (closed-range 1 2 #:comparator natural<=>)) + + (test-case "out-of-bounds syntax" + (check-exn #rx"out of bounds" + (λ () (syntax-line-range stx #:linemap (string-linemap "short")))))) + + (test-case (name-string linemap-line-start-position) + + (test-case "two-line string with ending newline" + (define map (string-linemap "hello\nworld\n")) + (check-equal? (linemap-line-start-position map 1) 0) + (check-equal? (linemap-line-start-position map 2) 6) + (check-equal? (linemap-line-start-position map 3) 12)) + + (test-case "multiple blank lines" + (define map (string-linemap "a\n\nb")) + (check-equal? (linemap-line-start-position map 1) 0) + (check-equal? (linemap-line-start-position map 2) 2) + (check-equal? (linemap-line-start-position map 3) 3))) + + (test-case (name-string linemap-line-end-position) + + (test-case "two-line string with ending newline" + (define map (string-linemap "hello\nworld\n")) + (check-equal? (linemap-line-end-position map 1) 5) + (check-equal? (linemap-line-end-position map 2) 11)) + + (test-case "multiple blank lines" + (define map (string-linemap "a\n\nb")) + (check-equal? (linemap-line-end-position map 1) 1) + (check-equal? (linemap-line-end-position map 2) 2) + (check-equal? (linemap-line-end-position map 3) 4))) + + (test-case (name-string linemap-position-to-start-of-line) + + (test-case "two-line string with ending newline" + (define map (string-linemap "hello\nworld\n")) + (check-equal? (linemap-position-to-start-of-line map 0) 0) + (check-equal? (linemap-position-to-start-of-line map 1) 0) + (check-equal? (linemap-position-to-start-of-line map 2) 0) + (check-equal? (linemap-position-to-start-of-line map 3) 0) + (check-equal? (linemap-position-to-start-of-line map 4) 0) + (check-equal? (linemap-position-to-start-of-line map 5) 0) + (check-equal? (linemap-position-to-start-of-line map 6) 6) + (check-equal? (linemap-position-to-start-of-line map 7) 6) + (check-equal? (linemap-position-to-start-of-line map 8) 6) + (check-equal? (linemap-position-to-start-of-line map 9) 6) + (check-equal? (linemap-position-to-start-of-line map 10) 6) + (check-equal? (linemap-position-to-start-of-line map 11) 6) + (check-equal? (linemap-position-to-start-of-line map 12) 12)) + + (test-case "multiple blank lines" + (define map (string-linemap "a\n\nb")) + (check-equal? (linemap-position-to-start-of-line map 0) 0) + (check-equal? (linemap-position-to-start-of-line map 1) 0) + (check-equal? (linemap-position-to-start-of-line map 2) 2) + (check-equal? (linemap-position-to-start-of-line map 3) 3) + (check-equal? (linemap-position-to-start-of-line map 4) 3))) + + (test-case (name-string linemap-position-to-end-of-line) + + (test-case "two-line string with ending newline" + (define map (string-linemap "hello\nworld\n")) + (check-equal? (linemap-position-to-end-of-line map 0) 5) + (check-equal? (linemap-position-to-end-of-line map 1) 5) + (check-equal? (linemap-position-to-end-of-line map 2) 5) + (check-equal? (linemap-position-to-end-of-line map 3) 5) + (check-equal? (linemap-position-to-end-of-line map 4) 5) + (check-equal? (linemap-position-to-end-of-line map 5) 5) + (check-equal? (linemap-position-to-end-of-line map 6) 11) + (check-equal? (linemap-position-to-end-of-line map 7) 11) + (check-equal? (linemap-position-to-end-of-line map 8) 11) + (check-equal? (linemap-position-to-end-of-line map 9) 11) + (check-equal? (linemap-position-to-end-of-line map 10) 11) + (check-equal? (linemap-position-to-end-of-line map 11) 11) + (check-equal? (linemap-position-to-end-of-line map 12) 12)) + + (test-case "multiple blank lines" + (define map (string-linemap "a\n\nb")) + (check-equal? (linemap-position-to-end-of-line map 0) 1) + (check-equal? (linemap-position-to-end-of-line map 1) 1) + (check-equal? (linemap-position-to-end-of-line map 2) 2) + (check-equal? (linemap-position-to-end-of-line map 3) 4) + (check-equal? (linemap-position-to-end-of-line map 4) 4)))) diff --git a/grimoire/linemap.scrbl b/grimoire/linemap.scrbl new file mode 100644 index 00000000..72171ec0 --- /dev/null +++ b/grimoire/linemap.scrbl @@ -0,0 +1,145 @@ +#lang scribble/manual + + +@(require (for-label racket/base + racket/contract/base + rebellion/base/comparator + rebellion/base/range + resyntax/grimoire/linemap + resyntax/grimoire/source) + scribble/example + (submod resyntax/private/scribble-evaluator-factory doc)) + + +@(define make-evaluator + (make-module-sharing-evaluator-factory + #:public (list 'resyntax/grimoire/linemap) + #:private (list 'racket/base))) + + +@title[#:tag "linemap"]{Linemaps} +@defmodule[resyntax/grimoire/linemap] + +A @deftech{linemap} is a precomputed index of a string's line structure that supports converting +between character positions and line numbers. Source code is frequently viewed from one of two +perspectives: + +@itemlist[ + @item{A @emph{human} perspective, either reader or writer, who looks at code as a 2D grid composed of + lines and columns.} + + @item{A @emph{machine} perspective, that looks at code as one linear sequence of characters (or + perhaps even just plain bytes).}] + +Tools that serve as a human-machine interface for code often have to juggle these two perspectives. +Linemaps are Resyntax's tool for doing so. They are used in various places where human concerns +related to viewing and describing source code and source edits come up, such as displaying or +consuming line-based diffs in the command-line interface. See @secref["cli"] for further details on +that matter. + +@bold{Positions in a linemap are zero-based, but line numbers are one-based.} Positions +are character indices into the string. This follows the same convention as Racket's string operations +such as @racket[string-ref], as well as Resyntax's conventions for @tech{string replacements}. Line +numbers, however, follow the conventions outlined in +@secref["linecol" #:doc '(lib "scribblings/reference/reference.scrbl")]: source files begin at line +@racket[1]. This matches @racket[syntax-line] and the conventions of code editors --- line numbers +are almost exclusively useful in user interfaces, where one-based numbering is expected. + +@bold{Beware that @racket[syntax-position] and file port positions are @emph{one-based}, unlike + linemap positions.} The @racket[syntax-line-range] operation performs that conversion itself, but +positions obtained from syntax objects by other means must be converted before use with a linemap. + +The lines of a string are the segments separated by newline characters. The terminating newline is +not part of a line's contents, but positions of newline characters belong to the lines they +terminate. A string that ends with a newline has a final empty line after it, and the empty string +consists of a single empty line. Note that there are multiple distinct byte sequences that Racket +treats as a newline when reading source code --- for further details on how Resyntax handles these +cases, see the notes in @racket[with-input-from-source]. + + +@defproc[(linemap? [v any/c]) boolean?]{ + A predicate that recognizes @tech{linemaps}.} + + +@defproc[(string-linemap [str string?]) linemap?]{ + Constructs a @tech{linemap} of the lines in @racket[str]. Only @racket[#\newline] characters are + treated as line separators. In particular, Windows-style @racket["\r\n"] line endings are not + understood. This never arises in practice, because Resyntax normalizes all newlines to + @racket[#\newline] when reading @tech{source code} --- see @racket[with-input-from-source] for + details on that normalization and why it matters. + + @(examples + #:eval (make-evaluator) #:once + (eval:no-prompt (define lines (string-linemap "hello\nworld\n"))) + (linemap-position-to-line lines 0) + (code:comment "The trailing newline creates an empty third line.") + (linemap-position-to-line lines 12))} + + +@defproc[(linemap-position-to-line [map linemap?] [position exact-nonnegative-integer?]) + exact-positive-integer?]{ + Returns the line number of the line containing @racket[position]. The position of a newline + character is considered contained by the line that the newline terminates. The position equal to + the length of the string --- the string's exclusive end position --- is allowed, and belongs to + the last line. Positions greater than that raise a contract error. + + @(examples + #:eval (make-evaluator) #:once + (eval:no-prompt (define lines (string-linemap "hello\nworld\n"))) + (code:comment "Position 5 is line 1's terminating newline, so it belongs to line 1.") + (linemap-position-to-line lines 5) + (linemap-position-to-line lines 6) + (code:comment "Positions past the end of the string are out of bounds.") + (eval:error (linemap-position-to-line lines 100)))} + + +@defproc[(linemap-position-to-start-of-line [map linemap?] [position exact-nonnegative-integer?]) + exact-nonnegative-integer?]{ + Returns the position of the first character of the line containing @racket[position]. If the + string ends with a newline and @racket[position] is on the final, empty line after it, that + line's start position is equal to the length of the string. Like + @racket[linemap-position-to-line], positions greater than the length of the string raise a + contract error. + + @(examples + #:eval (make-evaluator) #:once + (eval:no-prompt (define lines (string-linemap "hello\nworld\n"))) + (linemap-position-to-start-of-line lines 8) + (code:comment "The final empty line starts at the very end of the string.") + (linemap-position-to-start-of-line lines 12))} + + +@defproc[(linemap-position-to-end-of-line [map linemap?] [position exact-nonnegative-integer?]) + exact-nonnegative-integer?]{ + Returns the position just past the last character of the contents of the line containing + @racket[position] --- that is, the position of the line's terminating newline, or the length of + the string if the line is the last one. Like @racket[linemap-position-to-line], positions greater + than the length of the string raise a contract error. + + @(examples + #:eval (make-evaluator) #:once + (eval:no-prompt (define lines (string-linemap "hello\nworld\n"))) + (code:comment "Line 1's contents end just before its newline at position 5.") + (linemap-position-to-end-of-line lines 2) + (linemap-position-to-end-of-line lines 8))} + + +@defproc[(syntax-line-range [stx syntax?] [#:linemap map linemap?]) range?]{ + Returns a closed range (with @racket[natural<=>] as its comparator) containing the line numbers + of every line that @racket[stx] spans, from the line on which it begins to the line on which it + ends. The @tech[#:doc '(lib "scribblings/reference/reference.scrbl")]{source location} of + @racket[stx] must refer to positions within the string that @racket[map] was built from. If the + source location extends past the end of that string, a contract error is raised. This is only + partial protection against accidentally pairing a syntax object with a linemap built from some + other, unrelated string, but it's better than nothing. + + @(examples + #:eval (make-evaluator) #:once + (eval:no-prompt + (define src "(define (f x)\n (* x 2))\n") + (define in (open-input-string src)) + (port-count-lines! in) + (define stx (read-syntax 'example in))) + (syntax-line-range stx #:linemap (string-linemap src)) + (code:comment "Pairing a syntax object with the wrong linemap is caught (sometimes).") + (eval:error (syntax-line-range stx #:linemap (string-linemap "some other string"))))} diff --git a/grimoire/source.rkt b/grimoire/source.rkt index 8350f50a..a7bc1502 100644 --- a/grimoire/source.rkt +++ b/grimoire/source.rkt @@ -53,6 +53,16 @@ ;@---------------------------------------------------------------------------------------------------- +;; All source text flows through this port wrapper, which decodes it as UTF-8 and converts +;; newline sequences like \r\n into single \n characters. +(define (reencoded-source-input-port in) + (reencode-input-port in "UTF-8" #false #false (object-name in) #true)) + + +(define (string-normalize-newlines str) + (port->string (reencoded-source-input-port (open-input-string str)))) + + (struct source () #:transparent) @@ -66,12 +76,13 @@ (struct string-source unmodified-source (contents) #:transparent - #:guard (λ (contents _) (string->immutable-string contents))) + #:guard (λ (contents _) (string->immutable-string (string-normalize-newlines contents)))) (struct modified-source source (original contents) #:transparent - #:guard (λ (original contents _) (values original (string->immutable-string contents)))) + #:guard (λ (original contents _) + (values original (string->immutable-string (string-normalize-newlines contents))))) (define (source-name src) @@ -83,8 +94,7 @@ (define (with-input-from-source code proc) (define (call-proc-with-reencoded-input in) - (define reencoded-in (reencode-input-port in "UTF-8" #false #false (object-name in) #true)) - (parameterize ([current-input-port reencoded-in]) + (parameterize ([current-input-port (reencoded-source-input-port in)]) (proc))) (match code @@ -137,6 +147,20 @@ (module+ test + (test-case "string and modified sources normalize newlines upon construction" + (define cr (string #\return)) + (define lf (string #\newline)) + (define nel (string (integer->char #x85))) + (define ls (string (integer->char #x2028))) + (define normalized (string-source (string-append "a" lf "b" lf "c"))) + (check-equal? (string-source (string-append "a" cr lf "b" cr lf "c")) normalized) + (check-equal? (string-source (string-append "a" cr "b" cr "c")) normalized) + (check-equal? (string-source (string-append "a" nel "b" cr nel "c")) normalized) + (check-equal? (string-source (string-append "a" ls "b" ls "c")) normalized) + (define base (string-source "base")) + (check-equal? (modified-source base (string-append "a" cr lf "b")) + (modified-source base (string-append "a" lf "b")))) + (test-case "source-read-language" (check-equal? (source-read-language (string-source "#lang racket")) 'racket) (check-equal? (source-read-language (string-source "#lang at-exp racket")) 'at-exp) diff --git a/grimoire/source.scrbl b/grimoire/source.scrbl index 6984a381..e9fbb644 100644 --- a/grimoire/source.scrbl +++ b/grimoire/source.scrbl @@ -4,6 +4,7 @@ @(require (for-label racket/base racket/contract/base racket/path + racket/port rebellion/base/immutable-string rebellion/collection/range-set resyntax/grimoire/source @@ -63,7 +64,10 @@ stack of dependent changes to commit in series without actually mutating the fil @defproc[(string-source [contents string?]) string-source?]{ - Constructs a source string containing @racket[contents] directly.} + Constructs a source string containing @racket[contents] directly. The newlines of + @racket[contents] are normalized at construction time, in the same manner described in + @racket[with-input-from-source]. Normalizing eagerly ensures that two string sources denoting the + same text are @racket[equal?] even if they were constructed with different newline conventions.} @defproc[(modified-source? [v any/c]) boolean?]{ @@ -76,7 +80,8 @@ stack of dependent changes to commit in series without actually mutating the fil Constructs a modified source that replaces the contents of @racket[original] with @racket[new-contents]. This represents a whole-file replacement --- the @emph{complete} contents of @racket[original] are @emph{entirely} swapped out with @racket[new-contents]. Modified sources cannot - represent partial edits on their own.} + represent partial edits on their own. Like @racket[string-source], the newlines of + @racket[new-contents] are normalized at construction time.} @defproc[(source-name [code source?]) (or/c path? symbol?)]{ @@ -104,13 +109,32 @@ stack of dependent changes to commit in series without actually mutating the fil @defproc[(source->string [code source?]) immutable-string?]{ Returns the full text of @racket[code], reading it from the filesystem if necessary. For @racket[modified-source?] values, this returns the new, updated text rather than the original - unmodified text.} + unmodified text. The returned text has its newlines normalized, as described in + @racket[with-input-from-source].} @defproc[(with-input-from-source [code source?] [proc (-> any)]) any]{ Calls @racket[proc] with @racket[current-input-port] set to a freshly opened input port reading the contents of @racket[code]. For unmodified file sources, this opens a file port. For modified - sources and string sources, this opens a string port without interacting with the filesystem.} + sources and string sources, this opens a string port without interacting with the filesystem. + + The opened port decodes the source as UTF-8 and @bold{normalizes newlines}, as in + @racket[reencode-input-port]: Windows-style @racket["\r\n"] sequences (and other newline + conventions) are converted into single @racket[#\newline] characters. Every operation that reads + a source goes through this normalization, including @racket[source->string] and + @racket[source-read-syntax]. This normalization mirrors the normalization Racket performs when + reading from ports with line counting enabled, as described in + @secref["linecol" #:doc '(lib "scribblings/reference/reference.scrbl")]. + + Newline normalization avoids various problems in Resyntax. Primarily, it ensures that the character + positions of source text strings match with the position numbers in syntax object source locations. + Without normalization, replacements could be misapplied in files if they contain Windows-style + newlines --- an especially difficult problem to diagnose given that newline characters are invisible + and their conventions platform-specific. See @hyperlink["https://github.com/jackfirth/resyntax/issues/272"]{this motivating bug report} + for an example. + + Note that string sources and modified sources apply this normalization eagerly, when the source value + is constructed, so the port-level conversion only has a visible effect for unmodified file sources.} @section{Parsing, Expanding, and Compiling Sources} diff --git a/private/analysis.rkt b/private/analysis.rkt index e723a5be..9e214090 100644 --- a/private/analysis.rkt +++ b/private/analysis.rkt @@ -42,7 +42,7 @@ resyntax/default-recommendations/analyzers/ignored-result-values resyntax/default-recommendations/analyzers/variable-mutability resyntax/private/analyzer - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/private/logger resyntax/grimoire/source resyntax/private/string-indent diff --git a/private/line-replacement.rkt b/private/line-replacement.rkt index 639f74f2..37d8df23 100644 --- a/private/line-replacement.rkt +++ b/private/line-replacement.rkt @@ -27,7 +27,7 @@ rebellion/streaming/reducer rebellion/streaming/transducer rebellion/type/record - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/grimoire/string-replacement) @@ -77,16 +77,13 @@ (define new-lmap (string-linemap new-string)) (define start-line - (linemap-position-to-line orig-lmap (add1 (string-replacement-start replacement)))) + (linemap-position-to-line orig-lmap (string-replacement-start replacement))) (define start-pos - (sub1 - (linemap-position-to-start-of-line orig-lmap (add1 (string-replacement-start replacement))))) + (linemap-position-to-start-of-line orig-lmap (string-replacement-start replacement))) (define original-end-pos - (sub1 - (linemap-position-to-end-of-line orig-lmap - (add1 (string-replacement-original-end replacement))))) + (linemap-position-to-end-of-line orig-lmap (string-replacement-original-end replacement))) (define new-end-pos - (sub1 (linemap-position-to-end-of-line new-lmap (add1 (string-replacement-new-end replacement))))) + (linemap-position-to-end-of-line new-lmap (string-replacement-new-end replacement))) (define original-substr (substring original-string start-pos original-end-pos)) (define new-substr (substring new-string start-pos new-end-pos)) diff --git a/private/linemap.rkt b/private/linemap.rkt deleted file mode 100644 index f499ce64..00000000 --- a/private/linemap.rkt +++ /dev/null @@ -1,234 +0,0 @@ -#lang racket/base - - -(require racket/contract/base) - - -(provide - (contract-out - [string-linemap (-> string? linemap?)] - [linemap? (-> any/c boolean?)] - [linemap-lines (-> linemap? (vectorof (and/c string? immutable?) #:immutable #true #:flat? #true))] - [linemap-position-to-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [linemap-line-start-position (-> linemap? exact-positive-integer? exact-positive-integer?)] - [linemap-position-to-start-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [linemap-position-to-end-of-line (-> linemap? exact-positive-integer? exact-positive-integer?)] - [syntax-start-line-position (-> syntax? #:linemap linemap? exact-positive-integer?)] - [syntax-end-line-position (-> syntax? #:linemap linemap? exact-positive-integer?)] - [syntax-line-range (-> syntax? #:linemap linemap? range?)])) - - -(require rebellion/base/comparator - rebellion/base/option - rebellion/base/range - rebellion/collection/entry - rebellion/collection/sorted-map - rebellion/collection/vector/builder) - - -(module+ test - (require rackunit - rebellion/private/static-name)) - - -;@---------------------------------------------------------------------------------------------------- - - -(struct linemap (lines line-numbers-by-start-position start-positions-by-line-number) #:transparent) - - -(define (string-linemap str) - (define lines (make-vector-builder)) - (define line-numbers-by-start-position (make-sorted-map-builder natural<=>)) - (define start-positions-by-line-number (make-vector-builder)) - (define char-count (string-length str)) - (let loop ([line-number 1] [line-start-index 0] [index 0]) - (cond - [(= index char-count) - (define last-line (substring str line-start-index index)) - (vector-builder-add lines last-line) - (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number) - (vector-builder-add start-positions-by-line-number (add1 line-start-index))] - [(equal? (string-ref str index) #\newline) - (define next-line (substring str line-start-index index)) - (vector-builder-add lines next-line) - (sorted-map-builder-put line-numbers-by-start-position (add1 line-start-index) line-number) - (vector-builder-add start-positions-by-line-number (add1 line-start-index)) - (define next-index (add1 index)) - (loop (add1 line-number) next-index next-index)] - [else (loop line-number line-start-index (add1 index))])) - (linemap (build-vector lines) - (build-sorted-map line-numbers-by-start-position) - (build-vector start-positions-by-line-number))) - - -(define (linemap-line map line) - (vector-ref (linemap-lines map) (sub1 line))) - - -(define (linemap-position-to-line map position) - (define line-numbers (linemap-line-numbers-by-start-position map)) - (entry-value (present-value (sorted-map-entry-at-most line-numbers position)))) - - -(define (linemap-line-start-position map line) - (vector-ref (linemap-start-positions-by-line-number map) (sub1 line))) - - -(define (linemap-line-end-position map line) - (+ (linemap-line-start-position map line) - (string-length (linemap-line map line)))) - - -(define (linemap-position-to-start-of-line map position) - (linemap-line-start-position map (linemap-position-to-line map position))) - - -(define (linemap-position-to-end-of-line map position) - (linemap-line-end-position map (linemap-position-to-line map position))) - - -(define (syntax-start-line-position stx #:linemap map) - (linemap-position-to-start-of-line map (syntax-position stx))) - - -(define (syntax-end-line-position stx #:linemap map) - (linemap-position-to-end-of-line map (+ (syntax-position stx) (syntax-span stx)))) - - -(define (syntax-line-range stx #:linemap map) - (define first-line (syntax-line stx)) - (define last-line (linemap-position-to-line map (+ (syntax-position stx) (syntax-span stx)))) - (unless (<= first-line last-line) - (raise-arguments-error 'syntax-line-range - "syntax object's last line number is before its first line number" - "syntax" stx - "first line" first-line - "last line" last-line - "position" (syntax-position stx) - "span" (syntax-span stx))) - (closed-range first-line last-line #:comparator natural<=>)) - - -(module+ test - - (test-case (name-string string-linemap) - - (define (nat-map . args) - (apply sorted-map #:key-comparator natural<=> args)) - - (check-equal? (string-linemap "") (linemap #("") (nat-map 1 1) #(1))) - (check-equal? (string-linemap "a") (linemap #("a") (nat-map 1 1) #(1))) - (check-equal? (string-linemap "λ") (linemap #("λ") (nat-map 1 1) #(1))) - (check-equal? (string-linemap "a\n") (linemap #("a" "") (nat-map 1 1 3 2) #(1 3))) - (check-equal? (string-linemap "λ\n") (linemap #("λ" "") (nat-map 1 1 3 2) #(1 3))) - (check-equal? (string-linemap "aaa\n") (linemap #("aaa" "") (nat-map 1 1 5 2) #(1 5))) - (check-equal? (string-linemap "aaa\nbbb") (linemap #("aaa" "bbb") (nat-map 1 1 5 2) #(1 5))) - (check-equal? (string-linemap "aaa\nbbb\n") - (linemap #("aaa" "bbb" "") (nat-map 1 1 5 2 9 3) #(1 5 9))) - (check-equal? (string-linemap "a\n\n\nb") - (linemap #("a" "" "" "b") (nat-map 1 1 3 2 4 3 5 4) #(1 3 4 5)))) - - (test-case (name-string linemap-position-to-line) - - (test-case "two-line string with ending newline" - (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-position-to-line map 1) 1) - (check-equal? (linemap-position-to-line map 2) 1) - (check-equal? (linemap-position-to-line map 3) 1) - (check-equal? (linemap-position-to-line map 4) 1) - (check-equal? (linemap-position-to-line map 5) 1) - (check-equal? (linemap-position-to-line map 6) 1) - (check-equal? (linemap-position-to-line map 7) 2) - (check-equal? (linemap-position-to-line map 8) 2) - (check-equal? (linemap-position-to-line map 9) 2) - (check-equal? (linemap-position-to-line map 10) 2) - (check-equal? (linemap-position-to-line map 11) 2) - (check-equal? (linemap-position-to-line map 12) 2) - (check-equal? (linemap-position-to-line map 13) 3)) - - (test-case "multiple blank lines" - (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-position-to-line map 1) 1) - (check-equal? (linemap-position-to-line map 2) 1) - (check-equal? (linemap-position-to-line map 3) 2) - (check-equal? (linemap-position-to-line map 4) 3) - (check-equal? (linemap-position-to-line map 5) 3))) - - (test-case (name-string linemap-line-start-position) - - (test-case "two-line string with ending newline" - (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-line-start-position map 1) 1) - (check-equal? (linemap-line-start-position map 2) 7) - (check-equal? (linemap-line-start-position map 3) 13)) - - (test-case "multiple blank lines" - (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-line-start-position map 1) 1) - (check-equal? (linemap-line-start-position map 2) 3) - (check-equal? (linemap-line-start-position map 3) 4))) - - (test-case (name-string linemap-line-end-position) - - (test-case "two-line string with ending newline" - (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-line-end-position map 1) 6) - (check-equal? (linemap-line-end-position map 2) 12)) - - (test-case "multiple blank lines" - (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-line-end-position map 1) 2) - (check-equal? (linemap-line-end-position map 2) 3) - (check-equal? (linemap-line-end-position map 3) 5))) - - (test-case (name-string linemap-position-to-start-of-line) - - (test-case "two-line string with ending newline" - (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-position-to-start-of-line map 1) 1) - (check-equal? (linemap-position-to-start-of-line map 2) 1) - (check-equal? (linemap-position-to-start-of-line map 3) 1) - (check-equal? (linemap-position-to-start-of-line map 4) 1) - (check-equal? (linemap-position-to-start-of-line map 5) 1) - (check-equal? (linemap-position-to-start-of-line map 6) 1) - (check-equal? (linemap-position-to-start-of-line map 7) 7) - (check-equal? (linemap-position-to-start-of-line map 8) 7) - (check-equal? (linemap-position-to-start-of-line map 9) 7) - (check-equal? (linemap-position-to-start-of-line map 10) 7) - (check-equal? (linemap-position-to-start-of-line map 11) 7) - (check-equal? (linemap-position-to-start-of-line map 12) 7) - (check-equal? (linemap-position-to-start-of-line map 13) 13)) - - (test-case "multiple blank lines" - (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-position-to-start-of-line map 1) 1) - (check-equal? (linemap-position-to-start-of-line map 2) 1) - (check-equal? (linemap-position-to-start-of-line map 3) 3) - (check-equal? (linemap-position-to-start-of-line map 4) 4) - (check-equal? (linemap-position-to-start-of-line map 5) 4))) - - (test-case (name-string linemap-position-to-end-of-line) - - (test-case "two-line string with ending newline" - (define map (string-linemap "hello\nworld\n")) - (check-equal? (linemap-position-to-end-of-line map 1) 6) - (check-equal? (linemap-position-to-end-of-line map 2) 6) - (check-equal? (linemap-position-to-end-of-line map 3) 6) - (check-equal? (linemap-position-to-end-of-line map 4) 6) - (check-equal? (linemap-position-to-end-of-line map 5) 6) - (check-equal? (linemap-position-to-end-of-line map 6) 6) - (check-equal? (linemap-position-to-end-of-line map 7) 12) - (check-equal? (linemap-position-to-end-of-line map 8) 12) - (check-equal? (linemap-position-to-end-of-line map 9) 12) - (check-equal? (linemap-position-to-end-of-line map 10) 12) - (check-equal? (linemap-position-to-end-of-line map 11) 12) - (check-equal? (linemap-position-to-end-of-line map 12) 12)) - - (test-case "multiple blank lines" - (define map (string-linemap "a\n\nb")) - (check-equal? (linemap-position-to-end-of-line map 1) 2) - (check-equal? (linemap-position-to-end-of-line map 2) 2) - (check-equal? (linemap-position-to-end-of-line map 3) 3) - (check-equal? (linemap-position-to-end-of-line map 4) 5) - (check-equal? (linemap-position-to-end-of-line map 5) 5)))) diff --git a/private/refactoring-result.rkt b/private/refactoring-result.rkt index 23c17bb6..733d98cf 100644 --- a/private/refactoring-result.rkt +++ b/private/refactoring-result.rkt @@ -50,7 +50,7 @@ resyntax/private/code-snippet resyntax/private/commit resyntax/private/line-replacement - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/private/logger resyntax/grimoire/source resyntax/grimoire/string-replacement @@ -192,9 +192,9 @@ (define lmap (string-linemap full-orig-code)) (define start (string-replacement-start replacement)) (define end (string-replacement-original-end replacement)) - (define start-column (- (add1 start) (linemap-position-to-start-of-line lmap (add1 start)))) + (define start-column (- start (linemap-position-to-start-of-line lmap start))) (define raw-text (string->immutable-string (substring full-orig-code start end))) - (code-snippet raw-text start-column (linemap-position-to-line lmap (add1 start)))) + (code-snippet raw-text start-column (linemap-position-to-line lmap start))) (define (refactoring-result-new-code result) @@ -203,8 +203,8 @@ (source->string (syntax-replacement-source (refactoring-result-syntax-replacement result)))) (define lmap (string-linemap full-orig-code)) (define start (string-replacement-start replacement)) - (define original-line (linemap-position-to-line lmap (add1 start))) - (define original-column (- (add1 start) (linemap-position-to-start-of-line lmap (add1 start)))) + (define original-line (linemap-position-to-line lmap start)) + (define original-column (- start (linemap-position-to-start-of-line lmap start))) (define refactored-source-code (string-replacement-apply replacement full-orig-code)) (define new-code-string (substring refactored-source-code diff --git a/private/syntax-replacement.rkt b/private/syntax-replacement.rkt index 0789a78c..6a05271c 100644 --- a/private/syntax-replacement.rkt +++ b/private/syntax-replacement.rkt @@ -42,7 +42,7 @@ rebellion/collection/range-set rebellion/private/static-name rebellion/type/record - resyntax/private/linemap + resyntax/grimoire/linemap resyntax/private/logger resyntax/grimoire/source resyntax/private/string-indent @@ -264,14 +264,12 @@ ;; For single-line replacements, we need to account for trailing text after the replacement ;; to ensure the formatted code doesn't exceed the line length limit. + (define linemap (string-linemap original)) + (define orig-end (string-replacement-original-end replacement)) (define trailing-text-length - (let ([linemap (string-linemap original)] - [orig-end (string-replacement-original-end replacement)]) - ;; Convert from 0-based string indices to 1-based positions for linemap - (if (= (linemap-position-to-line linemap (add1 start)) - (linemap-position-to-line linemap (add1 orig-end))) - (- (linemap-position-to-end-of-line linemap (add1 orig-end)) (add1 orig-end)) - 0))) + (if (= (linemap-position-to-line linemap start) (linemap-position-to-line linemap orig-end)) + (- (linemap-position-to-end-of-line linemap orig-end) orig-end) + 0)) (define allowed-width (- base-allowed-width trailing-text-length)) (define formatted-code-substring