fix: keep glued lexemes intact when formatting

src/formatter/layout.ts

@@ -23,7 +23,7 @@ import {
   pivotSubClauses,
   windowSubClauses,
 } from "./phrases"
-import { gapBetween, isOperand, isSign, Piece } from "./spacing"
+import { gapBetween, isGluedWords, isOperand, isSign, Piece } from "./spacing"
 import { Statement } from "./statements"
 
 type Element = { leading: Doc; doc: Doc }
@@ -208,8 +208,20 @@ class StatementBuilder {
     return { leading, doc: text(this.textOf(piece)) }
   }
 
+  private phraseAt(phrases: Phrase[]): PhraseMatch | null {
+    const piece = this.peek()
+    if (
+      piece !== undefined &&
+      this.previous !== null &&
+      isGluedWords(this.previous, piece)
+    ) {
+      return null
+    }
+    return matchPhrase(this.tokens, this.index, phrases)
+  }
+
   private matchClause(current: Phrase | null): PhraseMatch | null {
-    const match = matchPhrase(this.tokens, this.index, this.phrases)
+    const match = this.phraseAt(this.phrases)
     if (match === null || continuesPhrase(current, match.phrase)) return null
     return match
   }
@@ -345,7 +357,7 @@ class StatementBuilder {
         items.length === 0 &&
         current === null &&
         ctx.subClauses.length > 0 &&
-        matchPhrase(this.tokens, this.index, ctx.subClauses) !== null
+        this.phraseAt(ctx.subClauses) !== null
       ) {
         subClauseStarted = true
       }
@@ -369,7 +381,7 @@ class StatementBuilder {
 
       const subClause =
         current !== null && endsOperand(this.previous)
-          ? matchPhrase(this.tokens, this.index, ctx.subClauses)
+          ? this.phraseAt(ctx.subClauses)
           : null
       if (subClause !== null) {
         closeItem()

src/formatter/spacing.ts

@@ -30,11 +30,16 @@ const noSpaceAfter: ReadonlySet<string> = new Set([
 const isComment = (piece: Piece) =>
   piece.token.kind === "lineComment" || piece.token.kind === "blockComment"
 
-const isNumber = (piece: Piece) => piece.token.tokenName === "NumberLiteral"
+const isNumber = (piece: Piece) => /^\d/.test(piece.token.image)
 
 const isUncertain = (piece: Piece) =>
   piece.token.kind === "opaque" || piece.token.kind === "tolerant"
 
+export const isGluedWords = (previous: Piece, next: Piece) =>
+  previous.token.kind === "word" &&
+  next.token.kind === "word" &&
+  next.gapBefore === ""
+
 export const isSign = (piece: Piece) =>
   piece.token.tokenName === "Minus" || piece.token.tokenName === "Plus"
 
@@ -53,6 +58,7 @@ export const gapBetween = (previous: Piece | null, next: Piece): string => {
     return preserved(next)
   }
   if (previous.unary && !isComment(next)) return ""
+  if (isGluedWords(previous, next)) return ""
   const previousName = previous.token.tokenName
   const nextName = next.token.tokenName
   if (nextName === "Dot" && isNumber(previous)) return " "

tests/formatter/lexemes.test.ts

@@ -0,0 +1,189 @@
+import { describe, expect, it } from "vitest"
+import { format } from "../../src/formatter/index"
+import {
+  assertPreserved,
+  assertSameCharacters,
+  assertSameLexemes,
+  assertSameStream,
+} from "./oracles"
+
+const splitLexemes = [
+  "0x1F",
+  "0xDEADbeef",
+  "16777217.0f",
+  "1.5f",
+  "5c",
+  "7b",
+  "1500ns",
+  "1.5L",
+  "2h30m",
+  "123abc",
+  "1e",
+]
+
+const contexts = [
+  (lexeme: string) => `SELECT id, ${lexeme} FROM t`,
+  (lexeme: string) => `SELECT f(${lexeme}) FROM t`,
+  (lexeme: string) => `SELECT * FROM t WHERE h = ${lexeme}`,
+  (lexeme: string) => `SELECT -${lexeme}`,
+  (lexeme: string) => `SELECT a[${lexeme}]`,
+  (lexeme: string) => `INSERT INTO t VALUES (1, ${lexeme})`,
+  (lexeme: string) => `CREATE TABLE g (h geohash(${lexeme}), ts timestamp)`,
+]
+
+describe("lexemes the lexer splits into several tokens", () => {
+  it.each(
+    splitLexemes.flatMap((lexeme) =>
+      contexts.map((context) => [context(lexeme)] as const),
+    ),
+  )("keeps %s glued", (sql) => {
+    assertPreserved(sql)
+  })
+
+  it("formats a hex literal without touching it", () => {
+    // Given
+    const sql = "SELECT id, 0x1F FROM t"
+
+    // When
+    const output = format(sql)
+
+    // Then
+    expect(output).toBe("SELECT id, 0x1F\nFROM t")
+  })
+
+  it("formats a geohash precision without touching it", () => {
+    // Given
+    const sql = "CREATE TABLE g (h geohash(5c), ts timestamp) timestamp(ts)"
+
+    // When
+    const output = format(sql)
+
+    // Then
+    expect(output).toBe(
+      "CREATE TABLE g (\n  h geohash(5c),\n  ts timestamp\n) timestamp(ts)",
+    )
+  })
+
+  it("does not start a clause inside a glued run", () => {
+    // Given
+    const sql = "SELECT 10.5mFROM FROM t"
+
+    // When
+    const output = format(sql)
+
+    // Then
+    expect(output).toBe("SELECT 10.5mFROM\nFROM t")
+  })
+
+  it("keeps a dot apart from a suffixed number", () => {
+    // Given
+    const sql = "SELECT t. 5m"
+
+    // When
+    const output = format(sql)
+
+    // Then
+    expect(output).toBe("SELECT t. 5m")
+  })
+})
+
+const lexemes = [
+  "SELECT",
+  "FROM",
+  "WHERE",
+  "AND",
+  "AS",
+  "NULL",
+  "key",
+  "a",
+  "x1F",
+  "f",
+  "café",
+  "0",
+  "1.5",
+  ".5",
+  "5.",
+  "1e5",
+  "1.5E+3",
+  "1_000",
+  "0x1F",
+  "1.5f",
+  "5c",
+  "1500ns",
+  "100L",
+  "10.5m",
+  "5m",
+  "1d",
+  "#sp052w92p1p8",
+  "##0101",
+  "#",
+  "'a'",
+  "'a''b'",
+  "'abc",
+  '"a"',
+  '"abc',
+  "+",
+  "-",
+  "*",
+  "/",
+  "%",
+  "=",
+  "!=",
+  "<>",
+  "<",
+  "<=",
+  ">",
+  ">=",
+  "<<",
+  "<<=",
+  ">>",
+  ">>=",
+  "||",
+  "|",
+  "&",
+  "^",
+  "~",
+  "!~",
+  "~=",
+  "::",
+  ":",
+  ":=",
+  "@",
+  "!",
+  "?",
+  "$",
+  "(",
+  ")",
+  "[",
+  "]",
+  ",",
+  ";",
+  ".",
+  "--c\n",
+  "/*c*/",
+  "/*c",
+  "@x",
+  ":name",
+  "$1",
+]
+
+const separators = ["", " ", "\n"]
+
+describe("adjacent lexeme pairs", () => {
+  it("never changes characters or lexemes for any pair", () => {
+    // Given
+    const pairs = lexemes.flatMap((left) =>
+      lexemes.flatMap((right) =>
+        separators.map((separator) => `${left}${separator}${right}`),
+      ),
+    )
+
+    // When / Then
+    for (const sql of pairs) {
+      const output = format(sql)
+      assertSameCharacters(sql, output)
+      assertSameLexemes(sql, output)
+      assertSameStream(sql, output)
+    }
+  })
+})

tests/formatter/oracles.ts

@@ -2,6 +2,7 @@ import { expect } from "vitest"
 import { parseToAst, toSql } from "../../src/index"
 import { format, FormatOptions } from "../../src/formatter/index"
 import { scan, StreamToken } from "../../src/formatter/lexer"
+import { tokenize } from "../../src/parser/lexer"
 
 const significant = (sql: string): StreamToken[] =>
   scan(sql).filter((token) => token.kind !== "whitespace")
@@ -36,6 +37,27 @@ export const assertSameOperatorAdjacency = (input: string, output: string) => {
   expect(adjacencyPairs(output)).toEqual(adjacencyPairs(input))
 }
 
+const lexemeSignature = (sql: string): string[] => {
+  const result = tokenize(sql)
+  return [
+    ...result.tokens.map((token) => `${token.tokenType.name}:${token.image}`),
+    ...result.errors.map(
+      (error) =>
+        `error:${sql.slice(error.offset, error.offset + error.length)}`,
+    ),
+  ]
+}
+
+export const assertSameLexemes = (input: string, output: string) => {
+  expect(lexemeSignature(output)).toEqual(lexemeSignature(input))
+}
+
+const withoutWhitespace = (sql: string) => sql.replace(/\s+/g, "")
+
+export const assertSameCharacters = (input: string, output: string) => {
+  expect(withoutWhitespace(output)).toBe(withoutWhitespace(input))
+}
+
 export const assertIdempotent = (sql: string, options?: FormatOptions) => {
   const once = format(sql, options)
   expect(format(once, options)).toBe(once)
@@ -52,6 +74,10 @@ export const assertPreserved = (input: string, options?: FormatOptions) => {
   const output = format(input, options)
   assertSameStream(input, output)
   assertSameOperatorAdjacency(input, output)
+  if (!options?.capitalize) {
+    assertSameCharacters(input, output)
+    assertSameLexemes(input, output)
+  }
   assertIdempotent(input, options)
   if (parsesCleanly(input)) assertSameAst(input, output)
 }