Use pinned gzip for localization generation

tools/go.mod

@@ -75,7 +75,7 @@ require (
 	github.com/iancoleman/strcase v0.3.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jedib0t/go-pretty v4.3.0+incompatible // indirect
-	github.com/klauspost/compress v1.18.4 // indirect
+	github.com/klauspost/compress v1.19.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect

tools/go.sum

@@ -172,8 +172,8 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf
 github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo=
 github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
-github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
-github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ=
+github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=

tsc/go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Microsoft/go-winio v0.6.2
 	github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68
 	github.com/google/go-cmp v0.7.0
+	github.com/klauspost/compress v1.19.0
 	github.com/mackerelio/go-osstat v0.2.7
 	github.com/peter-evans/patience v0.3.0
 	github.com/zeebo/xxh3 v1.1.0

tsc/go.sum

@@ -4,6 +4,8 @@ github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSy
 github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ=
+github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
 github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
 github.com/mackerelio/go-osstat v0.2.7 h1:TCavZi10wF49bT6iQZ9eT2keGZQpC69MTDfdJej5e94=

tsc/internal/diagnostics/diagnostics_test.go

@@ -143,3 +143,12 @@ func TestLocalize_ByKey(t *testing.T) {
 		})
 	}
 }
+
+func TestLoadAllLocales(t *testing.T) {
+	t.Parallel()
+
+	for i, load := range localeFuncs[1:] {
+		assert.Assert(t, load != nil, "locale loader %d is nil", i+1)
+		assert.Assert(t, len(load()) != 0, "locale %d has no messages", i+1)
+	}
+}

tsc/internal/diagnostics/generate.go

@@ -5,7 +5,6 @@ package main
 import (
 	"bytes"
 	"cmp"
-	"compress/gzip"
 	"encoding/xml"
 	"flag"
 	"fmt"
@@ -20,10 +19,12 @@ import (
 	"slices"
 	"strconv"
 	"strings"
+	"time"
 	"unicode"
 
-	"github.com/microsoft/TypeScript/tsc/internal/collections"
+	"github.com/klauspost/compress/gzip"
 	"github.com/microsoft/TypeScript/tsc/internal/json"
+	"github.com/zeebo/xxh3"
 	"golang.org/x/text/language"
 )
 
@@ -207,12 +208,10 @@ func generateLocalizations(knownKeys map[string]bool, locDir string) *bytes.Buff
 	slices.Sort(localeFiles)
 
 	type localeInfo struct {
-		varName   string
-		tgtCul    string
-		canonical string // canonical lowercase form (e.g., "zh-cn", "pt-br")
-		lang      string
-		messages  map[string]string
-		filename  string
+		varName  string
+		tgtCul   string
+		messages map[string]string
+		filename string
 	}
 
 	var locales []localeInfo
@@ -237,60 +236,81 @@ func generateLocalizations(knownKeys map[string]bool, locDir string) *bytes.Buff
 		// Convert locale code to valid Go identifier
 		localeVar := strings.ReplaceAll(strings.ReplaceAll(tgtCul, "-", ""), "_", "")
 
-		// Parse the locale using language.Tag to get canonical forms
-		tag, err := language.Parse(tgtCul)
+		// Validate the locale.
+		_, err := language.Parse(tgtCul)
 		if err != nil {
 			log.Printf("failed to parse locale %q: %v", tgtCul, err)
 			continue
 		}
 
-		base, _ := tag.Base()
-		lang := strings.ToLower(base.String())
-
-		// Get canonical form (lowercase with dash)
-		canonical := strings.ToLower(tgtCul)
-
 		// Filename for the JSON.gz file (use the original tgtCul as standard language tag)
 		filename := fmt.Sprintf("%s.json.gz", tgtCul)
 
-		// Write the JSON.gz file
-		// Convert map to OrderedMap with sorted keys for consistent ordering
-		keys := slices.Sorted(maps.Keys(localizedMessages))
-		var orderedMessages collections.OrderedMap[string, string]
-		for _, key := range keys {
-			orderedMessages.Set(key, localizedMessages[key])
+		locales = append(locales, localeInfo{
+			varName:  localeVar,
+			tgtCul:   tgtCul,
+			messages: localizedMessages,
+			filename: filename,
+		})
+	}
+
+	const missingLocaleMessage = "\x00"
+	allKeys := make(map[string]bool)
+	for _, loc := range locales {
+		for key, message := range loc.messages {
+			if message == missingLocaleMessage {
+				log.Fatalf("locale %s contains a message reserved for the missing-message sentinel", loc.tgtCul)
+			}
+			allKeys[key] = true
 		}
-		jsonData, err := json.Marshal(&orderedMessages)
+	}
+	localeKeys := slices.Sorted(maps.Keys(allKeys))
+	localeKeysHasher := xxh3.New()
+	for _, key := range localeKeys {
+		localeKeysHasher.WriteString(key)
+		localeKeysHasher.WriteString("\x00")
+	}
+	localeKeysHash128 := localeKeysHasher.Sum128()
+	localeKeysHash := fmt.Sprintf("%016x%016x", localeKeysHash128.Hi, localeKeysHash128.Lo)
+
+	for _, loc := range locales {
+		values := make([]string, len(localeKeys)+1)
+		values[0] = localeKeysHash
+		for i, key := range localeKeys {
+			if message, ok := loc.messages[key]; ok {
+				values[i+1] = message
+			} else {
+				values[i+1] = missingLocaleMessage
+			}
+		}
+		jsonData, err := json.Marshal(values)
 		if err != nil {
-			log.Fatalf("failed to marshal locale %s: %v", tgtCul, err)
+			log.Fatalf("failed to marshal locale %s: %v", loc.tgtCul, err)
 		}
-
 		var compressed bytes.Buffer
 		gzipWriter, err := gzip.NewWriterLevel(&compressed, gzip.BestCompression)
 		if err != nil {
-			log.Fatalf("failed to create gzip writer for locale %s: %v", tgtCul, err)
+			log.Fatalf("failed to create gzip writer for locale %s: %v", loc.tgtCul, err)
 		}
+		gzipWriter.ModTime = time.Unix(0, 0)
 		if _, err := gzipWriter.Write(jsonData); err != nil {
-			log.Fatalf("failed to compress locale %s: %v", tgtCul, err)
+			log.Fatalf("failed to compress locale %s: %v", loc.tgtCul, err)
 		}
 		if err := gzipWriter.Close(); err != nil {
-			log.Fatalf("failed to close gzip writer for locale %s: %v", tgtCul, err)
+			log.Fatalf("failed to close gzip writer for locale %s: %v", loc.tgtCul, err)
 		}
-
-		outputPath := filepath.Join(locDir, filename)
+		outputPath := filepath.Join(locDir, loc.filename)
 		if err := os.WriteFile(outputPath, compressed.Bytes(), 0o644); err != nil {
 			log.Fatalf("failed to write locale file %s: %v", outputPath, err)
 		}
+	}
 
-		locales = append(locales, localeInfo{
-			varName:   localeVar,
-			tgtCul:    tgtCul,
-			canonical: canonical,
-			lang:      lang,
-			messages:  localizedMessages,
-			filename:  filename,
-		})
+	buf.WriteString("\nvar localeKeys = [...]Key{\n")
+	for _, key := range localeKeys {
+		fmt.Fprintf(&buf, "\t%q,\n", key)
 	}
+	buf.WriteString("}\n")
+	fmt.Fprintf(&buf, "\nconst localeKeysHash = %q\n", localeKeysHash)
 
 	// Generate matcher with inlined tags
 	// English is first (index 0) as the default/fallback with no translation needed
@@ -317,10 +337,19 @@ func generateLocalizations(knownKeys map[string]bool, locDir string) *bytes.Buff
 	buf.WriteString("\t\tpanic(\"failed to create gzip reader: \" + err.Error())\n")
 	buf.WriteString("\t}\n")
 	buf.WriteString("\tdefer gr.Close()\n")
-	buf.WriteString("\tvar result map[Key]string\n")
-	buf.WriteString("\tif err := json.UnmarshalRead(gr, &result); err != nil {\n")
+	buf.WriteString("\tvar values []string\n")
+	buf.WriteString("\tif err := json.UnmarshalRead(gr, &values); err != nil {\n")
 	buf.WriteString("\t\tpanic(\"failed to unmarshal locale data: \" + err.Error())\n")
 	buf.WriteString("\t}\n")
+	buf.WriteString("\tif len(values) != len(localeKeys)+1 || values[0] != localeKeysHash {\n")
+	buf.WriteString("\t\tpanic(\"locale data does not match its key table\")\n")
+	buf.WriteString("\t}\n")
+	buf.WriteString("\tresult := make(map[Key]string, len(localeKeys))\n")
+	buf.WriteString("\tfor i, value := range values[1:] {\n")
+	fmt.Fprintf(&buf, "\t\tif value != %q {\n", missingLocaleMessage)
+	buf.WriteString("\t\t\tresult[localeKeys[i]] = value\n")
+	buf.WriteString("\t\t}\n")
+	buf.WriteString("\t}\n")
 	buf.WriteString("\treturn result\n")
 	buf.WriteString("}\n")
 

tsc/internal/diagnostics/generate_deps.go

@@ -0,0 +1,6 @@
+//go:build generate
+
+package diagnostics
+
+// Keep dependencies used by generate.go visible to go mod tidy.
+import _ "github.com/klauspost/compress/gzip"