From f7b077cbf4c9e01f1d26ae0978c2af57b72ebe1e Mon Sep 17 00:00:00 2001 From: Cheese Date: Mon, 17 Aug 2026 17:51:14 +0800 Subject: [PATCH] fix(fs): stabilize regions and mount output --- AGENTS.md | 17 ++- README.md | 6 +- docs/priciples.md | 4 +- e2e/installer_test.go | 33 ++++- internal/api/endpoints/endpoints.go | 152 ++++------------------- internal/api/endpoints/endpoints_test.go | 104 +++++----------- internal/cli/commands.go | 14 +-- internal/cli/root_test.go | 28 ++++- internal/config/region/region.go | 14 ++- internal/config/region/region_test.go | 10 +- internal/fs/drive9_companion.go | 2 +- internal/fs/drive9_companion_test.go | 53 ++++++++ internal/fs/fscred/credential.go | 11 +- internal/fs/fscred/credential_test.go | 34 +++++ internal/telemetrybackend/event.go | 17 +-- scripts/install.ps1 | 21 +--- scripts/install.sh | 44 +------ 17 files changed, 256 insertions(+), 308 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6473a10..7a7901e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -855,12 +855,11 @@ for SQL execution Basic Auth. Use `internal/api/endpoints` for Starter, IAM/account, and fs endpoint selection. Do not add service URLs to user config. The default Starter host is `https://serverless.tidbapi.com`; the default IAM host is -`https://iam.tidbapi.com`. The ti fs host is resolved from the hosted ti fs -region manifest at -`https://drive9.ai/manifest/regions/drive9-regions.json`, matching the active -profile's cloud provider and region against `tidb_cloud_native` entries. If the -manifest does not contain the profile placement, return a clear unsupported -endpoint error; do not add a user-facing raw server URL flag or config key. +`https://iam.tidbapi.com`. The ti fs host is resolved from the four-region +endpoint mapping owned by `internal/api/endpoints`. If the mapping does not +contain the profile placement, return a clear unsupported endpoint error; do +not add a user-facing raw server URL flag or config key or restore a runtime +manifest dependency. Tests may override the IAM base URL with `TI_TEST_IAM_BASE_URL` and the fs manifest URL with `TI_TEST_FS_MANIFEST_URL`, only when `TI_ALLOW_TEST_ENDPOINTS=1`; these are hidden test controls, not supported @@ -951,11 +950,11 @@ Supported MVP placement values: | `aws-eu-central-1` | AWS | Frankfurt | | `aws-ap-northeast-1` | AWS | Tokyo | | `aws-ap-southeast-1` | AWS | Singapore | -| `ali-ap-southeast-1` | Alibaba Cloud | Singapore | +| `alicloud-ap-southeast-1` | Alibaba Cloud | Singapore | The prefix before the first `-` is the cloud provider selector. `aws` maps to -internal provider `aws`; `ali` maps to internal provider `alibaba_cloud`. Keep -this mapping centralized in `internal/config/region`. +internal provider `aws`; `alicloud` maps to internal provider +`alibaba_cloud`. Keep this mapping centralized in `internal/config/region`. Do not store secrets in logs, telemetry, generated docs examples, or test fixtures. diff --git a/README.md b/README.md index 63d7368..0c95d14 100644 --- a/README.md +++ b/README.md @@ -102,9 +102,9 @@ Automation should move to `TI_*` and `TIDB_CLOUD_*` environment variables. The v ### Configure - Authentication: a TiDB Cloud Public Key and a Private Key from the [TiDB Cloud API Keys](https://tidbcloud.com/org-settings/api-keys) console. -- Default region: one of aws-us-east-1, aws-us-west-2, aws-eu-central-1, aws-ap-northeast-1, aws-ap-southeast-1, or ali-ap-southeast-1. - - Regions support TiDB Cloud Filesystem: aws-us-east-1, aws-us-west-2, aws-ap-southeast-1, or ali-ap-southeast-1. - - Regions support TiDB Cloud Starter: aws-us-east-1, aws-us-west-2, aws-eu-central-1, aws-ap-northeast-1, aws-ap-southeast-1, or ali-ap-southeast-1. +- Default region: one of aws-us-east-1, aws-us-west-2, aws-eu-central-1, aws-ap-northeast-1, aws-ap-southeast-1, or alicloud-ap-southeast-1. + - Regions supporting TiDB Cloud Filesystem: aws-us-east-1, aws-ap-southeast-1, aws-us-west-2, or alicloud-ap-southeast-1. These endpoints are built into `ti`; endpoint resolution does not download a Drive9 region manifest. + - Regions supporting TiDB Cloud Starter: aws-us-east-1, aws-us-west-2, aws-eu-central-1, aws-ap-northeast-1, aws-ap-southeast-1, or alicloud-ap-southeast-1. Set up a default profile with one command: diff --git a/docs/priciples.md b/docs/priciples.md index 8706dde..62d89f8 100644 --- a/docs/priciples.md +++ b/docs/priciples.md @@ -75,9 +75,9 @@ Supported TiDB Cloud Starter placement values are: | `aws-eu-central-1` | AWS | Frankfurt | | `aws-ap-northeast-1` | AWS | Tokyo | | `aws-ap-southeast-1` | AWS | Singapore | -| `ali-ap-southeast-1` | Alibaba Cloud | Singapore | +| `alicloud-ap-southeast-1` | Alibaba Cloud | Singapore | -`aws` maps to the internal provider `aws`; `ali` maps to `alibaba_cloud`. TiDB Cloud Filesystem availability is resolved from the hosted Drive9 region manifest and can be a subset of the Starter regions. +`aws` maps to the internal provider `aws`; `alicloud` maps to `alibaba_cloud`. TiDB Cloud Filesystem supports `aws-us-east-1`, `aws-ap-southeast-1`, `aws-us-west-2`, and `alicloud-ap-southeast-1` through endpoint mappings built into `ti`. ## TiDB Cloud Authentication diff --git a/e2e/installer_test.go b/e2e/installer_test.go index f518acb..ad6663c 100644 --- a/e2e/installer_test.go +++ b/e2e/installer_test.go @@ -173,7 +173,6 @@ case "$url" in */%s) source=%q ;; */checksums.txt) source=%q ;; */%s) source=%q ;; - */drive9-regions.json) printf '{"regions":[]}'; exit 0 ;; *) printf 'unexpected URL: %%s\n' "$url" >&2; exit 1 ;; esac if [ -n "$out" ]; then @@ -196,6 +195,14 @@ fi if !strings.Contains(string(output), "PATH shadowing detected") || !strings.Contains(string(output), shadow) { t.Fatalf("installer did not report the unrelated ti on PATH:\n%s", output) } + if strings.Contains(string(output), "ti fs companion installed to") { + t.Fatalf("installer exposed the companion installation path:\n%s", output) + } + for _, regionCode := range []string{"aws-us-east-1", "aws-ap-southeast-1", "aws-us-west-2", "alicloud-ap-southeast-1"} { + if !strings.Contains(string(output), regionCode) { + t.Fatalf("installer did not list ti fs region %q:\n%s", regionCode, output) + } + } if data, err := os.ReadFile(shadow); err != nil || !strings.Contains(string(data), "unrelated ti") { t.Fatalf("installer replaced PATH shadow: %q, %v", data, err) } @@ -246,6 +253,30 @@ func TestInstallersDoNotEscalatePrivileges(t *testing.T) { } } +func TestInstallersUseProductOwnedFSRegionsAndHideCompanionPath(t *testing.T) { + for _, path := range []string{ + filepath.Join("..", "scripts", "install.sh"), + filepath.Join("..", "scripts", "install.ps1"), + } { + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + content := string(data) + if strings.Contains(content, "drive9-regions.json") { + t.Fatalf("%s still downloads the Drive9 region manifest", path) + } + if strings.Contains(content, "ti fs companion installed to") { + t.Fatalf("%s still reports the companion installation path", path) + } + for _, regionCode := range []string{"aws-us-east-1", "aws-ap-southeast-1", "aws-us-west-2", "alicloud-ap-southeast-1"} { + if !strings.Contains(content, regionCode) { + t.Fatalf("%s does not list ti fs region %q", path, regionCode) + } + } + } +} + func TestInstallersExplainTelemetryControls(t *testing.T) { for _, path := range []string{ filepath.Join("..", "scripts", "install.sh"), diff --git a/internal/api/endpoints/endpoints.go b/internal/api/endpoints/endpoints.go index e23a344..f576443 100644 --- a/internal/api/endpoints/endpoints.go +++ b/internal/api/endpoints/endpoints.go @@ -2,14 +2,11 @@ package endpoints import ( "context" - "crypto/sha256" "encoding/json" - "errors" "fmt" "net/http" "net/url" "os" - "path/filepath" "sort" "strings" "time" @@ -29,24 +26,9 @@ const ( const ( DefaultStarterBaseURL = "https://serverless.tidbapi.com" DefaultIAMBaseURL = "https://iam.tidbapi.com" - DefaultFSManifestURL = "https://drive9.ai/manifest/regions/drive9-regions.json" DefaultFSMode = "tidb_cloud_native" ) -var errFSManifestUnavailable = errors.New("ti fs region manifest unavailable") - -const ( - fsManifestFetchAttempts = 3 - fsManifestRetryDelay = 200 * time.Millisecond - fsManifestCacheMaxAge = 24 * time.Hour -) - -type cachedFSRegionManifest struct { - ManifestURL string `json:"manifest_url"` - FetchedAt time.Time `json:"fetched_at"` - Manifest FSRegionManifest `json:"manifest"` -} - type ProviderRegion struct { Provider string Region string @@ -75,7 +57,6 @@ func NewResolver() Resolver { resolver := Resolver{ StarterBaseURL: DefaultStarterBaseURL, IAMBaseURL: DefaultIAMBaseURL, - FSManifestURL: DefaultFSManifestURL, FSMode: DefaultFSMode, } if os.Getenv("TI_ALLOW_TEST_ENDPOINTS") == "1" { @@ -217,10 +198,14 @@ func (r Resolver) fsManifest() (*FSRegionManifest, error) { return &manifest, nil } manifestURL := strings.TrimSpace(r.FSManifestURL) - if manifestURL == "" { - manifestURL = DefaultFSManifestURL + if manifestURL != "" { + return fetchFSManifest(context.Background(), manifestURL, r.FSManifestHTTPClient) } - return fetchFSManifest(context.Background(), manifestURL, r.FSManifestHTTPClient) + manifest := builtInFSRegionManifest() + if err := validateFSManifest(manifest); err != nil { + return nil, err + } + return manifest, nil } func fetchFSManifest(ctx context.Context, manifestURL string, client *http.Client) (*FSRegionManifest, error) { @@ -230,119 +215,38 @@ func fetchFSManifest(ctx context.Context, manifestURL string, client *http.Clien client = http.DefaultClient } - var lastErr error - for attempt := 0; attempt < fsManifestFetchAttempts; attempt++ { - manifest, retry, err := fetchFSManifestOnce(ctx, manifestURL, client) - if err == nil { - storeCachedFSManifest(manifestURL, manifest) - return manifest, nil - } - lastErr = err - if !retry || attempt == fsManifestFetchAttempts-1 { - break - } - timer := time.NewTimer(time.Duration(attempt+1) * fsManifestRetryDelay) - select { - case <-ctx.Done(): - timer.Stop() - lastErr = apperr.Wrap("api.fs_manifest_unavailable", "api", 1, fmt.Sprintf("%s: fetch %s", errFSManifestUnavailable, manifestURL), ctx.Err()) - if cached, cacheErr := loadCachedFSManifest(manifestURL); cacheErr == nil { - return cached, nil - } - return nil, lastErr - case <-timer.C: - } - } - if cached, cacheErr := loadCachedFSManifest(manifestURL); cacheErr == nil { - return cached, nil - } - return nil, lastErr -} - -func fetchFSManifestOnce(ctx context.Context, manifestURL string, client *http.Client) (*FSRegionManifest, bool, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, manifestURL, nil) if err != nil { - return nil, false, apperr.Wrap("api.fs_manifest_request", "api", 1, "build ti fs region manifest request", err) + return nil, apperr.Wrap("api.fs_manifest_request", "api", 1, "build test ti fs region manifest request", err) } res, err := client.Do(req) if err != nil { - return nil, true, apperr.Wrap("api.fs_manifest_unavailable", "api", 1, fmt.Sprintf("%s: fetch %s", errFSManifestUnavailable, manifestURL), err) + return nil, apperr.Wrap("api.fs_manifest_unavailable", "api", 1, fmt.Sprintf("test ti fs region manifest unavailable: fetch %s", manifestURL), err) } defer res.Body.Close() if res.StatusCode != http.StatusOK { - return nil, retryableFSManifestStatus(res.StatusCode), apperr.New("api.fs_manifest_unavailable", "api", 1, fmt.Sprintf("%s: fetch %s returned HTTP %d", errFSManifestUnavailable, manifestURL, res.StatusCode)) + return nil, apperr.New("api.fs_manifest_unavailable", "api", 1, fmt.Sprintf("test ti fs region manifest fetch %s returned HTTP %d", manifestURL, res.StatusCode)) } var manifest FSRegionManifest if err := json.NewDecoder(res.Body).Decode(&manifest); err != nil { - return nil, true, apperr.Wrap("api.fs_manifest_decode", "api", 1, "decode ti fs region manifest", err) + return nil, apperr.Wrap("api.fs_manifest_decode", "api", 1, "decode test ti fs region manifest", err) } - if err := validateFSManifest(&manifest); err != nil { - return nil, false, err - } - return &manifest, false, nil -} - -func retryableFSManifestStatus(statusCode int) bool { - return statusCode == http.StatusTooManyRequests || statusCode >= 500 -} - -func loadCachedFSManifest(manifestURL string) (*FSRegionManifest, error) { - cachePath, err := fsManifestCachePath(manifestURL) - if err != nil { - return nil, err - } - data, err := os.ReadFile(cachePath) - if err != nil { - return nil, err - } - var cached cachedFSRegionManifest - if err := json.Unmarshal(data, &cached); err != nil { - return nil, err - } - if cached.ManifestURL != manifestURL { - return nil, fmt.Errorf("cached ti fs manifest belongs to a different URL") - } - if cached.FetchedAt.IsZero() || time.Since(cached.FetchedAt) > fsManifestCacheMaxAge { - return nil, fmt.Errorf("cached ti fs manifest is expired") - } - manifest := cached.Manifest if err := validateFSManifest(&manifest); err != nil { return nil, err } return &manifest, nil } -func storeCachedFSManifest(manifestURL string, manifest *FSRegionManifest) { - if manifest == nil { - return +func builtInFSRegionManifest() *FSRegionManifest { + return &FSRegionManifest{ + Service: "ti-fs", + Regions: []FSRegionManifestEntry{ + {RegionCode: "aws-us-east-1", Mode: DefaultFSMode, ServerURL: "https://aws-us-east-1.drive9.ai", CloudProvider: "aws", TiDBRegion: "us-east-1"}, + {RegionCode: "aws-ap-southeast-1", Mode: DefaultFSMode, ServerURL: "https://aws-ap-southeast-1.drive9.ai", CloudProvider: "aws", TiDBRegion: "ap-southeast-1"}, + {RegionCode: "aws-us-west-2", Mode: DefaultFSMode, ServerURL: "https://aws-us-west-2.drive9.ai", CloudProvider: "aws", TiDBRegion: "us-west-2"}, + {RegionCode: "alicloud-ap-southeast-1", Mode: DefaultFSMode, ServerURL: "https://alicloud-ap-southeast-1.drive9.ai", CloudProvider: "alicloud", TiDBRegion: "ap-southeast-1"}, + }, } - cachePath, err := fsManifestCachePath(manifestURL) - if err != nil { - return - } - if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil { - return - } - cached := cachedFSRegionManifest{ - ManifestURL: manifestURL, - FetchedAt: time.Now().UTC(), - Manifest: *manifest, - } - data, err := json.MarshalIndent(cached, "", " ") - if err != nil { - return - } - _ = os.WriteFile(cachePath, data, 0o644) -} - -func fsManifestCachePath(manifestURL string) (string, error) { - home, err := os.UserHomeDir() - if err != nil || strings.TrimSpace(home) == "" { - return "", fmt.Errorf("resolve ti fs manifest cache home: %w", err) - } - sum := sha256.Sum256([]byte(manifestURL)) - name := fmt.Sprintf("fs-region-manifest-%x.json", sum[:8]) - return filepath.Join(home, ".ti", "cache", name), nil } func validateFSManifest(manifest *FSRegionManifest) error { @@ -428,11 +332,7 @@ func fsManifestEntryMatches(entry FSRegionManifestEntry, provider, apiProvider, } func fsRegionCode(provider, regionCode string) string { - prefix := APIProvider(provider) - if provider == region.ProviderAlibabaCloud { - prefix = "ali" - } - return prefix + "-" + regionCode + return APIProvider(provider) + "-" + regionCode } func supportedFSRegions(entries []FSRegionManifestEntry, mode string) string { @@ -442,15 +342,7 @@ func supportedFSRegions(entries []FSRegionManifestEntry, mode string) string { if strings.TrimSpace(entry.Mode) != mode { continue } - provider := entry.CloudProvider - if provider == "" { - provider = strings.SplitN(entry.RegionCode, "-", 2)[0] - } - regionCode := entry.TiDBRegion - if regionCode == "" { - regionCode = strings.TrimPrefix(entry.RegionCode, provider+"-") - } - value := provider + "/" + regionCode + value := entry.RegionCode if _, ok := seen[value]; ok { continue } diff --git a/internal/api/endpoints/endpoints_test.go b/internal/api/endpoints/endpoints_test.go index 0da32ed..4a77d6a 100644 --- a/internal/api/endpoints/endpoints_test.go +++ b/internal/api/endpoints/endpoints_test.go @@ -4,7 +4,6 @@ import ( "net/http" "net/http/httptest" "strings" - "sync/atomic" "testing" "github.com/tidbcloud/ti-cli/internal/apperr" @@ -148,6 +147,36 @@ func TestResolveFSFromRegionManifest(t *testing.T) { } } +func TestResolveFSUsesBuiltInFourRegionMapping(t *testing.T) { + tests := []struct { + name string + provider string + regionCode string + baseURL string + regionName string + }{ + {name: "AWS N. Virginia", provider: region.ProviderAWS, regionCode: "us-east-1", baseURL: "https://aws-us-east-1.drive9.ai", regionName: "aws-us-east-1"}, + {name: "AWS Singapore", provider: region.ProviderAWS, regionCode: "ap-southeast-1", baseURL: "https://aws-ap-southeast-1.drive9.ai", regionName: "aws-ap-southeast-1"}, + {name: "AWS Oregon", provider: region.ProviderAWS, regionCode: "us-west-2", baseURL: "https://aws-us-west-2.drive9.ai", regionName: "aws-us-west-2"}, + {name: "Alibaba Cloud Singapore", provider: region.ProviderAlibabaCloud, regionCode: "ap-southeast-1", baseURL: "https://alicloud-ap-southeast-1.drive9.ai", regionName: "alicloud-ap-southeast-1"}, + } + resolver := NewResolver() + if resolver.FSManifestURL != "" { + t.Fatalf("production resolver has external manifest URL %q", resolver.FSManifestURL) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + endpoint, err := resolver.ResolveFS(tt.provider, tt.regionCode) + if err != nil { + t.Fatalf("ResolveFS failed: %v", err) + } + if endpoint.BaseURL != tt.baseURL || endpoint.RegionName != tt.regionName { + t.Fatalf("endpoint = %#v, want base URL %q and region %q", endpoint, tt.baseURL, tt.regionName) + } + }) + } +} + func TestResolveFSUnsupportedManifestRegion(t *testing.T) { resolver := Resolver{ FSManifest: &FSRegionManifest{ @@ -171,7 +200,7 @@ func TestResolveFSUnsupportedManifestRegion(t *testing.T) { t.Fatalf("expected exit 2, got %d", got) } message := apperr.MessageFor(err) - if !strings.Contains(message, "ti fs is not available") || !strings.Contains(message, "aws/us-east-1") { + if !strings.Contains(message, "ti fs is not available") || !strings.Contains(message, "aws-us-east-1") { t.Fatalf("unexpected message: %q", message) } } @@ -202,77 +231,6 @@ func TestResolveFSFetchesManifestURL(t *testing.T) { } } -func TestResolveFSRetriesManifestFetch(t *testing.T) { - var calls int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if atomic.AddInt32(&calls, 1) == 1 { - http.Error(w, "temporary manifest failure", http.StatusInternalServerError) - return - } - _, _ = w.Write([]byte(`{ - "service": "drive9", - "regions": [ - { - "region_code": "aws-us-east-1", - "mode": "tidb_cloud_native", - "server_url": "https://aws-us-east-1.drive9.ai", - "cloud_provider": "aws", - "tidb_region": "us-east-1" - } - ] - }`)) - })) - defer server.Close() - - endpoint, err := (Resolver{FSManifestURL: server.URL}).ResolveFS(region.ProviderAWS, "us-east-1") - if err != nil { - t.Fatalf("ResolveFS failed after retry: %v", err) - } - if endpoint.BaseURL != "https://aws-us-east-1.drive9.ai" { - t.Fatalf("unexpected endpoint: %#v", endpoint) - } - if got := atomic.LoadInt32(&calls); got != 2 { - t.Fatalf("manifest calls = %d, want 2", got) - } -} - -func TestResolveFSUsesCachedManifestAfterFetchFailure(t *testing.T) { - t.Setenv("HOME", t.TempDir()) - var fail atomic.Bool - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if fail.Load() { - http.Error(w, "temporary manifest failure", http.StatusInternalServerError) - return - } - _, _ = w.Write([]byte(`{ - "service": "drive9", - "regions": [ - { - "region_code": "aws-us-east-1", - "mode": "tidb_cloud_native", - "server_url": "https://aws-us-east-1.drive9.ai", - "cloud_provider": "aws", - "tidb_region": "us-east-1" - } - ] - }`)) - })) - defer server.Close() - - resolver := Resolver{FSManifestURL: server.URL} - if _, err := resolver.ResolveFS(region.ProviderAWS, "us-east-1"); err != nil { - t.Fatalf("initial ResolveFS failed: %v", err) - } - fail.Store(true) - endpoint, err := resolver.ResolveFS(region.ProviderAWS, "us-east-1") - if err != nil { - t.Fatalf("ResolveFS did not fall back to cache: %v", err) - } - if endpoint.BaseURL != "https://aws-us-east-1.drive9.ai" { - t.Fatalf("unexpected endpoint from cache: %#v", endpoint) - } -} - func TestResolveUnsupportedRegionShowsValidRegions(t *testing.T) { _, err := NewResolver().ResolveStarter(region.ProviderAlibabaCloud, "us-east-1") if err == nil { diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 5789caa..9b819f5 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -617,7 +617,7 @@ func newDBExecuteSQLCommand(info version.Info) *cobra.Command { addSQLCredentialFlags(cmd) cmd.Flags().String("database", "", "Default database (aka. schema) name.") cmd.Flags().String("sql", "", "The SQL statement to execute.") - cmd.Flags().String("transport", "https", "SQL execution transport protocal: https or mysql.") + cmd.Flags().String("transport", "https", "SQL execution transport protocol: https or mysql.") markUsageRequired(cmd, "db-cluster-id", "sql") return cmd } @@ -1501,7 +1501,7 @@ func newFSCopyFileCommand(info version.Info) *cobra.Command { cmd.Flags().Bool("to-stdout", false, "Write --from-remote to stdout.") cmd.Flags().Bool("overwrite", false, "Replace an existing destination file.") cmd.Flags().Bool("create-parents", false, "Create missing local parent directories when copying from a TiDB Cloud file system.") - cmd.Flags().Bool("append", false, "Append a local file content to a file in the TiDB Cloudfile system.") + cmd.Flags().Bool("append", false, "Append a local file content to a file in the TiDB Cloud file system.") cmd.Flags().Bool("recursive", false, "Copy directory structure recursively.") cmd.Flags().Bool("resume", false, "Resume an active copy operation.") cmd.Flags().String("layer-id", "", "Write the copied file content into a file system layer instead of the base file system.") @@ -1871,7 +1871,7 @@ func newFSFindFilesCommand(info version.Info) *cobra.Command { cmd.Flags().String("file-name-pattern", "", "File name pattern filter, such as *.md.") cmd.Flags().String("resource-type", "", "Resource type filter: file or directory.") cmd.Flags().String("tag", "", "Tag filter.") - cmd.Flags().String("layer-id", "", "Search files and directorieswithin a specific file system layer.") + cmd.Flags().String("layer-id", "", "Search files and directories within a specific file system layer.") cmd.Flags().String("newer", "", "Only return files newer than the filter.") cmd.Flags().String("older", "", "Only return files older than the filter.") cmd.Flags().Int64("min-size-bytes", 0, "Minimum file size in bytes.") @@ -2131,7 +2131,7 @@ func newFSPackFileSystemCommand(info version.Info) *cobra.Command { }, }, info) cmd.Flags().String("local-root", "", "Local overlay root containing the overlay directory.") - cmd.Flags().String("remote-root", "/", "The TiDB Cloudfile system root represented by the local overlay.") + cmd.Flags().String("remote-root", "/", "The TiDB Cloud file system root represented by the local overlay.") cmd.Flags().String("mount-path", "", "The local mounted path.") cmd.Flags().String("mount-profile", "", "The mount profile: coding-agent, portable, or none. Default: none.") cmd.Flags().String("archive-path", "", "The path for the packed archive.") @@ -3049,7 +3049,7 @@ func newVaultCreateSecretCommand(info version.Info) *cobra.Command { func newVaultReplaceSecretCommand(info version.Info) *cobra.Command { cmd := newControlPlaneCommand(controlPlaneCommandSpec{ Use: "replace-secret", - Short: "Replace all fields in a file systemvault secret from a directory.", + Short: "Replace all fields in a file system vault secret from a directory.", Mutation: mutatingCommand, Permission: authz.FSVaultSecretUpdate, Run: func(ctx commandContext) (any, error) { @@ -3212,7 +3212,7 @@ func newVaultCreateGrantCommand(info version.Info) *cobra.Command { func newVaultDeleteGrantCommand(info version.Info) *cobra.Command { cmd := newControlPlaneCommand(controlPlaneCommandSpec{ Use: "delete-grant", - Short: "Delete a file systemvault grant.", + Short: "Delete a file system vault grant.", Mutation: mutatingCommand, Permission: authz.FSVaultGrantDelete, Run: func(ctx commandContext) (any, error) { @@ -3324,7 +3324,7 @@ func newVaultRunWithSecretCommand(info version.Info) *cobra.Command { func newVaultMountCommand(info version.Info) *cobra.Command { cmd := newControlPlaneCommand(controlPlaneCommandSpec{ Use: "mount-vault", - Short: "Mount readable file system vaule secrets as a local read-only FUSE filesystem.", + Short: "Mount readable file system vault secrets as a local read-only FUSE filesystem.", Mutation: mutatingCommand, Permission: authz.FSVaultSecretRead, Run: func(ctx commandContext) (any, error) { diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index cfc4963..fd37bcd 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/tidbcloud/ti-cli/internal/apperr" "github.com/tidbcloud/ti-cli/internal/authz" "github.com/tidbcloud/ti-cli/internal/config/store" @@ -65,6 +66,31 @@ func TestHelpCommands(t *testing.T) { } } +func TestCommandDescriptionsHaveNoKnownTypos(t *testing.T) { + root := NewRootCommand(testVersion()) + knownTypos := []string{ + "cloudfile", + "directorieswithin", + "protocal", + "systemvault", + "vaule", + } + visitCommands(root, func(command *cobra.Command) { + texts := []string{command.Short, command.Long} + command.LocalFlags().VisitAll(func(flag *pflag.Flag) { + texts = append(texts, flag.Usage) + }) + for _, value := range texts { + lower := strings.ToLower(value) + for _, typo := range knownTypos { + if strings.Contains(lower, typo) { + t.Errorf("%s contains typo %q in %q", command.CommandPath(), typo, value) + } + } + } + }) +} + func TestRootRequiresCommand(t *testing.T) { stdout, stderr, err := executeForTest() if err == nil { @@ -1108,7 +1134,7 @@ func TestRegionOverrideAllowsEnvironmentCredentialsWithoutEnvRegion(t *testing.T t.Setenv("TIDB_CLOUD_PUBLIC_KEY", "test-public") t.Setenv("TIDB_CLOUD_PRIVATE_KEY", "test-private") - stdout, _, err := executeForTest("--region", "ali-ap-southeast-1", "db", "create-db-cluster", "--db-cluster-name", "demo-cluster", "--db-cluster-type", "starter", "--dry-run") + stdout, _, err := executeForTest("--region", "alicloud-ap-southeast-1", "db", "create-db-cluster", "--db-cluster-name", "demo-cluster", "--db-cluster-type", "starter", "--dry-run") if err != nil { t.Fatalf("expected dry-run to succeed, got %v", err) } diff --git a/internal/config/region/region.go b/internal/config/region/region.go index 809b953..61983d5 100644 --- a/internal/config/region/region.go +++ b/internal/config/region/region.go @@ -10,8 +10,9 @@ const ( ProviderAWS = "aws" ProviderAlibabaCloud = "alibaba_cloud" - ProviderPrefixAWS = "aws" - ProviderPrefixAlibabaCloud = "ali" + ProviderPrefixAWS = "aws" + ProviderPrefixAlibabaCloud = "alicloud" + ProviderPrefixAlibabaCloudLegacy = "ali" ) type Region struct { @@ -42,8 +43,9 @@ var supported = map[string][]Region{ } var providerPrefixes = map[string]string{ - ProviderPrefixAWS: ProviderAWS, - ProviderPrefixAlibabaCloud: ProviderAlibabaCloud, + ProviderPrefixAWS: ProviderAWS, + ProviderPrefixAlibabaCloud: ProviderAlibabaCloud, + ProviderPrefixAlibabaCloudLegacy: ProviderAlibabaCloud, } var providerToPrefix = map[string]string{ @@ -134,8 +136,8 @@ func ParsePlacementCode(code string) (Placement, error) { } } return Placement{ - Code: prefix + "-" + nativeCode, - Prefix: prefix, + Code: providerToPrefix[provider] + "-" + nativeCode, + Prefix: providerToPrefix[provider], Provider: provider, NativeCode: nativeCode, RegionLabel: regionLabel, diff --git a/internal/config/region/region_test.go b/internal/config/region/region_test.go index e2cf6dc..3c8b1e7 100644 --- a/internal/config/region/region_test.go +++ b/internal/config/region/region_test.go @@ -46,12 +46,14 @@ func TestValidateRejectsUnsupportedProviderRegions(t *testing.T) { func TestParsePlacementCode(t *testing.T) { tests := []struct { code string + canonical string provider string nativeCode string }{ - {"aws-us-east-1", ProviderAWS, "us-east-1"}, - {"aws-ap-southeast-1", ProviderAWS, "ap-southeast-1"}, - {"ali-ap-southeast-1", ProviderAlibabaCloud, "ap-southeast-1"}, + {"aws-us-east-1", "aws-us-east-1", ProviderAWS, "us-east-1"}, + {"aws-ap-southeast-1", "aws-ap-southeast-1", ProviderAWS, "ap-southeast-1"}, + {"alicloud-ap-southeast-1", "alicloud-ap-southeast-1", ProviderAlibabaCloud, "ap-southeast-1"}, + {"ali-ap-southeast-1", "alicloud-ap-southeast-1", ProviderAlibabaCloud, "ap-southeast-1"}, } for _, tt := range tests { @@ -60,7 +62,7 @@ func TestParsePlacementCode(t *testing.T) { if err != nil { t.Fatalf("ParsePlacementCode failed: %v", err) } - if placement.Code != tt.code || placement.Provider != tt.provider || placement.NativeCode != tt.nativeCode { + if placement.Code != tt.canonical || placement.Provider != tt.provider || placement.NativeCode != tt.nativeCode { t.Fatalf("unexpected placement: %#v", placement) } }) diff --git a/internal/fs/drive9_companion.go b/internal/fs/drive9_companion.go index 900f0fa..9640b4b 100644 --- a/internal/fs/drive9_companion.go +++ b/internal/fs/drive9_companion.go @@ -1155,7 +1155,7 @@ func (s Service) drive9MountFileSystem(ctx context.Context, opts MountFileSystem args = append(args, "--no-auto-unpack") } args = append(args, drive9Remote(remotePath), opts.MountPath) - result, err := s.drive9Run(ctx, opts.Profile, args, false) + result, err := s.drive9Run(ctx, opts.Profile, args, !opts.Foreground) if err != nil { return MountResult{}, err } diff --git a/internal/fs/drive9_companion_test.go b/internal/fs/drive9_companion_test.go index 2bc1b86..902eb3f 100644 --- a/internal/fs/drive9_companion_test.go +++ b/internal/fs/drive9_companion_test.go @@ -2,6 +2,7 @@ package fs import ( "bufio" + "bytes" "context" "encoding/base64" "encoding/json" @@ -407,6 +408,49 @@ func TestDrive9MountLocatorRoutesDrainAndUnmountWithoutCredentials(t *testing.T) } } +func TestDrive9MountSuppressesCompanionSuccessChatter(t *testing.T) { + home := t.TempDir() + companion, _ := buildFakeDrive9(t) + var stdout bytes.Buffer + var stderr bytes.Buffer + service := testCompanionService(home, companion) + service.Stdout = &stdout + service.Stderr = &stderr + + result, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), + FileSystemName: "workspace", + MountPath: filepath.Join(t.TempDir(), "workspace"), + RemotePath: "/", + }) + if err != nil { + t.Fatalf("MountFileSystem failed: %v", err) + } + if stdout.Len() != 0 || stderr.Len() != 0 { + t.Fatalf("companion mount output leaked to ti output: stdout=%q stderr=%q", stdout.String(), stderr.String()) + } + if result.Driver != "webdav" || result.Status != "mounted" { + t.Fatalf("mount result did not retain captured companion state: %#v", result) + } +} + +func TestDrive9MountPreservesCapturedFailureDiagnostics(t *testing.T) { + companion, _ := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_MOUNT_FAIL", "1") + _, err := testCompanionService(t.TempDir(), companion).MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), + FileSystemName: "workspace", + MountPath: filepath.Join(t.TempDir(), "workspace"), + RemotePath: "/", + }) + if err == nil { + t.Fatal("expected mount failure") + } + if message := apperr.MessageFor(err); !strings.Contains(message, "background mount exited before becoming ready") { + t.Fatalf("mount failure lost companion diagnostics: %q", message) + } +} + func TestDrive9FailedUnmountPreservesMountLocator(t *testing.T) { home := t.TempDir() companion, recordPath := buildFakeDrive9(t) @@ -565,6 +609,15 @@ func main() { "region_code": os.Getenv("DRIVE9_REGION_CODE"), "server": os.Getenv("DRIVE9_SERVER"), }) + case args[0] == "mount" && (len(args) == 1 || args[1] != "drain"): + fmt.Fprintln(os.Stderr, "component: drive9 mount") + fmt.Fprintln(os.Stderr, "drive9: mount mode: webdav") + if os.Getenv("TI_FAKE_DRIVE9_MOUNT_FAIL") == "1" { + fmt.Fprintln(os.Stderr, "mount: drive9 mount: background mount exited before becoming ready") + os.Exit(1) + } + fmt.Fprintln(os.Stderr, "drive9: mount running in background") + fmt.Fprintln(os.Stderr, "drive9: unmount with drive9 umount /workspace") case len(args) >= 3 && args[0] == "admin" && args[1] == "tenant" && args[2] == "delete": if os.Getenv("TI_FAKE_DRIVE9_NOT_FOUND") == "1" { fmt.Fprintln(os.Stderr, "delete admin tenant: HTTP 404: tenant not found") diff --git a/internal/fs/fscred/credential.go b/internal/fs/fscred/credential.go index d456dad..14b4dad 100644 --- a/internal/fs/fscred/credential.go +++ b/internal/fs/fscred/credential.go @@ -505,8 +505,15 @@ func ResolveCredential(homeDir string, profile *config.Profile, opts ResolveCred if err != nil { return nil, Credential{}, apperr.Wrap("config.invalid_region", "config", 2, err.Error(), err) } - if found && credential.RegionCode != placement.Code { - return nil, Credential{}, apperr.New("fs.credential_region_mismatch", "config", 2, fmt.Sprintf("file system %q credentials are for %s, not %s", id, credential.RegionCode, placement.Code)) + if found { + storedPlacement, storedErr := region.ParsePlacementCode(credential.RegionCode) + if storedErr != nil { + return nil, Credential{}, apperr.Wrap("config.invalid_region", "config", 2, storedErr.Error(), storedErr) + } + if storedPlacement.Code != placement.Code { + return nil, Credential{}, apperr.New("fs.credential_region_mismatch", "config", 2, fmt.Sprintf("file system %q credentials are for %s, not %s", id, storedPlacement.Code, placement.Code)) + } + credential.RegionCode = storedPlacement.Code } metadataMatchesToken := found && subtle.ConstantTimeCompare([]byte(token), []byte(credential.APIKey)) == 1 credential.FileSystemID = id diff --git a/internal/fs/fscred/credential_test.go b/internal/fs/fscred/credential_test.go index 0557b3f..a93ff56 100644 --- a/internal/fs/fscred/credential_test.go +++ b/internal/fs/fscred/credential_test.go @@ -94,6 +94,40 @@ func TestOldCredentialWithoutTokenMetadataRemainsReadable(t *testing.T) { } } +func TestCredentialWithLegacyAlibabaRegionRemainsReadable(t *testing.T) { + home := t.TempDir() + profile := credentialTestProfile() + profile.PlacementRegionCode = "alicloud-ap-southeast-1" + profile.CloudProvider = "alibaba_cloud" + profile.RegionCode = "ap-southeast-1" + paths, err := CredentialPath(home, profile.Name, "tenant-legacy-region") + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(paths.Credentials), 0o700); err != nil { + t.Fatal(err) + } + if err := writeTOML(paths.Credentials, Credential{ + FileSystemID: "tenant-legacy-region", + RegionCode: "ali-ap-southeast-1", + APIKey: wrappedToken(t, "tenant-legacy-region"), + }, 0o600); err != nil { + t.Fatal(err) + } + + selected, credential, err := ResolveCredential(home, profile, ResolveCredentialOptions{ + FileSystemID: "tenant-legacy-region", + FileSystemIDExplicit: true, + TokenRequired: true, + }) + if err != nil { + t.Fatal(err) + } + if selected.FSPlacementRegionCode != "alicloud-ap-southeast-1" || credential.RegionCode != "alicloud-ap-southeast-1" { + t.Fatalf("legacy region was not normalized: selected=%#v credential=%#v", selected, credential) + } +} + func TestCredentialLockSerializesWriters(t *testing.T) { home := t.TempDir() ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) diff --git a/internal/telemetrybackend/event.go b/internal/telemetrybackend/event.go index c84866f..2c52b70 100644 --- a/internal/telemetrybackend/event.go +++ b/internal/telemetrybackend/event.go @@ -62,14 +62,15 @@ var disallowedFieldNames = map[string]struct{}{ } var allowedRegions = map[string]struct{}{ - "": {}, - "unknown": {}, - "aws-us-east-1": {}, - "aws-us-west-2": {}, - "aws-eu-central-1": {}, - "aws-ap-northeast-1": {}, - "aws-ap-southeast-1": {}, - "ali-ap-southeast-1": {}, + "": {}, + "unknown": {}, + "aws-us-east-1": {}, + "aws-us-west-2": {}, + "aws-eu-central-1": {}, + "aws-ap-northeast-1": {}, + "aws-ap-southeast-1": {}, + "ali-ap-southeast-1": {}, + "alicloud-ap-southeast-1": {}, } var allowedOperatingSystems = map[string]struct{}{ diff --git a/scripts/install.ps1 b/scripts/install.ps1 index c98c0e0..9babc20 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -91,26 +91,10 @@ function Print-Regions { Write-Output "" Write-Output " Config regions:" Write-Output " aws-us-east-1, aws-us-west-2, aws-eu-central-1, aws-ap-northeast-1, aws-ap-southeast-1" - Write-Output " ali-ap-southeast-1" + Write-Output " alicloud-ap-southeast-1" Write-Output "" Write-Output " ti fs regions:" - try { - $manifest = Invoke-RestMethod -Uri "https://drive9.ai/manifest/regions/drive9-regions.json" - $regions = @($manifest.regions | Where-Object { $_.mode -eq "tidb_cloud_native" } | ForEach-Object { - $prefix = $_.cloud_provider - if ($prefix -eq "alicloud" -or $prefix -eq "alibaba_cloud") { - $prefix = "ali" - } - " $prefix-$($_.tidb_region)" - } | Sort-Object -Unique) - if ($regions.Count -gt 0) { - $regions | ForEach-Object { Write-Output $_ } - return - } - } catch { - } - Write-Output " aws-us-east-1, aws-us-west-2, aws-ap-southeast-1, ali-ap-southeast-1" - Warn "Could not fetch the latest ti fs region manifest; run ti fs check-file-system after configure" + Write-Output " aws-us-east-1, aws-ap-southeast-1, aws-us-west-2, alicloud-ap-southeast-1" } function Print-NextSteps { @@ -248,7 +232,6 @@ try { Move-Item -Force -Path $CompanionPath -Destination $CompanionTarget & $Target --version Write-Output "ti installed to $Target" - Write-Output "ti fs companion installed to $CompanionTarget" Bootstrap-Config Report-PathStatus Print-Regions diff --git a/scripts/install.sh b/scripts/install.sh index 88dab7e..4e843ee 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -101,16 +101,6 @@ download() { fi } -download_quiet_stdout() { - if command -v curl >/dev/null 2>&1; then - curl -fsSL "$1" 2>/dev/null || true - elif command -v wget >/dev/null 2>&1; then - wget -q -O - "$1" 2>/dev/null || true - else - true - fi -} - case "$(uname -s)" in Darwin) OS="darwin" ;; Linux) OS="linux" ;; @@ -265,39 +255,10 @@ print_regions() { printf "\n" printf " ${BOLD}Config regions:${RESET}\n" printf " aws-us-east-1, aws-us-west-2, aws-eu-central-1, aws-ap-northeast-1, aws-ap-southeast-1\n" - printf " ali-ap-southeast-1\n" + printf " alicloud-ap-southeast-1\n" printf "\n" printf " ${BOLD}ti fs regions:${RESET}\n" - MANIFEST="$(download_quiet_stdout "https://drive9.ai/manifest/regions/drive9-regions.json")" - FS_REGIONS="$(printf "%s\n" "$MANIFEST" | awk ' - /"mode"[[:space:]]*:[[:space:]]*"tidb_cloud_native"/ { native=1 } - /"cloud_provider"[[:space:]]*:/ { - provider=$0 - sub(/^.*"cloud_provider"[[:space:]]*:[[:space:]]*"/, "", provider) - sub(/".*$/, "", provider) - } - /"tidb_region"[[:space:]]*:/ { - region=$0 - sub(/^.*"tidb_region"[[:space:]]*:[[:space:]]*"/, "", region) - sub(/".*$/, "", region) - } - /^[[:space:]]*}/ { - if (native && provider != "" && region != "") { - prefix=provider - if (prefix == "alicloud" || prefix == "alibaba_cloud") { - prefix="ali" - } - print " " prefix "-" region - } - native=0; provider=""; region="" - } - ' | sort -u)" - if [ -n "$FS_REGIONS" ]; then - printf "%s\n" "$FS_REGIONS" - else - printf " aws-us-east-1, aws-us-west-2, aws-ap-southeast-1, ali-ap-southeast-1\n" - warn "Could not fetch the latest ti fs region manifest; run ti fs check-file-system after configure" - fi + printf " aws-us-east-1, aws-ap-southeast-1, aws-us-west-2, alicloud-ap-southeast-1\n" } print_next_steps() { @@ -404,7 +365,6 @@ install_file "${TMP_DIR}/${COMPANION_ARTIFACT}" "$COMPANION_TARGET" "$TARGET" --version success "ti installed to ${TARGET}" -success "ti fs companion installed to ${COMPANION_TARGET}" bootstrap_config report_path_status print_regions