From 5aaef4971c0cbc525d911c38b9307abb78228126 Mon Sep 17 00:00:00 2001 From: "Jakub A. W" Date: Sat, 8 Aug 2026 20:18:12 +0200 Subject: [PATCH 1/4] feat(ext): add observability extension hooks --- config/metrics.go | 20 ++++ config/metrics_test.go | 18 +++ ext/ext.go | 8 +- ext/registry.go | 60 ++++++++-- ext/registry_test.go | 25 ++++ ext/upstream.go | 55 +++++++++ internal/app/app.go | 100 ++++++++++++++++ internal/app/app_test.go | 76 +++++++++++- internal/core/passthrough.go | 3 + internal/core/semantic.go | 3 + internal/core/semantic_test.go | 3 + internal/llmclient/client.go | 111 ++++++++++++++---- internal/llmclient/client_test.go | 74 ++++++++++++ internal/llmclient/hooks.go | 13 ++ internal/llmclient/hooks_test.go | 11 +- internal/llmclient/operations.go | 11 ++ internal/providers/anthropic/anthropic.go | 3 + internal/providers/anthropic/chat.go | 8 +- internal/providers/anthropic/chat_stream.go | 8 +- .../anthropic/passthrough_semantics.go | 2 +- internal/providers/anthropic/responses.go | 16 ++- internal/providers/bedrock/bedrock.go | 6 +- internal/providers/bedrock/bedrock_test.go | 41 +++++++ internal/providers/bedrock/chat.go | 8 +- internal/providers/bedrock/chat_stream.go | 10 +- internal/providers/bedrock/observation.go | 99 ++++++++++++++++ internal/providers/cohere/chat.go | 16 ++- internal/providers/cohere/cohere.go | 8 +- internal/providers/cohere/cohere_test.go | 11 +- internal/providers/cohere/embeddings.go | 8 +- .../providers/cohere/passthrough_semantics.go | 8 ++ .../cohere/passthrough_semantics_test.go | 22 ++++ internal/providers/config.go | 8 +- internal/providers/config_test.go | 3 + internal/providers/credentials.go | 1 + .../deepseek/passthrough_semantics.go | 4 +- internal/providers/factory.go | 35 +++++- internal/providers/factory_test.go | 20 +++- internal/providers/gemini/gemini.go | 40 ++++--- .../providers/kilo/passthrough_semantics.go | 2 +- internal/providers/ollama/ollama.go | 8 +- .../providers/openai/compatible_provider.go | 65 ++++++---- .../providers/openai/passthrough_semantics.go | 6 +- .../openai/passthrough_semantics_test.go | 7 ++ internal/providers/openrouter/openrouter.go | 2 +- .../openrouter/passthrough_semantics.go | 5 + .../openrouter/passthrough_semantics_test.go | 23 ++++ internal/providers/passthrough.go | 16 ++- .../providers/sglang/passthrough_semantics.go | 8 +- internal/providers/sglang/sglang.go | 3 + internal/providers/vertex/vertex.go | 8 +- internal/providers/vertex/vertex_test.go | 12 +- .../providers/vllm/passthrough_semantics.go | 8 +- internal/providers/vllm/vllm.go | 3 + .../providers/zai/passthrough_semantics.go | 5 + .../zai/passthrough_semantics_test.go | 23 ++++ internal/providers/zai/zai.go | 5 +- internal/server/handlers_test.go | 3 + internal/server/http.go | 23 ++-- internal/server/http_test.go | 5 +- .../passthrough_semantic_enrichment_test.go | 5 +- internal/server/passthrough_service.go | 3 + .../server/translated_inference_service.go | 12 +- 63 files changed, 1087 insertions(+), 149 deletions(-) create mode 100644 config/metrics_test.go create mode 100644 ext/upstream.go create mode 100644 internal/llmclient/operations.go create mode 100644 internal/providers/bedrock/observation.go create mode 100644 internal/providers/cohere/passthrough_semantics.go create mode 100644 internal/providers/cohere/passthrough_semantics_test.go create mode 100644 internal/providers/openrouter/passthrough_semantics.go create mode 100644 internal/providers/openrouter/passthrough_semantics_test.go create mode 100644 internal/providers/zai/passthrough_semantics.go create mode 100644 internal/providers/zai/passthrough_semantics_test.go diff --git a/config/metrics.go b/config/metrics.go index 2d53420ac..354dbbc0b 100644 --- a/config/metrics.go +++ b/config/metrics.go @@ -1,5 +1,10 @@ package config +import ( + "path" + "strings" +) + // MetricsConfig holds observability configuration for Prometheus metrics type MetricsConfig struct { // Enabled controls whether Prometheus metrics are collected and exposed @@ -10,3 +15,18 @@ type MetricsConfig struct { // Default: "/metrics" Endpoint string `yaml:"endpoint" env:"METRICS_ENDPOINT"` } + +// ResolveMetricsEndpoint returns the normalized, safe endpoint used by the +// HTTP server. Extensions should use the same value when excluding Prometheus +// scrapes from request instrumentation. +func ResolveMetricsEndpoint(endpoint string) string { + metricsPath := "/metrics" + if endpoint != "" { + metricsPath = path.Clean(endpoint) + } + if metricsPath == "/v1" || strings.HasPrefix(metricsPath, "/v1/") || + metricsPath == "/p" || strings.HasPrefix(metricsPath, "/p/") { + return "/metrics" + } + return metricsPath +} diff --git a/config/metrics_test.go b/config/metrics_test.go new file mode 100644 index 000000000..aeef572fa --- /dev/null +++ b/config/metrics_test.go @@ -0,0 +1,18 @@ +package config + +import "testing" + +func TestResolveMetricsEndpoint(t *testing.T) { + tests := map[string]string{ + "": "/metrics", + "/monitoring/metrics/": "/monitoring/metrics", + "/foo/../metrics-custom": "/metrics-custom", + "/v1/models": "/metrics", + "/p/internal": "/metrics", + } + for input, want := range tests { + if got := ResolveMetricsEndpoint(input); got != want { + t.Errorf("ResolveMetricsEndpoint(%q) = %q, want %q", input, got, want) + } + } +} diff --git a/ext/ext.go b/ext/ext.go index ef2bb92ea..a043ec372 100644 --- a/ext/ext.go +++ b/ext/ext.go @@ -1,9 +1,9 @@ // Package ext is the public extension API for building custom gateway // binaries on top of GoModel. External modules register request rewriters, -// HTTP middleware, extra routes, runtime settings, and a route selector on a -// Registry (usually ext.Default) before starting the gateway; core consumes -// an immutable snapshot of the registry at server construction. An empty -// registry adds zero request overhead. +// HTTP middleware, extra routes, runtime settings, upstream observers, and a +// route selector on a Registry (usually ext.Default) before startup. Core +// consumes an immutable snapshot at server construction; an empty registry +// adds zero request overhead. package ext import ( diff --git a/ext/registry.go b/ext/registry.go index 67af0b7dc..45062369c 100644 --- a/ext/registry.go +++ b/ext/registry.go @@ -11,14 +11,36 @@ import ( // Register everything before the server is constructed (before run.Run or // app.New); core snapshots each registration list during initialization. type Registry struct { - mu sync.Mutex - rewriters []RequestRewriter - middleware []echo.MiddlewareFunc - routes []func(*echo.Echo) - publicPaths []string - routeSelector RouteSelector - settings []RuntimeSetting - authenticators []RequestAuthenticator + mu sync.Mutex + rewriters []RequestRewriter + outerMiddleware []echo.MiddlewareFunc + middleware []echo.MiddlewareFunc + routes []func(*echo.Echo) + publicPaths []string + routeSelector RouteSelector + settings []RuntimeSetting + authenticators []RequestAuthenticator + observers []UpstreamObserver +} + +// UseOuterMiddleware adds middleware at the outer HTTP boundary, after +// credential-like request URI values are redacted and before request logging, +// recovery, limits, audit capture, and authentication. It is intended for +// observability and correlation middleware that must cover the whole request. +// It must not depend on an authenticated identity. +func (r *Registry) UseOuterMiddleware(m echo.MiddlewareFunc) { + r.mu.Lock() + defer r.mu.Unlock() + r.outerMiddleware = append(r.outerMiddleware, m) +} + +// RegisterUpstreamObserver adds an observer for logical provider calls. +// Observers run in registration order and may derive the context passed to +// later observers and to the provider request. +func (r *Registry) RegisterUpstreamObserver(observer UpstreamObserver) { + r.mu.Lock() + defer r.mu.Unlock() + r.observers = append(r.observers, observer) } // RegisterAuthenticator adds a request authentication mechanism. Core bearer @@ -93,6 +115,13 @@ func (r *Registry) Middleware() []echo.MiddlewareFunc { return slices.Clone(r.middleware) } +// OuterMiddleware returns a defensive copy of registered outer middleware. +func (r *Registry) OuterMiddleware() []echo.MiddlewareFunc { + r.mu.Lock() + defer r.mu.Unlock() + return slices.Clone(r.outerMiddleware) +} + // Routes returns a defensive copy of the registered route callbacks. func (r *Registry) Routes() []func(*echo.Echo) { r.mu.Lock() @@ -128,6 +157,13 @@ func (r *Registry) Authenticators() []RequestAuthenticator { return slices.Clone(r.authenticators) } +// UpstreamObservers returns a defensive copy of registered observers. +func (r *Registry) UpstreamObservers() []UpstreamObserver { + r.mu.Lock() + defer r.mu.Unlock() + return slices.Clone(r.observers) +} + // Default is the process-wide registry used by package-level helpers and, by // default, by run.Run. var Default = &Registry{} @@ -138,6 +174,9 @@ func RegisterRewriter(rw RequestRewriter) { Default.RegisterRewriter(rw) } // UseMiddleware registers middleware on the Default registry. func UseMiddleware(m echo.MiddlewareFunc) { Default.UseMiddleware(m) } +// UseOuterMiddleware registers outer HTTP middleware on the Default registry. +func UseOuterMiddleware(m echo.MiddlewareFunc) { Default.UseOuterMiddleware(m) } + // RegisterRoutes registers a route callback on the Default registry. func RegisterRoutes(fn func(e *echo.Echo)) { Default.RegisterRoutes(fn) } @@ -154,3 +193,8 @@ func RegisterSetting(setting RuntimeSetting) { Default.RegisterSetting(setting) func RegisterAuthenticator(authenticator RequestAuthenticator) { Default.RegisterAuthenticator(authenticator) } + +// RegisterUpstreamObserver registers an observer on the Default registry. +func RegisterUpstreamObserver(observer UpstreamObserver) { + Default.RegisterUpstreamObserver(observer) +} diff --git a/ext/registry_test.go b/ext/registry_test.go index e175dbef4..fd1032dbe 100644 --- a/ext/registry_test.go +++ b/ext/registry_test.go @@ -13,6 +13,14 @@ import ( type namedRewriter struct{ name string } +type namedObserver struct{ name string } + +func (o *namedObserver) Name() string { return o.name } +func (o *namedObserver) Start(ctx context.Context, _ UpstreamCall) context.Context { + return ctx +} +func (o *namedObserver) End(context.Context, UpstreamResult) {} + type namedAuthenticator struct{ name string } func (a *namedAuthenticator) Name() string { return a.name } @@ -71,9 +79,11 @@ func TestRegistrySnapshotsAreIsolated(t *testing.T) { func TestRegistryCollectsMiddlewareAndRoutes(t *testing.T) { reg := &Registry{} + reg.UseOuterMiddleware(func(next echo.HandlerFunc) echo.HandlerFunc { return next }) reg.UseMiddleware(func(next echo.HandlerFunc) echo.HandlerFunc { return next }) reg.RegisterRoutes(func(_ *echo.Echo) {}) + assert.Len(t, reg.OuterMiddleware(), 1) assert.Len(t, reg.Middleware(), 1) assert.Len(t, reg.Routes(), 1) } @@ -89,6 +99,17 @@ func TestRegistryCollectsRequestAuthenticators(t *testing.T) { assert.Len(t, snapshot, 1, "earlier snapshot must not grow") } +func TestRegistryCollectsUpstreamObservers(t *testing.T) { + reg := &Registry{} + reg.RegisterUpstreamObserver(&namedObserver{name: "otel"}) + + snapshot := reg.UpstreamObservers() + require.Len(t, snapshot, 1) + assert.Equal(t, "otel", snapshot[0].Name()) + reg.RegisterUpstreamObserver(&namedObserver{name: "other"}) + assert.Len(t, snapshot, 1, "earlier snapshot must not grow") +} + func TestRegistryCollectsRuntimeSettings(t *testing.T) { reg := &Registry{} reg.RegisterSetting(&testRuntimeSetting{value: "high"}) @@ -108,16 +129,20 @@ func TestRegistryConcurrentRegistration(t *testing.T) { for range workers { wg.Go(func() { reg.RegisterRewriter(&namedRewriter{name: "w"}) + reg.UseOuterMiddleware(func(next echo.HandlerFunc) echo.HandlerFunc { return next }) reg.UseMiddleware(func(next echo.HandlerFunc) echo.HandlerFunc { return next }) reg.AddPublicPaths("/p") + reg.RegisterUpstreamObserver(&namedObserver{name: "w"}) _ = reg.Rewriters() }) } wg.Wait() assert.Len(t, reg.Rewriters(), workers) + assert.Len(t, reg.OuterMiddleware(), workers) assert.Len(t, reg.Middleware(), workers) assert.Len(t, reg.PublicPaths(), workers) + assert.Len(t, reg.UpstreamObservers(), workers) } type namedSelector struct{ name string } diff --git a/ext/upstream.go b/ext/upstream.go new file mode 100644 index 000000000..490ed07df --- /dev/null +++ b/ext/upstream.go @@ -0,0 +1,55 @@ +package ext + +import ( + "context" + "time" +) + +// UpstreamCall describes one logical call from GoModel to a configured model +// provider. Transport retries are folded into the same call. +type UpstreamCall struct { + // Provider is the configured provider instance name. ProviderType is its + // implementation type (for example "openai" or "anthropic"). + Provider string + ProviderType string + Model string + // Operation is the semantic GenAI operation selected by the provider + // adapter (for example "chat", "generate_content", or "embeddings"). + // It is empty for calls that are not model inference operations. + Operation string + Endpoint string + Method string + Stream bool +} + +// UpstreamResult describes a completed logical provider call. For streaming +// calls completion means that the upstream stream was established, not that +// its response body was fully consumed. +type UpstreamResult struct { + UpstreamCall + StatusCode int + Duration time.Duration + Err error +} + +// UpstreamObserver observes calls to model providers without participating in +// request handling. Start may return a derived context (for example one that +// carries a trace span); the same context is passed to End and to the provider +// request. Implementations must be safe for concurrent use and should not +// block the request path. +// +// Core contains observer panics so optional instrumentation cannot fail model +// traffic. Every successful Start invocation is paired with one End call. +type UpstreamObserver interface { + Name() string + Start(ctx context.Context, call UpstreamCall) context.Context + End(ctx context.Context, result UpstreamResult) +} + +// UpstreamStreamObserver optionally observes the first response chunk of a +// successful streaming call. Duration is measured from request issuance until +// the first body read that returns bytes; calls that end without bytes are not +// reported. +type UpstreamStreamObserver interface { + FirstResponseChunk(ctx context.Context, result UpstreamResult) +} diff --git a/internal/app/app.go b/internal/app/app.go index 180e785d1..180835dc4 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -118,6 +118,7 @@ func applyExtensions(serverCfg *server.Config, extensions *ext.Registry) { return } serverCfg.RequestRewriters = extensions.Rewriters() + serverCfg.OuterMiddleware = extensions.OuterMiddleware() serverCfg.ExtraMiddleware = extensions.Middleware() serverCfg.ExtraRoutes = extensions.Routes() serverCfg.ExtraAuthSkipPaths = extensions.PublicPaths() @@ -167,6 +168,98 @@ func routeSelectorHooks(selector ext.RouteSelector) llmclient.Hooks { } } +// upstreamObserverHooks adapts the public extension observer contract to the +// internal provider client hooks. Optional observer code is isolated from the +// request path: a panic is logged with fixed metadata and the call continues. +func upstreamObserverHooks(observer ext.UpstreamObserver) llmclient.Hooks { + name := upstreamObserverLabel(observer) + hooks := llmclient.Hooks{ + OnRequestStart: func(ctx context.Context, info llmclient.RequestInfo) (next context.Context) { + next = ctx + defer func() { + if recover() != nil { + next = ctx + slog.Error("upstream observer panicked during observation", + "observer", name, "event", "call_start") + } + }() + if derived := observer.Start(ctx, upstreamCallFromRequest(info)); derived != nil { + next = derived + } + return next + }, + OnRequestEnd: func(ctx context.Context, info llmclient.ResponseInfo) { + defer func() { + if recover() != nil { + slog.Error("upstream observer panicked during observation", + "observer", name, "event", "call_end") + } + }() + observer.End(ctx, ext.UpstreamResult{ + UpstreamCall: upstreamCallFromResponse(info), + StatusCode: info.StatusCode, + Duration: info.Duration, + Err: info.Error, + }) + }, + } + streamObserver, ok := observer.(ext.UpstreamStreamObserver) + if !ok { + return hooks + } + hooks.OnStreamFirstChunk = func(ctx context.Context, info llmclient.ResponseInfo) { + defer func() { + if recover() != nil { + slog.Error("upstream observer panicked during observation", + "observer", name, "event", "first_response_chunk") + } + }() + streamObserver.FirstResponseChunk(ctx, ext.UpstreamResult{ + UpstreamCall: upstreamCallFromResponse(info), + StatusCode: info.StatusCode, + Duration: info.Duration, + Err: info.Error, + }) + } + return hooks +} + +func upstreamCallFromRequest(info llmclient.RequestInfo) ext.UpstreamCall { + return ext.UpstreamCall{ + Provider: info.Provider, + ProviderType: info.ProviderType, + Model: info.Model, + Operation: info.Operation, + Endpoint: info.Endpoint, + Method: info.Method, + Stream: info.Stream, + } +} + +func upstreamCallFromResponse(info llmclient.ResponseInfo) ext.UpstreamCall { + return ext.UpstreamCall{ + Provider: info.Provider, + ProviderType: info.ProviderType, + Model: info.Model, + Operation: info.Operation, + Endpoint: info.Endpoint, + Method: info.Method, + Stream: info.Stream, + } +} + +func upstreamObserverLabel(observer ext.UpstreamObserver) (name string) { + if observer == nil { + return "unknown" + } + defer func() { + if recover() != nil || name == "" { + name = "unknown" + } + }() + return observer.Name() +} + func routeAffinityContext(ctx context.Context) (source, sessionID string) { sessionID = core.SessionIDFromContext(ctx) workflow := core.GetWorkflow(ctx) @@ -290,6 +383,13 @@ func New(ctx context.Context, cfg Config) (*App, error) { if routeSelector != nil { cfg.Factory.AddHooks(routeSelectorHooks(routeSelector)) } + if cfg.Extensions != nil { + for _, observer := range cfg.Extensions.UpstreamObservers() { + if observer != nil { + cfg.Factory.AddHooks(upstreamObserverHooks(observer)) + } + } + } providerResult, err := providers.Init(ctx, cfg.AppConfig, cfg.Factory) if err != nil { diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 25b96119a..75069758a 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -29,6 +29,76 @@ type routeObservationSelector struct { outcome ext.RouteOutcome } +type upstreamObservation struct { + call ext.UpstreamCall + result ext.UpstreamResult + firstChunk ext.UpstreamResult +} + +func (*upstreamObservation) Name() string { return "test" } +func (o *upstreamObservation) Start(ctx context.Context, call ext.UpstreamCall) context.Context { + o.call = call + return context.WithValue(ctx, upstreamContextKey{}, "derived") +} +func (o *upstreamObservation) End(_ context.Context, result ext.UpstreamResult) { + o.result = result +} +func (o *upstreamObservation) FirstResponseChunk(_ context.Context, result ext.UpstreamResult) { + o.firstChunk = result +} + +type upstreamContextKey struct{} + +func TestUpstreamObserverHooksExposeProviderCall(t *testing.T) { + observer := &upstreamObservation{} + hooks := upstreamObserverHooks(observer) + ctx := hooks.OnRequestStart(t.Context(), llmclient.RequestInfo{ + Provider: "openai-eu", ProviderType: "openai", Model: "gpt-5", Operation: llmclient.OperationChat, Endpoint: "/chat/completions", Method: http.MethodPost, Stream: true, + }) + if got := ctx.Value(upstreamContextKey{}); got != "derived" { + t.Fatalf("derived context value = %v, want derived", got) + } + hooks.OnRequestEnd(ctx, llmclient.ResponseInfo{ + Provider: "openai-eu", ProviderType: "openai", Model: "gpt-5", Operation: llmclient.OperationChat, Endpoint: "/chat/completions", + Method: http.MethodPost, StatusCode: http.StatusOK, Duration: time.Second, Stream: true, + }) + hooks.OnStreamFirstChunk(ctx, llmclient.ResponseInfo{ + Provider: "openai-eu", ProviderType: "openai", Model: "gpt-5", Operation: llmclient.OperationChat, + Endpoint: "/chat/completions", Method: http.MethodPost, StatusCode: http.StatusOK, Duration: 2 * time.Second, Stream: true, + }) + + if observer.call.ProviderType != "openai" || observer.call.Method != http.MethodPost || !observer.call.Stream { + t.Fatalf("call = %+v, want POST streaming call", observer.call) + } + if observer.result.StatusCode != http.StatusOK || observer.result.Duration != time.Second || !observer.result.Stream { + t.Fatalf("result = %+v, want successful one-second streaming result", observer.result) + } + if observer.call.Operation != llmclient.OperationChat || observer.firstChunk.Duration != 2*time.Second { + t.Fatalf("operation/first chunk = %q/%v, want chat/2s", observer.call.Operation, observer.firstChunk.Duration) + } +} + +type panickingUpstreamObserver struct{} + +func (*panickingUpstreamObserver) Name() string { panic("name") } +func (*panickingUpstreamObserver) Start(context.Context, ext.UpstreamCall) context.Context { + panic("start") +} +func (*panickingUpstreamObserver) End(context.Context, ext.UpstreamResult) { panic("end") } +func (*panickingUpstreamObserver) FirstResponseChunk(context.Context, ext.UpstreamResult) { + panic("first chunk") +} + +func TestUpstreamObserverHooksContainExtensionPanics(t *testing.T) { + hooks := upstreamObserverHooks(&panickingUpstreamObserver{}) + ctx := t.Context() + if got := hooks.OnRequestStart(ctx, llmclient.RequestInfo{}); got != ctx { + t.Fatal("panicking observer must preserve the original context") + } + hooks.OnRequestEnd(ctx, llmclient.ResponseInfo{}) + hooks.OnStreamFirstChunk(ctx, llmclient.ResponseInfo{}) +} + func (*routeObservationSelector) Name() string { return "observer" } func (*routeObservationSelector) Select(ext.RouteRequest) (string, bool) { return "", false } func (*routeObservationSelector) OnAttemptStart(ext.RouteTarget) {} @@ -697,6 +767,7 @@ func TestUsagePricingRecalculationConfigured(t *testing.T) { func TestApplyExtensionsSnapshotsRegistryIntoServerConfig(t *testing.T) { reg := &ext.Registry{} reg.RegisterRewriter(&staticRewriter{name: "r1"}) + reg.UseOuterMiddleware(func(next echo.HandlerFunc) echo.HandlerFunc { return next }) reg.UseMiddleware(func(next echo.HandlerFunc) echo.HandlerFunc { return next }) reg.RegisterRoutes(func(_ *echo.Echo) {}) reg.AddPublicPaths("/sso/callback", "/sso/*") @@ -708,6 +779,9 @@ func TestApplyExtensionsSnapshotsRegistryIntoServerConfig(t *testing.T) { if len(serverCfg.RequestRewriters) != 1 || serverCfg.RequestRewriters[0].Name() != "r1" { t.Errorf("RequestRewriters not copied: %+v", serverCfg.RequestRewriters) } + if len(serverCfg.OuterMiddleware) != 1 { + t.Errorf("OuterMiddleware not copied: %d entries", len(serverCfg.OuterMiddleware)) + } if len(serverCfg.ExtraMiddleware) != 1 { t.Errorf("ExtraMiddleware not copied: %d entries", len(serverCfg.ExtraMiddleware)) } @@ -724,7 +798,7 @@ func TestApplyExtensionsSnapshotsRegistryIntoServerConfig(t *testing.T) { // A nil registry must leave the config untouched. empty := &server.Config{} applyExtensions(empty, nil) - if empty.RequestRewriters != nil || empty.ExtraMiddleware != nil || empty.ExtraRoutes != nil || empty.ExtraAuthSkipPaths != nil || empty.RequestAuthenticators != nil { + if empty.RequestRewriters != nil || empty.OuterMiddleware != nil || empty.ExtraMiddleware != nil || empty.ExtraRoutes != nil || empty.ExtraAuthSkipPaths != nil || empty.RequestAuthenticators != nil { t.Error("nil registry must not modify server config") } } diff --git a/internal/core/passthrough.go b/internal/core/passthrough.go index a31b741df..27a830884 100644 --- a/internal/core/passthrough.go +++ b/internal/core/passthrough.go @@ -10,6 +10,9 @@ import ( type PassthroughRequest struct { Method string Endpoint string + Operation string // optional semantic GenAI operation derived at ingress + Model string // optional model derived from the opaque request body + Stream bool // explicit streaming intent derived from the request body Body io.ReadCloser Headers http.Header ProviderName string // optional: concrete configured provider instance name for name-based routing diff --git a/internal/core/semantic.go b/internal/core/semantic.go index 6cb122682..e38ce3184 100644 --- a/internal/core/semantic.go +++ b/internal/core/semantic.go @@ -42,6 +42,8 @@ type PassthroughRouteInfo struct { RawEndpoint string NormalizedEndpoint string SemanticOperation string + GenAIOperation string // standard GenAI operation, if this is an inference call + Stream bool // explicit streaming intent derived from the request body AuditPath string Model string } @@ -280,6 +282,7 @@ func ApplyBodySelectorHints(env *WhiteBoxPrompt, model, provider string, stream if model != "" { cloned.Model = model } + cloned.Stream = stream CachePassthroughRouteInfo(env, &cloned) } } diff --git a/internal/core/semantic_test.go b/internal/core/semantic_test.go index e2599e145..302f1b2a8 100644 --- a/internal/core/semantic_test.go +++ b/internal/core/semantic_test.go @@ -133,6 +133,9 @@ func TestDeriveWhiteBoxPrompt_PassthroughRouteParams(t *testing.T) { if info.Model != "gpt-5-mini" { t.Fatalf("PassthroughRouteInfo.Model = %q, want gpt-5-mini", info.Model) } + if !info.Stream { + t.Fatal("PassthroughRouteInfo.Stream = false, want true") + } if info.AuditPath != "/p/openai/responses" { t.Fatalf("PassthroughRouteInfo.AuditPath = %q, want /p/openai/responses", info.AuditPath) } diff --git a/internal/llmclient/client.go b/internal/llmclient/client.go index 8895aa962..fd1ba704e 100644 --- a/internal/llmclient/client.go +++ b/internal/llmclient/client.go @@ -28,22 +28,27 @@ import ( // RequestInfo contains metadata about a request for observability hooks type RequestInfo struct { - Provider string // Provider name (e.g., "openai", "anthropic") - Model string // Model name (e.g., "gpt-4", "claude-3-opus") - Endpoint string // API endpoint (e.g., "/chat/completions", "/models") - Method string // HTTP method (e.g., "POST", "GET") - Stream bool // Whether this is a streaming request + Provider string // Configured provider name + ProviderType string // Provider implementation type (e.g., "openai", "anthropic") + Model string // Model name (e.g., "gpt-4", "claude-3-opus") + Operation string // Semantic GenAI operation; empty for non-inference calls + Endpoint string // API endpoint (e.g., "/chat/completions", "/models") + Method string // HTTP method (e.g., "POST", "GET") + Stream bool // Whether this is a streaming request } // ResponseInfo contains metadata about a response for observability hooks type ResponseInfo struct { - Provider string // Provider name - Model string // Model name - Endpoint string // API endpoint - StatusCode int // HTTP status code (0 if network error) - Duration time.Duration // Request duration - Stream bool // Whether this was a streaming request - Error error // Error if request failed (nil on success) + Provider string // Configured provider name + ProviderType string // Provider implementation type + Model string // Model name + Operation string // Semantic GenAI operation + Endpoint string // API endpoint + Method string // HTTP method + StatusCode int // HTTP status code (0 if network error) + Duration time.Duration // Request duration + Stream bool // Whether this was a streaming request + Error error // Error if request failed (nil on success) // CircuitState is the provider's circuit breaker state after this request // completed ("closed", "half-open", "open"); empty when the breaker is // disabled. It reflects the moment of completion, so metrics built from it @@ -61,6 +66,10 @@ type Hooks struct { // OnRequestEnd is called after a request completes (success or failure). // For streaming requests, this is called when the stream starts, not when it closes. OnRequestEnd func(ctx context.Context, info ResponseInfo) + + // OnStreamFirstChunk is called once when a successful streaming response + // body first returns bytes. It is not called for empty or unread streams. + OnStreamFirstChunk func(ctx context.Context, info ResponseInfo) } // Config holds configuration for the LLM client @@ -144,8 +153,13 @@ func (c *Client) BaseURL() string { type Request struct { Method string Endpoint string - Body any // Will be JSON marshaled if not nil - RawBody []byte // Used as-is (e.g., multipart form bodies). Mutually exclusive with Body and RawBodyReader. + Model string + // Operation explicitly identifies model inference semantics for + // observability. Leave empty for control-plane and other non-inference calls. + Operation string + Stream bool // explicit stream intent; Accept: text/event-stream remains a fallback + Body any // Will be JSON marshaled if not nil + RawBody []byte // Used as-is (e.g., multipart form bodies). Mutually exclusive with Body and RawBodyReader. // RawBodyReader streams the request body without buffering it in memory. // It is intended for one-shot passthrough requests and is not replayable for retries. RawBodyReader io.Reader @@ -195,11 +209,12 @@ func (c *Client) beginRequest(ctx context.Context, req Request, stream bool) (re ctx: ctx, startedAt: time.Now(), requestInfo: RequestInfo{ - Provider: c.config.ProviderName, - Model: extractModel(req.Body), - Endpoint: req.Endpoint, - Method: req.Method, - Stream: stream, + Provider: c.config.ProviderName, + Model: requestModel(req), + Operation: req.Operation, + Endpoint: req.Endpoint, + Method: req.Method, + Stream: stream, }, } @@ -221,6 +236,13 @@ func (c *Client) beginRequest(ctx context.Context, req Request, stream bool) (re return scope, nil } +func requestModel(req Request) string { + if model := strings.TrimSpace(req.Model); model != "" { + return model + } + return extractModel(req.Body) +} + func (c *Client) finishRequest(scope requestScope, statusCode int, err error) { if c.config.Hooks.OnRequestEnd == nil { return @@ -231,8 +253,11 @@ func (c *Client) finishRequest(scope requestScope, statusCode int, err error) { } c.config.Hooks.OnRequestEnd(scope.ctx, ResponseInfo{ Provider: c.config.ProviderName, + ProviderType: scope.requestInfo.ProviderType, Model: scope.requestInfo.Model, + Operation: scope.requestInfo.Operation, Endpoint: scope.requestInfo.Endpoint, + Method: scope.requestInfo.Method, StatusCode: statusCode, Duration: time.Since(scope.startedAt), Stream: scope.requestInfo.Stream, @@ -241,6 +266,49 @@ func (c *Client) finishRequest(scope requestScope, statusCode int, err error) { }) } +func (c *Client) finishStreamFirstChunk(scope requestScope, statusCode int) { + if c.config.Hooks.OnStreamFirstChunk == nil { + return + } + c.config.Hooks.OnStreamFirstChunk(scope.ctx, ResponseInfo{ + Provider: c.config.ProviderName, + ProviderType: scope.requestInfo.ProviderType, + Model: scope.requestInfo.Model, + Operation: scope.requestInfo.Operation, + Endpoint: scope.requestInfo.Endpoint, + Method: scope.requestInfo.Method, + StatusCode: statusCode, + Duration: time.Since(scope.startedAt), + Stream: true, + }) +} + +func (c *Client) observeFirstChunk(scope requestScope, resp *http.Response) { + if resp == nil || resp.Body == nil || !scope.requestInfo.Stream { + return + } + resp.Body = &firstChunkReadCloser{ + ReadCloser: resp.Body, + onFirstChunk: func() { + c.finishStreamFirstChunk(scope, resp.StatusCode) + }, + } +} + +type firstChunkReadCloser struct { + io.ReadCloser + once sync.Once + onFirstChunk func() +} + +func (r *firstChunkReadCloser) Read(p []byte) (int, error) { + n, err := r.ReadCloser.Read(p) + if n > 0 { + r.once.Do(r.onFirstChunk) + } + return n, err +} + // completeScope is the standard terminal step for a request that has passed // beginRequest. It records the circuit-breaker outcome (using cbErr to decide // whether the failure was transport-level) and emits the metrics observation. @@ -508,6 +576,7 @@ func (c *Client) DoStream(ctx context.Context, req Request) (io.ReadCloser, erro } c.completeScope(scope, resp.StatusCode, nil, nil) + c.observeFirstChunk(scope, resp) return resp.Body, nil } @@ -543,7 +612,7 @@ func hasIdempotencyKey(headers http.Header) bool { // DoPassthrough executes a request and returns the raw upstream HTTP response. // Unlike DoRaw, it preserves non-200 responses for the caller to proxy unchanged. func (c *Client) DoPassthrough(ctx context.Context, req Request) (*http.Response, error) { - stream := strings.Contains(strings.ToLower(strings.Join(req.Headers.Values("Accept"), ",")), "text/event-stream") + stream := req.Stream || strings.Contains(strings.ToLower(strings.Join(req.Headers.Values("Accept"), ",")), "text/event-stream") scope, err := c.beginRequest(ctx, req, stream) if err != nil { closeRawBodyReader(req) @@ -583,6 +652,7 @@ func (c *Client) DoPassthrough(ctx context.Context, req Request) (*http.Response if retryable { if scope.halfOpenProbe || attempt == maxAttempts-1 { c.completeScope(scope, resp.StatusCode, nil, nil) + c.observeFirstChunk(scope, resp) return resp, nil } _ = resp.Body.Close() @@ -590,6 +660,7 @@ func (c *Client) DoPassthrough(ctx context.Context, req Request) (*http.Response } c.completeScope(scope, resp.StatusCode, nil, nil) + c.observeFirstChunk(scope, resp) return resp, nil } diff --git a/internal/llmclient/client_test.go b/internal/llmclient/client_test.go index 03b1d4219..9d7e8ffe1 100644 --- a/internal/llmclient/client_test.go +++ b/internal/llmclient/client_test.go @@ -686,6 +686,80 @@ func TestClient_DoStream_Success(t *testing.T) { } } +func TestClient_DoPassthrough_FirstChunkHookUsesOpaqueStreamBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("data: passthrough\n\n")) + })) + defer server.Close() + + var firstChunk ResponseInfo + cfg := DefaultConfig("test", server.URL) + cfg.Hooks.OnStreamFirstChunk = func(_ context.Context, info ResponseInfo) { firstChunk = info } + client := New(cfg, nil) + resp, err := client.DoPassthrough(t.Context(), Request{ + Method: http.MethodPost, + Endpoint: "/v2/chat", + Operation: OperationChat, + Model: "command-r", + Stream: true, + }) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if firstChunk.Duration != 0 { + t.Fatal("first chunk hook fired at passthrough response headers") + } + if _, err := io.ReadAll(resp.Body); err != nil { + t.Fatal(err) + } + if firstChunk.Operation != OperationChat || firstChunk.Model != "command-r" || !firstChunk.Stream { + t.Fatalf("first chunk = %+v, want streaming command-r chat", firstChunk) + } +} + +func TestClient_DoStream_FirstChunkHookWaitsForBodyBytes(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("data: first\n\n")) + })) + defer server.Close() + + var firstChunks []ResponseInfo + cfg := DefaultConfig("test", server.URL) + cfg.Hooks.OnStreamFirstChunk = func(_ context.Context, info ResponseInfo) { + firstChunks = append(firstChunks, info) + } + client := New(cfg, nil) + stream, err := client.DoStream(t.Context(), Request{ + Method: http.MethodPost, + Endpoint: "/stream", + Operation: OperationChat, + Body: map[string]bool{"stream": true}, + }) + if err != nil { + t.Fatal(err) + } + defer stream.Close() + if len(firstChunks) != 0 { + t.Fatalf("first chunk hook fired at response headers: %+v", firstChunks) + } + buf := make([]byte, 1) + if _, err := stream.Read(buf); err != nil { + t.Fatal(err) + } + if len(firstChunks) != 1 || firstChunks[0].Operation != OperationChat || !firstChunks[0].Stream { + t.Fatalf("first chunk observations = %+v, want one streaming chat observation", firstChunks) + } + if _, err := io.ReadAll(stream); err != nil { + t.Fatal(err) + } + if len(firstChunks) != 1 { + t.Fatalf("first chunk hook fired %d times, want once", len(firstChunks)) + } +} + func TestClient_DoStream_Error(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusUnauthorized) diff --git a/internal/llmclient/hooks.go b/internal/llmclient/hooks.go index 4b29a4a5b..2b9d0f59d 100644 --- a/internal/llmclient/hooks.go +++ b/internal/llmclient/hooks.go @@ -8,6 +8,7 @@ import "context" func JoinHooks(hooks ...Hooks) Hooks { var starts []func(ctx context.Context, info RequestInfo) context.Context var ends []func(ctx context.Context, info ResponseInfo) + var firstChunks []func(ctx context.Context, info ResponseInfo) for _, h := range hooks { if h.OnRequestStart != nil { starts = append(starts, h.OnRequestStart) @@ -15,6 +16,9 @@ func JoinHooks(hooks ...Hooks) Hooks { if h.OnRequestEnd != nil { ends = append(ends, h.OnRequestEnd) } + if h.OnStreamFirstChunk != nil { + firstChunks = append(firstChunks, h.OnStreamFirstChunk) + } } joined := Hooks{} @@ -37,5 +41,14 @@ func JoinHooks(hooks ...Hooks) Hooks { } } } + if len(firstChunks) == 1 { + joined.OnStreamFirstChunk = firstChunks[0] + } else if len(firstChunks) > 1 { + joined.OnStreamFirstChunk = func(ctx context.Context, info ResponseInfo) { + for _, firstChunk := range firstChunks { + firstChunk(ctx, info) + } + } + } return joined } diff --git a/internal/llmclient/hooks_test.go b/internal/llmclient/hooks_test.go index d75f0513a..732d06775 100644 --- a/internal/llmclient/hooks_test.go +++ b/internal/llmclient/hooks_test.go @@ -17,6 +17,9 @@ func TestJoinHooksChainsCallbacks(t *testing.T) { OnRequestEnd: func(_ context.Context, _ ResponseInfo) { order = append(order, "end-1") }, + OnStreamFirstChunk: func(_ context.Context, _ ResponseInfo) { + order = append(order, "chunk-1") + }, } second := Hooks{ OnRequestStart: func(ctx context.Context, _ RequestInfo) context.Context { @@ -29,13 +32,17 @@ func TestJoinHooksChainsCallbacks(t *testing.T) { OnRequestEnd: func(_ context.Context, _ ResponseInfo) { order = append(order, "end-2") }, + OnStreamFirstChunk: func(_ context.Context, _ ResponseInfo) { + order = append(order, "chunk-2") + }, } joined := JoinHooks(first, Hooks{}, second) ctx := joined.OnRequestStart(t.Context(), RequestInfo{}) joined.OnRequestEnd(ctx, ResponseInfo{}) + joined.OnStreamFirstChunk(ctx, ResponseInfo{}) - want := []string{"start-1", "start-2", "end-1", "end-2"} + want := []string{"start-1", "start-2", "end-1", "end-2", "chunk-1", "chunk-2"} if len(order) != len(want) { t.Fatalf("callback order = %v, want %v", order, want) } @@ -48,7 +55,7 @@ func TestJoinHooksChainsCallbacks(t *testing.T) { func TestJoinHooksEmpty(t *testing.T) { joined := JoinHooks(Hooks{}, Hooks{}) - if joined.OnRequestStart != nil || joined.OnRequestEnd != nil { + if joined.OnRequestStart != nil || joined.OnRequestEnd != nil || joined.OnStreamFirstChunk != nil { t.Fatalf("JoinHooks of empty hooks should have nil callbacks") } } diff --git a/internal/llmclient/operations.go b/internal/llmclient/operations.go new file mode 100644 index 000000000..d5453e56f --- /dev/null +++ b/internal/llmclient/operations.go @@ -0,0 +1,11 @@ +package llmclient + +// Well-known GenAI operation names carried through observability hooks. +// Provider adapters select these explicitly so instrumentation never has to +// infer semantics from provider-specific URL shapes. +const ( + OperationChat = "chat" + OperationGenerateContent = "generate_content" + OperationTextCompletion = "text_completion" + OperationEmbeddings = "embeddings" +) diff --git a/internal/providers/anthropic/anthropic.go b/internal/providers/anthropic/anthropic.go index 818328faf..ad3b3e416 100644 --- a/internal/providers/anthropic/anthropic.go +++ b/internal/providers/anthropic/anthropic.go @@ -176,6 +176,9 @@ func (p *Provider) Passthrough(ctx context.Context, req *core.PassthroughRequest resp, err := p.client.DoPassthrough(ctx, llmclient.Request{ Method: req.Method, Endpoint: providers.PassthroughEndpoint(req.Endpoint), + Operation: req.Operation, + Model: req.Model, + Stream: req.Stream, RawBodyReader: req.Body, Headers: req.Headers, }) diff --git a/internal/providers/anthropic/chat.go b/internal/providers/anthropic/chat.go index 81caa20dc..f4a17cb52 100644 --- a/internal/providers/anthropic/chat.go +++ b/internal/providers/anthropic/chat.go @@ -75,9 +75,11 @@ func (p *Provider) ChatCompletion(ctx context.Context, req *core.ChatRequest) (* var anthropicResp anthropicResponse err = p.client.Do(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/messages", - Body: anthropicReq, + Method: http.MethodPost, + Endpoint: "/messages", + Operation: llmclient.OperationChat, + Model: req.Model, + Body: anthropicReq, }, &anthropicResp) if err != nil { return nil, err diff --git a/internal/providers/anthropic/chat_stream.go b/internal/providers/anthropic/chat_stream.go index da3d5be78..6fb88ebfe 100644 --- a/internal/providers/anthropic/chat_stream.go +++ b/internal/providers/anthropic/chat_stream.go @@ -23,9 +23,11 @@ func (p *Provider) StreamChatCompletion(ctx context.Context, req *core.ChatReque anthropicReq.Stream = true stream, err := p.client.DoStream(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/messages", - Body: anthropicReq, + Method: http.MethodPost, + Endpoint: "/messages", + Operation: llmclient.OperationChat, + Model: req.Model, + Body: anthropicReq, }) if err != nil { return nil, err diff --git a/internal/providers/anthropic/passthrough_semantics.go b/internal/providers/anthropic/passthrough_semantics.go index 623a9fc28..5e720ec80 100644 --- a/internal/providers/anthropic/passthrough_semantics.go +++ b/internal/providers/anthropic/passthrough_semantics.go @@ -3,6 +3,6 @@ package anthropic import "github.com/enterpilot/gomodel/internal/providers" var passthroughSemanticEnricher = providers.NewSemanticEnricher("anthropic", map[string]providers.PassthroughEndpointSemantics{ - "/messages": {Operation: "anthropic.messages", AuditPath: "/v1/messages"}, + "/messages": {Operation: "anthropic.messages", GenAIOperation: "chat", AuditPath: "/v1/messages"}, "/messages/batches": {Operation: "anthropic.messages_batches", AuditPath: "/v1/messages/batches"}, }) diff --git a/internal/providers/anthropic/responses.go b/internal/providers/anthropic/responses.go index 21a50f950..5b6b20c8c 100644 --- a/internal/providers/anthropic/responses.go +++ b/internal/providers/anthropic/responses.go @@ -82,9 +82,11 @@ func (p *Provider) Responses(ctx context.Context, req *core.ResponsesRequest) (* var anthropicResp anthropicResponse err = p.client.Do(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/messages", - Body: anthropicReq, + Method: http.MethodPost, + Endpoint: "/messages", + Operation: llmclient.OperationChat, + Model: req.Model, + Body: anthropicReq, }, &anthropicResp) if err != nil { return nil, err @@ -102,9 +104,11 @@ func (p *Provider) StreamResponses(ctx context.Context, req *core.ResponsesReque anthropicReq.Stream = true stream, err := p.client.DoStream(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/messages", - Body: anthropicReq, + Method: http.MethodPost, + Endpoint: "/messages", + Operation: llmclient.OperationChat, + Model: req.Model, + Body: anthropicReq, }) if err != nil { return nil, err diff --git a/internal/providers/bedrock/bedrock.go b/internal/providers/bedrock/bedrock.go index d31efca0c..971a0eac7 100644 --- a/internal/providers/bedrock/bedrock.go +++ b/internal/providers/bedrock/bedrock.go @@ -29,6 +29,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/bedrockruntime" "github.com/enterpilot/gomodel/internal/core" + "github.com/enterpilot/gomodel/internal/llmclient" "github.com/enterpilot/gomodel/internal/providers" ) @@ -54,6 +55,7 @@ type Provider struct { region string runtime *bedrockruntime.Client control *bedrock.Client + hooks llmclient.Hooks configErr error } @@ -62,8 +64,8 @@ type Provider struct { // BaseURL is interpreted as either an AWS region ("us-east-1") or a fully // qualified endpoint URL. When empty, the region is resolved from the // standard AWS environment variables / shared config. -func New(providerCfg providers.ProviderConfig, _ providers.ProviderOptions) core.Provider { - p := &Provider{} +func New(providerCfg providers.ProviderConfig, opts providers.ProviderOptions) core.Provider { + p := &Provider{hooks: opts.Hooks} region, endpoint := parseBaseURL(providerCfg.BaseURL) loadOpts := []func(*awsconfig.LoadOptions) error{} diff --git a/internal/providers/bedrock/bedrock_test.go b/internal/providers/bedrock/bedrock_test.go index 8d4595c76..9a02026da 100644 --- a/internal/providers/bedrock/bedrock_test.go +++ b/internal/providers/bedrock/bedrock_test.go @@ -1,8 +1,11 @@ package bedrock import ( + "context" "encoding/json" "errors" + "io" + "net/http" "strings" "testing" @@ -11,9 +14,47 @@ import ( brtypes "github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types" "github.com/enterpilot/gomodel/internal/core" + "github.com/enterpilot/gomodel/internal/llmclient" "github.com/enterpilot/gomodel/internal/providers" ) +func TestCallObservationCoversBedrockSDKAndFirstChunk(t *testing.T) { + var starts []llmclient.RequestInfo + var ends, chunks []llmclient.ResponseInfo + type contextKey struct{} + p := &Provider{hooks: llmclient.Hooks{ + OnRequestStart: func(ctx context.Context, info llmclient.RequestInfo) context.Context { + starts = append(starts, info) + return context.WithValue(ctx, contextKey{}, true) + }, + OnRequestEnd: func(_ context.Context, info llmclient.ResponseInfo) { + ends = append(ends, info) + }, + OnStreamFirstChunk: func(ctx context.Context, info llmclient.ResponseInfo) { + if ctx.Value(contextKey{}) != true { + t.Error("first chunk hook did not receive derived context") + } + chunks = append(chunks, info) + }, + }} + + observation := p.beginCallObservation(t.Context(), "anthropic.claude", true) + observation.end(http.StatusOK, nil) + stream := observedStream(io.NopCloser(strings.NewReader("data: first\n\n")), observation) + if len(chunks) != 0 { + t.Fatal("first chunk hook fired before stream read") + } + if _, err := io.ReadAll(stream); err != nil { + t.Fatal(err) + } + if len(starts) != 1 || len(ends) != 1 || len(chunks) != 1 { + t.Fatalf("hook counts = start:%d end:%d chunk:%d, want 1/1/1", len(starts), len(ends), len(chunks)) + } + if starts[0].Operation != llmclient.OperationChat || starts[0].Endpoint != converseEndpoint || !starts[0].Stream { + t.Fatalf("start info = %+v, want streaming Bedrock chat", starts[0]) + } +} + func TestParseBaseURL(t *testing.T) { cases := []struct { name string diff --git a/internal/providers/bedrock/chat.go b/internal/providers/bedrock/chat.go index 60bd1a018..01029df63 100644 --- a/internal/providers/bedrock/chat.go +++ b/internal/providers/bedrock/chat.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math" + "net/http" "strings" "time" @@ -32,6 +33,8 @@ func (p *Provider) ChatCompletion(ctx context.Context, req *core.ChatRequest) (* if err != nil { return nil, err } + observation := p.beginCallObservation(ctx, req.Model, false) + ctx = observation.ctx out, err := p.runtime.Converse(ctx, converseInput(parts)) if err != nil && partsHaveCachePoints(parts) && isCachePointValidationError(err) { @@ -39,8 +42,11 @@ func (p *Provider) ChatCompletion(ctx context.Context, req *core.ChatRequest) (* out, err = p.runtime.Converse(ctx, converseInput(parts)) } if err != nil { - return nil, mapAWSError(err) + mappedErr := mapAWSError(err) + observation.end(statusCodeFromError(mappedErr), mappedErr) + return nil, mappedErr } + observation.end(http.StatusOK, nil) return convertConverseOutput(req.Model, out), nil } diff --git a/internal/providers/bedrock/chat_stream.go b/internal/providers/bedrock/chat_stream.go index 085230435..19b92c0d9 100644 --- a/internal/providers/bedrock/chat_stream.go +++ b/internal/providers/bedrock/chat_stream.go @@ -3,6 +3,7 @@ package bedrock import ( "context" "io" + "net/http" "strconv" "sync" "time" @@ -28,6 +29,8 @@ func (p *Provider) StreamChatCompletion(ctx context.Context, req *core.ChatReque if err != nil { return nil, err } + observation := p.beginCallObservation(ctx, req.Model, true) + ctx = observation.ctx out, err := p.runtime.ConverseStream(ctx, converseStreamInput(parts)) if err != nil && partsHaveCachePoints(parts) && isCachePointValidationError(err) { @@ -35,10 +38,13 @@ func (p *Provider) StreamChatCompletion(ctx context.Context, req *core.ChatReque out, err = p.runtime.ConverseStream(ctx, converseStreamInput(parts)) } if err != nil { - return nil, mapAWSError(err) + mappedErr := mapAWSError(err) + observation.end(statusCodeFromError(mappedErr), mappedErr) + return nil, mappedErr } + observation.end(http.StatusOK, nil) - return newOpenAIStream(out, req.Model), nil + return observedStream(newOpenAIStream(out, req.Model), observation), nil } func converseStreamInput(parts converseParts) *bedrockruntime.ConverseStreamInput { diff --git a/internal/providers/bedrock/observation.go b/internal/providers/bedrock/observation.go new file mode 100644 index 000000000..ed21444e2 --- /dev/null +++ b/internal/providers/bedrock/observation.go @@ -0,0 +1,99 @@ +package bedrock + +import ( + "context" + "io" + "net/http" + "sync" + "time" + + "github.com/enterpilot/gomodel/internal/llmclient" +) + +const converseEndpoint = "Converse" + +type callObservation struct { + ctx context.Context + startedAt time.Time + info llmclient.RequestInfo + hooks llmclient.Hooks +} + +func (p *Provider) beginCallObservation(ctx context.Context, model string, stream bool) callObservation { + observation := callObservation{ + ctx: ctx, + startedAt: time.Now(), + info: llmclient.RequestInfo{ + Provider: providerName, + Model: model, + Operation: llmclient.OperationChat, + Endpoint: converseEndpoint, + Method: http.MethodPost, + Stream: stream, + }, + hooks: p.hooks, + } + if p.hooks.OnRequestStart != nil { + if derived := p.hooks.OnRequestStart(ctx, observation.info); derived != nil { + observation.ctx = derived + } + } + return observation +} + +func (o callObservation) end(statusCode int, err error) { + if o.hooks.OnRequestEnd == nil { + return + } + o.hooks.OnRequestEnd(o.ctx, o.responseInfo(statusCode, err)) +} + +func (o callObservation) firstResponseChunk() { + if o.hooks.OnStreamFirstChunk == nil { + return + } + o.hooks.OnStreamFirstChunk(o.ctx, o.responseInfo(http.StatusOK, nil)) +} + +func (o callObservation) responseInfo(statusCode int, err error) llmclient.ResponseInfo { + return llmclient.ResponseInfo{ + Provider: o.info.Provider, + Model: o.info.Model, + Operation: o.info.Operation, + Endpoint: o.info.Endpoint, + Method: o.info.Method, + StatusCode: statusCode, + Duration: time.Since(o.startedAt), + Stream: o.info.Stream, + Error: err, + } +} + +func observedStream(body io.ReadCloser, observation callObservation) io.ReadCloser { + return &observedReadCloser{ + ReadCloser: body, + onFirstRead: observation.firstResponseChunk, + } +} + +type observedReadCloser struct { + io.ReadCloser + once sync.Once + onFirstRead func() +} + +func (r *observedReadCloser) Read(p []byte) (int, error) { + n, err := r.ReadCloser.Read(p) + if n > 0 { + r.once.Do(r.onFirstRead) + } + return n, err +} + +func statusCodeFromError(err error) int { + type httpStatus interface{ HTTPStatusCode() int } + if status, ok := err.(httpStatus); ok { + return status.HTTPStatusCode() + } + return 0 +} diff --git a/internal/providers/cohere/chat.go b/internal/providers/cohere/chat.go index 746d27a6b..47922cc3c 100644 --- a/internal/providers/cohere/chat.go +++ b/internal/providers/cohere/chat.go @@ -23,9 +23,11 @@ func (p *Provider) ChatCompletion(ctx context.Context, req *core.ChatRequest) (* } var upstream chatResponse if err := p.client.Do(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/v2/chat", - Body: upstreamReq, + Method: http.MethodPost, + Endpoint: "/v2/chat", + Operation: llmclient.OperationChat, + Model: req.Model, + Body: upstreamReq, }, &upstream); err != nil { return nil, err } @@ -42,9 +44,11 @@ func (p *Provider) StreamChatCompletion(ctx context.Context, req *core.ChatReque return nil, err } body, err := p.client.DoStream(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/v2/chat", - Body: upstreamReq, + Method: http.MethodPost, + Endpoint: "/v2/chat", + Operation: llmclient.OperationChat, + Model: req.Model, + Body: upstreamReq, }) if err != nil { return nil, err diff --git a/internal/providers/cohere/cohere.go b/internal/providers/cohere/cohere.go index 6516f0d5b..269de37f3 100644 --- a/internal/providers/cohere/cohere.go +++ b/internal/providers/cohere/cohere.go @@ -16,8 +16,9 @@ const defaultBaseURL = "https://api.cohere.com" // Registration provides factory registration for the Cohere provider. var Registration = providers.Registration{ - Type: "cohere", - New: New, + Type: "cohere", + New: New, + PassthroughSemanticEnricher: passthroughSemanticEnricher, Discovery: providers.DiscoveryConfig{ DefaultBaseURL: defaultBaseURL, }, @@ -134,6 +135,9 @@ func (p *Provider) Passthrough(ctx context.Context, req *core.PassthroughRequest resp, err := p.client.DoPassthrough(ctx, llmclient.Request{ Method: req.Method, Endpoint: providers.PassthroughEndpoint(req.Endpoint), + Operation: req.Operation, + Model: req.Model, + Stream: req.Stream, RawBodyReader: req.Body, Headers: req.Headers, }) diff --git a/internal/providers/cohere/cohere_test.go b/internal/providers/cohere/cohere_test.go index 3fded8bd4..e12991d6f 100644 --- a/internal/providers/cohere/cohere_test.go +++ b/internal/providers/cohere/cohere_test.go @@ -19,6 +19,7 @@ import ( func TestChatCompletionTranslatesRequestAndResponse(t *testing.T) { var captured map[string]any + var operation string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/v2/chat" { t.Errorf("path = %q, want /v2/chat", r.URL.Path) @@ -57,7 +58,12 @@ func TestChatCompletionTranslatesRequestAndResponse(t *testing.T) { })) defer server.Close() - provider := NewWithHTTPClient("test-key", server.URL, server.Client(), llmclient.Hooks{}) + provider := NewWithHTTPClient("test-key", server.URL, server.Client(), llmclient.Hooks{ + OnRequestStart: func(ctx context.Context, info llmclient.RequestInfo) context.Context { + operation = info.Operation + return ctx + }, + }) req := &core.ChatRequest{ Model: "command-a-03-2025", Messages: []core.Message{ @@ -144,6 +150,9 @@ func TestChatCompletionTranslatesRequestAndResponse(t *testing.T) { if captured["k"] != float64(20) { t.Fatalf("k = %#v", captured["k"]) } + if operation != llmclient.OperationChat { + t.Fatalf("operation = %q, want chat", operation) + } responseFormat := captured["response_format"].(map[string]any) if responseFormat["type"] != "json_object" { t.Fatalf("response_format.type = %#v, want json_object", responseFormat["type"]) diff --git a/internal/providers/cohere/embeddings.go b/internal/providers/cohere/embeddings.go index fbe58a936..b6fc5d333 100644 --- a/internal/providers/cohere/embeddings.go +++ b/internal/providers/cohere/embeddings.go @@ -20,9 +20,11 @@ func (p *Provider) Embeddings(ctx context.Context, req *core.EmbeddingRequest) ( } var upstream embedResponse if err := p.client.Do(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/v2/embed", - Body: upstreamReq, + Method: http.MethodPost, + Endpoint: "/v2/embed", + Operation: llmclient.OperationEmbeddings, + Model: req.Model, + Body: upstreamReq, }, &upstream); err != nil { return nil, err } diff --git a/internal/providers/cohere/passthrough_semantics.go b/internal/providers/cohere/passthrough_semantics.go new file mode 100644 index 000000000..c9878f697 --- /dev/null +++ b/internal/providers/cohere/passthrough_semantics.go @@ -0,0 +1,8 @@ +package cohere + +import "github.com/enterpilot/gomodel/internal/providers" + +var passthroughSemanticEnricher = providers.NewSemanticEnricher("cohere", map[string]providers.PassthroughEndpointSemantics{ + "/v2/chat": {Operation: "cohere.chat", GenAIOperation: "chat", AuditPath: "/p/cohere/v2/chat"}, + "/v2/embed": {Operation: "cohere.embed", GenAIOperation: "embeddings", AuditPath: "/p/cohere/v2/embed"}, +}) diff --git a/internal/providers/cohere/passthrough_semantics_test.go b/internal/providers/cohere/passthrough_semantics_test.go new file mode 100644 index 000000000..f7fe36405 --- /dev/null +++ b/internal/providers/cohere/passthrough_semantics_test.go @@ -0,0 +1,22 @@ +package cohere + +import ( + "testing" + + "github.com/enterpilot/gomodel/internal/core" +) + +func TestPassthroughSemanticEnricherRecognizesCohereV2Inference(t *testing.T) { + tests := map[string]string{ + "v2/chat": "chat", + "v2/embed": "embeddings", + } + for endpoint, want := range tests { + info := passthroughSemanticEnricher.Enrich(nil, nil, &core.PassthroughRouteInfo{ + Provider: "cohere", RawEndpoint: endpoint, NormalizedEndpoint: endpoint, + }) + if info == nil || info.GenAIOperation != want { + t.Fatalf("GenAIOperation for %q = %+v, want %q", endpoint, info, want) + } + } +} diff --git a/internal/providers/config.go b/internal/providers/config.go index a946f765f..19cac57d8 100644 --- a/internal/providers/config.go +++ b/internal/providers/config.go @@ -15,6 +15,10 @@ import ( // ProviderConfig holds the fully resolved provider configuration after merging // global defaults with per-provider overrides. type ProviderConfig struct { + // Name is the configured provider instance name (for example "openai-eu"). + // It is populated by configuration resolution and may be empty in tests or + // direct constructor calls. + Name string Type string // APIKey is the provider's primary credential: the first entry of APIKeys, // or "" for keyless providers. Prefer APIKeys for anything that @@ -817,7 +821,9 @@ func HasResolvedProviderValue(value string) bool { func buildProviderConfigs(raw map[string]config.RawProviderConfig, global config.ResilienceConfig) map[string]ProviderConfig { result := make(map[string]ProviderConfig, len(raw)) for name, r := range raw { - result[name] = buildProviderConfig(r, global) + resolved := buildProviderConfig(r, global) + resolved.Name = name + result[name] = resolved } return result } diff --git a/internal/providers/config_test.go b/internal/providers/config_test.go index 953cd8883..0f32feb3f 100644 --- a/internal/providers/config_test.go +++ b/internal/providers/config_test.go @@ -295,6 +295,9 @@ func TestBuildProviderConfigs_MultipleProviders(t *testing.T) { if got["anthropic"].Resilience.Retry.MaxRetries != globalRetry.MaxRetries { t.Errorf("anthropic MaxRetries = %d, want %d (global)", got["anthropic"].Resilience.Retry.MaxRetries, globalRetry.MaxRetries) } + if got["openai"].Name != "openai" || got["anthropic"].Name != "anthropic" { + t.Fatalf("provider names = %q/%q, want map keys", got["openai"].Name, got["anthropic"].Name) + } } func TestBuildProviderConfigs_EmptyMap(t *testing.T) { diff --git a/internal/providers/credentials.go b/internal/providers/credentials.go index b2eaefa3f..bf81a142e 100644 --- a/internal/providers/credentials.go +++ b/internal/providers/credentials.go @@ -325,6 +325,7 @@ func (s *CredentialsService) buildProvider(row ManagedProviderCredential) (core. } cfg := buildProviderConfig(rawCfg, s.resilience) + cfg.Name = name provider, err := s.factory.Create(cfg) if err != nil { return nil, ProviderConfig{}, err diff --git a/internal/providers/deepseek/passthrough_semantics.go b/internal/providers/deepseek/passthrough_semantics.go index f08f7fc3c..03c9dcd87 100644 --- a/internal/providers/deepseek/passthrough_semantics.go +++ b/internal/providers/deepseek/passthrough_semantics.go @@ -3,6 +3,6 @@ package deepseek import "github.com/enterpilot/gomodel/internal/providers" var passthroughSemanticEnricher = providers.NewSemanticEnricher("deepseek", map[string]providers.PassthroughEndpointSemantics{ - "/chat/completions": {Operation: "deepseek.chat_completions", AuditPath: "/v1/chat/completions"}, - "/beta/completions": {Operation: "deepseek.fim_completions", AuditPath: "/beta/completions"}, + "/chat/completions": {Operation: "deepseek.chat_completions", GenAIOperation: "chat", AuditPath: "/v1/chat/completions"}, + "/beta/completions": {Operation: "deepseek.fim_completions", GenAIOperation: "text_completion", AuditPath: "/beta/completions"}, }) diff --git a/internal/providers/factory.go b/internal/providers/factory.go index 7e54a24be..25790cab4 100644 --- a/internal/providers/factory.go +++ b/internal/providers/factory.go @@ -2,6 +2,7 @@ package providers import ( + "context" "fmt" "maps" "sort" @@ -134,7 +135,7 @@ func (f *ProviderFactory) Create(cfg ProviderConfig) (core.Provider, error) { // One Keyring per provider instance: every client this provider builds // shares session affinity and the sessionless round-robin sequence. opts := ProviderOptions{ - Hooks: hooks, + Hooks: hooksWithProviderIdentity(hooks, cfg.Name, cfg.Type), Models: cfg.Models, Resilience: cfg.Resilience, Keys: NewKeyringWithSessionStickiness(cfg.SessionStickyKeys, cfg.APIKeys...), @@ -143,6 +144,38 @@ func (f *ProviderFactory) Create(cfg ProviderConfig) (core.Provider, error) { return builder(cfg, opts), nil } +func hooksWithProviderIdentity(hooks llmclient.Hooks, providerName, providerType string) llmclient.Hooks { + if hooks.OnRequestStart == nil && hooks.OnRequestEnd == nil && hooks.OnStreamFirstChunk == nil { + return hooks + } + setIdentity := func(provider *string, implementation *string) { + if providerName != "" { + *provider = providerName + } + *implementation = providerType + } + typed := llmclient.Hooks{} + if hooks.OnRequestStart != nil { + typed.OnRequestStart = func(ctx context.Context, info llmclient.RequestInfo) context.Context { + setIdentity(&info.Provider, &info.ProviderType) + return hooks.OnRequestStart(ctx, info) + } + } + if hooks.OnRequestEnd != nil { + typed.OnRequestEnd = func(ctx context.Context, info llmclient.ResponseInfo) { + setIdentity(&info.Provider, &info.ProviderType) + hooks.OnRequestEnd(ctx, info) + } + } + if hooks.OnStreamFirstChunk != nil { + typed.OnStreamFirstChunk = func(ctx context.Context, info llmclient.ResponseInfo) { + setIdentity(&info.Provider, &info.ProviderType) + hooks.OnStreamFirstChunk(ctx, info) + } + } + return typed +} + // discoveryConfigsSnapshot returns provider discovery metadata keyed by provider type. func (f *ProviderFactory) discoveryConfigsSnapshot() map[string]DiscoveryConfig { f.mu.RLock() diff --git a/internal/providers/factory_test.go b/internal/providers/factory_test.go index 9ce77ae45..60551c50a 100644 --- a/internal/providers/factory_test.go +++ b/internal/providers/factory_test.go @@ -239,11 +239,20 @@ func TestProviderFactory_Create_PassesResolvedProviderConfig(t *testing.T) { func TestProviderFactory_SetHooks(t *testing.T) { factory := NewProviderFactory() + var startName, startType, endName, endType, chunkName, chunkType string mockHooks := llmclient.Hooks{ OnRequestStart: func(ctx context.Context, info llmclient.RequestInfo) context.Context { + startName = info.Provider + startType = info.ProviderType return ctx }, + OnRequestEnd: func(_ context.Context, info llmclient.ResponseInfo) { + endName, endType = info.Provider, info.ProviderType + }, + OnStreamFirstChunk: func(_ context.Context, info llmclient.ResponseInfo) { + chunkName, chunkType = info.Provider, info.ProviderType + }, } factory.SetHooks(mockHooks) @@ -257,6 +266,7 @@ func TestProviderFactory_SetHooks(t *testing.T) { }) cfg := ProviderConfig{ + Name: "test-eu", Type: "test", APIKey: "test-key", } @@ -269,6 +279,14 @@ func TestProviderFactory_SetHooks(t *testing.T) { if receivedOpts.Hooks.OnRequestStart == nil { t.Error("expected hooks to be passed to builder via ProviderOptions") } + receivedOpts.Hooks.OnRequestStart(t.Context(), llmclient.RequestInfo{}) + receivedOpts.Hooks.OnRequestEnd(t.Context(), llmclient.ResponseInfo{}) + receivedOpts.Hooks.OnStreamFirstChunk(t.Context(), llmclient.ResponseInfo{}) + if startName != "test-eu" || endName != "test-eu" || chunkName != "test-eu" || + startType != "test" || endType != "test" || chunkType != "test" { + t.Fatalf("provider identities = %q/%q, %q/%q, %q/%q; want test-eu/test", + startName, startType, endName, endType, chunkName, chunkType) + } } func TestProviderFactory_HooksPassedToBuilder(t *testing.T) { @@ -327,7 +345,7 @@ func TestProviderFactory_ZeroHooks(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - if receivedOpts.Hooks.OnRequestStart != nil || receivedOpts.Hooks.OnRequestEnd != nil { + if receivedOpts.Hooks.OnRequestStart != nil || receivedOpts.Hooks.OnRequestEnd != nil || receivedOpts.Hooks.OnStreamFirstChunk != nil { t.Error("expected zero hooks when SetHooks not called") } } diff --git a/internal/providers/gemini/gemini.go b/internal/providers/gemini/gemini.go index a67645187..d4fc24670 100644 --- a/internal/providers/gemini/gemini.go +++ b/internal/providers/gemini/gemini.go @@ -462,9 +462,11 @@ func (p *Provider) ChatCompletion(ctx context.Context, req *core.ChatRequest) (* } var resp core.ChatResponse err = p.client.Do(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/chat/completions", - Body: body, + Method: http.MethodPost, + Endpoint: "/chat/completions", + Operation: llmclient.OperationChat, + Model: req.Model, + Body: body, }, &resp) if err != nil { return nil, err @@ -484,9 +486,11 @@ func (p *Provider) nativeChatCompletion(ctx context.Context, req *core.ChatReque p.prepareCachedContent(ctx, req, body) var geminiResp geminiGenerateContentResponse err = p.nativeClient.Do(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: nativeGenerateEndpoint(req.Model), - Body: body, + Method: http.MethodPost, + Endpoint: nativeGenerateEndpoint(req.Model), + Operation: llmclient.OperationGenerateContent, + Model: req.Model, + Body: body, }, &geminiResp) if err != nil { return nil, err @@ -511,9 +515,11 @@ func (p *Provider) StreamChatCompletion(ctx context.Context, req *core.ChatReque return nil, err } stream, err := p.client.DoStream(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/chat/completions", - Body: body, + Method: http.MethodPost, + Endpoint: "/chat/completions", + Operation: llmclient.OperationChat, + Model: req.Model, + Body: body, }) if err != nil { return nil, err @@ -531,9 +537,11 @@ func (p *Provider) nativeStreamChatCompletion(ctx context.Context, req *core.Cha } p.prepareCachedContent(ctx, req, body) stream, err := p.nativeClient.DoStream(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: nativeStreamEndpoint(req.Model), - Body: body, + Method: http.MethodPost, + Endpoint: nativeStreamEndpoint(req.Model), + Operation: llmclient.OperationGenerateContent, + Model: req.Model, + Body: body, }) if err != nil { return nil, err @@ -828,9 +836,11 @@ func (p *Provider) Embeddings(ctx context.Context, req *core.EmbeddingRequest) ( } var resp core.EmbeddingResponse err = p.client.Do(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/embeddings", - Body: body, + Method: http.MethodPost, + Endpoint: "/embeddings", + Operation: llmclient.OperationEmbeddings, + Model: req.Model, + Body: body, }, &resp) if err != nil { return nil, err diff --git a/internal/providers/kilo/passthrough_semantics.go b/internal/providers/kilo/passthrough_semantics.go index 6187d13a1..9be88666f 100644 --- a/internal/providers/kilo/passthrough_semantics.go +++ b/internal/providers/kilo/passthrough_semantics.go @@ -3,5 +3,5 @@ package kilo import "github.com/enterpilot/gomodel/internal/providers" var passthroughSemanticEnricher = providers.NewSemanticEnricher("kilo", map[string]providers.PassthroughEndpointSemantics{ - "/chat/completions": {Operation: "kilo.chat_completions", AuditPath: "/v1/chat/completions"}, + "/chat/completions": {Operation: "kilo.chat_completions", GenAIOperation: "chat", AuditPath: "/v1/chat/completions"}, }) diff --git a/internal/providers/ollama/ollama.go b/internal/providers/ollama/ollama.go index dea6ec8a2..c33157485 100644 --- a/internal/providers/ollama/ollama.go +++ b/internal/providers/ollama/ollama.go @@ -166,9 +166,11 @@ func (p *Provider) Embeddings(ctx context.Context, req *core.EmbeddingRequest) ( var ollamaResp ollamaEmbedResponse err := p.nativeClient.Do(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/api/embed", - Body: ollamaReq, + Method: http.MethodPost, + Endpoint: "/api/embed", + Operation: llmclient.OperationEmbeddings, + Model: req.Model, + Body: ollamaReq, }, &ollamaResp) if err != nil { return nil, err diff --git a/internal/providers/openai/compatible_provider.go b/internal/providers/openai/compatible_provider.go index 9de552776..ba1fde69f 100644 --- a/internal/providers/openai/compatible_provider.go +++ b/internal/providers/openai/compatible_provider.go @@ -155,6 +155,14 @@ func (p *CompatibleProvider) Do(ctx context.Context, req llmclient.Request, resu return p.client.Do(ctx, p.prepareRequest(req), result) } +func (p *CompatibleProvider) doModelRequest(ctx context.Context, req llmclient.Request, result any, responseModel *string) error { + if err := p.Do(ctx, req, result); err != nil { + return err + } + core.EnsureModel(responseModel, req.Model) + return nil +} + func (p *CompatibleProvider) ChatCompletion(ctx context.Context, req *core.ChatRequest) (*core.ChatResponse, error) { if req == nil { return nil, core.NewInvalidRequestError("chat request is required", nil) @@ -169,10 +177,12 @@ func (p *CompatibleProvider) ChatCompletion(ctx context.Context, req *core.ChatR return nil, err } err = p.Do(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/chat/completions", - Body: body, - Headers: p.chatHeaders(ctx, adapted), + Method: http.MethodPost, + Endpoint: "/chat/completions", + Operation: llmclient.OperationChat, + Model: req.Model, + Body: body, + Headers: p.chatHeaders(ctx, adapted), }, &resp) if err != nil { return nil, err @@ -195,10 +205,12 @@ func (p *CompatibleProvider) StreamChatCompletion(ctx context.Context, req *core return nil, err } stream, err := p.client.DoStream(ctx, p.prepareRequest(llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/chat/completions", - Body: body, - Headers: p.chatHeaders(ctx, streamReq), + Method: http.MethodPost, + Endpoint: "/chat/completions", + Operation: llmclient.OperationChat, + Model: req.Model, + Body: body, + Headers: p.chatHeaders(ctx, streamReq), })) if err != nil { return nil, err @@ -252,15 +264,16 @@ func (p *CompatibleProvider) Responses(ctx context.Context, req *core.ResponsesR return nil, core.NewInvalidRequestError("responses request is required", nil) } var resp core.ResponsesResponse - err := p.Do(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/responses", - Body: req, - }, &resp) + err := p.doModelRequest(ctx, llmclient.Request{ + Method: http.MethodPost, + Endpoint: "/responses", + Operation: llmclient.OperationChat, + Model: req.Model, + Body: req, + }, &resp, &resp.Model) if err != nil { return nil, err } - core.EnsureModel(&resp.Model, req.Model) return &resp, nil } @@ -269,9 +282,11 @@ func (p *CompatibleProvider) StreamResponses(ctx context.Context, req *core.Resp return nil, core.NewInvalidRequestError("responses request is required", nil) } stream, err := p.client.DoStream(ctx, p.prepareRequest(llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/responses", - Body: req.WithStreaming(), + Method: http.MethodPost, + Endpoint: "/responses", + Operation: llmclient.OperationChat, + Model: req.Model, + Body: req.WithStreaming(), })) if err != nil { return nil, err @@ -393,15 +408,16 @@ func (p *CompatibleProvider) Embeddings(ctx context.Context, req *core.Embedding return nil, core.NewInvalidRequestError("embedding request is required", nil) } var resp core.EmbeddingResponse - err := p.Do(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: "/embeddings", - Body: req, - }, &resp) + err := p.doModelRequest(ctx, llmclient.Request{ + Method: http.MethodPost, + Endpoint: "/embeddings", + Operation: llmclient.OperationEmbeddings, + Model: req.Model, + Body: req, + }, &resp, &resp.Model) if err != nil { return nil, err } - core.EnsureModel(&resp.Model, req.Model) return &resp, nil } @@ -413,6 +429,9 @@ func (p *CompatibleProvider) Passthrough(ctx context.Context, req *core.Passthro resp, err := p.client.DoPassthrough(ctx, p.prepareRequest(llmclient.Request{ Method: req.Method, Endpoint: providers.PassthroughEndpoint(req.Endpoint), + Operation: req.Operation, + Model: req.Model, + Stream: req.Stream, RawBodyReader: req.Body, Headers: req.Headers, })) diff --git a/internal/providers/openai/passthrough_semantics.go b/internal/providers/openai/passthrough_semantics.go index 3e170aece..e848635b2 100644 --- a/internal/providers/openai/passthrough_semantics.go +++ b/internal/providers/openai/passthrough_semantics.go @@ -2,8 +2,4 @@ package openai import "github.com/enterpilot/gomodel/internal/providers" -var passthroughSemanticEnricher = providers.NewSemanticEnricher("openai", map[string]providers.PassthroughEndpointSemantics{ - "/chat/completions": {Operation: "openai.chat_completions", AuditPath: "/v1/chat/completions"}, - "/responses": {Operation: "openai.responses", AuditPath: "/v1/responses"}, - "/embeddings": {Operation: "openai.embeddings", AuditPath: "/v1/embeddings"}, -}) +var passthroughSemanticEnricher = providers.NewOpenAICompatibleSemanticEnricher("openai") diff --git a/internal/providers/openai/passthrough_semantics_test.go b/internal/providers/openai/passthrough_semantics_test.go index 777545b52..a6bd6051a 100644 --- a/internal/providers/openai/passthrough_semantics_test.go +++ b/internal/providers/openai/passthrough_semantics_test.go @@ -13,24 +13,28 @@ func TestPassthroughSemanticEnricher_Enrich(t *testing.T) { name string info *core.PassthroughRouteInfo wantOperation string + wantGenAI string wantAuditPath string }{ { name: "responses", info: &core.PassthroughRouteInfo{Provider: "openai", RawEndpoint: "responses", NormalizedEndpoint: "responses"}, wantOperation: "openai.responses", + wantGenAI: "chat", wantAuditPath: "/v1/responses", }, { name: "chat completions", info: &core.PassthroughRouteInfo{Provider: "openai", RawEndpoint: "v1/chat/completions", NormalizedEndpoint: "chat/completions"}, wantOperation: "openai.chat_completions", + wantGenAI: "chat", wantAuditPath: "/v1/chat/completions", }, { name: "embeddings", info: &core.PassthroughRouteInfo{Provider: "openai", RawEndpoint: "embeddings", NormalizedEndpoint: "embeddings"}, wantOperation: "openai.embeddings", + wantGenAI: "embeddings", wantAuditPath: "/v1/embeddings", }, { @@ -51,6 +55,9 @@ func TestPassthroughSemanticEnricher_Enrich(t *testing.T) { if got.SemanticOperation != tt.wantOperation { t.Fatalf("SemanticOperation = %q, want %q", got.SemanticOperation, tt.wantOperation) } + if got.GenAIOperation != tt.wantGenAI { + t.Fatalf("GenAIOperation = %q, want %q", got.GenAIOperation, tt.wantGenAI) + } if got.AuditPath != tt.wantAuditPath { t.Fatalf("AuditPath = %q, want %q", got.AuditPath, tt.wantAuditPath) } diff --git a/internal/providers/openrouter/openrouter.go b/internal/providers/openrouter/openrouter.go index fce92fca6..c9437ee6a 100644 --- a/internal/providers/openrouter/openrouter.go +++ b/internal/providers/openrouter/openrouter.go @@ -20,7 +20,7 @@ const ( var Registration = providers.Registration{ Type: "openrouter", New: New, - PassthroughSemanticEnricher: openai.Registration.PassthroughSemanticEnricher, + PassthroughSemanticEnricher: passthroughSemanticEnricher, Discovery: providers.DiscoveryConfig{ DefaultBaseURL: defaultBaseURL, }, diff --git a/internal/providers/openrouter/passthrough_semantics.go b/internal/providers/openrouter/passthrough_semantics.go new file mode 100644 index 000000000..f7327fd21 --- /dev/null +++ b/internal/providers/openrouter/passthrough_semantics.go @@ -0,0 +1,5 @@ +package openrouter + +import "github.com/enterpilot/gomodel/internal/providers" + +var passthroughSemanticEnricher = providers.NewOpenAICompatibleSemanticEnricher("openrouter") diff --git a/internal/providers/openrouter/passthrough_semantics_test.go b/internal/providers/openrouter/passthrough_semantics_test.go new file mode 100644 index 000000000..d8fcecc81 --- /dev/null +++ b/internal/providers/openrouter/passthrough_semantics_test.go @@ -0,0 +1,23 @@ +package openrouter + +import ( + "testing" + + "github.com/enterpilot/gomodel/internal/core" +) + +func TestPassthroughSemanticEnricherUsesOpenRouterType(t *testing.T) { + enricher := Registration.PassthroughSemanticEnricher + if enricher == nil { + t.Fatal("registration passthrough enricher is nil") + } + if got := enricher.ProviderType(); got != "openrouter" { + t.Fatalf("ProviderType() = %q, want openrouter", got) + } + info := enricher.Enrich(nil, nil, &core.PassthroughRouteInfo{ + Provider: "openrouter", NormalizedEndpoint: "chat/completions", + }) + if info == nil || info.GenAIOperation != "chat" || info.SemanticOperation != "openrouter.chat_completions" { + t.Fatalf("enriched info = %+v, want OpenRouter chat semantics", info) + } +} diff --git a/internal/providers/passthrough.go b/internal/providers/passthrough.go index a47b06084..1f7e72fa2 100644 --- a/internal/providers/passthrough.go +++ b/internal/providers/passthrough.go @@ -57,8 +57,9 @@ func PassthroughEndpointPath(info *core.PassthroughRouteInfo) string { // PassthroughEndpointSemantics names the semantic operation and audit path // for one provider passthrough endpoint. type PassthroughEndpointSemantics struct { - Operation string - AuditPath string + Operation string + GenAIOperation string + AuditPath string } // SemanticEnricher implements core.PassthroughSemanticEnricher from a static @@ -75,6 +76,16 @@ func NewSemanticEnricher(providerType string, endpoints map[string]PassthroughEn return SemanticEnricher{providerType: providerType, endpoints: endpoints} } +// NewOpenAICompatibleSemanticEnricher describes the common inference surface +// while retaining the concrete provider type used to select the enricher. +func NewOpenAICompatibleSemanticEnricher(providerType string) SemanticEnricher { + return NewSemanticEnricher(providerType, map[string]PassthroughEndpointSemantics{ + "/chat/completions": {Operation: providerType + ".chat_completions", GenAIOperation: "chat", AuditPath: "/v1/chat/completions"}, + "/responses": {Operation: providerType + ".responses", GenAIOperation: "chat", AuditPath: "/v1/responses"}, + "/embeddings": {Operation: providerType + ".embeddings", GenAIOperation: "embeddings", AuditPath: "/v1/embeddings"}, + }) +} + // ProviderType returns the provider type this enricher serves. func (e SemanticEnricher) ProviderType() string { return e.providerType @@ -90,6 +101,7 @@ func (e SemanticEnricher) Enrich(_ *core.RequestSnapshot, _ *core.WhiteBoxPrompt normalizedEndpoint := strings.TrimLeft(strings.TrimSpace(PassthroughEndpointPath(&enriched)), "/") if semantics, ok := e.endpoints["/"+normalizedEndpoint]; ok { enriched.SemanticOperation = semantics.Operation + enriched.GenAIOperation = semantics.GenAIOperation enriched.AuditPath = semantics.AuditPath } else if strings.TrimSpace(enriched.AuditPath) == "" && normalizedEndpoint != "" { enriched.AuditPath = "/p/" + e.providerType + "/" + normalizedEndpoint diff --git a/internal/providers/sglang/passthrough_semantics.go b/internal/providers/sglang/passthrough_semantics.go index 246defce7..9ae5b22b2 100644 --- a/internal/providers/sglang/passthrough_semantics.go +++ b/internal/providers/sglang/passthrough_semantics.go @@ -3,8 +3,8 @@ package sglang import "github.com/enterpilot/gomodel/internal/providers" var passthroughSemanticEnricher = providers.NewSemanticEnricher("sglang", map[string]providers.PassthroughEndpointSemantics{ - "/chat/completions": {Operation: "sglang.chat_completions", AuditPath: "/v1/chat/completions"}, - "/responses": {Operation: "sglang.responses", AuditPath: "/v1/responses"}, - "/embeddings": {Operation: "sglang.embeddings", AuditPath: "/v1/embeddings"}, - "/completions": {Operation: "sglang.completions", AuditPath: "/v1/completions"}, + "/chat/completions": {Operation: "sglang.chat_completions", GenAIOperation: "chat", AuditPath: "/v1/chat/completions"}, + "/responses": {Operation: "sglang.responses", GenAIOperation: "chat", AuditPath: "/v1/responses"}, + "/embeddings": {Operation: "sglang.embeddings", GenAIOperation: "embeddings", AuditPath: "/v1/embeddings"}, + "/completions": {Operation: "sglang.completions", GenAIOperation: "text_completion", AuditPath: "/v1/completions"}, }) diff --git a/internal/providers/sglang/sglang.go b/internal/providers/sglang/sglang.go index 51f3fe3e0..3b6047261 100644 --- a/internal/providers/sglang/sglang.go +++ b/internal/providers/sglang/sglang.go @@ -134,6 +134,9 @@ func (p *Provider) Passthrough(ctx context.Context, req *core.PassthroughRequest resp, err := p.rootClient.DoPassthrough(ctx, llmclient.Request{ Method: req.Method, Endpoint: endpoint, + Operation: req.Operation, + Model: req.Model, + Stream: req.Stream, RawBodyReader: req.Body, Headers: req.Headers, }) diff --git a/internal/providers/vertex/vertex.go b/internal/providers/vertex/vertex.go index c96c6b771..c2c8ce78d 100644 --- a/internal/providers/vertex/vertex.go +++ b/internal/providers/vertex/vertex.go @@ -248,9 +248,11 @@ func (p *Provider) Embeddings(ctx context.Context, req *core.EmbeddingRequest) ( var resp vertexEmbeddingPredictResponse err = p.nativeClient.Do(ctx, llmclient.Request{ - Method: http.MethodPost, - Endpoint: vertexPredictEndpoint(req.Model), - Body: body, + Method: http.MethodPost, + Endpoint: vertexPredictEndpoint(req.Model), + Operation: llmclient.OperationEmbeddings, + Model: req.Model, + Body: body, }, &resp) if err != nil { return nil, err diff --git a/internal/providers/vertex/vertex_test.go b/internal/providers/vertex/vertex_test.go index 8b5afcfda..615928b0f 100644 --- a/internal/providers/vertex/vertex_test.go +++ b/internal/providers/vertex/vertex_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/enterpilot/gomodel/internal/core" + "github.com/enterpilot/gomodel/internal/llmclient" "github.com/enterpilot/gomodel/internal/providers" "github.com/enterpilot/gomodel/internal/providers/googlecommon" @@ -36,6 +37,7 @@ func TestProviderDoesNotExposeFilesOrBatches(t *testing.T) { } func TestEmbeddingsUsesNativePrediction(t *testing.T) { + var operation string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/v1/projects/prod-ai/locations/us-central1/publishers/google/models/text-embedding-005:predict" { t.Errorf("Path = %q, want Vertex native predict endpoint", r.URL.Path) @@ -74,7 +76,12 @@ func TestEmbeddingsUsesNativePrediction(t *testing.T) { dimensions := 3 cfg := testConfig() cfg.BaseURL = server.URL + "/v1/projects/prod-ai/locations/us-central1/publishers/google" - provider := newProvider(cfg, providers.ProviderOptions{}, authedTestClient(server.Client())) + provider := newProvider(cfg, providers.ProviderOptions{Hooks: llmclient.Hooks{ + OnRequestStart: func(ctx context.Context, info llmclient.RequestInfo) context.Context { + operation = info.Operation + return ctx + }, + }}, authedTestClient(server.Client())) resp, err := provider.Embeddings(context.Background(), &core.EmbeddingRequest{ Model: "google/text-embedding-005", @@ -96,6 +103,9 @@ func TestEmbeddingsUsesNativePrediction(t *testing.T) { if resp.Usage.PromptTokens != 9 || resp.Usage.TotalTokens != 9 { t.Fatalf("usage = %+v, want 9 prompt/total tokens", resp.Usage) } + if operation != llmclient.OperationEmbeddings { + t.Fatalf("operation = %q, want embeddings", operation) + } } func TestEmbeddingsRejectsEmptyStringInBatch(t *testing.T) { diff --git a/internal/providers/vllm/passthrough_semantics.go b/internal/providers/vllm/passthrough_semantics.go index e6a60225f..d259263a4 100644 --- a/internal/providers/vllm/passthrough_semantics.go +++ b/internal/providers/vllm/passthrough_semantics.go @@ -3,8 +3,8 @@ package vllm import "github.com/enterpilot/gomodel/internal/providers" var passthroughSemanticEnricher = providers.NewSemanticEnricher("vllm", map[string]providers.PassthroughEndpointSemantics{ - "/chat/completions": {Operation: "vllm.chat_completions", AuditPath: "/v1/chat/completions"}, - "/responses": {Operation: "vllm.responses", AuditPath: "/v1/responses"}, - "/embeddings": {Operation: "vllm.embeddings", AuditPath: "/v1/embeddings"}, - "/completions": {Operation: "vllm.completions", AuditPath: "/v1/completions"}, + "/chat/completions": {Operation: "vllm.chat_completions", GenAIOperation: "chat", AuditPath: "/v1/chat/completions"}, + "/responses": {Operation: "vllm.responses", GenAIOperation: "chat", AuditPath: "/v1/responses"}, + "/embeddings": {Operation: "vllm.embeddings", GenAIOperation: "embeddings", AuditPath: "/v1/embeddings"}, + "/completions": {Operation: "vllm.completions", GenAIOperation: "text_completion", AuditPath: "/v1/completions"}, }) diff --git a/internal/providers/vllm/vllm.go b/internal/providers/vllm/vllm.go index 074e8a2c2..859f8d644 100644 --- a/internal/providers/vllm/vllm.go +++ b/internal/providers/vllm/vllm.go @@ -126,6 +126,9 @@ func (p *Provider) Passthrough(ctx context.Context, req *core.PassthroughRequest resp, err := p.rootClient.DoPassthrough(ctx, llmclient.Request{ Method: req.Method, Endpoint: endpoint, + Operation: req.Operation, + Model: req.Model, + Stream: req.Stream, RawBodyReader: req.Body, Headers: req.Headers, }) diff --git a/internal/providers/zai/passthrough_semantics.go b/internal/providers/zai/passthrough_semantics.go new file mode 100644 index 000000000..68d47254d --- /dev/null +++ b/internal/providers/zai/passthrough_semantics.go @@ -0,0 +1,5 @@ +package zai + +import "github.com/enterpilot/gomodel/internal/providers" + +var passthroughSemanticEnricher = providers.NewOpenAICompatibleSemanticEnricher("zai") diff --git a/internal/providers/zai/passthrough_semantics_test.go b/internal/providers/zai/passthrough_semantics_test.go new file mode 100644 index 000000000..8f324ea5a --- /dev/null +++ b/internal/providers/zai/passthrough_semantics_test.go @@ -0,0 +1,23 @@ +package zai + +import ( + "testing" + + "github.com/enterpilot/gomodel/internal/core" +) + +func TestPassthroughSemanticEnricherUsesZAIType(t *testing.T) { + enricher := Registration.PassthroughSemanticEnricher + if enricher == nil { + t.Fatal("registration passthrough enricher is nil") + } + if got := enricher.ProviderType(); got != "zai" { + t.Fatalf("ProviderType() = %q, want zai", got) + } + info := enricher.Enrich(nil, nil, &core.PassthroughRouteInfo{ + Provider: "zai", NormalizedEndpoint: "embeddings", + }) + if info == nil || info.GenAIOperation != "embeddings" || info.SemanticOperation != "zai.embeddings" { + t.Fatalf("enriched info = %+v, want Z.ai embedding semantics", info) + } +} diff --git a/internal/providers/zai/zai.go b/internal/providers/zai/zai.go index 7bca72e55..a10175f74 100644 --- a/internal/providers/zai/zai.go +++ b/internal/providers/zai/zai.go @@ -14,8 +14,9 @@ const defaultBaseURL = "https://api.z.ai/api/paas/v4" // Registration provides factory registration for the Z.ai provider. var Registration = providers.Registration{ - Type: "zai", - New: New, + Type: "zai", + New: New, + PassthroughSemanticEnricher: passthroughSemanticEnricher, Discovery: providers.DiscoveryConfig{ DefaultBaseURL: defaultBaseURL, }, diff --git a/internal/server/handlers_test.go b/internal/server/handlers_test.go index 666042ed5..d3b4fa1f6 100644 --- a/internal/server/handlers_test.go +++ b/internal/server/handlers_test.go @@ -2189,6 +2189,9 @@ func TestChatCompletionStreaming_FastPathUsesPassthroughForOpenAICompatibleProvi if mock.lastPassthroughReq == nil { t.Fatal("lastPassthroughReq = nil, want passthrough request") } + if !mock.lastPassthroughReq.Stream { + t.Fatal("passthrough request lost explicit stream intent") + } if body := readPassthroughRequestBody(t, mock.lastPassthroughReq.Body); body != reqBody { t.Fatalf("passthrough body = %q, want %q", body, reqBody) } diff --git a/internal/server/http.go b/internal/server/http.go index 171f54c57..c6cc1ae8a 100644 --- a/internal/server/http.go +++ b/internal/server/http.go @@ -110,6 +110,7 @@ type Config struct { StorageProbe ReadinessProbe // Optional: primary storage connectivity check; failure makes /health/ready report not_ready (503) CacheProbe ReadinessProbe // Optional: Redis cache connectivity check; failure makes /health/ready report degraded (200, non-blocking) RequestRewriters []ext.RequestRewriter // Optional: raw-body rewriters invoked on inference ingress (post-auth, pre-workflow-resolution) + OuterMiddleware []echo.MiddlewareFunc // Optional: extension middleware after sensitive URI redaction, before logging/recovery/limits ExtraMiddleware []echo.MiddlewareFunc // Optional: extension middleware registered after audit, before gateway auth ExtraRoutes []func(*echo.Echo) // Optional: extension route registration callbacks invoked after core routes ExtraAuthSkipPaths []string // Optional: extension paths appended to the auth skip list ("/*" suffix matches a prefix) @@ -221,19 +222,15 @@ func New(provider core.RoutableProvider, cfg *Config) *Server { authSkipPaths := []string{"/health", "/health/ready"} // Determine metrics path - metricsPath := "/metrics" + metricsPath := config.ResolveMetricsEndpoint("") if cfg != nil && cfg.MetricsEnabled { - if cfg.MetricsEndpoint != "" { - // Normalize path to prevent traversal attacks - metricsPath = path.Clean(cfg.MetricsEndpoint) - } + configuredPath := path.Clean(cfg.MetricsEndpoint) + metricsPath = config.ResolveMetricsEndpoint(cfg.MetricsEndpoint) // Prevent metrics endpoint from shadowing API routes (security: auth bypass) - if metricsPath == "/v1" || strings.HasPrefix(metricsPath, "/v1/") || - metricsPath == "/p" || strings.HasPrefix(metricsPath, "/p/") { + if metricsPath != configuredPath && cfg.MetricsEndpoint != "" { slog.Warn("metrics endpoint conflicts with API routes, using /metrics instead", "configured", cfg.MetricsEndpoint, - "normalized", metricsPath) - metricsPath = "/metrics" + "normalized", configuredPath) } authSkipPaths = append(authSkipPaths, metricsPath) } @@ -261,6 +258,14 @@ func New(provider core.RoutableProvider, cfg *Config) *Server { // Scrub credential-like query values before the outer request logger // snapshots RequestURI. URL.RawQuery remains intact for handlers. e.Use(redactSensitiveRequestURI()) + // Outer extension middleware covers the complete HTTP request while still + // seeing the credential-redacted URI. It runs before request logging, + // recovery, limits, audit, and auth, so it must not depend on identity. + if cfg != nil { + for _, m := range cfg.OuterMiddleware { + e.Use(m) + } + } // Request logger with optional filtering for model-only interactions if cfg != nil && cfg.LogOnlyModelInteractions { e.Use(middleware.RequestLoggerWithConfig(middleware.RequestLoggerConfig{ diff --git a/internal/server/http_test.go b/internal/server/http_test.go index ce1869d15..5153b6a4d 100644 --- a/internal/server/http_test.go +++ b/internal/server/http_test.go @@ -1086,7 +1086,7 @@ func TestProviderPassthroughRoute_EnabledByDefault(t *testing.T) { } srv := New(mock, &Config{}) - req := httptest.NewRequest(http.MethodPost, "/p/openai/responses", strings.NewReader(`{"model":"gpt-5-mini"}`)) + req := httptest.NewRequest(http.MethodPost, "/p/openai/responses", strings.NewReader(`{"model":"gpt-5-mini","stream":true}`)) req.Header.Set("Content-Type", "application/json") rec := httptest.NewRecorder() @@ -1098,6 +1098,9 @@ func TestProviderPassthroughRoute_EnabledByDefault(t *testing.T) { if got := mock.lastPassthroughProvider; got != "openai" { t.Fatalf("provider = %q, want openai", got) } + if mock.lastPassthroughReq == nil || !mock.lastPassthroughReq.Stream { + t.Fatalf("passthrough stream intent = %+v, want true", mock.lastPassthroughReq) + } mock.lastPassthroughProvider = "" mock.lastPassthroughReq = nil diff --git a/internal/server/passthrough_semantic_enrichment_test.go b/internal/server/passthrough_semantic_enrichment_test.go index 5e6eb2fd0..8d4886b94 100644 --- a/internal/server/passthrough_semantic_enrichment_test.go +++ b/internal/server/passthrough_semantic_enrichment_test.go @@ -34,7 +34,7 @@ func TestPassthroughSemanticEnrichment_EnrichesPromptBeforeWorkflowResolution(t provider := &mockProvider{} e := echo.New() - req := httptest.NewRequest(http.MethodPost, "/p/openai/v1/responses", strings.NewReader(`{"model":"gpt-5-mini"}`)) + req := httptest.NewRequest(http.MethodPost, "/p/openai/v1/responses", strings.NewReader(`{"model":"gpt-5-mini","stream":true}`)) req.Header.Set("Content-Type", "application/json") rec := httptest.NewRecorder() c := e.NewContext(req, rec) @@ -64,4 +64,7 @@ func TestPassthroughSemanticEnrichment_EnrichesPromptBeforeWorkflowResolution(t if capturedWorkflow.Passthrough.AuditPath != "/v1/responses" { t.Fatalf("AuditPath = %q, want /v1/responses", capturedWorkflow.Passthrough.AuditPath) } + if !capturedWorkflow.Passthrough.Stream { + t.Fatal("passthrough workflow lost stream intent") + } } diff --git a/internal/server/passthrough_service.go b/internal/server/passthrough_service.go index eab203720..cbeec373f 100644 --- a/internal/server/passthrough_service.go +++ b/internal/server/passthrough_service.go @@ -51,6 +51,9 @@ func (s *passthroughService) ProviderPassthrough(c *echo.Context) error { resp, err := passthroughProvider.Passthrough(ctx, providerType, &core.PassthroughRequest{ Method: c.Request().Method, Endpoint: endpoint, + Operation: info.GenAIOperation, + Model: info.Model, + Stream: info.Stream, Body: c.Request().Body, Headers: buildPassthroughHeaders(ctx, c.Request().Header), ProviderName: providerName, diff --git a/internal/server/translated_inference_service.go b/internal/server/translated_inference_service.go index 4f9ef61ab..9bf71f29f 100644 --- a/internal/server/translated_inference_service.go +++ b/internal/server/translated_inference_service.go @@ -19,6 +19,7 @@ import ( "github.com/enterpilot/gomodel/internal/conversationstore" "github.com/enterpilot/gomodel/internal/core" "github.com/enterpilot/gomodel/internal/gateway" + "github.com/enterpilot/gomodel/internal/llmclient" "github.com/enterpilot/gomodel/internal/observability" "github.com/enterpilot/gomodel/internal/responsecache" "github.com/enterpilot/gomodel/internal/responsestore" @@ -449,10 +450,13 @@ func (s *translatedInferenceService) tryFastPathStreamingChatPassthrough(c *echo const endpoint = "/chat/completions" providerType := strings.TrimSpace(workflow.ProviderType) resp, err := passthroughProvider.Passthrough(ctx, providerType, &core.PassthroughRequest{ - Method: c.Request().Method, - Endpoint: endpoint, - Body: c.Request().Body, - Headers: buildPassthroughHeaders(ctx, c.Request().Header), + Method: c.Request().Method, + Endpoint: endpoint, + Operation: llmclient.OperationChat, + Model: resolvedModelFromWorkflow(workflow, req.Model), + Stream: req.Stream, + Body: c.Request().Body, + Headers: buildPassthroughHeaders(ctx, c.Request().Header), }) if err != nil { return true, handleError(c, err) From eb796eeb662962f9be93b2e75cec13e69e37ccd0 Mon Sep 17 00:00:00 2001 From: "Jakub A. W" Date: Sat, 8 Aug 2026 20:57:29 +0200 Subject: [PATCH 2/4] fix(ext): harden telemetry extension hooks --- config/metrics.go | 2 +- config/metrics_test.go | 3 + ext/registry.go | 48 +++++++++ ext/upstream.go | 3 + internal/app/app.go | 47 ++++---- internal/app/app_test.go | 56 +++++++++- internal/core/passthrough.go | 19 ++-- internal/core/semantic.go | 15 +++ internal/llmclient/client.go | 100 ++++++++++-------- internal/llmclient/client_test.go | 72 +++++++++++++ internal/providers/anthropic/anthropic.go | 15 +-- internal/providers/cohere/cohere.go | 15 +-- internal/providers/credentials_test.go | 43 ++++++++ .../providers/openai/compatible_provider.go | 15 +-- .../openrouter/passthrough_semantics_test.go | 2 +- internal/providers/sglang/sglang.go | 15 +-- internal/providers/vllm/vllm.go | 15 +-- .../zai/passthrough_semantics_test.go | 2 +- internal/server/handlers_test.go | 6 ++ internal/server/http.go | 10 +- internal/server/http_test.go | 65 ++++++++++++ internal/server/passthrough_service.go | 17 +-- internal/server/request_selector_peek.go | 1 + .../server/translated_inference_service.go | 15 +-- 24 files changed, 469 insertions(+), 132 deletions(-) diff --git a/config/metrics.go b/config/metrics.go index 354dbbc0b..d20362f47 100644 --- a/config/metrics.go +++ b/config/metrics.go @@ -22,7 +22,7 @@ type MetricsConfig struct { func ResolveMetricsEndpoint(endpoint string) string { metricsPath := "/metrics" if endpoint != "" { - metricsPath = path.Clean(endpoint) + metricsPath = path.Clean("/" + endpoint) } if metricsPath == "/v1" || strings.HasPrefix(metricsPath, "/v1/") || metricsPath == "/p" || strings.HasPrefix(metricsPath, "/p/") { diff --git a/config/metrics_test.go b/config/metrics_test.go index aeef572fa..373775ada 100644 --- a/config/metrics_test.go +++ b/config/metrics_test.go @@ -5,8 +5,11 @@ import "testing" func TestResolveMetricsEndpoint(t *testing.T) { tests := map[string]string{ "": "/metrics", + "metrics": "/metrics", "/monitoring/metrics/": "/monitoring/metrics", "/foo/../metrics-custom": "/metrics-custom", + "v1/models": "/metrics", + "../v1/models": "/metrics", "/v1/models": "/metrics", "/p/internal": "/metrics", } diff --git a/ext/registry.go b/ext/registry.go index 45062369c..2f47ce7df 100644 --- a/ext/registry.go +++ b/ext/registry.go @@ -1,12 +1,23 @@ package ext import ( + "fmt" "slices" "sync" "github.com/labstack/echo/v5" ) +// HTTPServerConfig exposes generation-specific HTTP settings needed when an +// extension constructs outer middleware. A new value is supplied on reload. +type HTTPServerConfig struct { + MetricsEndpoint string +} + +// OuterMiddlewareFactory constructs middleware for one server generation. +// It is intended for middleware whose configuration can change on reload. +type OuterMiddlewareFactory func(HTTPServerConfig) (echo.MiddlewareFunc, error) + // Registry collects extensions to be consumed by the gateway at startup. // Register everything before the server is constructed (before run.Run or // app.New); core snapshots each registration list during initialization. @@ -14,6 +25,7 @@ type Registry struct { mu sync.Mutex rewriters []RequestRewriter outerMiddleware []echo.MiddlewareFunc + outerFactories []OuterMiddlewareFactory middleware []echo.MiddlewareFunc routes []func(*echo.Echo) publicPaths []string @@ -34,6 +46,14 @@ func (r *Registry) UseOuterMiddleware(m echo.MiddlewareFunc) { r.outerMiddleware = append(r.outerMiddleware, m) } +// UseOuterMiddlewareFactory registers generation-specific outer middleware. +// Core invokes the factory whenever it constructs or reloads the HTTP server. +func (r *Registry) UseOuterMiddlewareFactory(factory OuterMiddlewareFactory) { + r.mu.Lock() + defer r.mu.Unlock() + r.outerFactories = append(r.outerFactories, factory) +} + // RegisterUpstreamObserver adds an observer for logical provider calls. // Observers run in registration order and may derive the context passed to // later observers and to the provider request. @@ -122,6 +142,29 @@ func (r *Registry) OuterMiddleware() []echo.MiddlewareFunc { return slices.Clone(r.outerMiddleware) } +// OuterMiddlewareFor returns static outer middleware followed by middleware +// constructed for the supplied server generation. +func (r *Registry) OuterMiddlewareFor(cfg HTTPServerConfig) ([]echo.MiddlewareFunc, error) { + r.mu.Lock() + middleware := slices.Clone(r.outerMiddleware) + factories := slices.Clone(r.outerFactories) + r.mu.Unlock() + + for i, factory := range factories { + if factory == nil { + continue + } + m, err := factory(cfg) + if err != nil { + return nil, fmt.Errorf("construct outer middleware %d: %w", i, err) + } + if m != nil { + middleware = append(middleware, m) + } + } + return middleware, nil +} + // Routes returns a defensive copy of the registered route callbacks. func (r *Registry) Routes() []func(*echo.Echo) { r.mu.Lock() @@ -177,6 +220,11 @@ func UseMiddleware(m echo.MiddlewareFunc) { Default.UseMiddleware(m) } // UseOuterMiddleware registers outer HTTP middleware on the Default registry. func UseOuterMiddleware(m echo.MiddlewareFunc) { Default.UseOuterMiddleware(m) } +// UseOuterMiddlewareFactory registers generation-specific outer HTTP middleware. +func UseOuterMiddlewareFactory(factory OuterMiddlewareFactory) { + Default.UseOuterMiddlewareFactory(factory) +} + // RegisterRoutes registers a route callback on the Default registry. func RegisterRoutes(fn func(e *echo.Echo)) { Default.RegisterRoutes(fn) } diff --git a/ext/upstream.go b/ext/upstream.go index 490ed07df..c143fc8f3 100644 --- a/ext/upstream.go +++ b/ext/upstream.go @@ -20,6 +20,9 @@ type UpstreamCall struct { Endpoint string Method string Stream bool + // StreamUncertain is true when a bounded opaque-body peek could not + // determine request intent. A later first-chunk event can still confirm SSE. + StreamUncertain bool } // UpstreamResult describes a completed logical provider call. For streaming diff --git a/internal/app/app.go b/internal/app/app.go index 180835dc4..56ff2c5f8 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -113,16 +113,23 @@ type Config struct { // applyExtensions snapshots a registered extension set into the server // configuration. A nil registry leaves the config untouched. -func applyExtensions(serverCfg *server.Config, extensions *ext.Registry) { +func applyExtensions(serverCfg *server.Config, extensions *ext.Registry) error { if extensions == nil { - return + return nil + } + outerMiddleware, err := extensions.OuterMiddlewareFor(ext.HTTPServerConfig{ + MetricsEndpoint: serverCfg.MetricsEndpoint, + }) + if err != nil { + return err } serverCfg.RequestRewriters = extensions.Rewriters() - serverCfg.OuterMiddleware = extensions.OuterMiddleware() + serverCfg.OuterMiddleware = outerMiddleware serverCfg.ExtraMiddleware = extensions.Middleware() serverCfg.ExtraRoutes = extensions.Routes() serverCfg.ExtraAuthSkipPaths = extensions.PublicPaths() serverCfg.RequestAuthenticators = extensions.Authenticators() + return nil } // routeSelectorHooks adapts upstream client lifecycle events into route @@ -226,25 +233,27 @@ func upstreamObserverHooks(observer ext.UpstreamObserver) llmclient.Hooks { func upstreamCallFromRequest(info llmclient.RequestInfo) ext.UpstreamCall { return ext.UpstreamCall{ - Provider: info.Provider, - ProviderType: info.ProviderType, - Model: info.Model, - Operation: info.Operation, - Endpoint: info.Endpoint, - Method: info.Method, - Stream: info.Stream, + Provider: info.Provider, + ProviderType: info.ProviderType, + Model: info.Model, + Operation: info.Operation, + Endpoint: info.Endpoint, + Method: info.Method, + Stream: info.Stream, + StreamUncertain: info.StreamUncertain, } } func upstreamCallFromResponse(info llmclient.ResponseInfo) ext.UpstreamCall { return ext.UpstreamCall{ - Provider: info.Provider, - ProviderType: info.ProviderType, - Model: info.Model, - Operation: info.Operation, - Endpoint: info.Endpoint, - Method: info.Method, - Stream: info.Stream, + Provider: info.Provider, + ProviderType: info.ProviderType, + Model: info.Model, + Operation: info.Operation, + Endpoint: info.Endpoint, + Method: info.Method, + Stream: info.Stream, + StreamUncertain: info.StreamUncertain, } } @@ -764,7 +773,9 @@ func New(ctx context.Context, cfg Config) (*App, error) { serverCfg.UsageSummarizer = usageReader } - applyExtensions(serverCfg, cfg.Extensions) + if err := applyExtensions(serverCfg, cfg.Extensions); err != nil { + return fail("failed to configure extensions", err) + } // Wire the readiness storage probe. Storage is a required dependency, so a // failed ping makes /health/ready report not_ready (503). When no storage diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 75069758a..e314bada2 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -8,6 +8,7 @@ import ( "log/slog" "net/http" "net/http/httptest" + "slices" "strings" "testing" "time" @@ -78,6 +79,21 @@ func TestUpstreamObserverHooksExposeProviderCall(t *testing.T) { } } +func TestUpstreamObserverHooksExposeUncertainStreamIntent(t *testing.T) { + observer := &upstreamObservation{} + hooks := upstreamObserverHooks(observer) + ctx := hooks.OnRequestStart(t.Context(), llmclient.RequestInfo{ + Provider: "openai", Operation: llmclient.OperationChat, StreamUncertain: true, + }) + hooks.OnRequestEnd(ctx, llmclient.ResponseInfo{ + Provider: "openai", Operation: llmclient.OperationChat, StreamUncertain: true, + }) + + if !observer.call.StreamUncertain || !observer.result.StreamUncertain { + t.Fatalf("stream uncertainty was not propagated: call=%+v result=%+v", observer.call, observer.result) + } +} + type panickingUpstreamObserver struct{} func (*panickingUpstreamObserver) Name() string { panic("name") } @@ -768,20 +784,30 @@ func TestApplyExtensionsSnapshotsRegistryIntoServerConfig(t *testing.T) { reg := &ext.Registry{} reg.RegisterRewriter(&staticRewriter{name: "r1"}) reg.UseOuterMiddleware(func(next echo.HandlerFunc) echo.HandlerFunc { return next }) + var factoryMetricsEndpoints []string + reg.UseOuterMiddlewareFactory(func(cfg ext.HTTPServerConfig) (echo.MiddlewareFunc, error) { + factoryMetricsEndpoints = append(factoryMetricsEndpoints, cfg.MetricsEndpoint) + return func(next echo.HandlerFunc) echo.HandlerFunc { return next }, nil + }) reg.UseMiddleware(func(next echo.HandlerFunc) echo.HandlerFunc { return next }) reg.RegisterRoutes(func(_ *echo.Echo) {}) reg.AddPublicPaths("/sso/callback", "/sso/*") reg.RegisterAuthenticator(&appTestAuthenticator{}) - serverCfg := &server.Config{} - applyExtensions(serverCfg, reg) + serverCfg := &server.Config{MetricsEndpoint: "/monitoring/metrics"} + if err := applyExtensions(serverCfg, reg); err != nil { + t.Fatal(err) + } if len(serverCfg.RequestRewriters) != 1 || serverCfg.RequestRewriters[0].Name() != "r1" { t.Errorf("RequestRewriters not copied: %+v", serverCfg.RequestRewriters) } - if len(serverCfg.OuterMiddleware) != 1 { + if len(serverCfg.OuterMiddleware) != 2 { t.Errorf("OuterMiddleware not copied: %d entries", len(serverCfg.OuterMiddleware)) } + if !slices.Equal(factoryMetricsEndpoints, []string{"/monitoring/metrics"}) { + t.Errorf("factory MetricsEndpoints = %q", factoryMetricsEndpoints) + } if len(serverCfg.ExtraMiddleware) != 1 { t.Errorf("ExtraMiddleware not copied: %d entries", len(serverCfg.ExtraMiddleware)) } @@ -795,14 +821,36 @@ func TestApplyExtensionsSnapshotsRegistryIntoServerConfig(t *testing.T) { t.Errorf("RequestAuthenticators not copied: %v", serverCfg.RequestAuthenticators) } + reloaded := &server.Config{MetricsEndpoint: "/new/metrics"} + if err := applyExtensions(reloaded, reg); err != nil { + t.Fatal(err) + } + if !slices.Equal(factoryMetricsEndpoints, []string{"/monitoring/metrics", "/new/metrics"}) { + t.Errorf("factory MetricsEndpoints after reload = %q", factoryMetricsEndpoints) + } + // A nil registry must leave the config untouched. empty := &server.Config{} - applyExtensions(empty, nil) + if err := applyExtensions(empty, nil); err != nil { + t.Fatal(err) + } if empty.RequestRewriters != nil || empty.OuterMiddleware != nil || empty.ExtraMiddleware != nil || empty.ExtraRoutes != nil || empty.ExtraAuthSkipPaths != nil || empty.RequestAuthenticators != nil { t.Error("nil registry must not modify server config") } } +func TestApplyExtensionsReturnsOuterMiddlewareFactoryError(t *testing.T) { + reg := &ext.Registry{} + reg.UseOuterMiddlewareFactory(func(ext.HTTPServerConfig) (echo.MiddlewareFunc, error) { + return nil, errors.New("factory failed") + }) + + err := applyExtensions(&server.Config{}, reg) + if err == nil || !strings.Contains(err.Error(), "factory failed") { + t.Fatalf("applyExtensions() error = %v", err) + } +} + type appTestAuthenticator struct{} func (*appTestAuthenticator) Name() string { return "test" } diff --git a/internal/core/passthrough.go b/internal/core/passthrough.go index 27a830884..193ea917c 100644 --- a/internal/core/passthrough.go +++ b/internal/core/passthrough.go @@ -8,14 +8,17 @@ import ( // PassthroughRequest is the transport-oriented request for opaque provider-native forwarding. type PassthroughRequest struct { - Method string - Endpoint string - Operation string // optional semantic GenAI operation derived at ingress - Model string // optional model derived from the opaque request body - Stream bool // explicit streaming intent derived from the request body - Body io.ReadCloser - Headers http.Header - ProviderName string // optional: concrete configured provider instance name for name-based routing + Method string + Endpoint string + Operation string // optional semantic GenAI operation derived at ingress + Model string // optional model derived from the opaque request body + Stream bool // explicit streaming intent derived from the request body + // StreamUncertain means ingress intentionally stopped its bounded body peek + // before it could prove whether this opaque request streams. + StreamUncertain bool + Body io.ReadCloser + Headers http.Header + ProviderName string // optional: concrete configured provider instance name for name-based routing } // PassthroughResponse is the raw upstream response for opaque forwarding. diff --git a/internal/core/semantic.go b/internal/core/semantic.go index e38ce3184..06d111ed8 100644 --- a/internal/core/semantic.go +++ b/internal/core/semantic.go @@ -44,6 +44,7 @@ type PassthroughRouteInfo struct { SemanticOperation string GenAIOperation string // standard GenAI operation, if this is an inference call Stream bool // explicit streaming intent derived from the request body + StreamUncertain bool // bounded opaque-body inspection could not determine stream intent AuditPath string Model string } @@ -283,6 +284,20 @@ func ApplyBodySelectorHints(env *WhiteBoxPrompt, model, provider string, stream cloned.Model = model } cloned.Stream = stream + cloned.StreamUncertain = false + CachePassthroughRouteInfo(env, &cloned) + } +} + +// MarkPassthroughStreamUncertain records that bounded opaque-body inspection +// stopped before it could determine explicit streaming intent. +func MarkPassthroughStreamUncertain(env *WhiteBoxPrompt) { + if env == nil { + return + } + if passthrough := env.CachedPassthroughRouteInfo(); passthrough != nil { + cloned := *passthrough + cloned.StreamUncertain = true CachePassthroughRouteInfo(env, &cloned) } } diff --git a/internal/llmclient/client.go b/internal/llmclient/client.go index fd1ba704e..16f8fa578 100644 --- a/internal/llmclient/client.go +++ b/internal/llmclient/client.go @@ -35,20 +35,24 @@ type RequestInfo struct { Endpoint string // API endpoint (e.g., "/chat/completions", "/models") Method string // HTTP method (e.g., "POST", "GET") Stream bool // Whether this is a streaming request + // StreamUncertain means a bounded opaque-body inspection could not + // determine intent before the upstream call began. + StreamUncertain bool } // ResponseInfo contains metadata about a response for observability hooks type ResponseInfo struct { - Provider string // Configured provider name - ProviderType string // Provider implementation type - Model string // Model name - Operation string // Semantic GenAI operation - Endpoint string // API endpoint - Method string // HTTP method - StatusCode int // HTTP status code (0 if network error) - Duration time.Duration // Request duration - Stream bool // Whether this was a streaming request - Error error // Error if request failed (nil on success) + Provider string // Configured provider name + ProviderType string // Provider implementation type + Model string // Model name + Operation string // Semantic GenAI operation + Endpoint string // API endpoint + Method string // HTTP method + StatusCode int // HTTP status code (0 if network error) + Duration time.Duration // Request duration + Stream bool // Whether this was a streaming request + StreamUncertain bool // Whether request stream intent was unknown at dispatch + Error error // Error if request failed (nil on success) // CircuitState is the provider's circuit breaker state after this request // completed ("closed", "half-open", "open"); empty when the breaker is // disabled. It reflects the moment of completion, so metrics built from it @@ -156,10 +160,11 @@ type Request struct { Model string // Operation explicitly identifies model inference semantics for // observability. Leave empty for control-plane and other non-inference calls. - Operation string - Stream bool // explicit stream intent; Accept: text/event-stream remains a fallback - Body any // Will be JSON marshaled if not nil - RawBody []byte // Used as-is (e.g., multipart form bodies). Mutually exclusive with Body and RawBodyReader. + Operation string + Stream bool // explicit stream intent; Accept: text/event-stream remains a fallback + StreamUncertain bool // bounded opaque-body inspection could not determine stream intent + Body any // Will be JSON marshaled if not nil + RawBody []byte // Used as-is (e.g., multipart form bodies). Mutually exclusive with Body and RawBodyReader. // RawBodyReader streams the request body without buffering it in memory. // It is intended for one-shot passthrough requests and is not replayable for retries. RawBodyReader io.Reader @@ -209,12 +214,13 @@ func (c *Client) beginRequest(ctx context.Context, req Request, stream bool) (re ctx: ctx, startedAt: time.Now(), requestInfo: RequestInfo{ - Provider: c.config.ProviderName, - Model: requestModel(req), - Operation: req.Operation, - Endpoint: req.Endpoint, - Method: req.Method, - Stream: stream, + Provider: c.config.ProviderName, + Model: requestModel(req), + Operation: req.Operation, + Endpoint: req.Endpoint, + Method: req.Method, + Stream: stream, + StreamUncertain: req.StreamUncertain, }, } @@ -252,17 +258,18 @@ func (c *Client) finishRequest(scope requestScope, statusCode int, err error) { circuitState = c.circuitBreaker.State() } c.config.Hooks.OnRequestEnd(scope.ctx, ResponseInfo{ - Provider: c.config.ProviderName, - ProviderType: scope.requestInfo.ProviderType, - Model: scope.requestInfo.Model, - Operation: scope.requestInfo.Operation, - Endpoint: scope.requestInfo.Endpoint, - Method: scope.requestInfo.Method, - StatusCode: statusCode, - Duration: time.Since(scope.startedAt), - Stream: scope.requestInfo.Stream, - Error: err, - CircuitState: circuitState, + Provider: c.config.ProviderName, + ProviderType: scope.requestInfo.ProviderType, + Model: scope.requestInfo.Model, + Operation: scope.requestInfo.Operation, + Endpoint: scope.requestInfo.Endpoint, + Method: scope.requestInfo.Method, + StatusCode: statusCode, + Duration: time.Since(scope.startedAt), + Stream: scope.requestInfo.Stream, + StreamUncertain: scope.requestInfo.StreamUncertain, + Error: err, + CircuitState: circuitState, }) } @@ -271,20 +278,21 @@ func (c *Client) finishStreamFirstChunk(scope requestScope, statusCode int) { return } c.config.Hooks.OnStreamFirstChunk(scope.ctx, ResponseInfo{ - Provider: c.config.ProviderName, - ProviderType: scope.requestInfo.ProviderType, - Model: scope.requestInfo.Model, - Operation: scope.requestInfo.Operation, - Endpoint: scope.requestInfo.Endpoint, - Method: scope.requestInfo.Method, - StatusCode: statusCode, - Duration: time.Since(scope.startedAt), - Stream: true, + Provider: c.config.ProviderName, + ProviderType: scope.requestInfo.ProviderType, + Model: scope.requestInfo.Model, + Operation: scope.requestInfo.Operation, + Endpoint: scope.requestInfo.Endpoint, + Method: scope.requestInfo.Method, + StatusCode: statusCode, + Duration: time.Since(scope.startedAt), + Stream: true, + StreamUncertain: scope.requestInfo.StreamUncertain, }) } -func (c *Client) observeFirstChunk(scope requestScope, resp *http.Response) { - if resp == nil || resp.Body == nil || !scope.requestInfo.Stream { +func (c *Client) observeFirstChunk(scope requestScope, resp *http.Response, stream bool) { + if resp == nil || resp.Body == nil || !stream { return } resp.Body = &firstChunkReadCloser{ @@ -576,7 +584,7 @@ func (c *Client) DoStream(ctx context.Context, req Request) (io.ReadCloser, erro } c.completeScope(scope, resp.StatusCode, nil, nil) - c.observeFirstChunk(scope, resp) + c.observeFirstChunk(scope, resp, true) return resp.Body, nil } @@ -652,7 +660,6 @@ func (c *Client) DoPassthrough(ctx context.Context, req Request) (*http.Response if retryable { if scope.halfOpenProbe || attempt == maxAttempts-1 { c.completeScope(scope, resp.StatusCode, nil, nil) - c.observeFirstChunk(scope, resp) return resp, nil } _ = resp.Body.Close() @@ -660,7 +667,10 @@ func (c *Client) DoPassthrough(ctx context.Context, req Request) (*http.Response } c.completeScope(scope, resp.StatusCode, nil, nil) - c.observeFirstChunk(scope, resp) + if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices { + responseStream := stream || strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream") + c.observeFirstChunk(scope, resp, responseStream) + } return resp, nil } diff --git a/internal/llmclient/client_test.go b/internal/llmclient/client_test.go index 9d7e8ffe1..2c5eae338 100644 --- a/internal/llmclient/client_test.go +++ b/internal/llmclient/client_test.go @@ -719,6 +719,78 @@ func TestClient_DoPassthrough_FirstChunkHookUsesOpaqueStreamBody(t *testing.T) { } } +func TestClient_DoPassthrough_FirstChunkHookUsesSuccessfulSSEContentType(t *testing.T) { + body := `{"padding":"` + strings.Repeat("x", 65*1024) + `","stream":true}` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, err := io.Copy(io.Discard, r.Body); err != nil { + t.Errorf("read request body: %v", err) + } + w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") + _, _ = w.Write([]byte("data: passthrough\n\n")) + })) + defer server.Close() + + var firstChunks []ResponseInfo + cfg := DefaultConfig("test", server.URL) + cfg.Hooks.OnStreamFirstChunk = func(_ context.Context, info ResponseInfo) { + firstChunks = append(firstChunks, info) + } + client := New(cfg, nil) + resp, err := client.DoPassthrough(t.Context(), Request{ + Method: http.MethodPost, + Endpoint: "/chat/completions", + Operation: OperationChat, + RawBodyReader: io.NopCloser(strings.NewReader(body)), + }) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if len(firstChunks) != 0 { + t.Fatalf("first chunk hook fired at response headers: %+v", firstChunks) + } + if _, err := io.ReadAll(resp.Body); err != nil { + t.Fatal(err) + } + if len(firstChunks) != 1 || !firstChunks[0].Stream || firstChunks[0].Operation != OperationChat { + t.Fatalf("first chunk observations = %+v, want one streaming chat observation", firstChunks) + } +} + +func TestClient_DoPassthrough_ErrorResponseDoesNotFireFirstChunkHook(t *testing.T) { + for _, status := range []int{http.StatusBadRequest, http.StatusServiceUnavailable} { + t.Run(http.StatusText(status), func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(status) + _, _ = w.Write([]byte("data: error\n\n")) + })) + defer server.Close() + + firstChunks := 0 + cfg := DefaultConfig("test", server.URL) + cfg.Retry.MaxRetries = 0 + cfg.Hooks.OnStreamFirstChunk = func(context.Context, ResponseInfo) { firstChunks++ } + client := New(cfg, nil) + resp, err := client.DoPassthrough(t.Context(), Request{ + Method: http.MethodPost, + Endpoint: "/chat/completions", + Stream: true, + }) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if _, err := io.ReadAll(resp.Body); err != nil { + t.Fatal(err) + } + if firstChunks != 0 { + t.Fatalf("first chunk hook calls = %d, want 0", firstChunks) + } + }) + } +} + func TestClient_DoStream_FirstChunkHookWaitsForBodyBytes(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "text/event-stream") diff --git a/internal/providers/anthropic/anthropic.go b/internal/providers/anthropic/anthropic.go index ad3b3e416..c25e6887d 100644 --- a/internal/providers/anthropic/anthropic.go +++ b/internal/providers/anthropic/anthropic.go @@ -174,13 +174,14 @@ func (p *Provider) Passthrough(ctx context.Context, req *core.PassthroughRequest } resp, err := p.client.DoPassthrough(ctx, llmclient.Request{ - Method: req.Method, - Endpoint: providers.PassthroughEndpoint(req.Endpoint), - Operation: req.Operation, - Model: req.Model, - Stream: req.Stream, - RawBodyReader: req.Body, - Headers: req.Headers, + Method: req.Method, + Endpoint: providers.PassthroughEndpoint(req.Endpoint), + Operation: req.Operation, + Model: req.Model, + Stream: req.Stream, + StreamUncertain: req.StreamUncertain, + RawBodyReader: req.Body, + Headers: req.Headers, }) if err != nil { return nil, err diff --git a/internal/providers/cohere/cohere.go b/internal/providers/cohere/cohere.go index 269de37f3..0d520b51c 100644 --- a/internal/providers/cohere/cohere.go +++ b/internal/providers/cohere/cohere.go @@ -133,13 +133,14 @@ func (p *Provider) Passthrough(ctx context.Context, req *core.PassthroughRequest return nil, core.NewInvalidRequestError("passthrough request is required", nil) } resp, err := p.client.DoPassthrough(ctx, llmclient.Request{ - Method: req.Method, - Endpoint: providers.PassthroughEndpoint(req.Endpoint), - Operation: req.Operation, - Model: req.Model, - Stream: req.Stream, - RawBodyReader: req.Body, - Headers: req.Headers, + Method: req.Method, + Endpoint: providers.PassthroughEndpoint(req.Endpoint), + Operation: req.Operation, + Model: req.Model, + Stream: req.Stream, + StreamUncertain: req.StreamUncertain, + RawBodyReader: req.Body, + Headers: req.Headers, }) if err != nil { return nil, err diff --git a/internal/providers/credentials_test.go b/internal/providers/credentials_test.go index 29081fd39..0c6437573 100644 --- a/internal/providers/credentials_test.go +++ b/internal/providers/credentials_test.go @@ -8,6 +8,7 @@ import ( "github.com/enterpilot/gomodel/config" "github.com/enterpilot/gomodel/internal/core" + "github.com/enterpilot/gomodel/internal/llmclient" ) // fakeCredentialStore is an in-memory CredentialStore for CredentialsService tests. @@ -77,6 +78,48 @@ func newCredentialsTestFactory(t *testing.T) *ProviderFactory { return factory } +func TestCredentialsService_BuildProviderPreservesManagedHookIdentity(t *testing.T) { + var start llmclient.RequestInfo + var end llmclient.ResponseInfo + factory := NewProviderFactory() + factory.SetHooks(llmclient.Hooks{ + OnRequestStart: func(ctx context.Context, info llmclient.RequestInfo) context.Context { + start = info + return ctx + }, + OnRequestEnd: func(_ context.Context, info llmclient.ResponseInfo) { + end = info + }, + }) + var providerHooks llmclient.Hooks + factory.Add(Registration{ + Type: "test", + New: func(_ ProviderConfig, opts ProviderOptions) core.Provider { + providerHooks = opts.Hooks + return ®istryMockProvider{} + }, + }) + + service, err := NewCredentialsService(t.Context(), factory, NewModelRegistry(), newFakeCredentialStore(), nil, config.ResilienceConfig{}) + if err != nil { + t.Fatal(err) + } + _, _, err = service.buildProvider(ManagedProviderCredential{ + Name: "managed-eu", + Type: "test", + APIKeys: []string{"sk-test"}, + Enabled: true, + }) + if err != nil { + t.Fatal(err) + } + providerHooks.OnRequestStart(t.Context(), llmclient.RequestInfo{}) + providerHooks.OnRequestEnd(t.Context(), llmclient.ResponseInfo{}) + if start.Provider != "managed-eu" || start.ProviderType != "test" || end.Provider != "managed-eu" || end.ProviderType != "test" { + t.Fatalf("hook identities = start %q/%q end %q/%q, want managed-eu/test", start.Provider, start.ProviderType, end.Provider, end.ProviderType) + } +} + func TestCredentialsService_UpsertRegistersAndRoutesImmediately(t *testing.T) { ctx := t.Context() factory := newCredentialsTestFactory(t) diff --git a/internal/providers/openai/compatible_provider.go b/internal/providers/openai/compatible_provider.go index ba1fde69f..dc972147e 100644 --- a/internal/providers/openai/compatible_provider.go +++ b/internal/providers/openai/compatible_provider.go @@ -427,13 +427,14 @@ func (p *CompatibleProvider) Passthrough(ctx context.Context, req *core.Passthro } resp, err := p.client.DoPassthrough(ctx, p.prepareRequest(llmclient.Request{ - Method: req.Method, - Endpoint: providers.PassthroughEndpoint(req.Endpoint), - Operation: req.Operation, - Model: req.Model, - Stream: req.Stream, - RawBodyReader: req.Body, - Headers: req.Headers, + Method: req.Method, + Endpoint: providers.PassthroughEndpoint(req.Endpoint), + Operation: req.Operation, + Model: req.Model, + Stream: req.Stream, + StreamUncertain: req.StreamUncertain, + RawBodyReader: req.Body, + Headers: req.Headers, })) if err != nil { return nil, err diff --git a/internal/providers/openrouter/passthrough_semantics_test.go b/internal/providers/openrouter/passthrough_semantics_test.go index d8fcecc81..1be4571c6 100644 --- a/internal/providers/openrouter/passthrough_semantics_test.go +++ b/internal/providers/openrouter/passthrough_semantics_test.go @@ -17,7 +17,7 @@ func TestPassthroughSemanticEnricherUsesOpenRouterType(t *testing.T) { info := enricher.Enrich(nil, nil, &core.PassthroughRouteInfo{ Provider: "openrouter", NormalizedEndpoint: "chat/completions", }) - if info == nil || info.GenAIOperation != "chat" || info.SemanticOperation != "openrouter.chat_completions" { + if info == nil || info.GenAIOperation != "chat" || info.SemanticOperation != "openrouter.chat_completions" || info.AuditPath != "/v1/chat/completions" { t.Fatalf("enriched info = %+v, want OpenRouter chat semantics", info) } } diff --git a/internal/providers/sglang/sglang.go b/internal/providers/sglang/sglang.go index 3b6047261..6d9ac7463 100644 --- a/internal/providers/sglang/sglang.go +++ b/internal/providers/sglang/sglang.go @@ -132,13 +132,14 @@ func (p *Provider) Passthrough(ctx context.Context, req *core.PassthroughRequest } resp, err := p.rootClient.DoPassthrough(ctx, llmclient.Request{ - Method: req.Method, - Endpoint: endpoint, - Operation: req.Operation, - Model: req.Model, - Stream: req.Stream, - RawBodyReader: req.Body, - Headers: req.Headers, + Method: req.Method, + Endpoint: endpoint, + Operation: req.Operation, + Model: req.Model, + Stream: req.Stream, + StreamUncertain: req.StreamUncertain, + RawBodyReader: req.Body, + Headers: req.Headers, }) if err != nil { return nil, err diff --git a/internal/providers/vllm/vllm.go b/internal/providers/vllm/vllm.go index 859f8d644..bd5df22aa 100644 --- a/internal/providers/vllm/vllm.go +++ b/internal/providers/vllm/vllm.go @@ -124,13 +124,14 @@ func (p *Provider) Passthrough(ctx context.Context, req *core.PassthroughRequest endpoint := providers.PassthroughEndpoint(req.Endpoint) if !usesV1PassthroughBase(endpoint) { resp, err := p.rootClient.DoPassthrough(ctx, llmclient.Request{ - Method: req.Method, - Endpoint: endpoint, - Operation: req.Operation, - Model: req.Model, - Stream: req.Stream, - RawBodyReader: req.Body, - Headers: req.Headers, + Method: req.Method, + Endpoint: endpoint, + Operation: req.Operation, + Model: req.Model, + Stream: req.Stream, + StreamUncertain: req.StreamUncertain, + RawBodyReader: req.Body, + Headers: req.Headers, }) if err != nil { return nil, err diff --git a/internal/providers/zai/passthrough_semantics_test.go b/internal/providers/zai/passthrough_semantics_test.go index 8f324ea5a..9a68c69e3 100644 --- a/internal/providers/zai/passthrough_semantics_test.go +++ b/internal/providers/zai/passthrough_semantics_test.go @@ -17,7 +17,7 @@ func TestPassthroughSemanticEnricherUsesZAIType(t *testing.T) { info := enricher.Enrich(nil, nil, &core.PassthroughRouteInfo{ Provider: "zai", NormalizedEndpoint: "embeddings", }) - if info == nil || info.GenAIOperation != "embeddings" || info.SemanticOperation != "zai.embeddings" { + if info == nil || info.GenAIOperation != "embeddings" || info.SemanticOperation != "zai.embeddings" || info.AuditPath != "/v1/embeddings" { t.Fatalf("enriched info = %+v, want Z.ai embedding semantics", info) } } diff --git a/internal/server/handlers_test.go b/internal/server/handlers_test.go index d3b4fa1f6..94c48de1c 100644 --- a/internal/server/handlers_test.go +++ b/internal/server/handlers_test.go @@ -2242,6 +2242,12 @@ func TestChatCompletionStreaming_FastPathUsageCarriesResolvedProviderName(t *tes if got := usageLog.entries[0].ProviderName; got != "openai_test" { t.Fatalf("ProviderName = %q, want openai_test", got) } + if mock.lastPassthroughReq == nil { + t.Fatal("lastPassthroughReq = nil, want passthrough request") + } + if got := mock.lastPassthroughReq.ProviderName; got != "openai_test" { + t.Fatalf("passthrough ProviderName = %q, want openai_test", got) + } } func TestChatCompletionStreaming_FastPathSkipsQualifiedModelRewrite(t *testing.T) { diff --git a/internal/server/http.go b/internal/server/http.go index c6cc1ae8a..975950bed 100644 --- a/internal/server/http.go +++ b/internal/server/http.go @@ -224,8 +224,11 @@ func New(provider core.RoutableProvider, cfg *Config) *Server { // Determine metrics path metricsPath := config.ResolveMetricsEndpoint("") if cfg != nil && cfg.MetricsEnabled { - configuredPath := path.Clean(cfg.MetricsEndpoint) + configuredPath := path.Clean("/" + cfg.MetricsEndpoint) metricsPath = config.ResolveMetricsEndpoint(cfg.MetricsEndpoint) + if cfg.PprofEnabled && (metricsPath == "/debug/pprof" || strings.HasPrefix(metricsPath, "/debug/pprof/")) { + metricsPath = "/metrics" + } // Prevent metrics endpoint from shadowing API routes (security: auth bypass) if metricsPath != configuredPath && cfg.MetricsEndpoint != "" { slog.Warn("metrics endpoint conflicts with API routes, using /metrics instead", @@ -258,9 +261,10 @@ func New(provider core.RoutableProvider, cfg *Config) *Server { // Scrub credential-like query values before the outer request logger // snapshots RequestURI. URL.RawQuery remains intact for handlers. e.Use(redactSensitiveRequestURI()) + e.Use(middleware.Recover()) // Outer extension middleware covers the complete HTTP request while still // seeing the credential-redacted URI. It runs before request logging, - // recovery, limits, audit, and auth, so it must not depend on identity. + // limits, audit, and auth, so it must not depend on identity. if cfg != nil { for _, m := range cfg.OuterMiddleware { e.Use(m) @@ -303,8 +307,6 @@ func New(provider core.RoutableProvider, cfg *Config) *Server { } else { e.Use(middleware.RequestLogger()) } - e.Use(middleware.Recover()) - // Body size limit (default: 10MB) bodySizeLimit := "10M" if cfg != nil && cfg.BodySizeLimit != "" { diff --git a/internal/server/http_test.go b/internal/server/http_test.go index 5153b6a4d..dbb6f27f3 100644 --- a/internal/server/http_test.go +++ b/internal/server/http_test.go @@ -257,6 +257,47 @@ func TestMetricsEndpoint(t *testing.T) { } } +func TestMetricsEndpointDoesNotCollideWithPprof(t *testing.T) { + srv := New(&mockProvider{}, &Config{ + MetricsEnabled: true, + MetricsEndpoint: "/debug/pprof/goroutine", + PprofEnabled: true, + }) + + metricsReq := httptest.NewRequest(http.MethodGet, "/metrics", nil) + metricsRec := httptest.NewRecorder() + srv.ServeHTTP(metricsRec, metricsReq) + if metricsRec.Code != http.StatusOK || !strings.Contains(metricsRec.Body.String(), "go_goroutines") { + t.Fatalf("fallback metrics response = %d %q", metricsRec.Code, metricsRec.Body.String()) + } + + pprofReq := httptest.NewRequest(http.MethodGet, "/debug/pprof/goroutine", nil) + pprofRec := httptest.NewRecorder() + srv.ServeHTTP(pprofRec, pprofReq) + if pprofRec.Code != http.StatusOK || strings.Contains(pprofRec.Body.String(), "# HELP go_") { + t.Fatalf("pprof response = %d, unexpectedly served metrics", pprofRec.Code) + } +} + +func TestOuterMiddlewarePanicIsRecovered(t *testing.T) { + srv := New(&mockProvider{}, &Config{ + OuterMiddleware: []echo.MiddlewareFunc{ + func(next echo.HandlerFunc) echo.HandlerFunc { + return func(*echo.Context) error { + panic("outer middleware panic") + } + }, + }, + }) + + req := httptest.NewRequest(http.MethodGet, "/health", nil) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusInternalServerError) + } +} + func TestBasePathStripsPrefixBeforeRouting(t *testing.T) { mock := &mockProvider{ modelsResponse: &core.ModelsResponse{ @@ -1126,6 +1167,30 @@ func TestProviderPassthroughRoute_EnabledByDefault(t *testing.T) { } } +func TestProviderPassthroughRoute_MarksOversizedStreamIntentUncertain(t *testing.T) { + mock := &mockProvider{ + passthroughResponse: &core.PassthroughResponse{ + StatusCode: http.StatusOK, + Headers: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"ok":true}`)), + }, + } + srv := New(mock, &Config{}) + body := `{"model":"gpt-5-mini","padding":"` + strings.Repeat("x", 65*1024) + `","stream":true}` + req := httptest.NewRequest(http.MethodPost, "/p/openai/responses", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + + srv.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) + } + if mock.lastPassthroughReq == nil || !mock.lastPassthroughReq.StreamUncertain { + t.Fatalf("passthrough stream metadata = %+v, want uncertain", mock.lastPassthroughReq) + } +} + func TestProviderPassthroughRoute_DisabledRequiresAuthBefore404(t *testing.T) { mock := &mockProvider{} srv := New(mock, &Config{ diff --git a/internal/server/passthrough_service.go b/internal/server/passthrough_service.go index cbeec373f..f9c010b44 100644 --- a/internal/server/passthrough_service.go +++ b/internal/server/passthrough_service.go @@ -49,14 +49,15 @@ func (s *passthroughService) ProviderPassthrough(c *echo.Context) error { ctx, _ := requestContextWithRequestID(c.Request()) c.SetRequest(c.Request().WithContext(ctx)) resp, err := passthroughProvider.Passthrough(ctx, providerType, &core.PassthroughRequest{ - Method: c.Request().Method, - Endpoint: endpoint, - Operation: info.GenAIOperation, - Model: info.Model, - Stream: info.Stream, - Body: c.Request().Body, - Headers: buildPassthroughHeaders(ctx, c.Request().Header), - ProviderName: providerName, + Method: c.Request().Method, + Endpoint: endpoint, + Operation: info.GenAIOperation, + Model: info.Model, + Stream: info.Stream, + StreamUncertain: info.StreamUncertain, + Body: c.Request().Body, + Headers: buildPassthroughHeaders(ctx, c.Request().Header), + ProviderName: providerName, }) if err != nil { return handleError(c, err) diff --git a/internal/server/request_selector_peek.go b/internal/server/request_selector_peek.go index b3697c689..94372507f 100644 --- a/internal/server/request_selector_peek.go +++ b/internal/server/request_selector_peek.go @@ -28,6 +28,7 @@ func seedRequestBodySelectorHints(req *http.Request, bodyMode core.BodyMode, env hints := peekRequestBodySelectorHints(req, requestSelectorPeekLimit) if !hints.parsed || !hints.complete { + core.MarkPassthroughStreamUncertain(env) return } core.ApplyBodySelectorHints(env, hints.model, hints.provider, hints.stream) diff --git a/internal/server/translated_inference_service.go b/internal/server/translated_inference_service.go index 9bf71f29f..eb4703712 100644 --- a/internal/server/translated_inference_service.go +++ b/internal/server/translated_inference_service.go @@ -450,13 +450,14 @@ func (s *translatedInferenceService) tryFastPathStreamingChatPassthrough(c *echo const endpoint = "/chat/completions" providerType := strings.TrimSpace(workflow.ProviderType) resp, err := passthroughProvider.Passthrough(ctx, providerType, &core.PassthroughRequest{ - Method: c.Request().Method, - Endpoint: endpoint, - Operation: llmclient.OperationChat, - Model: resolvedModelFromWorkflow(workflow, req.Model), - Stream: req.Stream, - Body: c.Request().Body, - Headers: buildPassthroughHeaders(ctx, c.Request().Header), + Method: c.Request().Method, + Endpoint: endpoint, + Operation: llmclient.OperationChat, + Model: resolvedModelFromWorkflow(workflow, req.Model), + Stream: req.Stream, + Body: c.Request().Body, + Headers: buildPassthroughHeaders(ctx, c.Request().Header), + ProviderName: providerNameFromWorkflow(workflow), }) if err != nil { return true, handleError(c, err) From bfe90afd26c18410383647846ed24447b048ee09 Mon Sep 17 00:00:00 2001 From: "Jakub A. W" Date: Mon, 10 Aug 2026 11:17:18 +0200 Subject: [PATCH 3/4] fix(ext): align telemetry metadata resolution --- config/metrics.go | 10 +++++ config/metrics_test.go | 22 ++++++++++ internal/app/app.go | 1 + internal/app/app_test.go | 32 ++++++++++++--- internal/providers/credentials_test.go | 12 +++++- internal/server/handlers_test.go | 19 ++++++++- internal/server/http.go | 5 +-- internal/server/request_selector_peek.go | 19 +++++---- internal/server/request_selector_peek_test.go | 41 +++++++++++++++++++ 9 files changed, 140 insertions(+), 21 deletions(-) diff --git a/config/metrics.go b/config/metrics.go index d20362f47..7696aca41 100644 --- a/config/metrics.go +++ b/config/metrics.go @@ -30,3 +30,13 @@ func ResolveMetricsEndpoint(endpoint string) string { } return metricsPath } + +// ResolveMetricsEndpointWithPprof also prevents the metrics route from +// shadowing an enabled pprof route. +func ResolveMetricsEndpointWithPprof(endpoint string, pprofEnabled bool) string { + metricsPath := ResolveMetricsEndpoint(endpoint) + if pprofEnabled && (metricsPath == "/debug/pprof" || strings.HasPrefix(metricsPath, "/debug/pprof/")) { + return "/metrics" + } + return metricsPath +} diff --git a/config/metrics_test.go b/config/metrics_test.go index 373775ada..01e1d24e4 100644 --- a/config/metrics_test.go +++ b/config/metrics_test.go @@ -19,3 +19,25 @@ func TestResolveMetricsEndpoint(t *testing.T) { } } } + +func TestResolveMetricsEndpointWithPprof(t *testing.T) { + tests := map[string]struct { + endpoint string + pprofEnabled bool + want string + }{ + "pprof disabled": {endpoint: "/debug/pprof", want: "/debug/pprof"}, + "pprof root conflict": {endpoint: "/debug/pprof", pprofEnabled: true, want: "/metrics"}, + "pprof child conflict": { + endpoint: "/debug/pprof/goroutine", pprofEnabled: true, want: "/metrics", + }, + "custom endpoint": {endpoint: "monitoring/metrics", pprofEnabled: true, want: "/monitoring/metrics"}, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + if got := ResolveMetricsEndpointWithPprof(test.endpoint, test.pprofEnabled); got != test.want { + t.Errorf("ResolveMetricsEndpointWithPprof(%q, %v) = %q, want %q", test.endpoint, test.pprofEnabled, got, test.want) + } + }) + } +} diff --git a/internal/app/app.go b/internal/app/app.go index 56ff2c5f8..34ad1a9cc 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -117,6 +117,7 @@ func applyExtensions(serverCfg *server.Config, extensions *ext.Registry) error { if extensions == nil { return nil } + serverCfg.MetricsEndpoint = config.ResolveMetricsEndpointWithPprof(serverCfg.MetricsEndpoint, serverCfg.PprofEnabled) outerMiddleware, err := extensions.OuterMiddlewareFor(ext.HTTPServerConfig{ MetricsEndpoint: serverCfg.MetricsEndpoint, }) diff --git a/internal/app/app_test.go b/internal/app/app_test.go index e314bada2..8242c2c2c 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -808,6 +808,9 @@ func TestApplyExtensionsSnapshotsRegistryIntoServerConfig(t *testing.T) { if !slices.Equal(factoryMetricsEndpoints, []string{"/monitoring/metrics"}) { t.Errorf("factory MetricsEndpoints = %q", factoryMetricsEndpoints) } + if serverCfg.MetricsEndpoint != "/monitoring/metrics" { + t.Errorf("server MetricsEndpoint = %q, want /monitoring/metrics", serverCfg.MetricsEndpoint) + } if len(serverCfg.ExtraMiddleware) != 1 { t.Errorf("ExtraMiddleware not copied: %d entries", len(serverCfg.ExtraMiddleware)) } @@ -821,12 +824,31 @@ func TestApplyExtensionsSnapshotsRegistryIntoServerConfig(t *testing.T) { t.Errorf("RequestAuthenticators not copied: %v", serverCfg.RequestAuthenticators) } - reloaded := &server.Config{MetricsEndpoint: "/new/metrics"} - if err := applyExtensions(reloaded, reg); err != nil { - t.Fatal(err) + endpointTests := []struct { + name string + endpoint string + pprofEnabled bool + want string + }{ + {name: "reload endpoint", endpoint: "/new/metrics", want: "/new/metrics"}, + {name: "default endpoint", want: "/metrics"}, + {name: "custom endpoint without leading slash", endpoint: "monitoring/custom", want: "/monitoring/custom"}, + {name: "API route conflict", endpoint: "/v1", want: "/metrics"}, + {name: "pprof conflict", endpoint: "/debug/pprof", pprofEnabled: true, want: "/metrics"}, } - if !slices.Equal(factoryMetricsEndpoints, []string{"/monitoring/metrics", "/new/metrics"}) { - t.Errorf("factory MetricsEndpoints after reload = %q", factoryMetricsEndpoints) + for _, test := range endpointTests { + t.Run(test.name, func(t *testing.T) { + cfg := &server.Config{MetricsEndpoint: test.endpoint, PprofEnabled: test.pprofEnabled} + if err := applyExtensions(cfg, reg); err != nil { + t.Fatal(err) + } + if got := factoryMetricsEndpoints[len(factoryMetricsEndpoints)-1]; got != test.want { + t.Errorf("factory MetricsEndpoint = %q, want %q", got, test.want) + } + if cfg.MetricsEndpoint != test.want { + t.Errorf("server MetricsEndpoint = %q, want %q", cfg.MetricsEndpoint, test.want) + } + }) } // A nil registry must leave the config untouched. diff --git a/internal/providers/credentials_test.go b/internal/providers/credentials_test.go index 0c6437573..6c2a5c4f6 100644 --- a/internal/providers/credentials_test.go +++ b/internal/providers/credentials_test.go @@ -81,6 +81,7 @@ func newCredentialsTestFactory(t *testing.T) *ProviderFactory { func TestCredentialsService_BuildProviderPreservesManagedHookIdentity(t *testing.T) { var start llmclient.RequestInfo var end llmclient.ResponseInfo + var firstChunk llmclient.ResponseInfo factory := NewProviderFactory() factory.SetHooks(llmclient.Hooks{ OnRequestStart: func(ctx context.Context, info llmclient.RequestInfo) context.Context { @@ -90,6 +91,9 @@ func TestCredentialsService_BuildProviderPreservesManagedHookIdentity(t *testing OnRequestEnd: func(_ context.Context, info llmclient.ResponseInfo) { end = info }, + OnStreamFirstChunk: func(_ context.Context, info llmclient.ResponseInfo) { + firstChunk = info + }, }) var providerHooks llmclient.Hooks factory.Add(Registration{ @@ -115,8 +119,12 @@ func TestCredentialsService_BuildProviderPreservesManagedHookIdentity(t *testing } providerHooks.OnRequestStart(t.Context(), llmclient.RequestInfo{}) providerHooks.OnRequestEnd(t.Context(), llmclient.ResponseInfo{}) - if start.Provider != "managed-eu" || start.ProviderType != "test" || end.Provider != "managed-eu" || end.ProviderType != "test" { - t.Fatalf("hook identities = start %q/%q end %q/%q, want managed-eu/test", start.Provider, start.ProviderType, end.Provider, end.ProviderType) + providerHooks.OnStreamFirstChunk(t.Context(), llmclient.ResponseInfo{}) + if start.Provider != "managed-eu" || start.ProviderType != "test" || + end.Provider != "managed-eu" || end.ProviderType != "test" || + firstChunk.Provider != "managed-eu" || firstChunk.ProviderType != "test" { + t.Fatalf("hook identities = start %q/%q end %q/%q first chunk %q/%q, want managed-eu/test", + start.Provider, start.ProviderType, end.Provider, end.ProviderType, firstChunk.Provider, firstChunk.ProviderType) } } diff --git a/internal/server/handlers_test.go b/internal/server/handlers_test.go index 4b0392fd2..e571b58aa 100644 --- a/internal/server/handlers_test.go +++ b/internal/server/handlers_test.go @@ -31,6 +31,7 @@ import ( "github.com/enterpilot/gomodel/internal/filestore" "github.com/enterpilot/gomodel/internal/gateway" "github.com/enterpilot/gomodel/internal/guardrails" + "github.com/enterpilot/gomodel/internal/llmclient" "github.com/enterpilot/gomodel/internal/observability" provideradapter "github.com/enterpilot/gomodel/internal/providers" "github.com/enterpilot/gomodel/internal/responsestore" @@ -2245,8 +2246,22 @@ func TestChatCompletionStreaming_FastPathUsageCarriesResolvedProviderName(t *tes if mock.lastPassthroughReq == nil { t.Fatal("lastPassthroughReq = nil, want passthrough request") } - if got := mock.lastPassthroughReq.ProviderName; got != "openai_test" { - t.Fatalf("passthrough ProviderName = %q, want openai_test", got) + checks := []struct { + name string + got any + want any + }{ + {name: "Operation", got: mock.lastPassthroughReq.Operation, want: llmclient.OperationChat}, + {name: "Model", got: mock.lastPassthroughReq.Model, want: "gpt-4o-mini"}, + {name: "Stream", got: mock.lastPassthroughReq.Stream, want: true}, + {name: "ProviderName", got: mock.lastPassthroughReq.ProviderName, want: "openai_test"}, + } + for _, check := range checks { + t.Run(check.name, func(t *testing.T) { + if check.got != check.want { + t.Errorf("passthrough %s = %v, want %v", check.name, check.got, check.want) + } + }) } } diff --git a/internal/server/http.go b/internal/server/http.go index 975950bed..99fd66e64 100644 --- a/internal/server/http.go +++ b/internal/server/http.go @@ -225,10 +225,7 @@ func New(provider core.RoutableProvider, cfg *Config) *Server { metricsPath := config.ResolveMetricsEndpoint("") if cfg != nil && cfg.MetricsEnabled { configuredPath := path.Clean("/" + cfg.MetricsEndpoint) - metricsPath = config.ResolveMetricsEndpoint(cfg.MetricsEndpoint) - if cfg.PprofEnabled && (metricsPath == "/debug/pprof" || strings.HasPrefix(metricsPath, "/debug/pprof/")) { - metricsPath = "/metrics" - } + metricsPath = config.ResolveMetricsEndpointWithPprof(cfg.MetricsEndpoint, cfg.PprofEnabled) // Prevent metrics endpoint from shadowing API routes (security: auth bypass) if metricsPath != configuredPath && cfg.MetricsEndpoint != "" { slog.Warn("metrics endpoint conflicts with API routes, using /metrics instead", diff --git a/internal/server/request_selector_peek.go b/internal/server/request_selector_peek.go index 94372507f..3bca4472a 100644 --- a/internal/server/request_selector_peek.go +++ b/internal/server/request_selector_peek.go @@ -14,11 +14,12 @@ import ( const requestSelectorPeekLimit int64 = 64 * 1024 type requestBodySelectorHints struct { - model string - provider string - stream bool - parsed bool - complete bool + model string + provider string + stream bool + streamParsed bool + parsed bool + complete bool } func seedRequestBodySelectorHints(req *http.Request, bodyMode core.BodyMode, env *core.WhiteBoxPrompt) { @@ -27,11 +28,12 @@ func seedRequestBodySelectorHints(req *http.Request, bodyMode core.BodyMode, env } hints := peekRequestBodySelectorHints(req, requestSelectorPeekLimit) - if !hints.parsed || !hints.complete { + if hints.parsed || hints.streamParsed { + core.ApplyBodySelectorHints(env, hints.model, hints.provider, hints.stream) + } + if !hints.streamParsed { core.MarkPassthroughStreamUncertain(env) - return } - core.ApplyBodySelectorHints(env, hints.model, hints.provider, hints.stream) } func shouldPeekRequestBodySelectors(req *http.Request, bodyMode core.BodyMode, env *core.WhiteBoxPrompt) bool { @@ -122,6 +124,7 @@ func decodeRequestBodySelectorHints(r io.Reader) requestBodySelectorHints { return requestBodySelectorHints{} } hints.stream = stream + hints.streamParsed = true default: if err := skipJSONValue(dec); err != nil { return requestBodySelectorHints{} diff --git a/internal/server/request_selector_peek_test.go b/internal/server/request_selector_peek_test.go index 30c2b8494..a92f97f36 100644 --- a/internal/server/request_selector_peek_test.go +++ b/internal/server/request_selector_peek_test.go @@ -65,3 +65,44 @@ func TestSeedRequestBodySelectorHintsDoesNotMarkModelOnlyPeekAsParsed(t *testing t.Fatalf("RouteHints.Model = %q, want empty", env.RouteHints.Model) } } + +func TestSeedRequestBodySelectorHintsTracksStreamConfidenceIndependently(t *testing.T) { + tests := []struct { + name string + body string + wantStream bool + wantUncertain bool + }{ + { + name: "stream before model and provider", + body: `{"stream":true,"model":"gpt-4o-mini","provider":"openai","padding":"` + strings.Repeat("x", 65*1024) + `"}`, + wantStream: true, + wantUncertain: false, + }, + { + name: "stream absent before bounded peek stops", + body: `{"model":"gpt-4o-mini","provider":"openai","padding":"` + strings.Repeat("x", 65*1024) + `"}`, + wantStream: false, + wantUncertain: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/p/openai/chat/completions", strings.NewReader(test.body)) + env := &core.WhiteBoxPrompt{} + core.CachePassthroughRouteInfo(env, &core.PassthroughRouteInfo{Provider: "openai"}) + + seedRequestBodySelectorHints(req, core.BodyModeJSON, env) + + info := env.CachedPassthroughRouteInfo() + if info == nil { + t.Fatal("CachedPassthroughRouteInfo() = nil") + } + if info.Stream != test.wantStream || info.StreamUncertain != test.wantUncertain { + t.Errorf("stream metadata = stream %v uncertain %v, want stream %v uncertain %v", + info.Stream, info.StreamUncertain, test.wantStream, test.wantUncertain) + } + }) + } +} From c90d92fbe965bb82723a098cf4605df0b7894e2d Mon Sep 17 00:00:00 2001 From: "Jakub A. W" Date: Mon, 10 Aug 2026 15:37:05 +0200 Subject: [PATCH 4/4] fix(telemetry): preserve passthrough metadata --- internal/auditlog/enrich.go | 3 + internal/core/semantic.go | 33 +++++++++++ internal/server/handlers_test.go | 20 +++++-- .../server/passthrough_execution_helpers.go | 1 + .../passthrough_execution_helpers_test.go | 3 + internal/server/passthrough_service.go | 2 +- internal/server/request_snapshot.go | 3 +- internal/server/request_snapshot_test.go | 55 +++++++++++++++++++ 8 files changed, 113 insertions(+), 7 deletions(-) diff --git a/internal/auditlog/enrich.go b/internal/auditlog/enrich.go index 6e335576f..e4df25123 100644 --- a/internal/auditlog/enrich.go +++ b/internal/auditlog/enrich.go @@ -451,6 +451,9 @@ func enrichEntryWithWorkflow(entry *LogEntry, workflow *core.Workflow) { if model := strings.TrimSpace(workflow.Passthrough.Model); model != "" { entry.RequestedModel = model } + if providerName := strings.TrimSpace(workflow.Passthrough.ProviderName); providerName != "" && !executedProviderName { + entry.ProviderName = providerName + } } if !executedProvider { if providerType := strings.TrimSpace(workflow.ProviderType); providerType != "" { diff --git a/internal/core/semantic.go b/internal/core/semantic.go index 06d111ed8..dfa0d2357 100644 --- a/internal/core/semantic.go +++ b/internal/core/semantic.go @@ -262,6 +262,39 @@ func DeriveWhiteBoxPrompt(snapshot *RequestSnapshot) *WhiteBoxPrompt { return env } +// RefreshWhiteBoxPrompt rebuilds request semantics after a deferred body read +// while retaining passthrough metadata added by provider-owned enrichment. +// Body-derived model and stream intent from the refreshed snapshot remain +// authoritative when the complete body parses successfully. +func RefreshWhiteBoxPrompt(snapshot *RequestSnapshot, previous *WhiteBoxPrompt) *WhiteBoxPrompt { + refreshed := DeriveWhiteBoxPrompt(snapshot) + if refreshed == nil || previous == nil { + return refreshed + } + + current := refreshed.CachedPassthroughRouteInfo() + prior := previous.CachedPassthroughRouteInfo() + if current == nil || prior == nil { + return refreshed + } + + merged := *prior + if current.Provider != "" && merged.Provider == "" { + merged.Provider = current.Provider + } + if current.RawEndpoint != "" { + merged.RawEndpoint = current.RawEndpoint + } + if refreshed.JSONBodyParsed { + merged.Model = current.Model + merged.Stream = current.Stream + merged.StreamUncertain = current.StreamUncertain + } + CachePassthroughRouteInfo(refreshed, &merged) + refreshed.StreamRequested = merged.Stream + return refreshed +} + // ApplyBodySelectorHints records selector hints parsed from a request body. // The hints are intentionally sparse and best-effort; canonical request decode // remains authoritative for translated JSON requests. diff --git a/internal/server/handlers_test.go b/internal/server/handlers_test.go index e571b58aa..4da44adb0 100644 --- a/internal/server/handlers_test.go +++ b/internal/server/handlers_test.go @@ -6801,18 +6801,20 @@ func TestProviderPassthrough_UsesPassthroughModelForAuditEntry(t *testing.T) { }, Body: io.NopCloser(strings.NewReader(`{"ok":true}`)), }, + providerTypes: map[string]string{"openai_test/gpt-5-mini": "openai"}, + providerNames: map[string]string{"openai_test/gpt-5-mini": "openai_test"}, } e := echo.New() handler := NewHandler(provider, nil, nil, nil) - req := httptest.NewRequest(http.MethodPost, "/p/openai/v1/chat/completions", strings.NewReader(`{"model":"gpt-5-mini"}`)) + req := httptest.NewRequest(http.MethodPost, "/p/openai_test/v1/chat/completions", strings.NewReader(`{"model":"gpt-5-mini"}`)) req.Header.Set("Content-Type", "application/json") req = req.WithContext(core.WithWorkflow(req.Context(), &core.Workflow{ Mode: core.ExecutionModePassthrough, ProviderType: "openai", Passthrough: &core.PassthroughRouteInfo{ - Provider: "openai", + Provider: "openai_test", RawEndpoint: "chat/completions", NormalizedEndpoint: "chat/completions", Model: "gpt-5-mini", @@ -6837,6 +6839,9 @@ func TestProviderPassthrough_UsesPassthroughModelForAuditEntry(t *testing.T) { if entry.Provider != "openai" { t.Fatalf("audit entry provider = %q, want openai", entry.Provider) } + if entry.ProviderName != "openai_test" { + t.Fatalf("audit entry provider name = %q, want openai_test", entry.ProviderName) + } } func TestProviderPassthrough_UsesConfiguredProviderNameForAccessValidation(t *testing.T) { @@ -7056,6 +7061,8 @@ func TestProviderPassthrough_OpenAIStreamWritesUsageEntry(t *testing.T) { "data: [DONE]\n\n", )), }, + providerTypes: map[string]string{"openai_test/gpt-5-mini": "openai"}, + providerNames: map[string]string{"openai_test/gpt-5-mini": "openai_test"}, } usageLog := &collectingUsageLogger{ config: usage.Config{Enabled: true}, @@ -7065,7 +7072,7 @@ func TestProviderPassthrough_OpenAIStreamWritesUsageEntry(t *testing.T) { handler := NewHandler(provider, nil, usageLog, nil) e.POST("/p/:provider/*", handler.ProviderPassthrough) - req := httptest.NewRequest(http.MethodPost, "/p/openai/responses", strings.NewReader(`{"model":"gpt-5-mini"}`)) + req := httptest.NewRequest(http.MethodPost, "/p/openai_test/responses", strings.NewReader(`{"model":"gpt-5-mini"}`)) req.Header.Set("Content-Type", "application/json") req.Header.Set("X-Request-ID", "req-pass-stream-usage") rec := httptest.NewRecorder() @@ -7082,8 +7089,11 @@ func TestProviderPassthrough_OpenAIStreamWritesUsageEntry(t *testing.T) { if entry.Provider != "openai" { t.Fatalf("Provider = %q, want openai", entry.Provider) } - if entry.Endpoint != "/p/openai/responses" { - t.Fatalf("Endpoint = %q, want /p/openai/responses", entry.Endpoint) + if entry.ProviderName != "openai_test" { + t.Fatalf("ProviderName = %q, want openai_test", entry.ProviderName) + } + if entry.Endpoint != "/p/openai_test/responses" { + t.Fatalf("Endpoint = %q, want /p/openai_test/responses", entry.Endpoint) } if entry.Model != "gpt-5-mini" { t.Fatalf("Model = %q, want gpt-5-mini", entry.Model) diff --git a/internal/server/passthrough_execution_helpers.go b/internal/server/passthrough_execution_helpers.go index bf375776c..8bb17d611 100644 --- a/internal/server/passthrough_execution_helpers.go +++ b/internal/server/passthrough_execution_helpers.go @@ -52,5 +52,6 @@ func passthroughExecutionTarget(c *echo.Context, provider core.RoutableProvider, } info.Provider = providerType + info.ProviderName = providerName return providerType, providerName, endpoint, info, nil } diff --git a/internal/server/passthrough_execution_helpers_test.go b/internal/server/passthrough_execution_helpers_test.go index 5888bcf3b..def35c0ce 100644 --- a/internal/server/passthrough_execution_helpers_test.go +++ b/internal/server/passthrough_execution_helpers_test.go @@ -99,4 +99,7 @@ func TestPassthroughExecutionTarget_ResolvesConfiguredProviderNameToType(t *test if info == nil || info.Provider != "openai" { t.Fatalf("info.Provider = %#v, want openai", info) } + if info.ProviderName != "openai_test" { + t.Fatalf("info.ProviderName = %q, want openai_test", info.ProviderName) + } } diff --git a/internal/server/passthrough_service.go b/internal/server/passthrough_service.go index f9c010b44..945c663a4 100644 --- a/internal/server/passthrough_service.go +++ b/internal/server/passthrough_service.go @@ -69,5 +69,5 @@ func (s *passthroughService) ProviderPassthrough(c *echo.Context) error { } else { auditlog.EnrichEntry(c, info.Model, providerType) } - return s.proxyPassthroughResponse(c, providerType, providerNameFromWorkflow(workflow), endpoint, info, resp) + return s.proxyPassthroughResponse(c, providerType, providerName, endpoint, info, resp) } diff --git a/internal/server/request_snapshot.go b/internal/server/request_snapshot.go index 220b7d502..b2b6846c7 100644 --- a/internal/server/request_snapshot.go +++ b/internal/server/request_snapshot.go @@ -251,11 +251,12 @@ func storeRequestBodySnapshot(c *echo.Context, bodyBytes []byte) { updated := snapshot.WithOwnedCapturedBody(capturedBody, bodyNotCaptured) ctx := core.WithRequestSnapshot(req.Context(), updated) + previous := core.GetWhiteBoxPrompt(req.Context()) semanticSnapshot := updated if bodyNotCaptured { semanticSnapshot = snapshot.WithOwnedCapturedBody(bodyBytes, false) } - if semantics := core.DeriveWhiteBoxPrompt(semanticSnapshot); semantics != nil { + if semantics := core.RefreshWhiteBoxPrompt(semanticSnapshot, previous); semantics != nil { ctx = core.WithWhiteBoxPrompt(ctx, semantics) } c.SetRequest(req.WithContext(ctx)) diff --git a/internal/server/request_snapshot_test.go b/internal/server/request_snapshot_test.go index ac4fd389f..a3c9cf328 100644 --- a/internal/server/request_snapshot_test.go +++ b/internal/server/request_snapshot_test.go @@ -173,6 +173,61 @@ func TestSemanticJSONBodyRefreshesPromptFromFullBody(t *testing.T) { assert.Nil(t, updated.CapturedBodyView()) } +func TestStoreRequestBodySnapshot_PreservesPassthroughEnrichment(t *testing.T) { + e := echo.New() + padding := strings.Repeat("x", int(requestSnapshotInlineBodyLimit)+1) + reqBody := `{"model":"stub-model","padding":"` + padding + `","stream":true}` + req := httptest.NewRequest(http.MethodPost, "/p/vllm/chat/completions", strings.NewReader(reqBody)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + + snapshot := core.NewRequestSnapshot( + http.MethodPost, + "/p/vllm/chat/completions", + nil, + nil, + nil, + "application/json", + nil, + false, + "", + nil, + ) + prompt := core.DeriveWhiteBoxPrompt(snapshot) + require.NotNil(t, prompt) + core.CachePassthroughRouteInfo(prompt, &core.PassthroughRouteInfo{ + Provider: "vllm", + ProviderName: "vllm-eu", + RawEndpoint: "chat/completions", + NormalizedEndpoint: "chat/completions", + SemanticOperation: "vllm.chat_completions", + GenAIOperation: "chat", + StreamUncertain: true, + AuditPath: "/v1/chat/completions", + Model: "stub-model", + }) + ctx := core.WithRequestSnapshot(req.Context(), snapshot) + ctx = core.WithWhiteBoxPrompt(ctx, prompt) + c.SetRequest(req.WithContext(ctx)) + + storeRequestBodySnapshot(c, []byte(reqBody)) + + refreshed := core.GetWhiteBoxPrompt(c.Request().Context()) + require.NotNil(t, refreshed) + info := refreshed.CachedPassthroughRouteInfo() + require.NotNil(t, info) + assert.Equal(t, "vllm", info.Provider) + assert.Equal(t, "vllm-eu", info.ProviderName) + assert.Equal(t, "chat/completions", info.NormalizedEndpoint) + assert.Equal(t, "vllm.chat_completions", info.SemanticOperation) + assert.Equal(t, "chat", info.GenAIOperation) + assert.Equal(t, "/v1/chat/completions", info.AuditPath) + assert.Equal(t, "stub-model", info.Model) + assert.True(t, info.Stream) + assert.False(t, info.StreamUncertain) +} + func TestRequestSnapshotCapture_NormalizesUserPathHeader(t *testing.T) { e := echo.New()