diff --git a/cmd/server/main.go b/cmd/server/main.go index 790c651..4e53e29 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -5,6 +5,7 @@ import ( "errors" "flag" "fmt" + "io" "log" "net" "net/http" @@ -13,6 +14,8 @@ import ( "os/signal" "syscall" "time" + + "github.com/appthrust/appserver-go/appserver/requestlog" ) const ( @@ -65,6 +68,14 @@ func main() { } } +func newHTTPServer(upstream *url.URL, ready func() bool, logOutput io.Writer) *http.Server { + return &http.Server{ + Handler: requestlog.Wrap(newHandler(upstream, ready), requestlog.Options{Service: "vision", Output: logOutput}), + ReadHeaderTimeout: 10 * time.Second, ReadTimeout: requestTimeout, + WriteTimeout: requestTimeout + 5*time.Second, IdleTimeout: 60 * time.Second, + } +} + func serve(ctx context.Context, listen string, cfg config) error { listener, err := net.Listen("tcp", listen) if err != nil { @@ -78,11 +89,7 @@ func serve(ctx context.Context, listen string, cfg config) error { defer close(childDone) s.Run(childCtx) }() - server := &http.Server{ - Handler: newHandler(cfg.upstream, s.Ready), - ReadHeaderTimeout: 10 * time.Second, ReadTimeout: requestTimeout, - WriteTimeout: requestTimeout + 5*time.Second, IdleTimeout: 60 * time.Second, - } + server := newHTTPServer(cfg.upstream, s.Ready, os.Stderr) served := make(chan error, 1) go func() { served <- server.Serve(listener) }() log.Printf("vision-server listening on %s (project %s)", listener.Addr(), cfg.projectID) diff --git a/cmd/server/server.go b/cmd/server/server.go index d4035de..644ff83 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -13,6 +13,7 @@ import ( "time" "github.com/appthrust/appserver-go/appserver/httpx" + "github.com/appthrust/appserver-go/appserver/requestlog" ) const ( @@ -44,6 +45,9 @@ func newHandler(upstream *url.URL, ready func() bool) http.Handler { }, FlushInterval: -1, // Forward SSE chunks immediately, without buffering a completion. ModifyResponse: func(r *http.Response) error { + if r.StatusCode >= http.StatusBadRequest && r.StatusCode < 600 && r.Request != nil { + requestlog.SetOutcome(r.Request, "upstream_error", "upstream_http_status") + } r.Header.Set("Cache-Control", "no-store") return nil }, @@ -55,6 +59,18 @@ func newHandler(upstream *url.URL, ready func() bool) http.Handler { } }, } + serveProxy := func(w http.ResponseWriter, r *http.Request) { + defer func() { + if p := recover(); p != nil { + // ReverseProxy aborts the connection on a body copy failure after headers. + if p == http.ErrAbortHandler { + requestlog.SetOutcome(r, "stream_aborted", "proxy_copy_failed") + } + panic(p) + } + }() + proxy.ServeHTTP(w, r) + } slots := make(chan struct{}, maxInFlight) modelJSON, _ := json.Marshal(servedModel) mux := http.NewServeMux() @@ -110,13 +126,13 @@ func newHandler(upstream *url.URL, ready func() bool) http.Handler { r = r.WithContext(ctx) r.Body = io.NopCloser(bytes.NewReader(payload)) r.ContentLength = int64(len(payload)) - proxy.ServeHTTP(w, r) + serveProxy(w, r) }) for _, path := range []string{"/v1/models", "/metrics"} { mux.HandleFunc("GET "+path, func(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), requestTimeout) defer cancel() - proxy.ServeHTTP(w, r.WithContext(ctx)) + serveProxy(w, r.WithContext(ctx)) }) } mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { diff --git a/cmd/server/server_test.go b/cmd/server/server_test.go index ba3c2c7..275b828 100644 --- a/cmd/server/server_test.go +++ b/cmd/server/server_test.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "bytes" "context" "encoding/json" "io" @@ -178,6 +179,38 @@ func TestProxyStreamsBeforeCompletion(t *testing.T) { require.Equal(t, "data: {\"choices\":[]}\n", line) } +func TestRequestLogRecordsAbortedVisionStream(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Content-Length", "100") + _, _ = io.WriteString(w, "data: private-value\n\n") + })) + defer upstream.Close() + u, err := url.Parse(upstream.URL) + require.NoError(t, err) + var output bytes.Buffer + server := newHTTPServer(u, func() bool { return true }, &output) + r := httptest.NewRequest(http.MethodPost, "/v1/chat/completions?private-value", strings.NewReader(`{"stream":true}`)) + // ReverseProxy propagates copy failures as ErrAbortHandler when serving an HTTP server. + r = r.WithContext(context.WithValue(r.Context(), http.ServerContextKey, server)) + w := httptest.NewRecorder() + require.PanicsWithValue(t, http.ErrAbortHandler, func() { server.Handler.ServeHTTP(w, r) }) + require.Equal(t, http.StatusOK, w.Code) + require.Equal(t, "data: private-value\n\n", w.Body.String()) + + var entry struct { + Service, Outcome, Reason string + Status int + } + require.NoError(t, json.Unmarshal(output.Bytes(), &entry)) + require.Equal(t, 1, bytes.Count(output.Bytes(), []byte("\n"))) + require.Equal(t, "vision", entry.Service) + require.Equal(t, http.StatusOK, entry.Status) + require.Equal(t, "stream_aborted", entry.Outcome) + require.Equal(t, "proxy_copy_failed", entry.Reason) + require.NotContains(t, output.String(), "private-value") +} + func TestModelsAndMetricsAreForwardedUnchanged(t *testing.T) { metrics := "# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"qwen3-vl-4b-instruct-fp8\"} 2\n" upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/go.mod b/go.mod index 238217b..c5bb616 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/appthrust/vision go 1.26.0 require ( - github.com/appthrust/appserver-go v0.3.0 + github.com/appthrust/appserver-go v0.7.0 github.com/stretchr/testify v1.11.1 ) diff --git a/go.sum b/go.sum index d059bbb..851a385 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/appthrust/appserver-go v0.3.0 h1:zxOz0hkspZSXjLq3KWGDLkZIz42Jag6wqRujZ8lTru0= -github.com/appthrust/appserver-go v0.3.0/go.mod h1:0KSf9RQZ6OxBcE0lxYzzjKF37nfwbKps3wXUGlE0oFo= +github.com/appthrust/appserver-go v0.7.0 h1:Fd1G+l/Ys0aAAiI2ZmMKqAz4qY6Dv7WwsF/rZtUgPUA= +github.com/appthrust/appserver-go v0.7.0/go.mod h1:0KSf9RQZ6OxBcE0lxYzzjKF37nfwbKps3wXUGlE0oFo= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=