From 041f0f7f02915acfe3b8cd21148b97bb94d9beb1 Mon Sep 17 00:00:00 2001 From: reoring Date: Thu, 19 Nov 2026 07:38:14 +0900 Subject: [PATCH 1/2] Log one structured line per HTTP request Wrap the server handler with appserver-go requestlog and record the failures that only wrote a response. (appserver-go pinned to the PR commit until v0.7.0 is tagged.) --- cmd/server/main.go | 16 +++++++++++----- cmd/server/server.go | 38 +++++++++++++++++++++++--------------- cmd/server/server_test.go | 37 +++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 74 insertions(+), 23 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 7d80f98..82f6d06 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -19,6 +19,7 @@ import ( "github.com/appthrust/appserver-go/apphealth" "github.com/appthrust/appserver-go/appserver/health" "github.com/appthrust/appserver-go/appserver/ollamasupervisor" + "github.com/appthrust/appserver-go/appserver/requestlog" ) const ( @@ -105,11 +106,8 @@ func serve(ctx context.Context, listen string, cfg config) error { defer close(childDone) s.Run(childCtx) }() - server := &http.Server{ - Handler: newHandler(cfg.model, cfg.maxPromptTokens, cfg.upstream, func() bool { return !draining.Load() && s.Ready() }, &outcomes), - ReadHeaderTimeout: 10 * time.Second, ReadTimeout: requestTimeout, - WriteTimeout: requestTimeout + 5*time.Second, IdleTimeout: 60 * time.Second, - } + handler := newHandler(cfg.model, cfg.maxPromptTokens, cfg.upstream, func() bool { return !draining.Load() && s.Ready() }, &outcomes) + server := newServer(handler) served := make(chan error, 1) go func() { served <- server.Serve(listener) }() log.Printf("llm-server listening on %s (project %s, model %s)", listener.Addr(), cfg.projectID, cfg.model) @@ -131,3 +129,11 @@ func serve(ctx context.Context, listen string, cfg config) error { } return err } + +func newServer(handler http.Handler) *http.Server { + return &http.Server{ + Handler: requestlog.Wrap(handler, requestlog.Options{Service: "llm"}), + ReadHeaderTimeout: 10 * time.Second, ReadTimeout: requestTimeout, + WriteTimeout: requestTimeout + 5*time.Second, IdleTimeout: 60 * time.Second, + } +} diff --git a/cmd/server/server.go b/cmd/server/server.go index 7e52e2e..e6f73df 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -15,6 +15,7 @@ import ( "github.com/appthrust/appserver-go/appserver/health" "github.com/appthrust/appserver-go/appserver/httpx" "github.com/appthrust/appserver-go/appserver/releaseidentity" + "github.com/appthrust/appserver-go/appserver/requestlog" ) const ( @@ -57,7 +58,7 @@ func newHandler(model string, maxPromptTokens int, upstream *url.URL, ready func mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, r *http.Request) { if !ready() { outcomes.Record(false, apphealth.ReasonDependencyUnavailable) - writeError(w, http.StatusServiceUnavailable, "llm_unavailable", "The chat model is not ready") + writeError(w, r, http.StatusServiceUnavailable, "llm_unavailable", "The chat model is not ready") return } select { @@ -65,7 +66,7 @@ func newHandler(model string, maxPromptTokens int, upstream *url.URL, ready func defer func() { <-slots }() default: outcomes.Record(false, apphealth.ReasonResourceExhausted) - writeError(w, http.StatusServiceUnavailable, "llm_busy", "At most two chat requests may run at once") + writeError(w, r, http.StatusServiceUnavailable, "llm_busy", "At most two chat requests may run at once") return } ctx, cancel := context.WithTimeout(r.Context(), requestTimeout) @@ -86,28 +87,28 @@ func newHandler(model string, maxPromptTokens int, upstream *url.URL, ready func if err != nil || body == nil { var tooLarge *http.MaxBytesError if errors.As(err, &tooLarge) { - writeError(w, http.StatusRequestEntityTooLarge, "request_too_large", "Chat request body must not exceed 4 MiB") + writeError(w, r, http.StatusRequestEntityTooLarge, "request_too_large", "Chat request body must not exceed 4 MiB") } else { - writeError(w, http.StatusBadRequest, "invalid_request", "Chat request must be one JSON object") + writeError(w, r, http.StatusBadRequest, "invalid_request", "Chat request must be one JSON object") } return } if raw, ok := body["model"]; ok { var requested string if err := json.Unmarshal(raw, &requested); err != nil || requested != model { - writeError(w, http.StatusBadRequest, "invalid_request", "model must match LLM_MODEL") + writeError(w, r, http.StatusBadRequest, "invalid_request", "model must match LLM_MODEL") return } } body["model"] = modelJSON payload, err := json.Marshal(body) if err != nil { - writeError(w, http.StatusInternalServerError, "internal_error", "The chat request could not be encoded") + writeError(w, r, http.StatusInternalServerError, "internal_error", "The chat request could not be encoded") return } request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload)) if err != nil { - writeError(w, http.StatusInternalServerError, "internal_error", "The chat request could not be created") + writeError(w, r, http.StatusInternalServerError, "internal_error", "The chat request could not be created") return } request.Header.Set("Content-Type", "application/json") @@ -115,9 +116,9 @@ func newHandler(model string, maxPromptTokens int, upstream *url.URL, ready func if err != nil { outcomes.Record(false, apphealth.ReasonUpstreamUnavailable) if errors.Is(err, context.DeadlineExceeded) { - writeError(w, http.StatusGatewayTimeout, "llm_timeout", "Chat request exceeded 300 seconds") + writeError(w, r, http.StatusGatewayTimeout, "llm_timeout", "Chat request exceeded 300 seconds") } else { - writeError(w, http.StatusBadGateway, "llm_upstream_unavailable", "The chat upstream could not complete this request") + writeError(w, r, http.StatusBadGateway, "llm_upstream_unavailable", "The chat upstream could not complete this request") } return } @@ -137,7 +138,13 @@ func newHandler(model string, maxPromptTokens int, upstream *url.URL, ready func if response.StatusCode == http.StatusTooManyRequests { class = apphealth.ReasonResourceExhausted } - outcomes.Record(copyErr == nil && response.StatusCode >= 200 && response.StatusCode < 300, class) + statusOK := response.StatusCode >= 200 && response.StatusCode < 300 + if copyErr != nil { + requestlog.SetOutcome(r, "interrupted", "response_copy_failed") + } else if !statusOK { + requestlog.SetOutcome(r, "failed", "upstream_http_status") + } + outcomes.Record(copyErr == nil && statusOK, class) }) mux.HandleFunc("GET /v1/models", func(w http.ResponseWriter, _ *http.Request) { httpx.WriteJSON(w, http.StatusOK, map[string]any{ @@ -151,16 +158,16 @@ func newHandler(model string, maxPromptTokens int, upstream *url.URL, ready func mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { httpx.WriteJSON(w, http.StatusOK, map[string]string{"status": "ok"}) }) - mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, _ *http.Request) { + mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { if !ready() { - writeError(w, http.StatusServiceUnavailable, "llm_unavailable", "The chat model is not ready") + writeError(w, r, http.StatusServiceUnavailable, "llm_unavailable", "The chat model is not ready") return } httpx.WriteJSON(w, http.StatusOK, map[string]string{"status": "ok"}) }) mux.Handle("GET "+health.Path, applicationHealth.Handler(releaseidentity.HealthIdentity("LLM"))) - mux.Handle("GET /releasez", releaseidentity.NewHandler("LLM", func(w http.ResponseWriter, _ *http.Request) { - writeError(w, http.StatusServiceUnavailable, "release_unavailable", "LLM release status unavailable") + mux.Handle("GET /releasez", releaseidentity.NewHandler("LLM", func(w http.ResponseWriter, r *http.Request) { + writeError(w, r, http.StatusServiceUnavailable, "release_unavailable", "LLM release status unavailable") })) mux.HandleFunc("GET /{$}", func(w http.ResponseWriter, _ *http.Request) { httpx.WriteJSON(w, http.StatusOK, map[string]any{ @@ -183,7 +190,8 @@ func (w flushWriter) Write(p []byte) (int, error) { return n, w.controller.Flush() } -func writeError(w http.ResponseWriter, status int, code, message string) { +func writeError(w http.ResponseWriter, r *http.Request, status int, code, message string) { + requestlog.SetError(r, httpx.Fail(status, code, message)) kind := "server_error" if status >= 400 && status < 500 { kind = "invalid_request_error" diff --git a/cmd/server/server_test.go b/cmd/server/server_test.go index 2895bef..47b3c9f 100644 --- a/cmd/server/server_test.go +++ b/cmd/server/server_test.go @@ -164,6 +164,43 @@ func TestStreamFlushesEachChunkAndPreservesDone(t *testing.T) { require.Equal(t, last, string(rest)) } +func TestRequestLogRecordsInterruptedChatStream(t *testing.T) { + const partial = "data: private-response\n\n" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Content-Length", "100") // EOF before the declared end of the stream. + _, _ = io.WriteString(w, partial) + })) + defer upstream.Close() + h := handlerForTest(t, upstream.URL, func() bool { return true }) + output, err := os.CreateTemp(t.TempDir(), "stderr") + require.NoError(t, err) + defer output.Close() + stderr := os.Stderr + os.Stderr = output + server := newServer(h) + os.Stderr = stderr + + w := requestForTest(server.Handler, http.MethodPost, "/v1/chat/completions?token=private-query", + `{"stream":true,"messages":[{"role":"user","content":"private-prompt"}]}`) + require.Equal(t, http.StatusOK, w.Code) + require.Equal(t, partial, w.Body.String()) + data, err := os.ReadFile(output.Name()) + require.NoError(t, err) + line := string(data) + require.Equal(t, 1, strings.Count(line, "\n")) + require.NotContains(t, line, "private-") + var entry struct { + Service, Outcome, Reason string + Status int + } + require.NoError(t, json.Unmarshal(data, &entry)) + require.Equal(t, "llm", entry.Service) + require.Equal(t, http.StatusOK, entry.Status) + require.Equal(t, "interrupted", entry.Outcome) + require.Equal(t, "response_copy_failed", entry.Reason) +} + func TestProxyRejectsThirdInFlightStream(t *testing.T) { entered, release := make(chan struct{}, 2), make(chan struct{}) upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { diff --git a/go.mod b/go.mod index 8ae2cf1..e621562 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/appthrust/llm go 1.26.0 require ( - github.com/appthrust/appserver-go v0.3.0 + github.com/appthrust/appserver-go v0.6.1-0.20261118223418-5f8bf82f7a56 github.com/stretchr/testify v1.11.1 ) diff --git a/go.sum b/go.sum index d059bbb..27d32d5 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/appthrust/appserver-go v0.3.0 h1:zxOz0hkspZSXjLq3KWGDLkZIz42Jag6wqRujZ8lTru0= -github.com/appthrust/appserver-go v0.3.0/go.mod h1:0KSf9RQZ6OxBcE0lxYzzjKF37nfwbKps3wXUGlE0oFo= +github.com/appthrust/appserver-go v0.6.1-0.20261118223418-5f8bf82f7a56 h1:JKydZpovuh/gTTwMH63iDRKtPa2gjnDEUAbCF7VMa54= +github.com/appthrust/appserver-go v0.6.1-0.20261118223418-5f8bf82f7a56/go.mod h1:0KSf9RQZ6OxBcE0lxYzzjKF37nfwbKps3wXUGlE0oFo= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= From 2f883cd6dea4520e5a96087b055d978ce70280d4 Mon Sep 17 00:00:00 2001 From: reoring Date: Thu, 19 Nov 2026 07:51:28 +0900 Subject: [PATCH 2/2] Pin appserver-go v0.7.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e621562..ffbf9be 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/appthrust/llm go 1.26.0 require ( - github.com/appthrust/appserver-go v0.6.1-0.20261118223418-5f8bf82f7a56 + github.com/appthrust/appserver-go v0.7.0 github.com/stretchr/testify v1.11.1 ) diff --git a/go.sum b/go.sum index 27d32d5..851a385 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/appthrust/appserver-go v0.6.1-0.20261118223418-5f8bf82f7a56 h1:JKydZpovuh/gTTwMH63iDRKtPa2gjnDEUAbCF7VMa54= -github.com/appthrust/appserver-go v0.6.1-0.20261118223418-5f8bf82f7a56/go.mod h1:0KSf9RQZ6OxBcE0lxYzzjKF37nfwbKps3wXUGlE0oFo= +github.com/appthrust/appserver-go v0.7.0 h1:Fd1G+l/Ys0aAAiI2ZmMKqAz4qY6Dv7WwsF/rZtUgPUA= +github.com/appthrust/appserver-go v0.7.0/go.mod h1:0KSf9RQZ6OxBcE0lxYzzjKF37nfwbKps3wXUGlE0oFo= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=