Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions cmd/server/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"flag"
"fmt"
"io"
"log"
"net"
"net/http"
Expand All @@ -13,6 +14,8 @@ import (
"os/signal"
"syscall"
"time"

"github.com/appthrust/appserver-go/appserver/requestlog"
)

const (
Expand Down Expand Up @@ -65,6 +68,14 @@ func main() {
}
}

func newHTTPServer(upstream *url.URL, ready func() bool, logOutput io.Writer) *http.Server {
return &http.Server{
Handler: requestlog.Wrap(newHandler(upstream, ready), requestlog.Options{Service: "vision", Output: logOutput}),
ReadHeaderTimeout: 10 * time.Second, ReadTimeout: requestTimeout,
WriteTimeout: requestTimeout + 5*time.Second, IdleTimeout: 60 * time.Second,
}
}

func serve(ctx context.Context, listen string, cfg config) error {
listener, err := net.Listen("tcp", listen)
if err != nil {
Expand All @@ -78,11 +89,7 @@ func serve(ctx context.Context, listen string, cfg config) error {
defer close(childDone)
s.Run(childCtx)
}()
server := &http.Server{
Handler: newHandler(cfg.upstream, s.Ready),
ReadHeaderTimeout: 10 * time.Second, ReadTimeout: requestTimeout,
WriteTimeout: requestTimeout + 5*time.Second, IdleTimeout: 60 * time.Second,
}
server := newHTTPServer(cfg.upstream, s.Ready, os.Stderr)
served := make(chan error, 1)
go func() { served <- server.Serve(listener) }()
log.Printf("vision-server listening on %s (project %s)", listener.Addr(), cfg.projectID)
Expand Down
20 changes: 18 additions & 2 deletions cmd/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"time"

"github.com/appthrust/appserver-go/appserver/httpx"
"github.com/appthrust/appserver-go/appserver/requestlog"
)

const (
Expand Down Expand Up @@ -44,6 +45,9 @@ func newHandler(upstream *url.URL, ready func() bool) http.Handler {
},
FlushInterval: -1, // Forward SSE chunks immediately, without buffering a completion.
ModifyResponse: func(r *http.Response) error {
if r.StatusCode >= http.StatusBadRequest && r.StatusCode < 600 && r.Request != nil {
requestlog.SetOutcome(r.Request, "upstream_error", "upstream_http_status")
}
r.Header.Set("Cache-Control", "no-store")
return nil
},
Expand All @@ -55,6 +59,18 @@ func newHandler(upstream *url.URL, ready func() bool) http.Handler {
}
},
}
serveProxy := func(w http.ResponseWriter, r *http.Request) {
defer func() {
if p := recover(); p != nil {
// ReverseProxy aborts the connection on a body copy failure after headers.
if p == http.ErrAbortHandler {
requestlog.SetOutcome(r, "stream_aborted", "proxy_copy_failed")
}
panic(p)
}
}()
proxy.ServeHTTP(w, r)
}
slots := make(chan struct{}, maxInFlight)
modelJSON, _ := json.Marshal(servedModel)
mux := http.NewServeMux()
Expand Down Expand Up @@ -110,13 +126,13 @@ func newHandler(upstream *url.URL, ready func() bool) http.Handler {
r = r.WithContext(ctx)
r.Body = io.NopCloser(bytes.NewReader(payload))
r.ContentLength = int64(len(payload))
proxy.ServeHTTP(w, r)
serveProxy(w, r)
})
for _, path := range []string{"/v1/models", "/metrics"} {
mux.HandleFunc("GET "+path, func(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), requestTimeout)
defer cancel()
proxy.ServeHTTP(w, r.WithContext(ctx))
serveProxy(w, r.WithContext(ctx))
})
}
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
Expand Down
33 changes: 33 additions & 0 deletions cmd/server/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package main

import (
"bufio"
"bytes"
"context"
"encoding/json"
"io"
Expand Down Expand Up @@ -178,6 +179,38 @@ func TestProxyStreamsBeforeCompletion(t *testing.T) {
require.Equal(t, "data: {\"choices\":[]}\n", line)
}

func TestRequestLogRecordsAbortedVisionStream(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Content-Length", "100")
_, _ = io.WriteString(w, "data: private-value\n\n")
}))
defer upstream.Close()
u, err := url.Parse(upstream.URL)
require.NoError(t, err)
var output bytes.Buffer
server := newHTTPServer(u, func() bool { return true }, &output)
r := httptest.NewRequest(http.MethodPost, "/v1/chat/completions?private-value", strings.NewReader(`{"stream":true}`))
// ReverseProxy propagates copy failures as ErrAbortHandler when serving an HTTP server.
r = r.WithContext(context.WithValue(r.Context(), http.ServerContextKey, server))
w := httptest.NewRecorder()
require.PanicsWithValue(t, http.ErrAbortHandler, func() { server.Handler.ServeHTTP(w, r) })
require.Equal(t, http.StatusOK, w.Code)
require.Equal(t, "data: private-value\n\n", w.Body.String())

var entry struct {
Service, Outcome, Reason string
Status int
}
require.NoError(t, json.Unmarshal(output.Bytes(), &entry))
require.Equal(t, 1, bytes.Count(output.Bytes(), []byte("\n")))
require.Equal(t, "vision", entry.Service)
require.Equal(t, http.StatusOK, entry.Status)
require.Equal(t, "stream_aborted", entry.Outcome)
require.Equal(t, "proxy_copy_failed", entry.Reason)
require.NotContains(t, output.String(), "private-value")
}

func TestModelsAndMetricsAreForwardedUnchanged(t *testing.T) {
metrics := "# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"qwen3-vl-4b-instruct-fp8\"} 2\n"
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/appthrust/vision
go 1.26.0

require (
github.com/appthrust/appserver-go v0.3.0
github.com/appthrust/appserver-go v0.7.0
github.com/stretchr/testify v1.11.1
)

Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
github.com/appthrust/appserver-go v0.3.0 h1:zxOz0hkspZSXjLq3KWGDLkZIz42Jag6wqRujZ8lTru0=
github.com/appthrust/appserver-go v0.3.0/go.mod h1:0KSf9RQZ6OxBcE0lxYzzjKF37nfwbKps3wXUGlE0oFo=
github.com/appthrust/appserver-go v0.7.0 h1:Fd1G+l/Ys0aAAiI2ZmMKqAz4qY6Dv7WwsF/rZtUgPUA=
github.com/appthrust/appserver-go v0.7.0/go.mod h1:0KSf9RQZ6OxBcE0lxYzzjKF37nfwbKps3wXUGlE0oFo=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
Expand Down