Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions cmd/server/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/appthrust/appserver-go/apphealth"
"github.com/appthrust/appserver-go/appserver/health"
"github.com/appthrust/appserver-go/appserver/ollamasupervisor"
"github.com/appthrust/appserver-go/appserver/requestlog"
)

const (
Expand Down Expand Up @@ -105,11 +106,8 @@ func serve(ctx context.Context, listen string, cfg config) error {
defer close(childDone)
s.Run(childCtx)
}()
server := &http.Server{
Handler: newHandler(cfg.model, cfg.maxPromptTokens, cfg.upstream, func() bool { return !draining.Load() && s.Ready() }, &outcomes),
ReadHeaderTimeout: 10 * time.Second, ReadTimeout: requestTimeout,
WriteTimeout: requestTimeout + 5*time.Second, IdleTimeout: 60 * time.Second,
}
handler := newHandler(cfg.model, cfg.maxPromptTokens, cfg.upstream, func() bool { return !draining.Load() && s.Ready() }, &outcomes)
server := newServer(handler)
served := make(chan error, 1)
go func() { served <- server.Serve(listener) }()
log.Printf("llm-server listening on %s (project %s, model %s)", listener.Addr(), cfg.projectID, cfg.model)
Expand All @@ -131,3 +129,11 @@ func serve(ctx context.Context, listen string, cfg config) error {
}
return err
}

func newServer(handler http.Handler) *http.Server {
return &http.Server{
Handler: requestlog.Wrap(handler, requestlog.Options{Service: "llm"}),
ReadHeaderTimeout: 10 * time.Second, ReadTimeout: requestTimeout,
WriteTimeout: requestTimeout + 5*time.Second, IdleTimeout: 60 * time.Second,
}
}
38 changes: 23 additions & 15 deletions cmd/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/appthrust/appserver-go/appserver/health"
"github.com/appthrust/appserver-go/appserver/httpx"
"github.com/appthrust/appserver-go/appserver/releaseidentity"
"github.com/appthrust/appserver-go/appserver/requestlog"
)

const (
Expand Down Expand Up @@ -57,15 +58,15 @@ func newHandler(model string, maxPromptTokens int, upstream *url.URL, ready func
mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
if !ready() {
outcomes.Record(false, apphealth.ReasonDependencyUnavailable)
writeError(w, http.StatusServiceUnavailable, "llm_unavailable", "The chat model is not ready")
writeError(w, r, http.StatusServiceUnavailable, "llm_unavailable", "The chat model is not ready")
return
}
select {
case slots <- struct{}{}:
defer func() { <-slots }()
default:
outcomes.Record(false, apphealth.ReasonResourceExhausted)
writeError(w, http.StatusServiceUnavailable, "llm_busy", "At most two chat requests may run at once")
writeError(w, r, http.StatusServiceUnavailable, "llm_busy", "At most two chat requests may run at once")
return
}
ctx, cancel := context.WithTimeout(r.Context(), requestTimeout)
Expand All @@ -86,38 +87,38 @@ func newHandler(model string, maxPromptTokens int, upstream *url.URL, ready func
if err != nil || body == nil {
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
writeError(w, http.StatusRequestEntityTooLarge, "request_too_large", "Chat request body must not exceed 4 MiB")
writeError(w, r, http.StatusRequestEntityTooLarge, "request_too_large", "Chat request body must not exceed 4 MiB")
} else {
writeError(w, http.StatusBadRequest, "invalid_request", "Chat request must be one JSON object")
writeError(w, r, http.StatusBadRequest, "invalid_request", "Chat request must be one JSON object")
}
return
}
if raw, ok := body["model"]; ok {
var requested string
if err := json.Unmarshal(raw, &requested); err != nil || requested != model {
writeError(w, http.StatusBadRequest, "invalid_request", "model must match LLM_MODEL")
writeError(w, r, http.StatusBadRequest, "invalid_request", "model must match LLM_MODEL")
return
}
}
body["model"] = modelJSON
payload, err := json.Marshal(body)
if err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "The chat request could not be encoded")
writeError(w, r, http.StatusInternalServerError, "internal_error", "The chat request could not be encoded")
return
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload))
if err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "The chat request could not be created")
writeError(w, r, http.StatusInternalServerError, "internal_error", "The chat request could not be created")
return
}
request.Header.Set("Content-Type", "application/json")
response, err := client.Do(request)
if err != nil {
outcomes.Record(false, apphealth.ReasonUpstreamUnavailable)
if errors.Is(err, context.DeadlineExceeded) {
writeError(w, http.StatusGatewayTimeout, "llm_timeout", "Chat request exceeded 300 seconds")
writeError(w, r, http.StatusGatewayTimeout, "llm_timeout", "Chat request exceeded 300 seconds")
} else {
writeError(w, http.StatusBadGateway, "llm_upstream_unavailable", "The chat upstream could not complete this request")
writeError(w, r, http.StatusBadGateway, "llm_upstream_unavailable", "The chat upstream could not complete this request")
}
return
}
Expand All @@ -137,7 +138,13 @@ func newHandler(model string, maxPromptTokens int, upstream *url.URL, ready func
if response.StatusCode == http.StatusTooManyRequests {
class = apphealth.ReasonResourceExhausted
}
outcomes.Record(copyErr == nil && response.StatusCode >= 200 && response.StatusCode < 300, class)
statusOK := response.StatusCode >= 200 && response.StatusCode < 300
if copyErr != nil {
requestlog.SetOutcome(r, "interrupted", "response_copy_failed")
} else if !statusOK {
requestlog.SetOutcome(r, "failed", "upstream_http_status")
}
outcomes.Record(copyErr == nil && statusOK, class)
})
mux.HandleFunc("GET /v1/models", func(w http.ResponseWriter, _ *http.Request) {
httpx.WriteJSON(w, http.StatusOK, map[string]any{
Expand All @@ -151,16 +158,16 @@ func newHandler(model string, maxPromptTokens int, upstream *url.URL, ready func
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
httpx.WriteJSON(w, http.StatusOK, map[string]string{"status": "ok"})
})
mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, _ *http.Request) {
mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) {
if !ready() {
writeError(w, http.StatusServiceUnavailable, "llm_unavailable", "The chat model is not ready")
writeError(w, r, http.StatusServiceUnavailable, "llm_unavailable", "The chat model is not ready")
return
}
httpx.WriteJSON(w, http.StatusOK, map[string]string{"status": "ok"})
})
mux.Handle("GET "+health.Path, applicationHealth.Handler(releaseidentity.HealthIdentity("LLM")))
mux.Handle("GET /releasez", releaseidentity.NewHandler("LLM", func(w http.ResponseWriter, _ *http.Request) {
writeError(w, http.StatusServiceUnavailable, "release_unavailable", "LLM release status unavailable")
mux.Handle("GET /releasez", releaseidentity.NewHandler("LLM", func(w http.ResponseWriter, r *http.Request) {
writeError(w, r, http.StatusServiceUnavailable, "release_unavailable", "LLM release status unavailable")
}))
mux.HandleFunc("GET /{$}", func(w http.ResponseWriter, _ *http.Request) {
httpx.WriteJSON(w, http.StatusOK, map[string]any{
Expand All @@ -183,7 +190,8 @@ func (w flushWriter) Write(p []byte) (int, error) {
return n, w.controller.Flush()
}

func writeError(w http.ResponseWriter, status int, code, message string) {
func writeError(w http.ResponseWriter, r *http.Request, status int, code, message string) {
requestlog.SetError(r, httpx.Fail(status, code, message))
kind := "server_error"
if status >= 400 && status < 500 {
kind = "invalid_request_error"
Expand Down
37 changes: 37 additions & 0 deletions cmd/server/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,43 @@ func TestStreamFlushesEachChunkAndPreservesDone(t *testing.T) {
require.Equal(t, last, string(rest))
}

func TestRequestLogRecordsInterruptedChatStream(t *testing.T) {
const partial = "data: private-response\n\n"
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Content-Length", "100") // EOF before the declared end of the stream.
_, _ = io.WriteString(w, partial)
}))
defer upstream.Close()
h := handlerForTest(t, upstream.URL, func() bool { return true })
output, err := os.CreateTemp(t.TempDir(), "stderr")
require.NoError(t, err)
defer output.Close()
stderr := os.Stderr
os.Stderr = output
server := newServer(h)
os.Stderr = stderr

w := requestForTest(server.Handler, http.MethodPost, "/v1/chat/completions?token=private-query",
`{"stream":true,"messages":[{"role":"user","content":"private-prompt"}]}`)
require.Equal(t, http.StatusOK, w.Code)
require.Equal(t, partial, w.Body.String())
data, err := os.ReadFile(output.Name())
require.NoError(t, err)
line := string(data)
require.Equal(t, 1, strings.Count(line, "\n"))
require.NotContains(t, line, "private-")
var entry struct {
Service, Outcome, Reason string
Status int
}
require.NoError(t, json.Unmarshal(data, &entry))
require.Equal(t, "llm", entry.Service)
require.Equal(t, http.StatusOK, entry.Status)
require.Equal(t, "interrupted", entry.Outcome)
require.Equal(t, "response_copy_failed", entry.Reason)
}

func TestProxyRejectsThirdInFlightStream(t *testing.T) {
entered, release := make(chan struct{}, 2), make(chan struct{})
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/appthrust/llm
go 1.26.0

require (
github.com/appthrust/appserver-go v0.3.0
github.com/appthrust/appserver-go v0.7.0
github.com/stretchr/testify v1.11.1
)

Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
github.com/appthrust/appserver-go v0.3.0 h1:zxOz0hkspZSXjLq3KWGDLkZIz42Jag6wqRujZ8lTru0=
github.com/appthrust/appserver-go v0.3.0/go.mod h1:0KSf9RQZ6OxBcE0lxYzzjKF37nfwbKps3wXUGlE0oFo=
github.com/appthrust/appserver-go v0.7.0 h1:Fd1G+l/Ys0aAAiI2ZmMKqAz4qY6Dv7WwsF/rZtUgPUA=
github.com/appthrust/appserver-go v0.7.0/go.mod h1:0KSf9RQZ6OxBcE0lxYzzjKF37nfwbKps3wXUGlE0oFo=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
Expand Down