diff --git a/.github/workflows/cohere-java-tests.yml b/.github/workflows/cohere-java-tests.yml new file mode 100644 index 000000000..34f2c91f2 --- /dev/null +++ b/.github/workflows/cohere-java-tests.yml @@ -0,0 +1,54 @@ +name: Cohere Java compatibility + +on: + pull_request: + paths: + - 'java/traceai-java-cohere/**' + - 'java/traceai-java-core/**' + - 'java/pom.xml' + - '.github/workflows/cohere-java-tests.yml' + push: + branches: [dev, main] + paths: + - 'java/traceai-java-cohere/**' + - 'java/traceai-java-core/**' + - 'java/pom.xml' + - '.github/workflows/cohere-java-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + compatibility: + name: Cohere ${{ matrix.sdk }} synthetic contracts + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + sdk: ['1.9.3', '1.12.0'] + defaults: + run: + working-directory: java + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + cache: maven + cache-dependency-path: 'java/**/pom.xml' + - name: Resolve dependencies and compile + run: mvn -B -ntp -pl traceai-java-cohere -am -Dcohere-java.version=${{ matrix.sdk }} -DskipTests package + - name: Run synthetic SDK and OTLP contracts + run: >- + mvn -B -ntp -pl traceai-java-cohere -am + -Dcohere-java.version=${{ matrix.sdk }} + -Dtest=CohereSurfaceTest,CohereOtlpTest,CohereRequestSafetyTest,TracedCohereTransportTest,TracedCohereStreamTest,TracedCohereClientPrivacyTest + -Dsurefire.failIfNoSpecifiedTests=false test + - uses: actions/upload-artifact@v4 + if: always() + with: + name: cohere-java-${{ matrix.sdk }}-reports + path: java/traceai-java-cohere/target/surefire-reports/ diff --git a/java/traceai-java-cohere/pom.xml b/java/traceai-java-cohere/pom.xml index 29eb559a5..e445b4d9f 100644 --- a/java/traceai-java-cohere/pom.xml +++ b/java/traceai-java-cohere/pom.xml @@ -25,11 +25,17 @@ com.cohere cohere-java - 1.9.3 + ${cohere-java.version} provided + + io.opentelemetry.proto + opentelemetry-proto + 1.0.0-alpha + test + org.junit.jupiter junit-jupiter @@ -56,4 +62,26 @@ test + + + + cohere-1.12-maven-transport + + + cohere-java.version + 1.12.0 + + + + + + com.squareup.okhttp3 + okhttp-jvm + 5.2.1 + test + + + + diff --git a/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/CohereAttributes.java b/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/CohereAttributes.java new file mode 100644 index 000000000..6b0af0d18 --- /dev/null +++ b/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/CohereAttributes.java @@ -0,0 +1,211 @@ +package ai.traceai.cohere; + +import ai.traceai.FITracer; +import ai.traceai.SemanticConventions; +import com.cohere.api.core.ObjectMappers; +import com.cohere.api.resources.v2.types.V2ChatStreamResponse; +import com.cohere.api.types.ApiMeta; +import com.cohere.api.types.NonStreamedChatResponse; +import com.cohere.api.types.StreamedChatResponse; +import com.cohere.api.types.Usage; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.opentelemetry.api.trace.Span; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** Shared, privacy-gated attributes for the additional SDK surfaces. */ +final class CohereAttributes { + private final FITracer tracer; + private final Span span; + // Never accumulate unbounded output or retain content when capture is disabled. + private final StringBuilder text = new StringBuilder(); + private boolean truncated; + private boolean failed; + private static final int MAX_TEXT = 32000; + + CohereAttributes(FITracer tracer, Span span) { + this.tracer = tracer; + this.span = span; + span.setAttribute(SemanticConventions.LLM_SYSTEM, "cohere"); + span.setAttribute(SemanticConventions.LLM_PROVIDER, "cohere"); + } + + void model(String model) { + span.setAttribute(SemanticConventions.LLM_MODEL_NAME, model); + span.setAttribute(SemanticConventions.LLM_REQUEST_MODEL, model); + } + + void chatInput(Object request) { + if (tracer.getConfig().isHideInputs() || tracer.getConfig().isHideInputMessages()) return; + JsonNode json = inputJson(request); + if (json == null) return; + List> messages = new ArrayList<>(); + if (json.has("messages")) { + for (JsonNode message : json.path("messages")) { + messages.add(FITracer.message(message.path("role").asText(), content(message.path("content")))); + } + } else { + if (json.hasNonNull("preamble")) messages.add(FITracer.message("system", json.get("preamble").asText())); + for (JsonNode message : json.path("chat_history")) { + String role = message.path("role").asText().toLowerCase(java.util.Locale.ROOT); + messages.add(FITracer.message("chatbot".equals(role) ? "assistant" : role, + message.path("message").asText())); + } + messages.add(FITracer.message("user", json.path("message").asText())); + } + tracer.setInputMessages(span, messages); + tracer.setInputValue(span, json.has("messages") ? json.get("messages").toString() : json.path("message").asText()); + tracer.setRawInput(span, json.toString()); + } + + void rawInput(Object request) { + if (tracer.getConfig().isHideInputs()) return; + JsonNode json = inputJson(request); + if (json != null) tracer.setRawInput(span, json.toString()); + } + + void rawOutput(Object response) { + if (tracer.getConfig().isHideOutputs()) return; + JsonNode json = json(response); + if (json != null) tracer.setRawOutput(span, json.toString()); + } + + void chatOutput(Object response) { + if (!captureOutput()) return; + JsonNode json = json(response); + if (json == null) return; + String result = json.has("text") ? json.get("text").asText() : content(json.path("message").path("content")); + tracer.setOutputValue(span, result); + tracer.setOutputMessages(span, List.of(FITracer.message("assistant", result))); + tracer.setRawOutput(span, json.toString()); + } + + void usage(Usage usage) { + usage.getTokens().ifPresent(tokens -> { + tokens.getInputTokens().ifPresent(value -> span.setAttribute(SemanticConventions.LLM_TOKEN_COUNT_PROMPT, value.longValue())); + tokens.getOutputTokens().ifPresent(value -> span.setAttribute(SemanticConventions.LLM_TOKEN_COUNT_COMPLETION, value.longValue())); + if (tokens.getInputTokens().isPresent() && tokens.getOutputTokens().isPresent()) { + span.setAttribute(SemanticConventions.LLM_TOKEN_COUNT_TOTAL, + tokens.getInputTokens().get().longValue() + tokens.getOutputTokens().get().longValue()); + } + }); + } + + void meta(ApiMeta meta) { + meta.getTokens().ifPresent(tokens -> { + tokens.getInputTokens().ifPresent(value -> span.setAttribute(SemanticConventions.LLM_TOKEN_COUNT_PROMPT, value.longValue())); + tokens.getOutputTokens().ifPresent(value -> span.setAttribute(SemanticConventions.LLM_TOKEN_COUNT_COMPLETION, value.longValue())); + if (tokens.getInputTokens().isPresent() && tokens.getOutputTokens().isPresent()) { + span.setAttribute(SemanticConventions.LLM_TOKEN_COUNT_TOTAL, + tokens.getInputTokens().get().longValue() + tokens.getOutputTokens().get().longValue()); + } + }); + meta.getBilledUnits().ifPresent(units -> { + units.getInputTokens().ifPresent(value -> span.setAttribute("cohere.billed_units.input_tokens", value.longValue())); + units.getSearchUnits().ifPresent(value -> span.setAttribute("cohere.rerank.search_units", value.longValue())); + }); + } + + void embeddingMeta(ApiMeta meta) { + meta(meta); + // Embeddings often report only billed input tokens. Keep actual usage + // when present (including zero), and never fabricate output or total counts. + if (meta.getTokens().flatMap(tokens -> tokens.getInputTokens()).isEmpty()) { + meta.getBilledUnits().flatMap(units -> units.getInputTokens()).ifPresent(value -> + span.setAttribute(SemanticConventions.LLM_TOKEN_COUNT_PROMPT, value.longValue())); + } + } + + TracedCohereStream.EventState v1Event(StreamedChatResponse event) { + event.getStreamStart().ifPresent(start -> span.setAttribute(SemanticConventions.LLM_RESPONSE_ID, start.getGenerationId())); + event.getTextGeneration().ifPresent(delta -> append(delta.getText())); + if (event.getStreamEnd().isEmpty()) return TracedCohereStream.EventState.CONTINUE; + var end = event.getStreamEnd().get(); + NonStreamedChatResponse response = end.getResponse(); + response.getGenerationId().ifPresent(id -> span.setAttribute(SemanticConventions.LLM_RESPONSE_ID, id)); + response.getMeta().ifPresent(this::meta); + if (captureOutput() && response.getText() != null) { + text.setLength(0); + truncated = false; + append(response.getText()); + } + return finishReason(end.getFinishReason().toString(), null); + } + + TracedCohereStream.EventState v2Event(V2ChatStreamResponse event) { + event.getMessageStart().flatMap(start -> start.getId()) + .ifPresent(id -> span.setAttribute(SemanticConventions.LLM_RESPONSE_ID, id)); + event.getContentDelta().flatMap(delta -> delta.getDelta()).flatMap(delta -> delta.getMessage()) + .flatMap(message -> message.getContent()).flatMap(content -> content.getText()).ifPresent(this::append); + if (event.getMessageEnd().isEmpty()) return TracedCohereStream.EventState.CONTINUE; + var end = event.getMessageEnd().get(); + end.getId().ifPresent(id -> span.setAttribute(SemanticConventions.LLM_RESPONSE_ID, id)); + end.getDelta().flatMap(delta -> delta.getUsage()).ifPresent(this::usage); + String reason = end.getDelta().flatMap(delta -> delta.getFinishReason()).map(Object::toString).orElse(""); + String error = end.getDelta().flatMap(delta -> delta.getError()).orElse(null); + return finishReason(reason, error); + } + + boolean hasError() { + return failed; + } + + TracedCohereStream.EventState finishReason(String reason, String error) { + if (!reason.isEmpty()) span.setAttribute(SemanticConventions.LLM_RESPONSE_FINISH_REASON, reason); + if (error != null || reason.toLowerCase(java.util.Locale.ROOT).startsWith("error")) { + failed = true; + tracer.setError(span, new IllegalStateException(error == null ? "Cohere generation failed" : error)); + return TracedCohereStream.EventState.ERROR; + } + return TracedCohereStream.EventState.COMPLETE; + } + + void streamOutput() { + if (!captureOutput()) return; + tracer.setOutputValue(span, text.toString()); + tracer.setOutputMessages(span, List.of(FITracer.message("assistant", text.toString()))); + if (truncated) span.setAttribute("cohere.stream.output_truncated", true); + } + + private void append(String value) { + if (!captureOutput()) return; + int remaining = MAX_TEXT - text.length(); + text.append(value, 0, Math.min(remaining, value.length())); + truncated |= value.length() > remaining; + } + + private boolean captureOutput() { + return !tracer.getConfig().isHideOutputs() && !tracer.getConfig().isHideOutputMessages(); + } + + private static String content(JsonNode content) { + if (content.isTextual()) return content.asText(); + StringBuilder result = new StringBuilder(); + if (content.isArray()) for (JsonNode part : content) result.append(part.path("text").asText("")); + return result.toString(); + } + + private static JsonNode inputJson(Object request) { + JsonNode json = json(request); + if (json == null) return null; + // Authentication is never trace content, even with input capture enabled. + // Only edit the serialized tracing copy; the SDK still needs these tokens. + for (JsonNode connector : json.path("connectors")) { + if (connector.isObject()) ((ObjectNode) connector).remove("user_access_token"); + } + return json; + } + + private static JsonNode json(Object value) { + try { + // Use the SDK's Jackson mapping, which understands its unions and Optional fields. + return ObjectMappers.JSON_MAPPER.valueToTree(value); + } catch (IllegalArgumentException error) { + // Unsupported user-supplied extension data must not break the provider call. + return null; + } + } +} diff --git a/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereClient.java b/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereClient.java index 142579d83..632e2dc51 100644 --- a/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereClient.java +++ b/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereClient.java @@ -16,7 +16,8 @@ /** * Instrumentation wrapper for Cohere Java client. - * Wraps the Cohere client to provide automatic tracing of all API calls. + * Traces v1 chat, embed, rerank and chat streams, plus the same operations through + * {@link #v2()}. Calls made directly on {@link #unwrap()} are not traced. * * Usage: * @@ -100,7 +101,8 @@ public NonStreamedChatResponse chat(ChatRequest request) { span.setAttribute("cohere.preamble", preamble)); } - tracer.setRawInput(span, request); + // Use the same credential-safe tracing copy as the streaming path. + new CohereAttributes(tracer, span).rawInput(request); // Execute request NonStreamedChatResponse response = client.chat(request); @@ -308,9 +310,53 @@ public RerankResponse rerank(RerankRequest request) { } /** - * Gets the underlying Cohere client. + * Returns a typed traced facade for v2 chat, embed, rerank and chat streaming. + * @return the traced v2 client + */ + public TracedCohereV2Client v2() { + return new TracedCohereV2Client(client.v2(), tracer); + } + + /** + * Opens a traced v1 chat stream. Always use try-with-resources when consuming it. + * @param request the stream request + * @return the closeable stream of unchanged SDK events + */ + public TracedCohereStream chatStream(ChatStreamRequest request) { + return chatStream(request, null); + } + + /** + * Opens a traced v1 chat stream with SDK request options. + * @param request the stream request + * @param options the SDK request options, or null for defaults + * @return the closeable stream of unchanged SDK events + */ + public TracedCohereStream chatStream( + ChatStreamRequest request, com.cohere.api.core.RequestOptions options) { + Span span = tracer.startSpan("Cohere Chat Stream", FISpanKind.LLM); + CohereAttributes attributes = new CohereAttributes(tracer, span); + try (Scope ignored = span.makeCurrent()) { + request.getModel().ifPresent(attributes::model); + request.getTemperature().ifPresent(value -> + span.setAttribute(SemanticConventions.LLM_REQUEST_TEMPERATURE, value.doubleValue())); + request.getMaxTokens().ifPresent(value -> + span.setAttribute(SemanticConventions.LLM_REQUEST_MAX_TOKENS, value.longValue())); + attributes.chatInput(request); + return new TracedCohereStream<>(client.chatStream(request, options), tracer, span, + attributes::v1Event, attributes::streamOutput); + } catch (RuntimeException | Error error) { + tracer.setError(span, error); + span.end(); + throw error; + } + } + + /** + * Gets the underlying, untraced Cohere client. Calls through this escape hatch + * bypass instrumentation; use {@link #v2()} for supported v2 operations. * - * @return the wrapped Cohere client + * @return the original SDK client, without tracing */ public Cohere unwrap() { return client; diff --git a/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereStream.java b/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereStream.java new file mode 100644 index 000000000..e86c81fb6 --- /dev/null +++ b/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereStream.java @@ -0,0 +1,145 @@ +package ai.traceai.cohere; + +import ai.traceai.FITracer; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Scope; + +import java.io.Closeable; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.function.Function; + +/** + * Single-use, closeable Cohere event stream. Use try-with-resources, especially when + * stopping iteration early. The original SDK events are returned unchanged. + * The span is current only while the SDK is opening or reading the stream, not + * while application code processes an event. Like the SDK stream, this is not thread-safe. + * + * @param the SDK event type + */ +public final class TracedCohereStream implements Iterable, Closeable { + enum EventState { CONTINUE, COMPLETE, ERROR } + + private final Iterable source; + private final FITracer tracer; + private final Span span; + private final Function observer; + private final Runnable captureOutput; + private boolean iterated; + private boolean ended; + private boolean closed; + + TracedCohereStream(Iterable source, FITracer tracer, Span span, + Function observer, Runnable captureOutput) { + this.source = source; + this.tracer = tracer; + this.span = span; + this.observer = observer; + this.captureOutput = captureOutput; + } + + @Override + public Iterator iterator() { + if (iterated) throw new IllegalStateException("Cohere streams can only be iterated once"); + iterated = true; + if (closed) return java.util.Collections.emptyIterator(); + final Iterator delegate; + try (Scope ignored = span.makeCurrent()) { + delegate = source.iterator(); + } catch (RuntimeException | Error error) { + fail(error); + throw error; + } + return new Iterator<>() { + @Override + public boolean hasNext() { + if (closed) return false; + try (Scope ignored = span.makeCurrent()) { + if (delegate.hasNext()) return true; + // EOF without the provider's terminal event is incomplete, not success. + close(); + return false; + } catch (IOException error) { + throw new UncheckedIOException(error); + } catch (RuntimeException | Error error) { + fail(error); + throw error; + } + } + + @Override + public T next() { + if (!hasNext()) throw new NoSuchElementException(); + try (Scope ignored = span.makeCurrent()) { + T event = delegate.next(); + EventState state = observer.apply(event); + if (state != EventState.CONTINUE) { + closeSource(); + finish(state == EventState.COMPLETE ? "completed" : "error", null); + } + return event; + } catch (IOException error) { + fail(error); + throw new UncheckedIOException(error); + } catch (RuntimeException | Error error) { + fail(error); + throw error; + } + } + }; + } + + @Override + public void close() throws IOException { + if (closed) return; + try (Scope ignored = span.makeCurrent()) { + closeSource(); + finish("incomplete", null); + } catch (IOException | RuntimeException | Error error) { + finish("error", error); + throw error; + } + } + + private void closeSource() throws IOException { + if (closed) return; + closed = true; + if (source instanceof Closeable) { + ((Closeable) source).close(); + } else if (source instanceof AutoCloseable) { + try { + ((AutoCloseable) source).close(); + } catch (IOException | RuntimeException error) { + throw error; + } catch (Exception error) { + throw new IOException(error); + } + } + } + + private void fail(Throwable error) { + try { + closeSource(); + } catch (IOException | RuntimeException | Error closeError) { + if (closeError != error) error.addSuppressed(closeError); + } finally { + finish("error", error); + } + } + + private void finish(String outcome, Throwable error) { + if (ended) return; + ended = true; + try { + captureOutput.run(); + span.setAttribute("cohere.stream.outcome", outcome); + if (error != null) tracer.setError(span, error); + else if ("completed".equals(outcome)) span.setStatus(StatusCode.OK); + } finally { + span.end(); + } + } +} diff --git a/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereV2Client.java b/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereV2Client.java new file mode 100644 index 000000000..4cd864518 --- /dev/null +++ b/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereV2Client.java @@ -0,0 +1,150 @@ +package ai.traceai.cohere; + +import ai.traceai.FISpanKind; +import ai.traceai.FITracer; +import ai.traceai.SemanticConventions; +import com.cohere.api.core.RequestOptions; +import com.cohere.api.resources.v2.V2Client; +import com.cohere.api.resources.v2.requests.V2ChatRequest; +import com.cohere.api.resources.v2.requests.V2ChatStreamRequest; +import com.cohere.api.resources.v2.requests.V2EmbedRequest; +import com.cohere.api.resources.v2.requests.V2RerankRequest; +import com.cohere.api.resources.v2.types.V2ChatResponse; +import com.cohere.api.resources.v2.types.V2ChatStreamResponse; +import com.cohere.api.resources.v2.types.V2RerankResponse; +import com.cohere.api.types.EmbedByTypeResponse; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Scope; + +import java.util.List; +import java.util.function.BiFunction; + +/** Typed tracing for Cohere v2 chat, chat streaming, embed and rerank. */ +public final class TracedCohereV2Client { + private final V2Client client; + private final FITracer tracer; + + /** + * Wraps an SDK v2 client without changing its requests, options or responses. + * @param client SDK v2 client + * @param tracer tracer used for the supported operations + */ + public TracedCohereV2Client(V2Client client, FITracer tracer) { + this.client = client; + this.tracer = tracer; + } + + public V2ChatResponse chat(V2ChatRequest request) { + return chat(request, null); + } + + public V2ChatResponse chat(V2ChatRequest request, RequestOptions options) { + return call("Cohere V2 Chat", FISpanKind.LLM, (span, attributes) -> { + attributes.model(request.getModel()); + request.getTemperature().ifPresent(value -> span.setAttribute(SemanticConventions.LLM_REQUEST_TEMPERATURE, value.doubleValue())); + request.getMaxTokens().ifPresent(value -> span.setAttribute(SemanticConventions.LLM_REQUEST_MAX_TOKENS, value.longValue())); + attributes.chatInput(request); + V2ChatResponse response = client.chat(request, options); + span.setAttribute(SemanticConventions.LLM_RESPONSE_ID, response.getId()); + attributes.finishReason(response.getFinishReason().toString(), null); + response.getUsage().ifPresent(attributes::usage); + attributes.chatOutput(response); + return response; + }); + } + + public EmbedByTypeResponse embed(V2EmbedRequest request) { + return embed(request, null); + } + + public EmbedByTypeResponse embed(V2EmbedRequest request, RequestOptions options) { + return call("Cohere V2 Embed", FISpanKind.EMBEDDING, (span, attributes) -> { + attributes.model(request.getModel()); + span.setAttribute(SemanticConventions.EMBEDDING_MODEL_NAME, request.getModel()); + span.setAttribute("cohere.input_type", request.getInputType().toString()); + request.getTexts().ifPresent(texts -> tracer.setInputValue(span, tracer.toJson(texts))); + attributes.rawInput(request); + EmbedByTypeResponse response = client.embed(request, options); + var vectors = response.getEmbeddings(); + // Prefer a non-packed encoding when multiple formats were requested. + List extends List extends Number>> values = vectors.getFloat().orElse(null); + if (values == null) values = vectors.getInt8().orElse(null); + if (values == null) values = vectors.getUint8().orElse(null); + if (values != null) { + span.setAttribute(SemanticConventions.EMBEDDING_VECTOR_COUNT, (long) values.size()); + if (!values.isEmpty()) span.setAttribute(SemanticConventions.EMBEDDING_DIMENSIONS, (long) values.get(0).size()); + } else { + // Binary vectors are packed; their element count is not the dimension. + vectors.getBinary().ifPresent(value -> span.setAttribute(SemanticConventions.EMBEDDING_VECTOR_COUNT, (long) value.size())); + vectors.getUbinary().ifPresent(value -> span.setAttribute(SemanticConventions.EMBEDDING_VECTOR_COUNT, (long) value.size())); + vectors.getBase64().ifPresent(value -> span.setAttribute(SemanticConventions.EMBEDDING_VECTOR_COUNT, (long) value.size())); + } + response.getMeta().ifPresent(attributes::embeddingMeta); + attributes.rawOutput(response); + return response; + }); + } + + public V2RerankResponse rerank(V2RerankRequest request) { + return rerank(request, null); + } + + public V2RerankResponse rerank(V2RerankRequest request, RequestOptions options) { + return call("Cohere V2 Rerank", FISpanKind.RERANKER, (span, attributes) -> { + attributes.model(request.getModel()); + tracer.setInputValue(span, request.getQuery()); + if (!tracer.getConfig().isHideInputs()) span.setAttribute(SemanticConventions.GEN_AI_RERANKER_QUERY, request.getQuery()); + span.setAttribute(SemanticConventions.GEN_AI_RERANKER_INPUT_DOCUMENTS, (long) request.getDocuments().size()); + request.getTopN().ifPresent(value -> span.setAttribute(SemanticConventions.GEN_AI_RERANKER_TOP_N, value.longValue())); + attributes.rawInput(request); + V2RerankResponse response = client.rerank(request, options); + span.setAttribute(SemanticConventions.GEN_AI_RERANKER_OUTPUT_DOCUMENTS, (long) response.getResults().size()); + if (!response.getResults().isEmpty()) { + var first = response.getResults().get(0); + span.setAttribute("cohere.rerank.top_score", (double) first.getRelevanceScore()); + span.setAttribute("cohere.rerank.top_index", (long) first.getIndex()); + } + response.getMeta().ifPresent(attributes::meta); + attributes.rawOutput(response); + return response; + }); + } + + /** Use try-with-resources so early termination also closes the SDK stream and span. */ + public TracedCohereStream chatStream(V2ChatStreamRequest request) { + return chatStream(request, null); + } + + public TracedCohereStream chatStream(V2ChatStreamRequest request, RequestOptions options) { + Span span = tracer.startSpan("Cohere V2 Chat Stream", FISpanKind.LLM); + CohereAttributes attributes = new CohereAttributes(tracer, span); + try (Scope ignored = span.makeCurrent()) { + attributes.model(request.getModel()); + request.getTemperature().ifPresent(value -> span.setAttribute(SemanticConventions.LLM_REQUEST_TEMPERATURE, value.doubleValue())); + request.getMaxTokens().ifPresent(value -> span.setAttribute(SemanticConventions.LLM_REQUEST_MAX_TOKENS, value.longValue())); + attributes.chatInput(request); + return new TracedCohereStream<>(client.chatStream(request, options), tracer, span, + attributes::v2Event, attributes::streamOutput); + } catch (RuntimeException | Error error) { + tracer.setError(span, error); + span.end(); + throw error; + } + } + + private T call(String name, FISpanKind kind, BiFunction operation) { + Span span = tracer.startSpan(name, kind); + try (Scope ignored = span.makeCurrent()) { + CohereAttributes attributes = new CohereAttributes(tracer, span); + T response = operation.apply(span, attributes); + if (!attributes.hasError()) span.setStatus(StatusCode.OK); + return response; + } catch (RuntimeException | Error error) { + tracer.setError(span, error); + throw error; + } finally { + span.end(); + } + } +} diff --git a/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereOtlpTest.java b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereOtlpTest.java new file mode 100644 index 000000000..2111a6ed9 --- /dev/null +++ b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereOtlpTest.java @@ -0,0 +1,104 @@ +package ai.traceai.cohere; + +import ai.traceai.FITracer; +import ai.traceai.SemanticConventions; +import ai.traceai.TraceConfig; +import com.cohere.api.resources.v2.requests.V2ChatRequest; +import com.cohere.api.resources.v2.requests.V2EmbedRequest; +import com.cohere.api.resources.v2.requests.V2RerankRequest; +import com.sun.net.httpserver.HttpServer; +import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.trace.v1.Span; +import io.opentelemetry.proto.trace.v1.Status; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.net.InetSocketAddress; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static ai.traceai.cohere.TracedCohereTransportTest.*; +import static org.assertj.core.api.Assertions.*; + +class CohereOtlpTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void exportsOneOtlpSpanPerOperationWithPrivacyAndUsage(boolean hide) throws Exception { + var bodies = new LinkedBlockingQueue(); + var contentTypes = new LinkedBlockingQueue(); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/v1/traces", exchange -> { + contentTypes.add(exchange.getRequestHeaders().getFirst("Content-Type")); + bodies.add(exchange.getRequestBody().readAllBytes()); + exchange.sendResponseHeaders(200, -1); + exchange.close(); + }); + server.start(); + try (var provider = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create( + OtlpHttpSpanExporter.builder().setEndpoint("http://127.0.0.1:" + server.getAddress().getPort() + "/v1/traces") + .setTimeout(Duration.ofSeconds(5)).build())).build()) { + var transport = new FakeTransport(CHAT_RESPONSE); + var config = TraceConfig.builder().hideInputs(hide).hideOutputs(hide).hideErrorDetails(hide).build(); + var traced = new TracedCohereClient(transport.client(), new FITracer(provider.get("cohere-wire"), config)); + traced.v2().chat(json(CHAT_INPUT, V2ChatRequest.class)); + transport.body = V2_STREAM; + try (var stream = stream(traced, true)) { stream.forEach(event -> {}); } + transport.body = V1_STREAM; + try (var stream = stream(traced, false)) { stream.forEach(event -> {}); } + transport.body = "{\"id\":\"e\",\"embeddings\":{\"float\":[[0.1,0.2]]}}"; + traced.v2().embed(json("{\"model\":\"embed\",\"input_type\":\"search_query\",\"texts\":[\"private-input\"]}", V2EmbedRequest.class)); + transport.body = "{\"results\":[{\"index\":0,\"relevance_score\":0.9}]}"; + traced.v2().rerank(json("{\"model\":\"rerank\",\"query\":\"private-query\",\"documents\":[\"private-doc\"]}", V2RerankRequest.class)); + transport.body = "{\"message\":\"private-error\"}"; + transport.code = 401; + assertThatThrownBy(() -> traced.v2().chat(json(CHAT_INPUT, V2ChatRequest.class))) + .isInstanceOf(com.cohere.api.errors.UnauthorizedError.class); + assertThat(provider.forceFlush().join(10, TimeUnit.SECONDS).isSuccess()).isTrue(); + List spans = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + byte[] bytes = bodies.poll(5, TimeUnit.SECONDS); + assertThat(bytes).isNotNull(); + assertThat(contentTypes.poll()).isEqualTo("application/x-protobuf"); + var request = ExportTraceServiceRequest.parseFrom(bytes); + request.getResourceSpansList().forEach(resource -> resource.getScopeSpansList() + .forEach(scope -> spans.addAll(scope.getSpansList()))); + if (hide) assertThat(request.toString()).doesNotContain("private-"); + } + assertThat(bodies).isEmpty(); + assertThat(spans).hasSize(6); + assertThat(spans.stream().map(Span::getSpanId).distinct()).hasSize(6); + var chats = spans.stream().filter(span -> span.getName().contains("Chat") + && span.getStatus().getCode() == Status.StatusCode.STATUS_CODE_OK).toList(); + assertThat(chats).hasSize(3); + for (Span span : chats) { + assertThat(span.getStatus().getCode()).isEqualTo(Status.StatusCode.STATUS_CODE_OK); + assertThat(attribute(span, SemanticConventions.LLM_TOKEN_COUNT_TOTAL).getIntValue()).isEqualTo(8); + assertThat(attribute(span, SemanticConventions.LLM_PROVIDER).getStringValue()).isEqualTo("cohere"); + assertThat(span.getEndTimeUnixNano()).isGreaterThanOrEqualTo(span.getStartTimeUnixNano()); + if (!hide) assertThat(attribute(span, SemanticConventions.OUTPUT_VALUE).getStringValue()).isEqualTo("private-output"); + } + Span error = spans.stream().filter(span -> span.getStatus().getCode() == Status.StatusCode.STATUS_CODE_ERROR) + .findFirst().orElseThrow(); + assertThat(error.getStatus().getCode()).isEqualTo(Status.StatusCode.STATUS_CODE_ERROR); + assertThat(error.getEventsCount()).isEqualTo(1); + if (hide) { + assertThat(error.getStatus().getMessage()).isEmpty(); + assertThat(error.getEvents(0).getAttributesList()).hasSize(1); + assertThat(error.getEvents(0).getAttributes(0).getKey()).isEqualTo("exception.type"); + } + } finally { + server.stop(0); + } + } + + private io.opentelemetry.proto.common.v1.AnyValue attribute(Span span, String key) { + return span.getAttributesList().stream().filter(attribute -> attribute.getKey().equals(key)) + .findFirst().orElseThrow().getValue(); + } +} diff --git a/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereRequestSafetyTest.java b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereRequestSafetyTest.java new file mode 100644 index 000000000..86f492616 --- /dev/null +++ b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereRequestSafetyTest.java @@ -0,0 +1,193 @@ +package ai.traceai.cohere; + +import ai.traceai.FITracer; +import ai.traceai.SemanticConventions; +import ai.traceai.TraceConfig; +import com.cohere.api.Cohere; +import com.cohere.api.core.ObjectMappers; +import com.cohere.api.requests.ChatRequest; +import com.cohere.api.requests.ChatStreamRequest; +import com.cohere.api.resources.v2.requests.V2EmbedRequest; +import com.sun.net.httpserver.HttpServer; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; +import io.opentelemetry.sdk.trace.data.SpanData; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.*; + +/** Real SDK requests over loopback, with synthetic credentials and no provider access. */ +class CohereRequestSafetyTest { + @RegisterExtension + static final OpenTelemetryExtension otel = OpenTelemetryExtension.create(); + + private static final String CONNECTOR_TOKEN = "synthetic-connector-bearer-token"; + private static final String SECOND_TOKEN = "synthetic-second-connector-token"; + private static final String CONNECTOR_REQUEST = """ + {"model":"test-model","message":"visible-input","connectors":[ + {"id":"internal-docs","user_access_token":"%s"}, + {"id":"other-docs","user_access_token":"%s"}, + {"id":"public-docs"}]} + """.formatted(CONNECTOR_TOKEN, SECOND_TOKEN); + private static final String V1_RESPONSE = "{\"text\":\"visible-output\",\"generation_id\":\"response-id\",\"finish_reason\":\"COMPLETE\"}"; + + @ParameterizedTest + @CsvSource({ + "true,200,0", "true,401,0", "false,200,0", "false,401,0", + "true,200,1", "true,401,1", "false,200,1", "false,401,1", + "true,200,2", "true,401,2", "false,200,2", "false,401,2" + }) + void connectorCredentialsAreTransportOnly(boolean streaming, int status, int privacyFlag) throws Exception { + String body = status == 401 ? "{\"message\":\"synthetic-provider-failure\"}" + : streaming ? streamResponse(V1_RESPONSE) : V1_RESPONSE; + try (var server = new ProviderServer(status, body)) { + var config = TraceConfig.builder().hideInputs(privacyFlag == 1) + .hideInputMessages(privacyFlag == 2).build(); + var traced = server.traced(config); + Class> requestType = streaming ? ChatStreamRequest.class : ChatRequest.class; + Object request = ObjectMappers.JSON_MAPPER.readValue(CONNECTOR_REQUEST, requestType); + org.assertj.core.api.ThrowableAssert.ThrowingCallable call = () -> { + if (streaming) { + try (var stream = traced.chatStream((ChatStreamRequest) request)) { + stream.forEach(event -> {}); + } + } else { + assertThat(traced.chat((ChatRequest) request).getText()).isEqualTo("visible-output"); + } + }; + if (status == 401) { + assertThatThrownBy(call).isInstanceOf(RuntimeException.class); + } else { + assertThatCode(call).doesNotThrowAnyException(); + } + + // Sanitization must not mutate either the SDK request or the bytes on the wire. + assertCredentials(ObjectMappers.JSON_MAPPER.valueToTree(request).toString()); + assertCredentials(server.request.get()); + SpanData span = onlySpan(); + assertThat(span.getStatus().getStatusCode()).isEqualTo(status == 401 ? StatusCode.ERROR : StatusCode.OK); + assertThat(span.getAttributes().toString()).doesNotContain(CONNECTOR_TOKEN, SECOND_TOKEN); + assertThat(span.getEvents().toString()).doesNotContain(CONNECTOR_TOKEN, SECOND_TOKEN); + assertThat(span.getStatus().getDescription()).doesNotContain(CONNECTOR_TOKEN, SECOND_TOKEN); + if (privacyFlag == 0) { + var raw = ObjectMappers.JSON_MAPPER.readTree(span.getAttributes().get(AttributeKey.stringKey(SemanticConventions.RAW_INPUT))); + assertThat(raw.path("message").asText()).isEqualTo("visible-input"); + assertThat(raw.path("connectors")).hasSize(3); + assertThat(raw.path("connectors").get(0).path("id").asText()).isEqualTo("internal-docs"); + for (var connector : raw.path("connectors")) { + assertThat(connector.has("user_access_token")).isFalse(); + } + } + } + } + + @ParameterizedTest + @MethodSource("embeddingUsage") + void embeddingUsageUsesBilledInputOnlyAsFallback(String meta, Long input, Long billed, Long output, Long total) throws Exception { + String response = "{\"id\":\"embed-id\",\"embeddings\":{\"float\":[[0.1,0.2]]},\"meta\":" + meta + "}"; + try (var server = new ProviderServer(200, response)) { + var request = ObjectMappers.JSON_MAPPER.readValue(""" + {"model":"embed-model","input_type":"search_document","texts":["visible-input"]} + """, V2EmbedRequest.class); + var result = server.traced(TraceConfig.builder().build()).v2().embed(request); + assertThat(result.getEmbeddings().getFloat().orElseThrow()).hasSize(1); + SpanData span = onlySpan(); + assertThat(span.getAttributes().get(AttributeKey.longKey("cohere.billed_units.input_tokens"))).isEqualTo(billed); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_PROMPT))).isEqualTo(input); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_COMPLETION))).isEqualTo(output); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_TOTAL))).isEqualTo(total); + } + } + + static Stream embeddingUsage() { + return Stream.of( + Arguments.of("{\"billed_units\":{\"input_tokens\":4}}", 4L, 4L, null, null), + Arguments.of("{\"tokens\":{},\"billed_units\":{\"input_tokens\":4}}", 4L, 4L, null, null), + Arguments.of("{\"tokens\":{\"input_tokens\":9},\"billed_units\":{\"input_tokens\":4}}", 9L, 4L, null, null), + Arguments.of("{\"tokens\":{\"input_tokens\":0},\"billed_units\":{\"input_tokens\":4}}", 0L, 4L, null, null), + Arguments.of("{\"tokens\":{\"input_tokens\":9,\"output_tokens\":2},\"billed_units\":{\"input_tokens\":4}}", 9L, 4L, 2L, 11L), + Arguments.of("{\"tokens\":{\"output_tokens\":2},\"billed_units\":{\"input_tokens\":4}}", 4L, 4L, 2L, null), + Arguments.of("{\"tokens\":{\"input_tokens\":9}}", 9L, null, null, null), + Arguments.of("{\"billed_units\":{\"input_tokens\":0}}", 0L, 0L, null, null), + Arguments.of("{}", null, null, null, null) + ); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void chatNeverSubstitutesBilledUnitsForActualTokenUsage(boolean hasTokens) throws Exception { + String meta = "{\"billed_units\":{\"input_tokens\":4}" + + (hasTokens ? ",\"tokens\":{\"input_tokens\":9,\"output_tokens\":2}" : "") + "}"; + String response = "{\"text\":\"visible-output\",\"meta\":" + meta + "}"; + try (var server = new ProviderServer(200, streamResponse(response))) { + try (var stream = server.traced(TraceConfig.builder().build()).chatStream( + ChatStreamRequest.builder().message("visible-input").build())) { + stream.forEach(event -> {}); + } + SpanData span = onlySpan(); + assertThat(span.getAttributes().get(AttributeKey.longKey("cohere.billed_units.input_tokens"))).isEqualTo(4L); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_PROMPT))).isEqualTo(hasTokens ? 9L : null); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_COMPLETION))).isEqualTo(hasTokens ? 2L : null); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_TOTAL))).isEqualTo(hasTokens ? 11L : null); + } + } + + private static String streamResponse(String response) { + return "{\"event_type\":\"stream-end\",\"finish_reason\":\"COMPLETE\",\"response\":" + response + "}\n"; + } + + private static void assertCredentials(String json) throws Exception { + var connectors = ObjectMappers.JSON_MAPPER.readTree(json).path("connectors"); + assertThat(connectors.get(0).path("user_access_token").asText()).isEqualTo(CONNECTOR_TOKEN); + assertThat(connectors.get(1).path("user_access_token").asText()).isEqualTo(SECOND_TOKEN); + assertThat(connectors.get(2).has("user_access_token")).isFalse(); + } + + private static SpanData onlySpan() { + assertThat(otel.getSpans()).hasSize(1); + return otel.getSpans().get(0); + } + + private static final class ProviderServer implements AutoCloseable { + private final HttpServer server; + private final AtomicReference request = new AtomicReference<>(); + + ProviderServer(int status, String body) throws Exception { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + try { + request.set(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8)); + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, bytes.length); + exchange.getResponseBody().write(bytes); + } finally { + exchange.close(); + } + }); + server.start(); + } + + TracedCohereClient traced(TraceConfig config) { + var client = Cohere.builder().token("synthetic-provider-key").maxRetries(0) + .url("http://127.0.0.1:" + server.getAddress().getPort()).build(); + return new TracedCohereClient(client, new FITracer(otel.getOpenTelemetry().getTracer("cohere-safety-test"), config)); + } + + @Override + public void close() { + server.stop(0); + } + } +} diff --git a/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereSurfaceTest.java b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereSurfaceTest.java new file mode 100644 index 000000000..8eff84caa --- /dev/null +++ b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereSurfaceTest.java @@ -0,0 +1,20 @@ +package ai.traceai.cohere; + +import com.cohere.api.requests.ChatStreamRequest; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class CohereSurfaceTest { + @Test + void exposesTracedV2Facade() throws Exception { + assertThat(TracedCohereClient.class.getMethod("v2").getReturnType().getSimpleName()) + .isEqualTo("TracedCohereV2Client"); + } + + @Test + void exposesCloseableV1Stream() throws Exception { + assertThat(AutoCloseable.class.isAssignableFrom(TracedCohereClient.class + .getMethod("chatStream", ChatStreamRequest.class).getReturnType())).isTrue(); + } +} diff --git a/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/TracedCohereStreamTest.java b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/TracedCohereStreamTest.java new file mode 100644 index 000000000..e1a61c5b6 --- /dev/null +++ b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/TracedCohereStreamTest.java @@ -0,0 +1,138 @@ +package ai.traceai.cohere; + +import ai.traceai.FISpanKind; +import ai.traceai.FITracer; +import ai.traceai.SemanticConventions; +import ai.traceai.TraceConfig; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +import static ai.traceai.cohere.TracedCohereTransportTest.*; +import static org.assertj.core.api.Assertions.*; + +class TracedCohereStreamTest { + @RegisterExtension + static final OpenTelemetryExtension otel = OpenTelemetryExtension.create(); + + @ParameterizedTest + @ValueSource(strings = {"iterator", "hasNext", "next"}) + void iteratorFailuresKeepOriginalExceptionAndCloseExactlyOnce(String stage) throws Exception { + var source = new Source(); + source.failureStage = stage; + var stream = stream(source); + assertThatThrownBy(() -> stream.iterator().next()).isSameAs(source.failure); + stream.close(); + assertThat(source.closes).isEqualTo(1); + assertThat(otel.getSpans()).hasSize(1); + assertThat(otel.getSpans().get(0).getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + } + + @Test + void closeFailureDoesNotMaskIterationFailure() throws Exception { + var source = new Source(); + source.failureStage = "next"; + source.closeFailure = new IOException("private-close-error"); + var stream = stream(source); + assertThatThrownBy(() -> stream.iterator().next()).isSameAs(source.failure); + assertThat(source.failure.getSuppressed()).containsExactly(source.closeFailure); + stream.close(); + assertThat(source.closes).isEqualTo(1); + assertThat(otel.getSpans()).hasSize(1); + } + + @Test + void explicitCloseFailureEndsSpanAndPropagates() { + var source = new Source(); + source.closeFailure = new IOException("private-close-error"); + var stream = stream(source); + assertThatThrownBy(stream::close).isSameAs(source.closeFailure); + assertThat(otel.getSpans()).hasSize(1); + assertThat(otel.getSpans().get(0).getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); + } + + @Test + void exhaustedIncompleteStreamClosesAndReturnsSameEvent() throws Exception { + var source = new Source(); + var stream = stream(source); + var iterator = stream.iterator(); + assertThat(iterator.next()).isSameAs(source.event); + assertThat(iterator.hasNext()).isFalse(); + assertThat(iterator.hasNext()).isFalse(); + stream.close(); + assertThat(source.closes).isEqualTo(1); + assertThat(otel.getSpans()).hasSize(1); + assertThat(value(otel.getSpans().get(0), "cohere.stream.outcome")).isEqualTo("incomplete"); + assertThatThrownBy(stream::iterator).isInstanceOf(IllegalStateException.class); + } + + @Test + void unopenedStreamCanBeClosedWithoutCreatingAnIterator() throws Exception { + var source = new Source(); + source.failureStage = "iterator"; + var stream = stream(source); + stream.close(); + assertThat(stream.iterator().hasNext()).isFalse(); + assertThat(source.closes).isEqualTo(1); + assertThat(otel.getSpans()).hasSize(1); + } + + @Test + void streamedTextIsBounded() throws Exception { + var transport = new FakeTransport(V2_STREAM.replace("private-output", "x".repeat(100000))); + var traced = new TracedCohereClient(transport.client(), tracer()); + try (var stream = TracedCohereTransportTest.stream(traced, true)) { stream.forEach(event -> {}); } + assertThat(value(otel.getSpans().get(0), SemanticConventions.OUTPUT_VALUE)).hasSize(32000); + assertThat(otel.getSpans().get(0).getAttributes().get( + io.opentelemetry.api.common.AttributeKey.booleanKey("cohere.stream.output_truncated"))).isTrue(); + } + + private FITracer tracer() { + return new FITracer(otel.getOpenTelemetry().getTracer("stream-test"), TraceConfig.builder().hideErrorDetails(true).build()); + } + + private TracedCohereStream stream(Source source) { + FITracer tracer = tracer(); + return new TracedCohereStream<>(source, tracer, tracer.startSpan("test stream", FISpanKind.LLM), + event -> TracedCohereStream.EventState.CONTINUE, () -> {}); + } + + static final class Source implements Iterable, Closeable { + final Object event = new Object(); + final RuntimeException failure = new IllegalStateException("private-error"); + String failureStage; + IOException closeFailure; + int closes; + + @Override + public Iterator iterator() { + if ("iterator".equals(failureStage)) throw failure; + Iterator delegate = List.of(event).iterator(); + return new Iterator<>() { + @Override public boolean hasNext() { + if ("hasNext".equals(failureStage)) throw failure; + return delegate.hasNext(); + } + @Override public Object next() { + if ("next".equals(failureStage)) throw failure; + return delegate.next(); + } + }; + } + + @Override public void close() throws IOException { + closes++; + if (closeFailure != null) throw closeFailure; + } + } +} diff --git a/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/TracedCohereTransportTest.java b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/TracedCohereTransportTest.java new file mode 100644 index 000000000..4d9271e97 --- /dev/null +++ b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/TracedCohereTransportTest.java @@ -0,0 +1,320 @@ +package ai.traceai.cohere; + +import ai.traceai.FITracer; +import ai.traceai.SemanticConventions; +import ai.traceai.TraceConfig; +import com.cohere.api.Cohere; +import com.cohere.api.core.ObjectMappers; +import com.cohere.api.core.RequestOptions; +import com.cohere.api.requests.ChatStreamRequest; +import com.cohere.api.resources.v2.requests.V2ChatRequest; +import com.cohere.api.resources.v2.requests.V2ChatStreamRequest; +import com.cohere.api.resources.v2.requests.V2EmbedRequest; +import com.cohere.api.resources.v2.requests.V2RerankRequest; +import com.cohere.api.resources.v2.types.V2ChatStreamResponse; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; +import io.opentelemetry.sdk.trace.data.SpanData; +import okhttp3.Interceptor; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okio.Buffer; +import okio.BufferedSource; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.*; + +class TracedCohereTransportTest { + @RegisterExtension + static final OpenTelemetryExtension otel = OpenTelemetryExtension.create(); + + static final String CHAT_INPUT = "{\"model\":\"test-model\",\"messages\":[{\"role\":\"user\",\"content\":\"private-input\"}]}"; + static final String CHAT_RESPONSE = """ + {"id":"response-id","finish_reason":"COMPLETE","message":{"role":"assistant", + "content":[{"type":"text","text":"private-output"}], + "tool_calls":[{"id":"call-id","type":"function","function":{"name":"private-tool","arguments":"{}"}}]}, + "usage":{"tokens":{"input_tokens":3,"output_tokens":5}}} + """; + static final String V2_STREAM = sse("message-start", "{\"id\":\"response-id\",\"delta\":{\"message\":{\"role\":\"assistant\"}}}") + + sse("content-delta", "{\"index\":0,\"delta\":{\"message\":{\"content\":{\"text\":\"private-output\"}}}}") + + sse("message-end", "{\"delta\":{\"finish_reason\":\"COMPLETE\",\"usage\":{\"tokens\":{\"input_tokens\":3,\"output_tokens\":5}}}}"); + static final String V1_STREAM = """ + {"event_type":"stream-start","generation_id":"response-id"} + {"event_type":"text-generation","text":"private-output"} + {"event_type":"stream-end","finish_reason":"COMPLETE","response":{"text":"private-output","generation_id":"response-id","meta":{"tokens":{"input_tokens":3,"output_tokens":5}}}} + """; + + @Test + void tracesV2ChatThroughRealSdkAndPreservesOptions() throws Exception { + FakeTransport transport = new FakeTransport(CHAT_RESPONSE); + TracedCohereClient traced = traced(transport, TraceConfig.builder().build()); + var request = json(CHAT_INPUT, V2ChatRequest.class); + var response = traced.v2().chat(request, RequestOptions.builder().addHeader("X-Test-Option", "kept").build()); + assertThat(response.getId()).isEqualTo("response-id"); + assertThat(response.getMessage().getToolCalls()).isPresent(); + assertThat(transport.request.url().encodedPath()).isEqualTo("/v2/chat"); + assertThat(transport.request.header("X-Test-Option")).isEqualTo("kept"); + assertThat(transport.sentBody).contains("private-input", "test-model"); + SpanData span = onlySpan(); + assertThat(transport.spanId).isEqualTo(span.getSpanId()); + assertThat(value(span, SemanticConventions.RAW_INPUT)).contains("\"role\":\"user\""); + assertThat(value(span, SemanticConventions.RAW_OUTPUT)).contains("private-tool"); + assertThat(value(span, SemanticConventions.OUTPUT_VALUE)).isEqualTo("private-output"); + assertUsage(span); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + } + + @Test + void tracesV2EmbedAndRerankThroughRealSdk() throws Exception { + FakeTransport transport = new FakeTransport(""" + {"id":"embedding-id","embeddings":{"float":[[0.1,0.2],[0.3,0.4]]}, + "meta":{"billed_units":{"input_tokens":4}}} + """); + var v2 = traced(transport, TraceConfig.builder().build()).v2(); + var embedded = v2.embed(json(""" + {"model":"embed-model","input_type":"search_document","texts":["private-a","private-b"]} + """, V2EmbedRequest.class)); + assertThat(embedded.getEmbeddings().getFloat().orElseThrow()).hasSize(2); + SpanData span = onlySpan(); + assertThat(value(span, SemanticConventions.FI_SPAN_KIND)).isEqualTo("EMBEDDING"); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.EMBEDDING_VECTOR_COUNT))).isEqualTo(2L); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.EMBEDDING_DIMENSIONS))).isEqualTo(2L); + assertThat(transport.request.url().encodedPath()).isEqualTo("/v2/embed"); + transport.body = "{\"results\":[{\"index\":1,\"relevance_score\":0.9}]}"; + var reranked = v2.rerank(json(""" + {"model":"rerank-model","query":"private-query","documents":["private-a","private-b"],"top_n":1} + """, V2RerankRequest.class), RequestOptions.builder().addHeader("X-Test-Option", "rerank").build()); + assertThat(reranked.getResults().get(0).getIndex()).isEqualTo(1); + assertThat(transport.request.url().encodedPath()).isEqualTo("/v2/rerank"); + assertThat(transport.request.header("X-Test-Option")).isEqualTo("rerank"); + assertThat(otel.getSpans()).hasSize(2); + span = otel.getSpans().get(1); + assertThat(value(span, SemanticConventions.FI_SPAN_KIND)).isEqualTo("RERANKER"); + assertThat(value(span, SemanticConventions.GEN_AI_RERANKER_QUERY)).isEqualTo("private-query"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void streamsUntilTerminalEventAndRestoresParent(boolean v2) throws Exception { + FakeTransport transport = new FakeTransport(v2 ? V2_STREAM : V1_STREAM); + var traced = traced(transport, TraceConfig.builder().build()); + Span parent = otel.getOpenTelemetry().getTracer("parent").spanBuilder("parent").startSpan(); + try (Scope ignored = parent.makeCurrent(); var stream = stream(traced, v2)) { + assertThat(otel.getSpans()).isEmpty(); + assertThat(Span.current().getSpanContext()).isEqualTo(parent.getSpanContext()); + var iterator = stream.iterator(); + iterator.next(); + assertThat(otel.getSpans()).isEmpty(); + iterator.next(); + assertThat(otel.getSpans()).isEmpty(); + iterator.next(); // terminal event completes even without another hasNext() + assertThat(transport.closed).isTrue(); + assertThat(iterator.hasNext()).isFalse(); + assertThatThrownBy(iterator::next).isInstanceOf(java.util.NoSuchElementException.class); + SpanData span = onlySpan(); + assertThat(span.getParentSpanId()).isEqualTo(parent.getSpanContext().getSpanId()); + assertThat(value(span, "cohere.stream.outcome")).isEqualTo("completed"); + assertThat(span.getStatus().getStatusCode()).isEqualTo(StatusCode.OK); + assertThat(value(span, SemanticConventions.OUTPUT_VALUE)).isEqualTo("private-output"); + assertUsage(span); + assertThat(Span.current().getSpanContext()).isEqualTo(parent.getSpanContext()); + } finally { + parent.end(); + } + assertThat(otel.getSpans()).hasSize(2); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void earlyCloseEndsIncompleteSpanAndClosesTransportOnce(boolean v2) throws Exception { + FakeTransport transport = new FakeTransport(v2 ? V2_STREAM : V1_STREAM); + var stream = stream(traced(transport, TraceConfig.builder().build()), v2); + stream.iterator().next(); + stream.close(); + stream.close(); + assertThat(transport.closed).isTrue(); + assertThat(value(onlySpan(), "cohere.stream.outcome")).isEqualTo("incomplete"); + assertThat(onlySpan().getStatus().getStatusCode()).isEqualTo(StatusCode.UNSET); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void malformedStreamsDoNotProduceSuccessfulSpans(boolean v2) throws Exception { + FakeTransport transport = new FakeTransport(v2 ? "event: content-delta\ndata: {not-json}\n\n" : "{not-json}\n"); + var stream = stream(traced(transport, TraceConfig.builder().hideErrorDetails(true).build()), v2); + assertThatThrownBy(() -> stream.iterator().next()).isInstanceOf(RuntimeException.class); + assertThat(transport.closed).isTrue(); + assertThat(onlySpan().getStatus().getDescription()).isEmpty(); + if (v2) { + // Both supported SDKs swallow malformed SSE events and expose EOF. + assertThat(value(onlySpan(), "cohere.stream.outcome")).isEqualTo("incomplete"); + assertThat(onlySpan().getStatus().getStatusCode()).isEqualTo(StatusCode.UNSET); + } else { + assertThat(onlySpan().getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); + assertThat(onlySpan().getEvents()).hasSize(1); + assertThat(onlySpan().getEvents().get(0).getAttributes().asMap()) + .containsOnlyKeys(AttributeKey.stringKey("exception.type")); + } + stream.close(); + assertThat(otel.getSpans()).hasSize(1); + } + + @ParameterizedTest + @ValueSource(strings = {"chat", "embed", "rerank", "v1stream", "v2stream"}) + void transportErrorsPreserveSdkException(String operation) throws Exception { + FakeTransport transport = new FakeTransport("{\"message\":\"private-error\"}"); + transport.code = 401; + var traced = traced(transport, TraceConfig.builder().hideInputs(true).hideOutputs(true).hideErrorDetails(true).build()); + assertThatThrownBy(() -> { + switch (operation) { + case "chat": traced.v2().chat(json(CHAT_INPUT, V2ChatRequest.class)); break; + case "embed": traced.v2().embed(json("{\"model\":\"m\",\"input_type\":\"search_query\",\"texts\":[\"private-input\"]}", V2EmbedRequest.class)); break; + case "rerank": traced.v2().rerank(json("{\"model\":\"m\",\"query\":\"private-input\",\"documents\":[\"d\"]}", V2RerankRequest.class)); break; + default: stream(traced, operation.equals("v2stream")); + } + }).isInstanceOf(com.cohere.api.errors.UnauthorizedError.class); + assertThat(onlySpan().getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); + assertThat(onlySpan().getAttributes().toString()).doesNotContain("private-"); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + } + + @ParameterizedTest + @ValueSource(ints = {1, 2, 4, 8, 15}) + void newChatSurfacesRespectContentAndMessagePrivacy(int flags) throws Exception { + TraceConfig config = TraceConfig.builder().hideInputs((flags & 1) != 0).hideOutputs((flags & 2) != 0) + .hideInputMessages((flags & 4) != 0).hideOutputMessages((flags & 8) != 0).build(); + FakeTransport transport = new FakeTransport(CHAT_RESPONSE); + var traced = traced(transport, config); + traced.v2().chat(json(CHAT_INPUT, V2ChatRequest.class)); + transport.body = V2_STREAM; + try (var stream = stream(traced, true)) { stream.forEach(event -> {}); } + transport.body = V1_STREAM; + try (var stream = stream(traced, false)) { stream.forEach(event -> {}); } + assertThat(otel.getSpans()).hasSize(3); + for (SpanData span : otel.getSpans()) { + if ((flags & 5) != 0) assertThat(span.getAttributes().toString()).doesNotContain("private-input"); + if ((flags & 10) != 0) assertThat(span.getAttributes().toString()).doesNotContain("private-output", "private-tool"); + assertUsage(span); + } + } + + @Test + void terminalProviderErrorIsNotReportedAsSuccess() throws Exception { + FakeTransport transport = new FakeTransport(sse("message-end", + "{\"delta\":{\"finish_reason\":\"ERROR\",\"error\":\"private-error\"}}")); + try (var stream = stream(traced(transport, TraceConfig.builder().hideErrorDetails(true).build()), true)) { + stream.forEach(event -> {}); + } + assertThat(onlySpan().getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); + assertThat(value(onlySpan(), "cohere.stream.outcome")).isEqualTo("error"); + assertThat(onlySpan().getAttributes().toString()).doesNotContain("private-error"); + } + + @Test + void synchronousProviderErrorIsNotReportedAsSuccess() throws Exception { + FakeTransport transport = new FakeTransport(CHAT_RESPONSE.replace("COMPLETE", "ERROR")); + var traced = traced(transport, TraceConfig.builder().hideErrorDetails(true).build()); + var response = traced.v2().chat(json(CHAT_INPUT, V2ChatRequest.class)); + assertThat(response.getFinishReason().toString()).isEqualTo("ERROR"); + assertThat(onlySpan().getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); + } + + @Test + void absentUsageDoesNotInventZeroCounts() throws Exception { + FakeTransport transport = new FakeTransport("{\"id\":\"r\",\"finish_reason\":\"COMPLETE\",\"message\":{\"role\":\"assistant\"}}"); + traced(transport, TraceConfig.builder().build()).v2().chat(json(CHAT_INPUT, V2ChatRequest.class)); + assertThat(onlySpan().getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_TOTAL))).isNull(); + } + + @Test + void unwrapRemainsOriginalUntracedEscapeHatch() throws Exception { + FakeTransport transport = new FakeTransport(CHAT_RESPONSE); + var traced = traced(transport, TraceConfig.builder().build()); + traced.unwrap().v2().chat(json(CHAT_INPUT, V2ChatRequest.class)); + assertThat(otel.getSpans()).isEmpty(); + } + + static TracedCohereStream> stream(TracedCohereClient traced, boolean v2) throws IOException { + if (v2) return traced.v2().chatStream(json(CHAT_INPUT, V2ChatStreamRequest.class)); + return traced.chatStream(ChatStreamRequest.builder().message("private-input").model("test-model").build()); + } + + private TracedCohereClient traced(FakeTransport transport, TraceConfig config) { + return new TracedCohereClient(transport.client(), new FITracer(otel.getOpenTelemetry().getTracer("cohere-test"), config)); + } + + static T json(String value, Class type) throws IOException { + return ObjectMappers.JSON_MAPPER.readValue(value, type); + } + + static String sse(String type, String data) { + return "event: " + type + "\ndata: {\"type\":\"" + type + "\"," + data.substring(1) + "\n\n"; + } + + private SpanData onlySpan() { + assertThat(otel.getSpans()).hasSize(1); + return otel.getSpans().get(0); + } + + static String value(SpanData span, String key) { + return span.getAttributes().get(AttributeKey.stringKey(key)); + } + + private void assertUsage(SpanData span) { + assertThat(value(span, SemanticConventions.LLM_PROVIDER)).isEqualTo("cohere"); + assertThat(value(span, SemanticConventions.LLM_REQUEST_MODEL)).isEqualTo("test-model"); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_PROMPT))).isEqualTo(3L); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_COMPLETION))).isEqualTo(5L); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_TOTAL))).isEqualTo(8L); + } + + /** All provider traffic ends here: no DNS, sockets, credentials or vendor API calls. */ + static final class FakeTransport implements Interceptor { + String body; + int code = 200; + Request request; + String sentBody; + String spanId; + boolean closed; + + FakeTransport(String body) { this.body = body; } + + Cohere client() { + return Cohere.builder().token("synthetic-not-a-credential") + .httpClient(new OkHttpClient.Builder().addInterceptor(this).build()).build(); + } + + @Override + public Response intercept(Chain chain) throws IOException { + request = chain.request(); + Buffer sent = new Buffer(); + request.body().writeTo(sent); + sentBody = sent.readUtf8(); + spanId = Span.current().getSpanContext().getSpanId(); + Buffer buffer = new Buffer().writeUtf8(body); + ResponseBody responseBody = new ResponseBody() { + @Override public MediaType contentType() { return MediaType.parse("application/json"); } + @Override public long contentLength() { return -1; } + @Override public BufferedSource source() { return buffer; } + @Override public void close() { closed = true; super.close(); } + }; + return new Response.Builder().request(request).protocol(Protocol.HTTP_1_1) + .code(code).message("synthetic").body(responseBody).build(); + } + } +}
Usage:
@@ -100,7 +101,8 @@ public NonStreamedChatResponse chat(ChatRequest request) { span.setAttribute("cohere.preamble", preamble)); } - tracer.setRawInput(span, request); + // Use the same credential-safe tracing copy as the streaming path. + new CohereAttributes(tracer, span).rawInput(request); // Execute request NonStreamedChatResponse response = client.chat(request); @@ -308,9 +310,53 @@ public RerankResponse rerank(RerankRequest request) { } /** - * Gets the underlying Cohere client. + * Returns a typed traced facade for v2 chat, embed, rerank and chat streaming. + * @return the traced v2 client + */ + public TracedCohereV2Client v2() { + return new TracedCohereV2Client(client.v2(), tracer); + } + + /** + * Opens a traced v1 chat stream. Always use try-with-resources when consuming it. + * @param request the stream request + * @return the closeable stream of unchanged SDK events + */ + public TracedCohereStream chatStream(ChatStreamRequest request) { + return chatStream(request, null); + } + + /** + * Opens a traced v1 chat stream with SDK request options. + * @param request the stream request + * @param options the SDK request options, or null for defaults + * @return the closeable stream of unchanged SDK events + */ + public TracedCohereStream chatStream( + ChatStreamRequest request, com.cohere.api.core.RequestOptions options) { + Span span = tracer.startSpan("Cohere Chat Stream", FISpanKind.LLM); + CohereAttributes attributes = new CohereAttributes(tracer, span); + try (Scope ignored = span.makeCurrent()) { + request.getModel().ifPresent(attributes::model); + request.getTemperature().ifPresent(value -> + span.setAttribute(SemanticConventions.LLM_REQUEST_TEMPERATURE, value.doubleValue())); + request.getMaxTokens().ifPresent(value -> + span.setAttribute(SemanticConventions.LLM_REQUEST_MAX_TOKENS, value.longValue())); + attributes.chatInput(request); + return new TracedCohereStream<>(client.chatStream(request, options), tracer, span, + attributes::v1Event, attributes::streamOutput); + } catch (RuntimeException | Error error) { + tracer.setError(span, error); + span.end(); + throw error; + } + } + + /** + * Gets the underlying, untraced Cohere client. Calls through this escape hatch + * bypass instrumentation; use {@link #v2()} for supported v2 operations. * - * @return the wrapped Cohere client + * @return the original SDK client, without tracing */ public Cohere unwrap() { return client; diff --git a/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereStream.java b/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereStream.java new file mode 100644 index 000000000..e86c81fb6 --- /dev/null +++ b/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereStream.java @@ -0,0 +1,145 @@ +package ai.traceai.cohere; + +import ai.traceai.FITracer; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Scope; + +import java.io.Closeable; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.function.Function; + +/** + * Single-use, closeable Cohere event stream. Use try-with-resources, especially when + * stopping iteration early. The original SDK events are returned unchanged. + * The span is current only while the SDK is opening or reading the stream, not + * while application code processes an event. Like the SDK stream, this is not thread-safe. + * + * @param the SDK event type + */ +public final class TracedCohereStream implements Iterable, Closeable { + enum EventState { CONTINUE, COMPLETE, ERROR } + + private final Iterable source; + private final FITracer tracer; + private final Span span; + private final Function observer; + private final Runnable captureOutput; + private boolean iterated; + private boolean ended; + private boolean closed; + + TracedCohereStream(Iterable source, FITracer tracer, Span span, + Function observer, Runnable captureOutput) { + this.source = source; + this.tracer = tracer; + this.span = span; + this.observer = observer; + this.captureOutput = captureOutput; + } + + @Override + public Iterator iterator() { + if (iterated) throw new IllegalStateException("Cohere streams can only be iterated once"); + iterated = true; + if (closed) return java.util.Collections.emptyIterator(); + final Iterator delegate; + try (Scope ignored = span.makeCurrent()) { + delegate = source.iterator(); + } catch (RuntimeException | Error error) { + fail(error); + throw error; + } + return new Iterator<>() { + @Override + public boolean hasNext() { + if (closed) return false; + try (Scope ignored = span.makeCurrent()) { + if (delegate.hasNext()) return true; + // EOF without the provider's terminal event is incomplete, not success. + close(); + return false; + } catch (IOException error) { + throw new UncheckedIOException(error); + } catch (RuntimeException | Error error) { + fail(error); + throw error; + } + } + + @Override + public T next() { + if (!hasNext()) throw new NoSuchElementException(); + try (Scope ignored = span.makeCurrent()) { + T event = delegate.next(); + EventState state = observer.apply(event); + if (state != EventState.CONTINUE) { + closeSource(); + finish(state == EventState.COMPLETE ? "completed" : "error", null); + } + return event; + } catch (IOException error) { + fail(error); + throw new UncheckedIOException(error); + } catch (RuntimeException | Error error) { + fail(error); + throw error; + } + } + }; + } + + @Override + public void close() throws IOException { + if (closed) return; + try (Scope ignored = span.makeCurrent()) { + closeSource(); + finish("incomplete", null); + } catch (IOException | RuntimeException | Error error) { + finish("error", error); + throw error; + } + } + + private void closeSource() throws IOException { + if (closed) return; + closed = true; + if (source instanceof Closeable) { + ((Closeable) source).close(); + } else if (source instanceof AutoCloseable) { + try { + ((AutoCloseable) source).close(); + } catch (IOException | RuntimeException error) { + throw error; + } catch (Exception error) { + throw new IOException(error); + } + } + } + + private void fail(Throwable error) { + try { + closeSource(); + } catch (IOException | RuntimeException | Error closeError) { + if (closeError != error) error.addSuppressed(closeError); + } finally { + finish("error", error); + } + } + + private void finish(String outcome, Throwable error) { + if (ended) return; + ended = true; + try { + captureOutput.run(); + span.setAttribute("cohere.stream.outcome", outcome); + if (error != null) tracer.setError(span, error); + else if ("completed".equals(outcome)) span.setStatus(StatusCode.OK); + } finally { + span.end(); + } + } +} diff --git a/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereV2Client.java b/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereV2Client.java new file mode 100644 index 000000000..4cd864518 --- /dev/null +++ b/java/traceai-java-cohere/src/main/java/ai/traceai/cohere/TracedCohereV2Client.java @@ -0,0 +1,150 @@ +package ai.traceai.cohere; + +import ai.traceai.FISpanKind; +import ai.traceai.FITracer; +import ai.traceai.SemanticConventions; +import com.cohere.api.core.RequestOptions; +import com.cohere.api.resources.v2.V2Client; +import com.cohere.api.resources.v2.requests.V2ChatRequest; +import com.cohere.api.resources.v2.requests.V2ChatStreamRequest; +import com.cohere.api.resources.v2.requests.V2EmbedRequest; +import com.cohere.api.resources.v2.requests.V2RerankRequest; +import com.cohere.api.resources.v2.types.V2ChatResponse; +import com.cohere.api.resources.v2.types.V2ChatStreamResponse; +import com.cohere.api.resources.v2.types.V2RerankResponse; +import com.cohere.api.types.EmbedByTypeResponse; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Scope; + +import java.util.List; +import java.util.function.BiFunction; + +/** Typed tracing for Cohere v2 chat, chat streaming, embed and rerank. */ +public final class TracedCohereV2Client { + private final V2Client client; + private final FITracer tracer; + + /** + * Wraps an SDK v2 client without changing its requests, options or responses. + * @param client SDK v2 client + * @param tracer tracer used for the supported operations + */ + public TracedCohereV2Client(V2Client client, FITracer tracer) { + this.client = client; + this.tracer = tracer; + } + + public V2ChatResponse chat(V2ChatRequest request) { + return chat(request, null); + } + + public V2ChatResponse chat(V2ChatRequest request, RequestOptions options) { + return call("Cohere V2 Chat", FISpanKind.LLM, (span, attributes) -> { + attributes.model(request.getModel()); + request.getTemperature().ifPresent(value -> span.setAttribute(SemanticConventions.LLM_REQUEST_TEMPERATURE, value.doubleValue())); + request.getMaxTokens().ifPresent(value -> span.setAttribute(SemanticConventions.LLM_REQUEST_MAX_TOKENS, value.longValue())); + attributes.chatInput(request); + V2ChatResponse response = client.chat(request, options); + span.setAttribute(SemanticConventions.LLM_RESPONSE_ID, response.getId()); + attributes.finishReason(response.getFinishReason().toString(), null); + response.getUsage().ifPresent(attributes::usage); + attributes.chatOutput(response); + return response; + }); + } + + public EmbedByTypeResponse embed(V2EmbedRequest request) { + return embed(request, null); + } + + public EmbedByTypeResponse embed(V2EmbedRequest request, RequestOptions options) { + return call("Cohere V2 Embed", FISpanKind.EMBEDDING, (span, attributes) -> { + attributes.model(request.getModel()); + span.setAttribute(SemanticConventions.EMBEDDING_MODEL_NAME, request.getModel()); + span.setAttribute("cohere.input_type", request.getInputType().toString()); + request.getTexts().ifPresent(texts -> tracer.setInputValue(span, tracer.toJson(texts))); + attributes.rawInput(request); + EmbedByTypeResponse response = client.embed(request, options); + var vectors = response.getEmbeddings(); + // Prefer a non-packed encoding when multiple formats were requested. + List extends List extends Number>> values = vectors.getFloat().orElse(null); + if (values == null) values = vectors.getInt8().orElse(null); + if (values == null) values = vectors.getUint8().orElse(null); + if (values != null) { + span.setAttribute(SemanticConventions.EMBEDDING_VECTOR_COUNT, (long) values.size()); + if (!values.isEmpty()) span.setAttribute(SemanticConventions.EMBEDDING_DIMENSIONS, (long) values.get(0).size()); + } else { + // Binary vectors are packed; their element count is not the dimension. + vectors.getBinary().ifPresent(value -> span.setAttribute(SemanticConventions.EMBEDDING_VECTOR_COUNT, (long) value.size())); + vectors.getUbinary().ifPresent(value -> span.setAttribute(SemanticConventions.EMBEDDING_VECTOR_COUNT, (long) value.size())); + vectors.getBase64().ifPresent(value -> span.setAttribute(SemanticConventions.EMBEDDING_VECTOR_COUNT, (long) value.size())); + } + response.getMeta().ifPresent(attributes::embeddingMeta); + attributes.rawOutput(response); + return response; + }); + } + + public V2RerankResponse rerank(V2RerankRequest request) { + return rerank(request, null); + } + + public V2RerankResponse rerank(V2RerankRequest request, RequestOptions options) { + return call("Cohere V2 Rerank", FISpanKind.RERANKER, (span, attributes) -> { + attributes.model(request.getModel()); + tracer.setInputValue(span, request.getQuery()); + if (!tracer.getConfig().isHideInputs()) span.setAttribute(SemanticConventions.GEN_AI_RERANKER_QUERY, request.getQuery()); + span.setAttribute(SemanticConventions.GEN_AI_RERANKER_INPUT_DOCUMENTS, (long) request.getDocuments().size()); + request.getTopN().ifPresent(value -> span.setAttribute(SemanticConventions.GEN_AI_RERANKER_TOP_N, value.longValue())); + attributes.rawInput(request); + V2RerankResponse response = client.rerank(request, options); + span.setAttribute(SemanticConventions.GEN_AI_RERANKER_OUTPUT_DOCUMENTS, (long) response.getResults().size()); + if (!response.getResults().isEmpty()) { + var first = response.getResults().get(0); + span.setAttribute("cohere.rerank.top_score", (double) first.getRelevanceScore()); + span.setAttribute("cohere.rerank.top_index", (long) first.getIndex()); + } + response.getMeta().ifPresent(attributes::meta); + attributes.rawOutput(response); + return response; + }); + } + + /** Use try-with-resources so early termination also closes the SDK stream and span. */ + public TracedCohereStream chatStream(V2ChatStreamRequest request) { + return chatStream(request, null); + } + + public TracedCohereStream chatStream(V2ChatStreamRequest request, RequestOptions options) { + Span span = tracer.startSpan("Cohere V2 Chat Stream", FISpanKind.LLM); + CohereAttributes attributes = new CohereAttributes(tracer, span); + try (Scope ignored = span.makeCurrent()) { + attributes.model(request.getModel()); + request.getTemperature().ifPresent(value -> span.setAttribute(SemanticConventions.LLM_REQUEST_TEMPERATURE, value.doubleValue())); + request.getMaxTokens().ifPresent(value -> span.setAttribute(SemanticConventions.LLM_REQUEST_MAX_TOKENS, value.longValue())); + attributes.chatInput(request); + return new TracedCohereStream<>(client.chatStream(request, options), tracer, span, + attributes::v2Event, attributes::streamOutput); + } catch (RuntimeException | Error error) { + tracer.setError(span, error); + span.end(); + throw error; + } + } + + private T call(String name, FISpanKind kind, BiFunction operation) { + Span span = tracer.startSpan(name, kind); + try (Scope ignored = span.makeCurrent()) { + CohereAttributes attributes = new CohereAttributes(tracer, span); + T response = operation.apply(span, attributes); + if (!attributes.hasError()) span.setStatus(StatusCode.OK); + return response; + } catch (RuntimeException | Error error) { + tracer.setError(span, error); + throw error; + } finally { + span.end(); + } + } +} diff --git a/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereOtlpTest.java b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereOtlpTest.java new file mode 100644 index 000000000..2111a6ed9 --- /dev/null +++ b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereOtlpTest.java @@ -0,0 +1,104 @@ +package ai.traceai.cohere; + +import ai.traceai.FITracer; +import ai.traceai.SemanticConventions; +import ai.traceai.TraceConfig; +import com.cohere.api.resources.v2.requests.V2ChatRequest; +import com.cohere.api.resources.v2.requests.V2EmbedRequest; +import com.cohere.api.resources.v2.requests.V2RerankRequest; +import com.sun.net.httpserver.HttpServer; +import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.trace.v1.Span; +import io.opentelemetry.proto.trace.v1.Status; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.net.InetSocketAddress; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static ai.traceai.cohere.TracedCohereTransportTest.*; +import static org.assertj.core.api.Assertions.*; + +class CohereOtlpTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void exportsOneOtlpSpanPerOperationWithPrivacyAndUsage(boolean hide) throws Exception { + var bodies = new LinkedBlockingQueue(); + var contentTypes = new LinkedBlockingQueue(); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/v1/traces", exchange -> { + contentTypes.add(exchange.getRequestHeaders().getFirst("Content-Type")); + bodies.add(exchange.getRequestBody().readAllBytes()); + exchange.sendResponseHeaders(200, -1); + exchange.close(); + }); + server.start(); + try (var provider = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create( + OtlpHttpSpanExporter.builder().setEndpoint("http://127.0.0.1:" + server.getAddress().getPort() + "/v1/traces") + .setTimeout(Duration.ofSeconds(5)).build())).build()) { + var transport = new FakeTransport(CHAT_RESPONSE); + var config = TraceConfig.builder().hideInputs(hide).hideOutputs(hide).hideErrorDetails(hide).build(); + var traced = new TracedCohereClient(transport.client(), new FITracer(provider.get("cohere-wire"), config)); + traced.v2().chat(json(CHAT_INPUT, V2ChatRequest.class)); + transport.body = V2_STREAM; + try (var stream = stream(traced, true)) { stream.forEach(event -> {}); } + transport.body = V1_STREAM; + try (var stream = stream(traced, false)) { stream.forEach(event -> {}); } + transport.body = "{\"id\":\"e\",\"embeddings\":{\"float\":[[0.1,0.2]]}}"; + traced.v2().embed(json("{\"model\":\"embed\",\"input_type\":\"search_query\",\"texts\":[\"private-input\"]}", V2EmbedRequest.class)); + transport.body = "{\"results\":[{\"index\":0,\"relevance_score\":0.9}]}"; + traced.v2().rerank(json("{\"model\":\"rerank\",\"query\":\"private-query\",\"documents\":[\"private-doc\"]}", V2RerankRequest.class)); + transport.body = "{\"message\":\"private-error\"}"; + transport.code = 401; + assertThatThrownBy(() -> traced.v2().chat(json(CHAT_INPUT, V2ChatRequest.class))) + .isInstanceOf(com.cohere.api.errors.UnauthorizedError.class); + assertThat(provider.forceFlush().join(10, TimeUnit.SECONDS).isSuccess()).isTrue(); + List spans = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + byte[] bytes = bodies.poll(5, TimeUnit.SECONDS); + assertThat(bytes).isNotNull(); + assertThat(contentTypes.poll()).isEqualTo("application/x-protobuf"); + var request = ExportTraceServiceRequest.parseFrom(bytes); + request.getResourceSpansList().forEach(resource -> resource.getScopeSpansList() + .forEach(scope -> spans.addAll(scope.getSpansList()))); + if (hide) assertThat(request.toString()).doesNotContain("private-"); + } + assertThat(bodies).isEmpty(); + assertThat(spans).hasSize(6); + assertThat(spans.stream().map(Span::getSpanId).distinct()).hasSize(6); + var chats = spans.stream().filter(span -> span.getName().contains("Chat") + && span.getStatus().getCode() == Status.StatusCode.STATUS_CODE_OK).toList(); + assertThat(chats).hasSize(3); + for (Span span : chats) { + assertThat(span.getStatus().getCode()).isEqualTo(Status.StatusCode.STATUS_CODE_OK); + assertThat(attribute(span, SemanticConventions.LLM_TOKEN_COUNT_TOTAL).getIntValue()).isEqualTo(8); + assertThat(attribute(span, SemanticConventions.LLM_PROVIDER).getStringValue()).isEqualTo("cohere"); + assertThat(span.getEndTimeUnixNano()).isGreaterThanOrEqualTo(span.getStartTimeUnixNano()); + if (!hide) assertThat(attribute(span, SemanticConventions.OUTPUT_VALUE).getStringValue()).isEqualTo("private-output"); + } + Span error = spans.stream().filter(span -> span.getStatus().getCode() == Status.StatusCode.STATUS_CODE_ERROR) + .findFirst().orElseThrow(); + assertThat(error.getStatus().getCode()).isEqualTo(Status.StatusCode.STATUS_CODE_ERROR); + assertThat(error.getEventsCount()).isEqualTo(1); + if (hide) { + assertThat(error.getStatus().getMessage()).isEmpty(); + assertThat(error.getEvents(0).getAttributesList()).hasSize(1); + assertThat(error.getEvents(0).getAttributes(0).getKey()).isEqualTo("exception.type"); + } + } finally { + server.stop(0); + } + } + + private io.opentelemetry.proto.common.v1.AnyValue attribute(Span span, String key) { + return span.getAttributesList().stream().filter(attribute -> attribute.getKey().equals(key)) + .findFirst().orElseThrow().getValue(); + } +} diff --git a/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereRequestSafetyTest.java b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereRequestSafetyTest.java new file mode 100644 index 000000000..86f492616 --- /dev/null +++ b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereRequestSafetyTest.java @@ -0,0 +1,193 @@ +package ai.traceai.cohere; + +import ai.traceai.FITracer; +import ai.traceai.SemanticConventions; +import ai.traceai.TraceConfig; +import com.cohere.api.Cohere; +import com.cohere.api.core.ObjectMappers; +import com.cohere.api.requests.ChatRequest; +import com.cohere.api.requests.ChatStreamRequest; +import com.cohere.api.resources.v2.requests.V2EmbedRequest; +import com.sun.net.httpserver.HttpServer; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; +import io.opentelemetry.sdk.trace.data.SpanData; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.*; + +/** Real SDK requests over loopback, with synthetic credentials and no provider access. */ +class CohereRequestSafetyTest { + @RegisterExtension + static final OpenTelemetryExtension otel = OpenTelemetryExtension.create(); + + private static final String CONNECTOR_TOKEN = "synthetic-connector-bearer-token"; + private static final String SECOND_TOKEN = "synthetic-second-connector-token"; + private static final String CONNECTOR_REQUEST = """ + {"model":"test-model","message":"visible-input","connectors":[ + {"id":"internal-docs","user_access_token":"%s"}, + {"id":"other-docs","user_access_token":"%s"}, + {"id":"public-docs"}]} + """.formatted(CONNECTOR_TOKEN, SECOND_TOKEN); + private static final String V1_RESPONSE = "{\"text\":\"visible-output\",\"generation_id\":\"response-id\",\"finish_reason\":\"COMPLETE\"}"; + + @ParameterizedTest + @CsvSource({ + "true,200,0", "true,401,0", "false,200,0", "false,401,0", + "true,200,1", "true,401,1", "false,200,1", "false,401,1", + "true,200,2", "true,401,2", "false,200,2", "false,401,2" + }) + void connectorCredentialsAreTransportOnly(boolean streaming, int status, int privacyFlag) throws Exception { + String body = status == 401 ? "{\"message\":\"synthetic-provider-failure\"}" + : streaming ? streamResponse(V1_RESPONSE) : V1_RESPONSE; + try (var server = new ProviderServer(status, body)) { + var config = TraceConfig.builder().hideInputs(privacyFlag == 1) + .hideInputMessages(privacyFlag == 2).build(); + var traced = server.traced(config); + Class> requestType = streaming ? ChatStreamRequest.class : ChatRequest.class; + Object request = ObjectMappers.JSON_MAPPER.readValue(CONNECTOR_REQUEST, requestType); + org.assertj.core.api.ThrowableAssert.ThrowingCallable call = () -> { + if (streaming) { + try (var stream = traced.chatStream((ChatStreamRequest) request)) { + stream.forEach(event -> {}); + } + } else { + assertThat(traced.chat((ChatRequest) request).getText()).isEqualTo("visible-output"); + } + }; + if (status == 401) { + assertThatThrownBy(call).isInstanceOf(RuntimeException.class); + } else { + assertThatCode(call).doesNotThrowAnyException(); + } + + // Sanitization must not mutate either the SDK request or the bytes on the wire. + assertCredentials(ObjectMappers.JSON_MAPPER.valueToTree(request).toString()); + assertCredentials(server.request.get()); + SpanData span = onlySpan(); + assertThat(span.getStatus().getStatusCode()).isEqualTo(status == 401 ? StatusCode.ERROR : StatusCode.OK); + assertThat(span.getAttributes().toString()).doesNotContain(CONNECTOR_TOKEN, SECOND_TOKEN); + assertThat(span.getEvents().toString()).doesNotContain(CONNECTOR_TOKEN, SECOND_TOKEN); + assertThat(span.getStatus().getDescription()).doesNotContain(CONNECTOR_TOKEN, SECOND_TOKEN); + if (privacyFlag == 0) { + var raw = ObjectMappers.JSON_MAPPER.readTree(span.getAttributes().get(AttributeKey.stringKey(SemanticConventions.RAW_INPUT))); + assertThat(raw.path("message").asText()).isEqualTo("visible-input"); + assertThat(raw.path("connectors")).hasSize(3); + assertThat(raw.path("connectors").get(0).path("id").asText()).isEqualTo("internal-docs"); + for (var connector : raw.path("connectors")) { + assertThat(connector.has("user_access_token")).isFalse(); + } + } + } + } + + @ParameterizedTest + @MethodSource("embeddingUsage") + void embeddingUsageUsesBilledInputOnlyAsFallback(String meta, Long input, Long billed, Long output, Long total) throws Exception { + String response = "{\"id\":\"embed-id\",\"embeddings\":{\"float\":[[0.1,0.2]]},\"meta\":" + meta + "}"; + try (var server = new ProviderServer(200, response)) { + var request = ObjectMappers.JSON_MAPPER.readValue(""" + {"model":"embed-model","input_type":"search_document","texts":["visible-input"]} + """, V2EmbedRequest.class); + var result = server.traced(TraceConfig.builder().build()).v2().embed(request); + assertThat(result.getEmbeddings().getFloat().orElseThrow()).hasSize(1); + SpanData span = onlySpan(); + assertThat(span.getAttributes().get(AttributeKey.longKey("cohere.billed_units.input_tokens"))).isEqualTo(billed); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_PROMPT))).isEqualTo(input); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_COMPLETION))).isEqualTo(output); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_TOTAL))).isEqualTo(total); + } + } + + static Stream embeddingUsage() { + return Stream.of( + Arguments.of("{\"billed_units\":{\"input_tokens\":4}}", 4L, 4L, null, null), + Arguments.of("{\"tokens\":{},\"billed_units\":{\"input_tokens\":4}}", 4L, 4L, null, null), + Arguments.of("{\"tokens\":{\"input_tokens\":9},\"billed_units\":{\"input_tokens\":4}}", 9L, 4L, null, null), + Arguments.of("{\"tokens\":{\"input_tokens\":0},\"billed_units\":{\"input_tokens\":4}}", 0L, 4L, null, null), + Arguments.of("{\"tokens\":{\"input_tokens\":9,\"output_tokens\":2},\"billed_units\":{\"input_tokens\":4}}", 9L, 4L, 2L, 11L), + Arguments.of("{\"tokens\":{\"output_tokens\":2},\"billed_units\":{\"input_tokens\":4}}", 4L, 4L, 2L, null), + Arguments.of("{\"tokens\":{\"input_tokens\":9}}", 9L, null, null, null), + Arguments.of("{\"billed_units\":{\"input_tokens\":0}}", 0L, 0L, null, null), + Arguments.of("{}", null, null, null, null) + ); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void chatNeverSubstitutesBilledUnitsForActualTokenUsage(boolean hasTokens) throws Exception { + String meta = "{\"billed_units\":{\"input_tokens\":4}" + + (hasTokens ? ",\"tokens\":{\"input_tokens\":9,\"output_tokens\":2}" : "") + "}"; + String response = "{\"text\":\"visible-output\",\"meta\":" + meta + "}"; + try (var server = new ProviderServer(200, streamResponse(response))) { + try (var stream = server.traced(TraceConfig.builder().build()).chatStream( + ChatStreamRequest.builder().message("visible-input").build())) { + stream.forEach(event -> {}); + } + SpanData span = onlySpan(); + assertThat(span.getAttributes().get(AttributeKey.longKey("cohere.billed_units.input_tokens"))).isEqualTo(4L); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_PROMPT))).isEqualTo(hasTokens ? 9L : null); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_COMPLETION))).isEqualTo(hasTokens ? 2L : null); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_TOTAL))).isEqualTo(hasTokens ? 11L : null); + } + } + + private static String streamResponse(String response) { + return "{\"event_type\":\"stream-end\",\"finish_reason\":\"COMPLETE\",\"response\":" + response + "}\n"; + } + + private static void assertCredentials(String json) throws Exception { + var connectors = ObjectMappers.JSON_MAPPER.readTree(json).path("connectors"); + assertThat(connectors.get(0).path("user_access_token").asText()).isEqualTo(CONNECTOR_TOKEN); + assertThat(connectors.get(1).path("user_access_token").asText()).isEqualTo(SECOND_TOKEN); + assertThat(connectors.get(2).has("user_access_token")).isFalse(); + } + + private static SpanData onlySpan() { + assertThat(otel.getSpans()).hasSize(1); + return otel.getSpans().get(0); + } + + private static final class ProviderServer implements AutoCloseable { + private final HttpServer server; + private final AtomicReference request = new AtomicReference<>(); + + ProviderServer(int status, String body) throws Exception { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + try { + request.set(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8)); + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, bytes.length); + exchange.getResponseBody().write(bytes); + } finally { + exchange.close(); + } + }); + server.start(); + } + + TracedCohereClient traced(TraceConfig config) { + var client = Cohere.builder().token("synthetic-provider-key").maxRetries(0) + .url("http://127.0.0.1:" + server.getAddress().getPort()).build(); + return new TracedCohereClient(client, new FITracer(otel.getOpenTelemetry().getTracer("cohere-safety-test"), config)); + } + + @Override + public void close() { + server.stop(0); + } + } +} diff --git a/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereSurfaceTest.java b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereSurfaceTest.java new file mode 100644 index 000000000..8eff84caa --- /dev/null +++ b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/CohereSurfaceTest.java @@ -0,0 +1,20 @@ +package ai.traceai.cohere; + +import com.cohere.api.requests.ChatStreamRequest; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class CohereSurfaceTest { + @Test + void exposesTracedV2Facade() throws Exception { + assertThat(TracedCohereClient.class.getMethod("v2").getReturnType().getSimpleName()) + .isEqualTo("TracedCohereV2Client"); + } + + @Test + void exposesCloseableV1Stream() throws Exception { + assertThat(AutoCloseable.class.isAssignableFrom(TracedCohereClient.class + .getMethod("chatStream", ChatStreamRequest.class).getReturnType())).isTrue(); + } +} diff --git a/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/TracedCohereStreamTest.java b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/TracedCohereStreamTest.java new file mode 100644 index 000000000..e1a61c5b6 --- /dev/null +++ b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/TracedCohereStreamTest.java @@ -0,0 +1,138 @@ +package ai.traceai.cohere; + +import ai.traceai.FISpanKind; +import ai.traceai.FITracer; +import ai.traceai.SemanticConventions; +import ai.traceai.TraceConfig; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +import static ai.traceai.cohere.TracedCohereTransportTest.*; +import static org.assertj.core.api.Assertions.*; + +class TracedCohereStreamTest { + @RegisterExtension + static final OpenTelemetryExtension otel = OpenTelemetryExtension.create(); + + @ParameterizedTest + @ValueSource(strings = {"iterator", "hasNext", "next"}) + void iteratorFailuresKeepOriginalExceptionAndCloseExactlyOnce(String stage) throws Exception { + var source = new Source(); + source.failureStage = stage; + var stream = stream(source); + assertThatThrownBy(() -> stream.iterator().next()).isSameAs(source.failure); + stream.close(); + assertThat(source.closes).isEqualTo(1); + assertThat(otel.getSpans()).hasSize(1); + assertThat(otel.getSpans().get(0).getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + } + + @Test + void closeFailureDoesNotMaskIterationFailure() throws Exception { + var source = new Source(); + source.failureStage = "next"; + source.closeFailure = new IOException("private-close-error"); + var stream = stream(source); + assertThatThrownBy(() -> stream.iterator().next()).isSameAs(source.failure); + assertThat(source.failure.getSuppressed()).containsExactly(source.closeFailure); + stream.close(); + assertThat(source.closes).isEqualTo(1); + assertThat(otel.getSpans()).hasSize(1); + } + + @Test + void explicitCloseFailureEndsSpanAndPropagates() { + var source = new Source(); + source.closeFailure = new IOException("private-close-error"); + var stream = stream(source); + assertThatThrownBy(stream::close).isSameAs(source.closeFailure); + assertThat(otel.getSpans()).hasSize(1); + assertThat(otel.getSpans().get(0).getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); + } + + @Test + void exhaustedIncompleteStreamClosesAndReturnsSameEvent() throws Exception { + var source = new Source(); + var stream = stream(source); + var iterator = stream.iterator(); + assertThat(iterator.next()).isSameAs(source.event); + assertThat(iterator.hasNext()).isFalse(); + assertThat(iterator.hasNext()).isFalse(); + stream.close(); + assertThat(source.closes).isEqualTo(1); + assertThat(otel.getSpans()).hasSize(1); + assertThat(value(otel.getSpans().get(0), "cohere.stream.outcome")).isEqualTo("incomplete"); + assertThatThrownBy(stream::iterator).isInstanceOf(IllegalStateException.class); + } + + @Test + void unopenedStreamCanBeClosedWithoutCreatingAnIterator() throws Exception { + var source = new Source(); + source.failureStage = "iterator"; + var stream = stream(source); + stream.close(); + assertThat(stream.iterator().hasNext()).isFalse(); + assertThat(source.closes).isEqualTo(1); + assertThat(otel.getSpans()).hasSize(1); + } + + @Test + void streamedTextIsBounded() throws Exception { + var transport = new FakeTransport(V2_STREAM.replace("private-output", "x".repeat(100000))); + var traced = new TracedCohereClient(transport.client(), tracer()); + try (var stream = TracedCohereTransportTest.stream(traced, true)) { stream.forEach(event -> {}); } + assertThat(value(otel.getSpans().get(0), SemanticConventions.OUTPUT_VALUE)).hasSize(32000); + assertThat(otel.getSpans().get(0).getAttributes().get( + io.opentelemetry.api.common.AttributeKey.booleanKey("cohere.stream.output_truncated"))).isTrue(); + } + + private FITracer tracer() { + return new FITracer(otel.getOpenTelemetry().getTracer("stream-test"), TraceConfig.builder().hideErrorDetails(true).build()); + } + + private TracedCohereStream stream(Source source) { + FITracer tracer = tracer(); + return new TracedCohereStream<>(source, tracer, tracer.startSpan("test stream", FISpanKind.LLM), + event -> TracedCohereStream.EventState.CONTINUE, () -> {}); + } + + static final class Source implements Iterable, Closeable { + final Object event = new Object(); + final RuntimeException failure = new IllegalStateException("private-error"); + String failureStage; + IOException closeFailure; + int closes; + + @Override + public Iterator iterator() { + if ("iterator".equals(failureStage)) throw failure; + Iterator delegate = List.of(event).iterator(); + return new Iterator<>() { + @Override public boolean hasNext() { + if ("hasNext".equals(failureStage)) throw failure; + return delegate.hasNext(); + } + @Override public Object next() { + if ("next".equals(failureStage)) throw failure; + return delegate.next(); + } + }; + } + + @Override public void close() throws IOException { + closes++; + if (closeFailure != null) throw closeFailure; + } + } +} diff --git a/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/TracedCohereTransportTest.java b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/TracedCohereTransportTest.java new file mode 100644 index 000000000..4d9271e97 --- /dev/null +++ b/java/traceai-java-cohere/src/test/java/ai/traceai/cohere/TracedCohereTransportTest.java @@ -0,0 +1,320 @@ +package ai.traceai.cohere; + +import ai.traceai.FITracer; +import ai.traceai.SemanticConventions; +import ai.traceai.TraceConfig; +import com.cohere.api.Cohere; +import com.cohere.api.core.ObjectMappers; +import com.cohere.api.core.RequestOptions; +import com.cohere.api.requests.ChatStreamRequest; +import com.cohere.api.resources.v2.requests.V2ChatRequest; +import com.cohere.api.resources.v2.requests.V2ChatStreamRequest; +import com.cohere.api.resources.v2.requests.V2EmbedRequest; +import com.cohere.api.resources.v2.requests.V2RerankRequest; +import com.cohere.api.resources.v2.types.V2ChatStreamResponse; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; +import io.opentelemetry.sdk.trace.data.SpanData; +import okhttp3.Interceptor; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okio.Buffer; +import okio.BufferedSource; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.*; + +class TracedCohereTransportTest { + @RegisterExtension + static final OpenTelemetryExtension otel = OpenTelemetryExtension.create(); + + static final String CHAT_INPUT = "{\"model\":\"test-model\",\"messages\":[{\"role\":\"user\",\"content\":\"private-input\"}]}"; + static final String CHAT_RESPONSE = """ + {"id":"response-id","finish_reason":"COMPLETE","message":{"role":"assistant", + "content":[{"type":"text","text":"private-output"}], + "tool_calls":[{"id":"call-id","type":"function","function":{"name":"private-tool","arguments":"{}"}}]}, + "usage":{"tokens":{"input_tokens":3,"output_tokens":5}}} + """; + static final String V2_STREAM = sse("message-start", "{\"id\":\"response-id\",\"delta\":{\"message\":{\"role\":\"assistant\"}}}") + + sse("content-delta", "{\"index\":0,\"delta\":{\"message\":{\"content\":{\"text\":\"private-output\"}}}}") + + sse("message-end", "{\"delta\":{\"finish_reason\":\"COMPLETE\",\"usage\":{\"tokens\":{\"input_tokens\":3,\"output_tokens\":5}}}}"); + static final String V1_STREAM = """ + {"event_type":"stream-start","generation_id":"response-id"} + {"event_type":"text-generation","text":"private-output"} + {"event_type":"stream-end","finish_reason":"COMPLETE","response":{"text":"private-output","generation_id":"response-id","meta":{"tokens":{"input_tokens":3,"output_tokens":5}}}} + """; + + @Test + void tracesV2ChatThroughRealSdkAndPreservesOptions() throws Exception { + FakeTransport transport = new FakeTransport(CHAT_RESPONSE); + TracedCohereClient traced = traced(transport, TraceConfig.builder().build()); + var request = json(CHAT_INPUT, V2ChatRequest.class); + var response = traced.v2().chat(request, RequestOptions.builder().addHeader("X-Test-Option", "kept").build()); + assertThat(response.getId()).isEqualTo("response-id"); + assertThat(response.getMessage().getToolCalls()).isPresent(); + assertThat(transport.request.url().encodedPath()).isEqualTo("/v2/chat"); + assertThat(transport.request.header("X-Test-Option")).isEqualTo("kept"); + assertThat(transport.sentBody).contains("private-input", "test-model"); + SpanData span = onlySpan(); + assertThat(transport.spanId).isEqualTo(span.getSpanId()); + assertThat(value(span, SemanticConventions.RAW_INPUT)).contains("\"role\":\"user\""); + assertThat(value(span, SemanticConventions.RAW_OUTPUT)).contains("private-tool"); + assertThat(value(span, SemanticConventions.OUTPUT_VALUE)).isEqualTo("private-output"); + assertUsage(span); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + } + + @Test + void tracesV2EmbedAndRerankThroughRealSdk() throws Exception { + FakeTransport transport = new FakeTransport(""" + {"id":"embedding-id","embeddings":{"float":[[0.1,0.2],[0.3,0.4]]}, + "meta":{"billed_units":{"input_tokens":4}}} + """); + var v2 = traced(transport, TraceConfig.builder().build()).v2(); + var embedded = v2.embed(json(""" + {"model":"embed-model","input_type":"search_document","texts":["private-a","private-b"]} + """, V2EmbedRequest.class)); + assertThat(embedded.getEmbeddings().getFloat().orElseThrow()).hasSize(2); + SpanData span = onlySpan(); + assertThat(value(span, SemanticConventions.FI_SPAN_KIND)).isEqualTo("EMBEDDING"); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.EMBEDDING_VECTOR_COUNT))).isEqualTo(2L); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.EMBEDDING_DIMENSIONS))).isEqualTo(2L); + assertThat(transport.request.url().encodedPath()).isEqualTo("/v2/embed"); + transport.body = "{\"results\":[{\"index\":1,\"relevance_score\":0.9}]}"; + var reranked = v2.rerank(json(""" + {"model":"rerank-model","query":"private-query","documents":["private-a","private-b"],"top_n":1} + """, V2RerankRequest.class), RequestOptions.builder().addHeader("X-Test-Option", "rerank").build()); + assertThat(reranked.getResults().get(0).getIndex()).isEqualTo(1); + assertThat(transport.request.url().encodedPath()).isEqualTo("/v2/rerank"); + assertThat(transport.request.header("X-Test-Option")).isEqualTo("rerank"); + assertThat(otel.getSpans()).hasSize(2); + span = otel.getSpans().get(1); + assertThat(value(span, SemanticConventions.FI_SPAN_KIND)).isEqualTo("RERANKER"); + assertThat(value(span, SemanticConventions.GEN_AI_RERANKER_QUERY)).isEqualTo("private-query"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void streamsUntilTerminalEventAndRestoresParent(boolean v2) throws Exception { + FakeTransport transport = new FakeTransport(v2 ? V2_STREAM : V1_STREAM); + var traced = traced(transport, TraceConfig.builder().build()); + Span parent = otel.getOpenTelemetry().getTracer("parent").spanBuilder("parent").startSpan(); + try (Scope ignored = parent.makeCurrent(); var stream = stream(traced, v2)) { + assertThat(otel.getSpans()).isEmpty(); + assertThat(Span.current().getSpanContext()).isEqualTo(parent.getSpanContext()); + var iterator = stream.iterator(); + iterator.next(); + assertThat(otel.getSpans()).isEmpty(); + iterator.next(); + assertThat(otel.getSpans()).isEmpty(); + iterator.next(); // terminal event completes even without another hasNext() + assertThat(transport.closed).isTrue(); + assertThat(iterator.hasNext()).isFalse(); + assertThatThrownBy(iterator::next).isInstanceOf(java.util.NoSuchElementException.class); + SpanData span = onlySpan(); + assertThat(span.getParentSpanId()).isEqualTo(parent.getSpanContext().getSpanId()); + assertThat(value(span, "cohere.stream.outcome")).isEqualTo("completed"); + assertThat(span.getStatus().getStatusCode()).isEqualTo(StatusCode.OK); + assertThat(value(span, SemanticConventions.OUTPUT_VALUE)).isEqualTo("private-output"); + assertUsage(span); + assertThat(Span.current().getSpanContext()).isEqualTo(parent.getSpanContext()); + } finally { + parent.end(); + } + assertThat(otel.getSpans()).hasSize(2); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void earlyCloseEndsIncompleteSpanAndClosesTransportOnce(boolean v2) throws Exception { + FakeTransport transport = new FakeTransport(v2 ? V2_STREAM : V1_STREAM); + var stream = stream(traced(transport, TraceConfig.builder().build()), v2); + stream.iterator().next(); + stream.close(); + stream.close(); + assertThat(transport.closed).isTrue(); + assertThat(value(onlySpan(), "cohere.stream.outcome")).isEqualTo("incomplete"); + assertThat(onlySpan().getStatus().getStatusCode()).isEqualTo(StatusCode.UNSET); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void malformedStreamsDoNotProduceSuccessfulSpans(boolean v2) throws Exception { + FakeTransport transport = new FakeTransport(v2 ? "event: content-delta\ndata: {not-json}\n\n" : "{not-json}\n"); + var stream = stream(traced(transport, TraceConfig.builder().hideErrorDetails(true).build()), v2); + assertThatThrownBy(() -> stream.iterator().next()).isInstanceOf(RuntimeException.class); + assertThat(transport.closed).isTrue(); + assertThat(onlySpan().getStatus().getDescription()).isEmpty(); + if (v2) { + // Both supported SDKs swallow malformed SSE events and expose EOF. + assertThat(value(onlySpan(), "cohere.stream.outcome")).isEqualTo("incomplete"); + assertThat(onlySpan().getStatus().getStatusCode()).isEqualTo(StatusCode.UNSET); + } else { + assertThat(onlySpan().getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); + assertThat(onlySpan().getEvents()).hasSize(1); + assertThat(onlySpan().getEvents().get(0).getAttributes().asMap()) + .containsOnlyKeys(AttributeKey.stringKey("exception.type")); + } + stream.close(); + assertThat(otel.getSpans()).hasSize(1); + } + + @ParameterizedTest + @ValueSource(strings = {"chat", "embed", "rerank", "v1stream", "v2stream"}) + void transportErrorsPreserveSdkException(String operation) throws Exception { + FakeTransport transport = new FakeTransport("{\"message\":\"private-error\"}"); + transport.code = 401; + var traced = traced(transport, TraceConfig.builder().hideInputs(true).hideOutputs(true).hideErrorDetails(true).build()); + assertThatThrownBy(() -> { + switch (operation) { + case "chat": traced.v2().chat(json(CHAT_INPUT, V2ChatRequest.class)); break; + case "embed": traced.v2().embed(json("{\"model\":\"m\",\"input_type\":\"search_query\",\"texts\":[\"private-input\"]}", V2EmbedRequest.class)); break; + case "rerank": traced.v2().rerank(json("{\"model\":\"m\",\"query\":\"private-input\",\"documents\":[\"d\"]}", V2RerankRequest.class)); break; + default: stream(traced, operation.equals("v2stream")); + } + }).isInstanceOf(com.cohere.api.errors.UnauthorizedError.class); + assertThat(onlySpan().getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); + assertThat(onlySpan().getAttributes().toString()).doesNotContain("private-"); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + } + + @ParameterizedTest + @ValueSource(ints = {1, 2, 4, 8, 15}) + void newChatSurfacesRespectContentAndMessagePrivacy(int flags) throws Exception { + TraceConfig config = TraceConfig.builder().hideInputs((flags & 1) != 0).hideOutputs((flags & 2) != 0) + .hideInputMessages((flags & 4) != 0).hideOutputMessages((flags & 8) != 0).build(); + FakeTransport transport = new FakeTransport(CHAT_RESPONSE); + var traced = traced(transport, config); + traced.v2().chat(json(CHAT_INPUT, V2ChatRequest.class)); + transport.body = V2_STREAM; + try (var stream = stream(traced, true)) { stream.forEach(event -> {}); } + transport.body = V1_STREAM; + try (var stream = stream(traced, false)) { stream.forEach(event -> {}); } + assertThat(otel.getSpans()).hasSize(3); + for (SpanData span : otel.getSpans()) { + if ((flags & 5) != 0) assertThat(span.getAttributes().toString()).doesNotContain("private-input"); + if ((flags & 10) != 0) assertThat(span.getAttributes().toString()).doesNotContain("private-output", "private-tool"); + assertUsage(span); + } + } + + @Test + void terminalProviderErrorIsNotReportedAsSuccess() throws Exception { + FakeTransport transport = new FakeTransport(sse("message-end", + "{\"delta\":{\"finish_reason\":\"ERROR\",\"error\":\"private-error\"}}")); + try (var stream = stream(traced(transport, TraceConfig.builder().hideErrorDetails(true).build()), true)) { + stream.forEach(event -> {}); + } + assertThat(onlySpan().getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); + assertThat(value(onlySpan(), "cohere.stream.outcome")).isEqualTo("error"); + assertThat(onlySpan().getAttributes().toString()).doesNotContain("private-error"); + } + + @Test + void synchronousProviderErrorIsNotReportedAsSuccess() throws Exception { + FakeTransport transport = new FakeTransport(CHAT_RESPONSE.replace("COMPLETE", "ERROR")); + var traced = traced(transport, TraceConfig.builder().hideErrorDetails(true).build()); + var response = traced.v2().chat(json(CHAT_INPUT, V2ChatRequest.class)); + assertThat(response.getFinishReason().toString()).isEqualTo("ERROR"); + assertThat(onlySpan().getStatus().getStatusCode()).isEqualTo(StatusCode.ERROR); + } + + @Test + void absentUsageDoesNotInventZeroCounts() throws Exception { + FakeTransport transport = new FakeTransport("{\"id\":\"r\",\"finish_reason\":\"COMPLETE\",\"message\":{\"role\":\"assistant\"}}"); + traced(transport, TraceConfig.builder().build()).v2().chat(json(CHAT_INPUT, V2ChatRequest.class)); + assertThat(onlySpan().getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_TOTAL))).isNull(); + } + + @Test + void unwrapRemainsOriginalUntracedEscapeHatch() throws Exception { + FakeTransport transport = new FakeTransport(CHAT_RESPONSE); + var traced = traced(transport, TraceConfig.builder().build()); + traced.unwrap().v2().chat(json(CHAT_INPUT, V2ChatRequest.class)); + assertThat(otel.getSpans()).isEmpty(); + } + + static TracedCohereStream> stream(TracedCohereClient traced, boolean v2) throws IOException { + if (v2) return traced.v2().chatStream(json(CHAT_INPUT, V2ChatStreamRequest.class)); + return traced.chatStream(ChatStreamRequest.builder().message("private-input").model("test-model").build()); + } + + private TracedCohereClient traced(FakeTransport transport, TraceConfig config) { + return new TracedCohereClient(transport.client(), new FITracer(otel.getOpenTelemetry().getTracer("cohere-test"), config)); + } + + static T json(String value, Class type) throws IOException { + return ObjectMappers.JSON_MAPPER.readValue(value, type); + } + + static String sse(String type, String data) { + return "event: " + type + "\ndata: {\"type\":\"" + type + "\"," + data.substring(1) + "\n\n"; + } + + private SpanData onlySpan() { + assertThat(otel.getSpans()).hasSize(1); + return otel.getSpans().get(0); + } + + static String value(SpanData span, String key) { + return span.getAttributes().get(AttributeKey.stringKey(key)); + } + + private void assertUsage(SpanData span) { + assertThat(value(span, SemanticConventions.LLM_PROVIDER)).isEqualTo("cohere"); + assertThat(value(span, SemanticConventions.LLM_REQUEST_MODEL)).isEqualTo("test-model"); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_PROMPT))).isEqualTo(3L); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_COMPLETION))).isEqualTo(5L); + assertThat(span.getAttributes().get(AttributeKey.longKey(SemanticConventions.LLM_TOKEN_COUNT_TOTAL))).isEqualTo(8L); + } + + /** All provider traffic ends here: no DNS, sockets, credentials or vendor API calls. */ + static final class FakeTransport implements Interceptor { + String body; + int code = 200; + Request request; + String sentBody; + String spanId; + boolean closed; + + FakeTransport(String body) { this.body = body; } + + Cohere client() { + return Cohere.builder().token("synthetic-not-a-credential") + .httpClient(new OkHttpClient.Builder().addInterceptor(this).build()).build(); + } + + @Override + public Response intercept(Chain chain) throws IOException { + request = chain.request(); + Buffer sent = new Buffer(); + request.body().writeTo(sent); + sentBody = sent.readUtf8(); + spanId = Span.current().getSpanContext().getSpanId(); + Buffer buffer = new Buffer().writeUtf8(body); + ResponseBody responseBody = new ResponseBody() { + @Override public MediaType contentType() { return MediaType.parse("application/json"); } + @Override public long contentLength() { return -1; } + @Override public BufferedSource source() { return buffer; } + @Override public void close() { closed = true; super.close(); } + }; + return new Response.Builder().request(request).protocol(Protocol.HTTP_1_1) + .code(code).message("synthetic").body(responseBody).build(); + } + } +}