diff --git a/.github/workflows/elasticsearch-java-docs.yml b/.github/workflows/elasticsearch-java-docs.yml new file mode 100644 index 000000000..cd315a524 --- /dev/null +++ b/.github/workflows/elasticsearch-java-docs.yml @@ -0,0 +1,71 @@ +name: Elasticsearch Java documentation + +on: + pull_request: + branches: [main] + paths: + - src/pages/docs/integrations/traceai/java/vector-databases.mdx + - scripts/java-elasticsearch/** + - scripts/crewai-ingestion-check/** + - .github/workflows/elasticsearch-java-docs.yml + +permissions: + contents: read + +jobs: + native-tracing: + name: Native tracing (Java ${{ matrix.java }}, Elasticsearch ${{ matrix.elasticsearch }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + java: ['17', '21'] + elasticsearch: ['8.12.0', '9.5.5'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: ${{ matrix.java }} + - uses: actions/setup-node@v4 + with: + node-version: '20' + - name: Check out the production ingestion contract + uses: actions/checkout@v4 + with: + repository: future-agi/future-agi + ref: 215b19543a47de8c7ae7e8dc452dc267440e5c8d + path: .test-deps/future-agi + sparse-checkout: fi-collector + persist-credentials: false + - uses: actions/setup-go@v5 + with: + go-version-file: .test-deps/future-agi/fi-collector/go.mod + cache-dependency-path: .test-deps/future-agi/fi-collector/go.sum + - name: Resolve production converter dependencies + working-directory: .test-deps/future-agi/fi-collector + run: go mod download + - name: Extract executable snippets and check the page contract + run: node scripts/java-elasticsearch/extract.mjs + - name: Resolve released dependencies before denying outbound networking + run: | + # sudo changes Java's user.home even when HOME is preserved. + export MAVEN_REPO="$RUNNER_TEMP/elasticsearch-m2" + echo "MAVEN_REPO=$MAVEN_REPO" >> "$GITHUB_ENV" + mvn -B -Dmaven.repo.local="$MAVEN_REPO" -f scripts/java-elasticsearch/pom.xml -Delasticsearch.version=${{ matrix.elasticsearch }} dependency:go-offline + mvn -B -Dmaven.repo.local="$MAVEN_REPO" -f scripts/java-elasticsearch/target/native-consumer/pom.xml -Delasticsearch.version=${{ matrix.elasticsearch }} dependency:go-offline + - name: Package the exact native example and test with loopback networking only + env: + ES_VERSION: ${{ matrix.elasticsearch }} + run: | + export INGESTION_CHECK="$RUNNER_TEMP/elasticsearch-ingestion-check" + sudo --preserve-env=JAVA_HOME,ES_VERSION,INGESTION_CHECK,MAVEN_REPO env PATH="$PATH" HOME="$HOME" \ + unshare --net sh -ec ' + ip link set lo up + ROOT="$PWD" + (cd .test-deps/future-agi/fi-collector && GOPROXY=off go build -o "$INGESTION_CHECK" "$ROOT/scripts/crewai-ingestion-check/main.go") + mvn -o -B -Dmaven.repo.local="$MAVEN_REPO" -f scripts/java-elasticsearch/target/native-consumer/pom.xml -Delasticsearch.version="$ES_VERSION" -DskipTests package dependency:build-classpath -Dmdep.outputFile=target/classpath.txt + mvn -o -B -Dmaven.repo.local="$MAVEN_REPO" -f scripts/java-elasticsearch/pom.xml -Delasticsearch.version="$ES_VERSION" test + node scripts/java-elasticsearch/check-ingestion.mjs "$INGESTION_CHECK" "$ES_VERSION" + ' diff --git a/scripts/java-elasticsearch/.gitignore b/scripts/java-elasticsearch/.gitignore new file mode 100644 index 000000000..b83d22266 --- /dev/null +++ b/scripts/java-elasticsearch/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/scripts/java-elasticsearch/check-ingestion.mjs b/scripts/java-elasticsearch/check-ingestion.mjs new file mode 100644 index 000000000..16704932c --- /dev/null +++ b/scripts/java-elasticsearch/check-ingestion.mjs @@ -0,0 +1,51 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +// Use the production converter, not a duplicate of its semantic mapping. +// Authentication/project resolution are tested separately by the collector; +// this contract verifies that the exact guide's wire spans survive conversion. +const [binary, version] = process.argv.slice(2); +assert.ok(binary && version, 'Usage: node check-ingestion.mjs CONVERTER ELASTICSEARCH_VERSION'); +const examples = ['NativeElasticsearchExample', 'AsyncElasticsearchExample', 'KnnElasticsearchExample']; +if (version === '8.12.0') examples.push('OptionalElasticsearchExample'); +for (const example of examples) { + const filename = fileURLToPath(new URL(`./target/wire/${version}/${example}.pb`, import.meta.url)); + const output = execFileSync(binary, [filename], { encoding: 'utf8', timeout: 30_000 }); + const rows = output.trim().split('\n').map(line => JSON.parse(line)); + const optional = example.startsWith('Optional'); + assert.equal(rows.length, optional ? 2 : 1, example); + const database = rows.find(row => row.name === 'search'); + assert.ok(database, example); + assert.equal(database.observation_type, 'unknown', 'native database spans are not semantic retrievers'); + assert.equal(database.attrs_string['db.system.name'] ?? database.attrs_string['db.system'], 'elasticsearch'); + assert.equal(database.attrs_string['db.operation.name'] ?? database.attrs_string['db.operation'], 'search'); + assert.equal(database.attrs_string['db.elasticsearch.path_parts.index'], 'documents'); + for (const row of rows) { + assert.equal(row.service_name, 'elasticsearch-search'); + assert.equal(row.resource_attrs.project_name, 'elasticsearch-native-test'); + assert.equal(row.resource_attrs.project_type, 'observe'); + assert.match(row.id, /^[0-9a-f]{16}$/); + assert.match(row.trace_id, /^[0-9a-f-]{36}$/); + assert.ok(row.latency_ms >= 0); + assert.equal(row.total_tokens, 0); + assert.equal(row.cost, 0); + assert.equal(row.input, ''); + assert.equal(row.output, ''); + } + if (optional) { + const parent = rows.find(row => row.observation_type === 'retriever'); + assert.ok(parent); + assert.equal(database.parent_span_id, parent.id); + assert.equal(database.trace_id, parent.trace_id); + assert.equal(parent.attrs_number['retriever.top_k'], 5); + assert.equal(parent.attrs_number['db.vector.results.count'], 1); + assert.equal(parent.parent_span_id, ''); + } else { + assert.equal(database.parent_span_id, ''); + } + for (const canary of ['synthetic-fi-key', 'synthetic-fi-secret', 'synthetic-es-key', 'synthetic-response-content', 'observability']) { + assert.ok(!output.includes(canary), `Unexpected payload content: ${canary}`); + } + console.log(`PASS ${version} ${example}: ${rows.length} production span rows`); +} diff --git a/scripts/java-elasticsearch/extract.mjs b/scripts/java-elasticsearch/extract.mjs new file mode 100644 index 000000000..4a320bcf1 --- /dev/null +++ b/scripts/java-elasticsearch/extract.mjs @@ -0,0 +1,78 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + +const page = readFileSync(new URL('../../src/pages/docs/integrations/traceai/java/vector-databases.mdx', import.meta.url), 'utf8'); +const section = page.split('## Elasticsearch\n')[1].split('## Common span attributes')[0]; +const native = section.split('### Optional retrieval metadata')[0]; +assert.match(native, /new OpenTelemetryForElasticsearch\(sdk, false\)/); +assert.match(native, /"project_name"/); +assert.match(native, /"project_type", "observe"/); +assert.match(native, /"X-Api-Key"/); +assert.match(native, /"X-Secret-Key"/); +assert.match(native, /\/tracer\/v1\/traces/); +assert.doesNotMatch(section, /main-SNAPSHOT|com\.github\.future-agi/); +assert.match(section, /com\.futureagi:traceai-java-elasticsearch:1\.0\.1/); +assert.match(section, /RETRIEVER.*parent.*native database child/); + +const output = new URL('./target/generated-sources/', import.meta.url); +mkdirSync(output, { recursive: true }); +function snippet(name) { + const code = section.split(`\x60\x60\x60java ${name}\n`)[1]?.split('\x60\x60\x60')[0]; + assert.ok(code, `Missing executable snippet: ${name}`); + return code; +} +const main = snippet('NativeElasticsearchExample.java'); +writeFileSync(new URL('NativeElasticsearchExample.java', output), main); +const imports = main.split('\n').filter(line => line.startsWith('import ')).join('\n'); +writeFileSync(new URL('OptionalRetrieval.java', output), `${imports} +public class OptionalRetrieval { + public static void run(ElasticsearchClient client, OpenTelemetrySdk sdk) throws Exception { +${snippet('OptionalRetrieval.java')} + } +} +`); +writeFileSync(new URL('AsyncSearch.java', output), `${imports} +public class AsyncSearch { + public static void run(RestClientTransport transport) throws Exception { +${snippet('AsyncSearch.java')} + } +} +`); +// Build the native example separately with only the dependencies from the page. +// This prevents the optional wrapper or test libraries from masking missing dependencies. +const consumer = new URL('./target/native-consumer/', import.meta.url); +mkdirSync(new URL('src/main/java/', consumer), { recursive: true }); +writeFileSync(new URL('src/main/java/NativeElasticsearchExample.java', consumer), main); +const searchBlock = /var client = new ElasticsearchClient\(transport\);[\s\S]*?System\.out\.println\("Matches returned: " \+ response\.hits\(\)\.hits\(\)\.size\(\)\);/; +assert.match(main, searchBlock); +const asyncMain = main.replace('class NativeElasticsearchExample', 'class AsyncElasticsearchExample') + .replace(searchBlock, snippet('AsyncSearch.java')); +writeFileSync(new URL('src/main/java/AsyncElasticsearchExample.java', consumer), asyncMain); +const knnMain = main.replace('class NativeElasticsearchExample', 'class KnnElasticsearchExample') + .replace(searchBlock, snippet('NativeKnnSearch.java')); +writeFileSync(new URL('src/main/java/KnnElasticsearchExample.java', consumer), knnMain); +const optionalMain = main.replace('class NativeElasticsearchExample', 'class OptionalElasticsearchExample') + .replace(searchBlock, 'var client = new ElasticsearchClient(transport);\n' + snippet('OptionalRetrieval.java')); +writeFileSync(new URL('OptionalElasticsearchExample.java', output), optionalMain); +const dependencies = native.split('\x60\x60\x60xml\n')[1].split('\x60\x60\x60')[0]; +writeFileSync(new URL('pom.xml', consumer), ` + + 4.0.0 + com.futureagi.docs + native-consumer + 1.0.0-SNAPSHOT + + 17 + UTF-8 + 8.12.0 + + ${dependencies.replaceAll('8.12.0', '${elasticsearch.version}')} + + org.apache.maven.pluginsmaven-compiler-plugin3.12.1 + org.apache.maven.pluginsmaven-dependency-plugin3.8.1 + org.apache.maven.pluginsmaven-jar-plugin3.3.0 + + +`); +assert.match(section, /not binary-compatible with the 9\.x KNN API/); +console.log('Elasticsearch page contract and snippet extraction passed.'); diff --git a/scripts/java-elasticsearch/pom.xml b/scripts/java-elasticsearch/pom.xml new file mode 100644 index 000000000..a90b7bd20 --- /dev/null +++ b/scripts/java-elasticsearch/pom.xml @@ -0,0 +1,94 @@ + + + 4.0.0 + com.futureagi.docs + elasticsearch-native-example + 1.0.0-SNAPSHOT + + 17 + UTF-8 + 8.12.0 + 1.35.0 + + + + co.elastic.clients + elasticsearch-java + ${elasticsearch.version} + + + org.elasticsearch.client + elasticsearch-rest-client + ${elasticsearch.version} + + + com.fasterxml.jackson.core + jackson-databind + 2.16.1 + + + io.opentelemetry + opentelemetry-sdk + ${opentelemetry.version} + + + io.opentelemetry + opentelemetry-exporter-otlp + ${opentelemetry.version} + + + com.futureagi + traceai-java-elasticsearch + 1.0.1 + + + io.opentelemetry + opentelemetry-sdk-testing + ${opentelemetry.version} + test + + + io.opentelemetry.proto + opentelemetry-proto + 1.0.0-alpha + test + + + org.junit.jupiter + junit-jupiter + 5.10.1 + test + + + + ${project.build.directory}/generated-sources + + + org.apache.maven.plugins + maven-compiler-plugin + 3.12.1 + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.3 + + + + org.apache.maven.surefire + surefire-junit-platform + 3.2.3 + + + + + -Djava.net.preferIPv4Stack=true + + ${elasticsearch.version} + + + + + + diff --git a/scripts/java-elasticsearch/src/test/java/NativeElasticsearchTest.java b/scripts/java-elasticsearch/src/test/java/NativeElasticsearchTest.java new file mode 100644 index 000000000..11c0fa068 --- /dev/null +++ b/scripts/java-elasticsearch/src/test/java/NativeElasticsearchTest.java @@ -0,0 +1,345 @@ +import ai.traceai.FITracer; +import ai.traceai.elasticsearch.TracedElasticsearchClient; +import co.elastic.clients.elasticsearch.ElasticsearchAsyncClient; +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.instrumentation.OpenTelemetryForElasticsearch; +import co.elastic.clients.transport.rest_client.RestClientTransport; +import com.sun.net.httpserver.HttpServer; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import org.apache.http.HttpHost; +import org.elasticsearch.client.RestClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.ByteArrayInputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.GZIPInputStream; + +import static org.junit.jupiter.api.Assertions.*; + +class NativeElasticsearchTest { + private static final String RESPONSE = """ + {"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0}, + "hits":{"total":{"value":1,"relation":"eq"},"max_score":0.9, + "hits":[{"_index":"documents","_id":"1","_score":0.9, + "_source":{"text":"synthetic-response-content"}}]}} + """; + private HttpServer server; + private String url; + private InMemorySpanExporter memory; + private OpenTelemetrySdk sdk; + private final AtomicInteger searches = new AtomicInteger(); + private final BlockingQueue exports = new LinkedBlockingQueue<>(); + private volatile String searchBody; + private volatile String authorization; + private volatile boolean error; + + record WireExport(String contentType, String encoding, String apiKey, String secretKey, + ExportTraceServiceRequest request) {} + + @BeforeEach + void start() throws Exception { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/documents/_search", exchange -> { + searches.incrementAndGet(); + authorization = exchange.getRequestHeaders().getFirst("Authorization"); + searchBody = new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8); + byte[] body = (error + ? "{\"error\":{\"type\":\"index_not_found_exception\",\"reason\":\"synthetic missing index\"},\"status\":404}" + : RESPONSE).getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.getResponseHeaders().add("X-Elastic-Product", "Elasticsearch"); + exchange.sendResponseHeaders(error ? 404 : 200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.createContext("/tracer/v1/traces", exchange -> { + var headers = exchange.getRequestHeaders(); + byte[] body = exchange.getRequestBody().readAllBytes(); + String encoding = headers.getFirst("Content-Encoding"); + if ("gzip".equals(encoding)) { + try (var gzip = new GZIPInputStream(new ByteArrayInputStream(body))) { + body = gzip.readAllBytes(); + } + } + exports.add(new WireExport(headers.getFirst("Content-Type"), encoding, + headers.getFirst("X-Api-Key"), headers.getFirst("X-Secret-Key"), + ExportTraceServiceRequest.parseFrom(body))); + exchange.getResponseHeaders().add("Content-Type", "application/x-protobuf"); + exchange.sendResponseHeaders(200, -1); + exchange.close(); + }); + server.start(); + url = "http://127.0.0.1:" + server.getAddress().getPort(); + memory = InMemorySpanExporter.create(); + sdk = OpenTelemetrySdk.builder().setTracerProvider(SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(memory)).build()).build(); + } + + @AfterEach + void stop() { + sdk.close(); + server.stop(0); + GlobalOpenTelemetry.resetForTest(); + } + + private RestClientTransport transport() { + return new RestClientTransport(RestClient.builder(HttpHost.create(url)).build(), + new JacksonJsonpMapper(), null, new OpenTelemetryForElasticsearch(sdk, false)); + } + + private static String attribute(SpanData span, String key) { + return span.getAttributes().get(AttributeKey.stringKey(key)); + } + + private SpanData assertNativeSpan() throws InterruptedException { + assertEquals(1, searches.get()); + // A completed async response can wake its caller before the native + // instrumentation's completion callback has finished exporting the span. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + var spans = memory.getFinishedSpanItems(); + while (spans.isEmpty() && System.nanoTime() < deadline) { + TimeUnit.MILLISECONDS.sleep(5); + spans = memory.getFinishedSpanItems(); + } + assertEquals(1, spans.size(), "native-only setup must finish exactly one database span"); + var span = spans.get(0); + assertEquals("search", span.getName()); + assertEquals(SpanKind.CLIENT, span.getKind()); + assertEquals("elasticsearch-api", span.getInstrumentationScopeInfo().getName()); + assertTrue("elasticsearch".equals(attribute(span, "db.system.name")) + || "elasticsearch".equals(attribute(span, "db.system"))); + assertTrue("search".equals(attribute(span, "db.operation.name")) + || "search".equals(attribute(span, "db.operation"))); + assertNull(attribute(span, "fi.span.kind")); + assertNull(attribute(span, "db.statement")); + assertNull(attribute(span, "db.query.text")); + assertFalse(span.getAttributes().toString().contains("synthetic-query-content")); + assertFalse(span.getAttributes().toString().contains("synthetic-response-content")); + return span; + } + + @Test + void explicitSdkTracesRealSynchronousClientWithoutWrapper() throws Exception { + try (var transport = transport()) { + var response = new ElasticsearchClient(transport).search(s -> s.index("documents") + .query(q -> q.match(m -> m.field("text").query("synthetic-query-content"))), Map.class); + assertEquals(1, response.hits().hits().size()); + assertTrue(searchBody.contains("synthetic-query-content")); + } + assertNativeSpan(); + } + + @Test + void explicitSdkTracesRealAsynchronousClientWithoutWrapper() throws Exception { + try (var transport = transport()) { + var response = new ElasticsearchAsyncClient(transport).search(s -> s.index("documents") + .query(q -> q.match(m -> m.field("text").query("synthetic-query-content"))), Map.class) + .get(10, TimeUnit.SECONDS); + assertEquals(1, response.hits().hits().size()); + } + assertNativeSpan(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void nativeSearchPreservesApplicationParent(boolean asynchronous) throws Exception { + var parent = sdk.getTracer("application").spanBuilder("retrieve-documents").startSpan(); + try (var scope = parent.makeCurrent(); var transport = transport()) { + if (asynchronous) { + new ElasticsearchAsyncClient(transport).search(s -> s.index("documents"), Map.class) + .get(10, TimeUnit.SECONDS); + } else { + new ElasticsearchClient(transport).search(s -> s.index("documents"), Map.class); + } + var child = assertNativeSpan(); + assertEquals(parent.getSpanContext().getSpanId(), child.getParentSpanId()); + assertEquals(parent.getSpanContext().getTraceId(), child.getTraceId()); + } finally { + parent.end(); + } + assertEquals(2, memory.getFinishedSpanItems().size()); + } + + @Test + void nativeAsyncErrorsPreserveFailureAndFinishOneSpan() throws Exception { + error = true; + try (var transport = transport()) { + var failure = assertThrows(java.util.concurrent.ExecutionException.class, + () -> new ElasticsearchAsyncClient(transport).search(s -> s.index("documents"), Map.class) + .get(10, TimeUnit.SECONDS)); + assertInstanceOf(ElasticsearchException.class, failure.getCause()); + } + assertEquals(StatusCode.ERROR, assertNativeSpan().getStatus().getStatusCode()); + } + + @Test + void defaultTransportUsesSdkRegisteredBeforeConstruction() throws Exception { + GlobalOpenTelemetry.set(sdk); + try (var transport = new RestClientTransport(RestClient.builder(HttpHost.create(url)).build(), + new JacksonJsonpMapper())) { + new ElasticsearchClient(transport).search(s -> s.index("documents"), Map.class); + } + assertNativeSpan(); + } + + @Test + void nativeErrorsPreserveClientFailureAndFinishOneSpan() throws Exception { + error = true; + try (var transport = transport()) { + assertThrows(ElasticsearchException.class, + () -> new ElasticsearchClient(transport).search(s -> s.index("documents"), Map.class)); + } + var span = assertNativeSpan(); + assertEquals(StatusCode.ERROR, span.getStatus().getStatusCode()); + assertFalse(span.getEvents().isEmpty(), "native errors may include details, as documented"); + } + + @Test + void publishedOptionalWrapperCompatibilityBoundaryMatchesGuide() throws Exception { + try (var transport = transport()) { + var client = new ElasticsearchClient(transport); + var traced = new TracedElasticsearchClient(client, new FITracer(sdk.getTracer("elasticsearch-retrieval"))); + if (!"8.12.0".equals(System.getProperty("elasticsearch.version"))) { + // The released wrapper supplies an 8.x KnnQuery builder lambda to 9.x KnnSearch. + var failure = assertThrows(ClassCastException.class, + () -> traced.knnSearch("documents", new float[]{0.1f, 0.2f, 0.3f}, 5, 100, "embedding")); + assertTrue(failure.getMessage().contains("KnnSearch$Builder") + && failure.getMessage().contains("KnnQuery$Builder")); + assertEquals(0, searches.get(), "9.x users must use the native path instead"); + return; + } + var response = traced.knnSearch("documents", new float[]{0.1f, 0.2f, 0.3f}, 5, 100, "embedding"); + assertEquals(1, response.hits().hits().size()); + assertSame(client, traced.unwrap()); + } + assertEquals(1, searches.get()); + var spans = memory.getFinishedSpanItems(); + assertEquals(2, spans.size()); + var child = spans.stream().filter(s -> s.getKind() == SpanKind.CLIENT).findFirst().orElseThrow(); + var parent = spans.stream().filter(s -> "RETRIEVER".equals(attribute(s, "fi.span.kind"))) + .findFirst().orElseThrow(); + assertEquals(parent.getSpanId(), child.getParentSpanId()); + assertEquals(parent.getTraceId(), child.getTraceId()); + assertEquals(5L, parent.getAttributes().get(AttributeKey.longKey("retriever.top_k"))); + assertEquals(3L, parent.getAttributes().get(AttributeKey.longKey("embedding.dimensions"))); + assertEquals(1L, parent.getAttributes().get(AttributeKey.longKey("db.vector.results.count"))); + assertEquals(0.9, parent.getAttributes().get(AttributeKey.doubleKey("elasticsearch.top_score"))); + } + + @ParameterizedTest + @ValueSource(strings = {"none", "gzip"}) + void nativeSpansExportAsProtobufWithPlainAndGzipTransport(String compression) throws Exception { + var exporter = OtlpHttpSpanExporter.builder().setEndpoint(url + "/tracer/v1/traces") + .setCompression(compression).build(); + try (var exportingSdk = OpenTelemetrySdk.builder().setTracerProvider(SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)).build()).build(); + var transport = new RestClientTransport(RestClient.builder(HttpHost.create(url)).build(), + new JacksonJsonpMapper(), null, new OpenTelemetryForElasticsearch(exportingSdk, false))) { + new ElasticsearchClient(transport).search(s -> s.index("documents") + .query(q -> q.match(m -> m.field("text").query("synthetic-query-content"))), Map.class); + } + var exported = exports.poll(10, TimeUnit.SECONDS); + assertNotNull(exported); + assertEquals("application/x-protobuf", exported.contentType()); + assertEquals("gzip".equals(compression) ? "gzip" : null, exported.encoding()); + var resource = exported.request().getResourceSpans(0); + assertEquals(1, resource.getScopeSpans(0).getSpansCount()); + var span = resource.getScopeSpans(0).getSpans(0); + assertEquals("search", span.getName()); + assertEquals(io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT, span.getKind()); + assertFalse(exported.request().toString().contains("synthetic-query-content")); + assertFalse(exported.request().toString().contains("synthetic-response-content")); + } + + static List documentedExamples() { + return "8.12.0".equals(System.getProperty("elasticsearch.version")) + ? List.of("NativeElasticsearchExample", "AsyncElasticsearchExample", "KnnElasticsearchExample", "OptionalElasticsearchExample") + : List.of("NativeElasticsearchExample", "AsyncElasticsearchExample", "KnnElasticsearchExample"); + } + + @ParameterizedTest + @org.junit.jupiter.params.provider.MethodSource("documentedExamples") + void exactDocumentedExamplesExportHeadersResourcesAndExpectedSpans(String mainClass) throws Exception { + boolean optional = mainClass.startsWith("Optional"); + var consumer = Path.of("target", "native-consumer", "target"); + // The native examples run from a packaged JAR with only the page's dependencies. + String classpath = optional ? System.getProperty("java.class.path") + : consumer.resolve("native-consumer-1.0.0-SNAPSHOT.jar").toAbsolutePath() + + java.io.File.pathSeparator + java.nio.file.Files.readString(consumer.resolve("classpath.txt")).trim(); + var process = new ProcessBuilder(Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-Djava.net.preferIPv4Stack=true", "-cp", classpath, mainClass); + // No inherited credentials or external endpoints, including from the developer's shell. + process.environment().clear(); + process.environment().putAll(Map.of( + "FI_API_KEY", "synthetic-fi-key", "FI_SECRET_KEY", "synthetic-fi-secret", + "FI_PROJECT_NAME", "elasticsearch-native-test", "ELASTICSEARCH_URL", url, + "ELASTICSEARCH_API_KEY", "synthetic-es-key", "ELASTICSEARCH_INDEX", "documents", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", url + "/tracer/v1/traces")); + process.redirectErrorStream(true); + var child = process.start(); + try { + assertTrue(child.waitFor(30, TimeUnit.SECONDS), "documented process timed out"); + String output = new String(child.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertEquals(0, child.exitValue(), output); + assertTrue(output.contains("Matches returned: 1"), output); + } finally { + child.destroyForcibly(); + } + assertEquals("ApiKey synthetic-es-key", authorization); + var exported = exports.poll(10, TimeUnit.SECONDS); + assertNotNull(exported, "SDK close must drain the documented batch exporter"); + assertEquals("synthetic-fi-key", exported.apiKey()); + assertEquals("synthetic-fi-secret", exported.secretKey()); + assertEquals("application/x-protobuf", exported.contentType()); + var resource = exported.request().getResourceSpans(0); + assertEquals("elasticsearch-native-test", stringValue(resource.getResource().getAttributesList(), "project_name")); + assertEquals("observe", stringValue(resource.getResource().getAttributesList(), "project_type")); + assertEquals("elasticsearch-search", stringValue(resource.getResource().getAttributesList(), "service.name")); + var scope = resource.getScopeSpansList().stream() + .filter(s -> s.getScope().getName().equals("elasticsearch-api")).findFirst().orElseThrow(); + assertEquals(1, scope.getSpansCount()); + assertEquals("search", scope.getSpans(0).getName()); + int spanCount = exported.request().getResourceSpansList().stream() + .flatMap(r -> r.getScopeSpansList().stream()).mapToInt(s -> s.getSpansCount()).sum(); + assertEquals(optional ? 2 : 1, spanCount); + assertFalse(exported.request().toString().contains("synthetic-es-key")); + assertFalse(exported.request().toString().contains("synthetic-fi-secret")); + assertFalse(exported.request().toString().contains("observability")); + assertFalse(exported.request().toString().contains("synthetic-response-content")); + assertEquals(1, searches.get()); + assertTrue(exports.isEmpty(), "one pipeline must not export the same operation twice"); + var payloads = Path.of("target", "wire", System.getProperty("elasticsearch.version")); + java.nio.file.Files.createDirectories(payloads); + java.nio.file.Files.write(payloads.resolve(mainClass + ".pb"), exported.request().toByteArray()); + } + + private static String stringValue(List attributes, String name) { + return attributes.stream().filter(a -> a.getKey().equals(name)).findFirst().orElseThrow() + .getValue().getStringValue(); + } +} diff --git a/src/pages/docs/integrations/traceai/java/vector-databases.mdx b/src/pages/docs/integrations/traceai/java/vector-databases.mdx index 6e9610bc5..014aa694b 100644 --- a/src/pages/docs/integrations/traceai/java/vector-databases.mdx +++ b/src/pages/docs/integrations/traceai/java/vector-databases.mdx @@ -4,15 +4,16 @@ description: "Trace vector database operations in Java. Qdrant, Milvus, ChromaDB --- -- 9 vector database integrations, all following the same `Traced(client)` pattern -- Search/query operations use `RETRIEVER` span kind -- Write operations (upsert, insert, delete) use `VECTOR_DB` span kind +- Elasticsearch uses native OpenTelemetry by default; optional wrappers add retrieval metadata +- Other integrations below use the `Traced(client)` pattern +- Wrapper search/query operations use `RETRIEVER` span kind +- Wrapper write operations (upsert, insert, delete) use `VECTOR_DB` span kind - Span attributes vary by database and operation ## Prerequisites -Complete the [Java SDK setup](/docs/integrations/traceai/java) first. For Pinecone, see the [dedicated Pinecone page](/docs/integrations/traceai/java/pinecone). +For Elasticsearch, follow the self-contained [native setup](#elasticsearch) below. For the other wrappers, complete the [Java SDK setup](/docs/integrations/traceai/java) first. For Pinecone, see the [dedicated Pinecone page](/docs/integrations/traceai/java/pinecone). --- @@ -402,47 +403,200 @@ Extra attributes: `azure_search.search_mode` (vector/hybrid/text), `azure_search ## Elasticsearch -Wraps `co.elastic.clients.elasticsearch.ElasticsearchClient`. KNN search with optional query filtering. +Use the [Elasticsearch Java client's native OpenTelemetry instrumentation](https://www.elastic.co/docs/reference/elasticsearch/clients/java/setup/opentelemetry) to send synchronous and asynchronous client operations to Future AGI. You do not need a `TracedElasticsearchClient` wrapper for native database spans. + +### Install dependencies + +Use Java 17 or newer and an Elasticsearch index you can search. The example uses the 8.12.0 client; match the client major version to your Elasticsearch cluster. For a 9.x cluster, set both `elasticsearch-java` and `elasticsearch-rest-client` to `9.5.5`. This example explicitly supplies the REST transport dependency so the same `RestClientTransport` setup works on both client versions. + +Add these dependencies inside your Maven project's `` element. All are available from Maven Central; no JitPack repository is needed. + +```xml + + co.elastic.clients + elasticsearch-java + 8.12.0 + + + org.elasticsearch.client + elasticsearch-rest-client + 8.12.0 + + + com.fasterxml.jackson.core + jackson-databind + 2.16.1 + + + io.opentelemetry + opentelemetry-sdk + 1.35.0 + + + io.opentelemetry + opentelemetry-exporter-otlp + 1.35.0 + +``` + +For Gradle, use the same coordinates with `mavenCentral()`: + +```groovy +implementation 'co.elastic.clients:elasticsearch-java:8.12.0' +implementation 'org.elasticsearch.client:elasticsearch-rest-client:8.12.0' +implementation 'com.fasterxml.jackson.core:jackson-databind:2.16.1' +implementation 'io.opentelemetry:opentelemetry-sdk:1.35.0' +implementation 'io.opentelemetry:opentelemetry-exporter-otlp:1.35.0' +``` + +### Configure credentials + +Set `FI_API_KEY` and `FI_SECRET_KEY` to your [Future AGI API credentials](/docs/admin-settings/api-keys). The Elasticsearch API key is separate and must have read access to the selected index. + +```bash +export FI_API_KEY="your-future-agi-api-key" +export FI_SECRET_KEY="your-future-agi-secret-key" +export FI_PROJECT_NAME="elasticsearch-search" +export ELASTICSEARCH_URL="https://your-elasticsearch-host:9200" +export ELASTICSEARCH_API_KEY="your-encoded-elasticsearch-api-key" +export ELASTICSEARCH_INDEX="documents" +export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="https://api.futureagi.com/tracer/v1/traces" +``` + +For self-hosted Future AGI, change the traces endpoint to your deployment's full `/tracer/v1/traces` URL. Keep the Elasticsearch URL free of embedded credentials and use your JVM's trusted certificate configuration; do not disable TLS verification. + +### Run a search + +Save this as `src/main/java/NativeElasticsearchExample.java` in your Maven project. Run it using your IDE or your application's normal Java launch command with the dependencies on the classpath. + +```java NativeElasticsearchExample.java +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.instrumentation.OpenTelemetryForElasticsearch; +import co.elastic.clients.transport.rest_client.RestClientTransport; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import org.apache.http.HttpHost; +import org.apache.http.message.BasicHeader; +import org.elasticsearch.client.RestClient; +import java.util.Map; + +public class NativeElasticsearchExample { + public static void main(String[] args) throws Exception { + String project = System.getenv("FI_PROJECT_NAME"); + var exporter = OtlpHttpSpanExporter.builder() + .setEndpoint(System.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")) + .addHeader("X-Api-Key", System.getenv("FI_API_KEY")) + .addHeader("X-Secret-Key", System.getenv("FI_SECRET_KEY")) + .build(); + var provider = SdkTracerProvider.builder() + .setResource(Resource.getDefault().merge(Resource.create(Attributes.builder() + .put("service.name", "elasticsearch-search") + .put("project_name", project) + .put("project_type", "observe") + .build()))) + .addSpanProcessor(BatchSpanProcessor.builder(exporter).build()) + .build(); + + // This SDK exports traces only and does not replace the global SDK. + try (var sdk = OpenTelemetrySdk.builder().setTracerProvider(provider).build()) { + var restClient = RestClient.builder(HttpHost.create(System.getenv("ELASTICSEARCH_URL"))) + .setDefaultHeaders(new BasicHeader[] { + new BasicHeader("Authorization", "ApiKey " + System.getenv("ELASTICSEARCH_API_KEY")) + }) + .build(); + // false keeps search request bodies out of native span attributes. + try (var transport = new RestClientTransport(restClient, new JacksonJsonpMapper(), + null, new OpenTelemetryForElasticsearch(sdk, false))) { + var client = new ElasticsearchClient(transport); + var response = client.search(s -> s + .index(System.getenv("ELASTICSEARCH_INDEX")) + .query(q -> q.match(m -> m.field("text").query("observability"))) + .size(5), Map.class); + System.out.println("Matches returned: " + response.hits().hits().size()); + } + } // Closing the SDK drains the batch processor before this process exits. + } +} +``` + +For a KNN search, replace the client and search inside the transport block with the following. Your index needs a three-dimensional `embedding` dense-vector field; use a vector with the dimensions configured in your own index. + +```java NativeKnnSearch.java +var client = new ElasticsearchClient(transport); +var response = client.search(s -> s + .index(System.getenv("ELASTICSEARCH_INDEX")) + .knn(k -> k.field("embedding") + .queryVector(java.util.List.of(0.1f, 0.2f, 0.3f)) + .k(5).numCandidates(100)) + .size(5), Map.class); +System.out.println("Matches returned: " + response.hits().hits().size()); +``` + +The native transport also traces `ElasticsearchAsyncClient`. To use it, replace the synchronous client and search inside the transport block with: + +```java AsyncSearch.java +var client = new co.elastic.clients.elasticsearch.ElasticsearchAsyncClient(transport); +var response = client.search(s -> s + .index(System.getenv("ELASTICSEARCH_INDEX")) + .query(q -> q.match(m -> m.field("text").query("observability"))) + .size(5), Map.class).get(30, java.util.concurrent.TimeUnit.SECONDS); +System.out.println("Matches returned: " + response.hits().hits().size()); +``` + +If your application already owns an OpenTelemetry SDK, reuse it and configure its exporter instead of creating another provider. Pass it to `new OpenTelemetryForElasticsearch(existingSdk, false)`. Alternatively, register your SDK globally **before constructing the transport** and use the two-argument `RestClientTransport(restClient, mapper)` constructor; native tracing uses that global SDK by default. Do not call `TraceAI.init()` to register a second global provider in an already-instrumented application. + + +Disabling search-body capture does not redact all metadata: native spans include index names, request URLs, and error details. Keep secrets out of URL/query parameters and review error data before exporting sensitive workloads. TraceAI's content flags do not control native Elasticsearch spans. + + +### Inspect your traces + +Open **Observe → Traces** in your Future AGI project. Native spans use the `elasticsearch-api` instrumentation scope and the `CLIENT` OpenTelemetry kind. The 8.12.0 client records `db.system=elasticsearch` and `db.operation=search`; 9.5.5 uses `db.system.name` and `db.operation.name`. These are database operations, not automatically `RETRIEVER` spans with top-k or result-score metadata. + +### Optional retrieval metadata + +With the **8.12.0 client**, keep `TracedElasticsearchClient` when you need the additional semantic retrieval span for KNN searches. The published wrapper `1.0.1` targets 8.12.0 and is not binary-compatible with the 9.x KNN API; use native tracing alone on 9.x. Add the published package from Maven Central: ```xml Maven - com.github.future-agi.traceAI + com.futureagi traceai-java-elasticsearch - main-SNAPSHOT + 1.0.1 ``` ```groovy Gradle -implementation 'com.github.future-agi.traceAI:traceai-java-elasticsearch:main-SNAPSHOT' +implementation 'com.futureagi:traceai-java-elasticsearch:1.0.1' ``` -```java -import ai.traceai.elasticsearch.TracedElasticsearchClient; -import co.elastic.clients.elasticsearch.ElasticsearchClient; - -ElasticsearchClient client = /* build with RestClient */; -TracedElasticsearchClient traced = new TracedElasticsearchClient(client); +Inside the example's transport block, use the same SDK for both the wrapper and native client. Replace the search with the following code; the index must have a three-dimensional `embedding` dense-vector field: -// KNN search -var results = traced.knnSearch("my-index", queryVectorArray, 10, 100, "embedding"); - -// KNN with filter -var filtered = traced.knnSearchWithFilter("my-index", queryVectorArray, 10, 100, "embedding", filterQuery); +```java OptionalRetrieval.java +var traced = new ai.traceai.elasticsearch.TracedElasticsearchClient( + client, new ai.traceai.FITracer(sdk.getTracer("elasticsearch-retrieval"))); +var results = traced.knnSearch(System.getenv("ELASTICSEARCH_INDEX"), + new float[] {0.1f, 0.2f, 0.3f}, 5, 100, "embedding"); +System.out.println("Matches returned: " + results.hits().hits().size()); +``` -// Index document -traced.index("my-index", "doc-1", Map.of("text", "hello", "embedding", vectorArray)); +This produces a semantic `RETRIEVER` parent and a native database child for one search. The parent adds `retriever.top_k`, `embedding.dimensions`, `db.vector.results.count`, and `elasticsearch.top_score`. It is optional enrichment, not a replacement for native tracing of the rest of the client API. Do not add another manual retrieval span around the wrapper for the same operation. -// Bulk index -traced.bulkIndex("my-index", documents); -``` +Existing constructors and `knnSearchWithFilter`, `index`, `bulkIndex`, `delete`, `createIndex`, and `unwrap` remain available. Native tracing alone is sufficient when you do not need these semantic wrapper spans. -**Spans:** "Elasticsearch KNN Search" (RETRIEVER), "Elasticsearch KNN Search with Filter" (RETRIEVER), "Elasticsearch Index Document" (VECTOR_DB), "Elasticsearch Bulk Index" (VECTOR_DB), "Elasticsearch Delete Document" (VECTOR_DB), "Elasticsearch Create Index" (VECTOR_DB) +### Troubleshooting -Extra attributes: `elasticsearch.num_candidates`, `elasticsearch.total_hits`, `elasticsearch.took_ms`, `elasticsearch.field` +- **No traces:** check both Future AGI headers, the `project_name` resource, and the full traces endpoint. Wait for the batch to export, and close the SDK on shutdown. With a global SDK, initialize it before creating your Elasticsearch transport and do not disable `OTEL_INSTRUMENTATION_ELASTICSEARCH_ENABLED`. +- **Two search-related spans:** the optional wrapper intentionally adds a retrieval parent. Use the native client alone for one database span per request. If you also use a Java agent, avoid configuring overlapping Elasticsearch instrumentation or a second exporter pipeline. +- **Elasticsearch rejects the request:** match your client major version to the server, confirm API-key read access, and check the index name and vector mapping. ---