Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions .github/workflows/elasticsearch-java-docs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
name: Elasticsearch Java documentation

on:
pull_request:
branches: [main]
paths:
- src/pages/docs/integrations/traceai/java/vector-databases.mdx
- scripts/java-elasticsearch/**
- scripts/crewai-ingestion-check/**
- .github/workflows/elasticsearch-java-docs.yml

permissions:
contents: read

jobs:
native-tracing:
name: Native tracing (Java ${{ matrix.java }}, Elasticsearch ${{ matrix.elasticsearch }})
runs-on: ubuntu-latest
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
java: ['17', '21']
elasticsearch: ['8.12.0', '9.5.5']
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
distribution: temurin
java-version: ${{ matrix.java }}
- uses: actions/setup-node@v4
with:
node-version: '20'
- name: Check out the production ingestion contract
uses: actions/checkout@v4
with:
repository: future-agi/future-agi
ref: 215b19543a47de8c7ae7e8dc452dc267440e5c8d
path: .test-deps/future-agi
sparse-checkout: fi-collector
persist-credentials: false
- uses: actions/setup-go@v5
with:
go-version-file: .test-deps/future-agi/fi-collector/go.mod
cache-dependency-path: .test-deps/future-agi/fi-collector/go.sum
- name: Resolve production converter dependencies
working-directory: .test-deps/future-agi/fi-collector
run: go mod download
- name: Extract executable snippets and check the page contract
run: node scripts/java-elasticsearch/extract.mjs
- name: Resolve released dependencies before denying outbound networking
run: |
# sudo changes Java's user.home even when HOME is preserved.
export MAVEN_REPO="$RUNNER_TEMP/elasticsearch-m2"
echo "MAVEN_REPO=$MAVEN_REPO" >> "$GITHUB_ENV"
mvn -B -Dmaven.repo.local="$MAVEN_REPO" -f scripts/java-elasticsearch/pom.xml -Delasticsearch.version=${{ matrix.elasticsearch }} dependency:go-offline
mvn -B -Dmaven.repo.local="$MAVEN_REPO" -f scripts/java-elasticsearch/target/native-consumer/pom.xml -Delasticsearch.version=${{ matrix.elasticsearch }} dependency:go-offline
- name: Package the exact native example and test with loopback networking only
env:
ES_VERSION: ${{ matrix.elasticsearch }}
run: |
export INGESTION_CHECK="$RUNNER_TEMP/elasticsearch-ingestion-check"
sudo --preserve-env=JAVA_HOME,ES_VERSION,INGESTION_CHECK,MAVEN_REPO env PATH="$PATH" HOME="$HOME" \
unshare --net sh -ec '
ip link set lo up
ROOT="$PWD"
(cd .test-deps/future-agi/fi-collector && GOPROXY=off go build -o "$INGESTION_CHECK" "$ROOT/scripts/crewai-ingestion-check/main.go")
mvn -o -B -Dmaven.repo.local="$MAVEN_REPO" -f scripts/java-elasticsearch/target/native-consumer/pom.xml -Delasticsearch.version="$ES_VERSION" -DskipTests package dependency:build-classpath -Dmdep.outputFile=target/classpath.txt
mvn -o -B -Dmaven.repo.local="$MAVEN_REPO" -f scripts/java-elasticsearch/pom.xml -Delasticsearch.version="$ES_VERSION" test
node scripts/java-elasticsearch/check-ingestion.mjs "$INGESTION_CHECK" "$ES_VERSION"
'
1 change: 1 addition & 0 deletions scripts/java-elasticsearch/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/target/
51 changes: 51 additions & 0 deletions scripts/java-elasticsearch/check-ingestion.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';

// Use the production converter, not a duplicate of its semantic mapping.
// Authentication/project resolution are tested separately by the collector;
// this contract verifies that the exact guide's wire spans survive conversion.
const [binary, version] = process.argv.slice(2);
assert.ok(binary && version, 'Usage: node check-ingestion.mjs CONVERTER ELASTICSEARCH_VERSION');
const examples = ['NativeElasticsearchExample', 'AsyncElasticsearchExample', 'KnnElasticsearchExample'];
if (version === '8.12.0') examples.push('OptionalElasticsearchExample');
for (const example of examples) {
const filename = fileURLToPath(new URL(`./target/wire/${version}/${example}.pb`, import.meta.url));
const output = execFileSync(binary, [filename], { encoding: 'utf8', timeout: 30_000 });
const rows = output.trim().split('\n').map(line => JSON.parse(line));
const optional = example.startsWith('Optional');
assert.equal(rows.length, optional ? 2 : 1, example);
const database = rows.find(row => row.name === 'search');
assert.ok(database, example);
assert.equal(database.observation_type, 'unknown', 'native database spans are not semantic retrievers');
assert.equal(database.attrs_string['db.system.name'] ?? database.attrs_string['db.system'], 'elasticsearch');
assert.equal(database.attrs_string['db.operation.name'] ?? database.attrs_string['db.operation'], 'search');
assert.equal(database.attrs_string['db.elasticsearch.path_parts.index'], 'documents');
for (const row of rows) {
assert.equal(row.service_name, 'elasticsearch-search');
assert.equal(row.resource_attrs.project_name, 'elasticsearch-native-test');
assert.equal(row.resource_attrs.project_type, 'observe');
assert.match(row.id, /^[0-9a-f]{16}$/);
assert.match(row.trace_id, /^[0-9a-f-]{36}$/);
assert.ok(row.latency_ms >= 0);
assert.equal(row.total_tokens, 0);
assert.equal(row.cost, 0);
assert.equal(row.input, '');
assert.equal(row.output, '');
}
if (optional) {
const parent = rows.find(row => row.observation_type === 'retriever');
assert.ok(parent);
assert.equal(database.parent_span_id, parent.id);
assert.equal(database.trace_id, parent.trace_id);
assert.equal(parent.attrs_number['retriever.top_k'], 5);
assert.equal(parent.attrs_number['db.vector.results.count'], 1);
assert.equal(parent.parent_span_id, '');
} else {
assert.equal(database.parent_span_id, '');
}
for (const canary of ['synthetic-fi-key', 'synthetic-fi-secret', 'synthetic-es-key', 'synthetic-response-content', 'observability']) {
assert.ok(!output.includes(canary), `Unexpected payload content: ${canary}`);
}
console.log(`PASS ${version} ${example}: ${rows.length} production span rows`);
}
78 changes: 78 additions & 0 deletions scripts/java-elasticsearch/extract.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
import assert from 'node:assert/strict';
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';

const page = readFileSync(new URL('../../src/pages/docs/integrations/traceai/java/vector-databases.mdx', import.meta.url), 'utf8');
const section = page.split('## Elasticsearch\n')[1].split('## Common span attributes')[0];
const native = section.split('### Optional retrieval metadata')[0];
assert.match(native, /new OpenTelemetryForElasticsearch\(sdk, false\)/);
assert.match(native, /"project_name"/);
assert.match(native, /"project_type", "observe"/);
assert.match(native, /"X-Api-Key"/);
assert.match(native, /"X-Secret-Key"/);
assert.match(native, /\/tracer\/v1\/traces/);
assert.doesNotMatch(section, /main-SNAPSHOT|com\.github\.future-agi/);
assert.match(section, /com\.futureagi:traceai-java-elasticsearch:1\.0\.1/);
assert.match(section, /RETRIEVER.*parent.*native database child/);

const output = new URL('./target/generated-sources/', import.meta.url);
mkdirSync(output, { recursive: true });
function snippet(name) {
const code = section.split(`\x60\x60\x60java ${name}\n`)[1]?.split('\x60\x60\x60')[0];
assert.ok(code, `Missing executable snippet: ${name}`);
return code;
}
const main = snippet('NativeElasticsearchExample.java');
writeFileSync(new URL('NativeElasticsearchExample.java', output), main);
const imports = main.split('\n').filter(line => line.startsWith('import ')).join('\n');
writeFileSync(new URL('OptionalRetrieval.java', output), `${imports}
public class OptionalRetrieval {
public static void run(ElasticsearchClient client, OpenTelemetrySdk sdk) throws Exception {
${snippet('OptionalRetrieval.java')}
}
}
`);
writeFileSync(new URL('AsyncSearch.java', output), `${imports}
public class AsyncSearch {
public static void run(RestClientTransport transport) throws Exception {
${snippet('AsyncSearch.java')}
}
}
`);
// Build the native example separately with only the dependencies from the page.
// This prevents the optional wrapper or test libraries from masking missing dependencies.
const consumer = new URL('./target/native-consumer/', import.meta.url);
mkdirSync(new URL('src/main/java/', consumer), { recursive: true });
writeFileSync(new URL('src/main/java/NativeElasticsearchExample.java', consumer), main);
const searchBlock = /var client = new ElasticsearchClient\(transport\);[\s\S]*?System\.out\.println\("Matches returned: " \+ response\.hits\(\)\.hits\(\)\.size\(\)\);/;
assert.match(main, searchBlock);
const asyncMain = main.replace('class NativeElasticsearchExample', 'class AsyncElasticsearchExample')
.replace(searchBlock, snippet('AsyncSearch.java'));
writeFileSync(new URL('src/main/java/AsyncElasticsearchExample.java', consumer), asyncMain);
const knnMain = main.replace('class NativeElasticsearchExample', 'class KnnElasticsearchExample')
.replace(searchBlock, snippet('NativeKnnSearch.java'));
writeFileSync(new URL('src/main/java/KnnElasticsearchExample.java', consumer), knnMain);
const optionalMain = main.replace('class NativeElasticsearchExample', 'class OptionalElasticsearchExample')
.replace(searchBlock, 'var client = new ElasticsearchClient(transport);\n' + snippet('OptionalRetrieval.java'));
writeFileSync(new URL('OptionalElasticsearchExample.java', output), optionalMain);
const dependencies = native.split('\x60\x60\x60xml\n')[1].split('\x60\x60\x60')[0];
writeFileSync(new URL('pom.xml', consumer), `
<project xmlns="http://maven.apache.org/POM/4.0.0">
<modelVersion>4.0.0</modelVersion>
<groupId>com.futureagi.docs</groupId>
<artifactId>native-consumer</artifactId>
<version>1.0.0-SNAPSHOT</version>
<properties>
<maven.compiler.release>17</maven.compiler.release>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<elasticsearch.version>8.12.0</elasticsearch.version>
</properties>
<dependencies>${dependencies.replaceAll('<version>8.12.0</version>', '<version>${elasticsearch.version}</version>')}</dependencies>
<build><plugins>
<plugin><groupId>org.apache.maven.plugins</groupId><artifactId>maven-compiler-plugin</artifactId><version>3.12.1</version></plugin>
<plugin><groupId>org.apache.maven.plugins</groupId><artifactId>maven-dependency-plugin</artifactId><version>3.8.1</version></plugin>
<plugin><groupId>org.apache.maven.plugins</groupId><artifactId>maven-jar-plugin</artifactId><version>3.3.0</version></plugin>
</plugins></build>
</project>
`);
assert.match(section, /not binary-compatible with the 9\.x KNN API/);
console.log('Elasticsearch page contract and snippet extraction passed.');
94 changes: 94 additions & 0 deletions scripts/java-elasticsearch/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.futureagi.docs</groupId>
<artifactId>elasticsearch-native-example</artifactId>
<version>1.0.0-SNAPSHOT</version>
<properties>
<maven.compiler.release>17</maven.compiler.release>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<elasticsearch.version>8.12.0</elasticsearch.version>
<opentelemetry.version>1.35.0</opentelemetry.version>
</properties>
<dependencies>
<dependency>
<groupId>co.elastic.clients</groupId>
<artifactId>elasticsearch-java</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.16.1</version>
</dependency>
<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-sdk</artifactId>
<version>${opentelemetry.version}</version>
</dependency>
<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-exporter-otlp</artifactId>
<version>${opentelemetry.version}</version>
</dependency>
<dependency>
<groupId>com.futureagi</groupId>
<artifactId>traceai-java-elasticsearch</artifactId>
<version>1.0.1</version>
</dependency>
<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-sdk-testing</artifactId>
<version>${opentelemetry.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.opentelemetry.proto</groupId>
<artifactId>opentelemetry-proto</artifactId>
<version>1.0.0-alpha</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>5.10.1</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<sourceDirectory>${project.build.directory}/generated-sources</sourceDirectory>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.12.1</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.2.3</version>
<dependencies>
<!-- Resolve the dynamically selected provider before tests lose network access. -->
<dependency>
<groupId>org.apache.maven.surefire</groupId>
<artifactId>surefire-junit-platform</artifactId>
<version>3.2.3</version>
</dependency>
</dependencies>
<configuration>
<!-- Match the loopback-only sandbox without IPv4-mapped IPv6 sockets. -->
<argLine>-Djava.net.preferIPv4Stack=true</argLine>
<systemPropertyVariables>
<elasticsearch.version>${elasticsearch.version}</elasticsearch.version>
</systemPropertyVariables>
</configuration>
</plugin>
</plugins>
</build>
</project>
Loading
Loading