diff --git a/.github/integration-ci/check_results.py b/.github/integration-ci/check_results.py new file mode 100644 index 000000000..1ea3e7ee0 --- /dev/null +++ b/.github/integration-ci/check_results.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Fail if a planned matrix was skipped, failed or cancelled (no empty green gate).""" +import json +import os + + +def check(needs): + if needs['plan']['result'] != 'success': + raise ValueError('integration planning did not succeed') + for language in ['typescript', 'go', 'java']: + matrix = json.loads(needs['plan']['outputs'][language]) + if not isinstance(matrix, list): + raise ValueError(f'{language}: missing/invalid matrix') + expected = 'success' if matrix else 'skipped' + actual = needs[language]['result'] + if actual != expected: + raise ValueError(f'{language}: planned {len(matrix)} packages, got {actual}, expected {expected}') + + +if __name__ == '__main__': + check(json.loads(os.environ['NEEDS_JSON'])) + print('Every planned integration built and passed its tests.') diff --git a/.github/integration-ci/install_maven.sh b/.github/integration-ci/install_maven.sh new file mode 100644 index 000000000..7285dfa00 --- /dev/null +++ b/.github/integration-ci/install_maven.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Exact Maven toolchain, verified before extracting; no third-party setup action. +set -euo pipefail +: "${RUNNER_TEMP:?}" +: "${GITHUB_PATH:?}" +archive="$RUNNER_TEMP/apache-maven-3.9.9-bin.tar.gz" +curl --fail --show-error --location --retry 3 --max-time 120 \ + https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.tar.gz \ + --output "$archive" +printf '%s %s\n' \ + a555254d6b53d267965a3404ecb14e53c3827c09c3b94b5678835887ab404556bfaf78dcfe03ba76fa2508649dca8531c74bca4d5846513522404d48e8c4ac8b \ + "$archive" | sha512sum --check --strict +tar -xzf "$archive" -C "$RUNNER_TEMP" +printf '%s\n' "$RUNNER_TEMP/apache-maven-3.9.9/bin" >> "$GITHUB_PATH" diff --git a/.github/integration-ci/integration_plan.py b/.github/integration-ci/integration_plan.py new file mode 100644 index 000000000..918a384ea --- /dev/null +++ b/.github/integration-ci/integration_plan.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Plan the Codex/Pi, Go and Java integration checks, without changing Python CI. + +Uses the existing Python planner's NUL-safe, rename-aware git diff reader. The +same files can land independently with each integration: absent target directories +are ignored, but an existing target may not silently lose its manifest or tests. +CI changes run every present target; TS shared dependencies run both TS targets. +""" +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / '.github/scripts')) +from python_test_plan import PlanError, changed_paths, commit_count + +TARGETS = ( + ('codex-sdk', 'typescript', 'typescript/packages/traceai_codex_sdk', 'package.json'), + ('pi-coding-agent', 'typescript', 'typescript/packages/traceai_pi_coding_agent', 'package.json'), + ('openai', 'go', 'go/openai', 'go.mod'), + ('a2a', 'go', 'go/a2a', 'go.mod'), + ('eino', 'go', 'go/eino', 'go.mod'), + ('arconia', 'java', 'java/examples/arconia-example', 'pom.xml'), + ('quarkus-langchain4j', 'java', 'java/examples/quarkus-langchain4j-example', 'pom.xml'), +) +WORKFLOW = '.github/workflows/integration-tests.yml' +CI_PREFIX = '.github/integration-ci/' +SHARED_DIFF_READER = '.github/scripts/python_test_plan.py' +TS_SHARED_FILES = { + 'typescript/package.json', 'typescript/pnpm-lock.yaml', + 'typescript/pnpm-workspace.yaml', 'typescript/.npmrc', +} +TS_SHARED_PREFIXES = ('typescript/packages/fi-core/', 'typescript/packages/fi-semantic-conventions/') +TEST_GLOBS = {'typescript': 'src/__tests__/**/*.test.ts', 'go': '**/*_test.go', 'java': 'src/test/java/**/*Test.java'} + + +def plan(changed, root=ROOT, all_targets=False): + changed = set(changed) + run_all = all_targets or WORKFLOW in changed or SHARED_DIFF_READER in changed or any(p.startswith(CI_PREFIX) for p in changed) + ts_shared = any(p in TS_SHARED_FILES or p.startswith(TS_SHARED_PREFIXES) or + (p.startswith('typescript/tsconfig') and p.endswith('.json')) for p in changed) + result = {'typescript': [], 'go': [], 'java': []} + for name, language, directory, manifest in TARGETS: + path = root / directory + if not path.is_dir(): + # A not-yet-landed or entirely deleted integration is not a target. + continue + if not (path / manifest).is_file(): + raise PlanError(f'{directory} exists but is missing {manifest}') + if not any(path.glob(TEST_GLOBS[language])): + raise PlanError(f'{directory} has no tests; refusing an empty test gate') + if run_all or (language == 'typescript' and ts_shared) or any(p.startswith(directory + '/') for p in changed): + result[language].append({'name': name, 'dir': directory}) + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + args = parser.add_mutually_exclusive_group() + args.add_argument('--diff') + args.add_argument('--all', action='store_true') + options = parser.parse_args(argv) + try: + if options.diff: + paths = changed_paths(options.diff, ROOT) + if not paths and commit_count(options.diff, ROOT) == 0: + raise PlanError('no commits in diff; check base/head wiring') + elif options.all: + paths = changed_paths(None, ROOT) + if not paths: + raise PlanError('no tracked files; check checkout') + else: + paths = [line.strip() for line in sys.stdin if line.strip()] + print(json.dumps(plan(paths, all_targets=options.all), sort_keys=True)) + except PlanError as exc: + print(f'integration_plan: {exc}', file=sys.stderr) + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/integration-ci/test_check_results.py b/.github/integration-ci/test_check_results.py new file mode 100644 index 000000000..349c9099f --- /dev/null +++ b/.github/integration-ci/test_check_results.py @@ -0,0 +1,42 @@ +import copy +import unittest +from check_results import check + + +class ResultsTests(unittest.TestCase): + def setUp(self): + self.needs = {'plan': {'result': 'success', 'outputs': { + 'typescript': '[]', 'go': '[{"name":"openai"}]', 'java': '[]'}}, + 'typescript': {'result': 'skipped'}, 'go': {'result': 'success'}, + 'java': {'result': 'skipped'}} + + def test_success(self): + check(self.needs) + + def test_failed_or_skipped_planner_fails(self): + for result in ['failure', 'cancelled', 'skipped']: + needs = copy.deepcopy(self.needs) + needs['plan']['result'] = result + with self.assertRaises(ValueError): + check(needs) + + def test_planned_matrix_must_succeed(self): + for result in ['failure', 'cancelled', 'skipped']: + needs = copy.deepcopy(self.needs) + needs['go']['result'] = result + with self.assertRaises(ValueError): + check(needs) + + def test_unplanned_matrix_must_not_claim_success(self): + self.needs['java']['result'] = 'success' + with self.assertRaises(ValueError): + check(self.needs) + + def test_missing_or_invalid_matrix_fails(self): + for value in ['', 'null', '{}', 'false']: + self.needs['plan']['outputs']['go'] = value + with self.assertRaises(ValueError): + check(self.needs) + del self.needs['plan']['outputs']['go'] + with self.assertRaises(KeyError): + check(self.needs) diff --git a/.github/integration-ci/test_integration_plan.py b/.github/integration-ci/test_integration_plan.py new file mode 100644 index 000000000..91c6e5c3b --- /dev/null +++ b/.github/integration-ci/test_integration_plan.py @@ -0,0 +1,109 @@ +"""Planner regression checks: every new path, shared dependencies and safe skips.""" +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +from integration_plan import TARGETS, WORKFLOW, PlanError, plan, changed_paths + + +class IntegrationPlanTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.tests = {} + for name, language, directory, manifest in TARGETS: + package = self.root / directory + package.mkdir(parents=True) + (package / manifest).write_text('{}') + test = {'typescript': 'src/__tests__/tracer.test.ts', 'go': 'tracer_test.go', + 'java': 'src/test/java/example/ExportTest.java'}[language] + test_path = package / test + test_path.parent.mkdir(parents=True, exist_ok=True) + test_path.touch() + self.tests[name] = test_path + + def names(self, paths, **kwargs): + return {p['name'] for matrix in plan(paths, self.root, **kwargs).values() for p in matrix} + + def test_every_owned_path_selects_its_own_target(self): + for name, _, directory, manifest in TARGETS: + for suffix in [manifest, 'README.md', 'example/main.go', 'src/changed.ts']: + with self.subTest(name=name, suffix=suffix): + self.assertEqual(self.names([directory + '/' + suffix]), {name}) + + def test_ci_edits_and_all_select_every_present_target(self): + expected = {t[0] for t in TARGETS} + for path in [WORKFLOW, '.github/integration-ci/integration_plan.py', + '.github/integration-ci/test_integration_plan.py', + '.github/integration-ci/install_maven.sh', '.github/scripts/python_test_plan.py']: + self.assertEqual(self.names([path]), expected) + self.assertEqual(self.names([], all_targets=True), expected) + + def test_shared_typescript_inputs_cover_both_consumers(self): + for path in ['typescript/pnpm-lock.yaml', 'typescript/package.json', + 'typescript/pnpm-workspace.yaml', 'typescript/.npmrc', + 'typescript/tsconfig.base.json', 'typescript/packages/fi-core/src/index.ts', + 'typescript/packages/fi-semantic-conventions/src/index.ts']: + with self.subTest(path=path): + self.assertEqual(self.names([path]), {'codex-sdk', 'pi-coding-agent'}) + + def test_unrelated_paths_do_not_claim_coverage(self): + self.assertEqual(self.names(['README.md', 'python/frameworks/openai/tracer.py', + '.github/workflows/python-tests.yml', + 'typescript/packages/traceai_openai/src/index.ts', + 'java/traceai-core/pom.xml', 'go/openai-other/main.go']), set()) + + def test_target_not_yet_landed_is_ignored(self): + for _, _, directory, _ in TARGETS[1:]: + shutil.rmtree(self.root / directory) + self.assertEqual(self.names([WORKFLOW]), {'codex-sdk'}) + self.assertEqual(self.names(['go/a2a/interceptor.go']), set()) + + def test_entirely_deleted_target_is_ignored(self): + shutil.rmtree(self.root / 'go/openai') + self.assertEqual(self.names(['go/openai/go.mod']), set()) + + def test_missing_manifest_fails_closed(self): + (self.root / 'go/openai/go.mod').unlink() + with self.assertRaisesRegex(PlanError, 'missing go.mod'): + plan(['go/openai/tracer.go'], self.root) + + def test_no_tests_fails_closed_for_each_language(self): + for name in ['codex-sdk', 'openai', 'arconia']: + with self.subTest(name=name): + self.tests[name].unlink() + with self.assertRaisesRegex(PlanError, 'no tests'): + plan([WORKFLOW], self.root) + self.tests[name].touch() + + def test_duplicate_paths_do_not_duplicate_jobs(self): + result = plan(['go/openai/go.mod'] * 5, self.root) + self.assertEqual(len(result['go']), 1) + + def test_moves_plan_both_old_and_new_owner_using_existing_diff_reader(self): + # Exercise real git's --no-renames/-z path handling, not just an artificial list. + def git(*args): + return subprocess.check_output(['git', '-C', str(self.root), *args], stderr=subprocess.STDOUT) + git('init', '-q') + git('add', '.') + env = __import__('os').environ.copy() + env.update(GIT_AUTHOR_NAME='CI test', GIT_COMMITTER_NAME='CI test', + GIT_AUTHOR_EMAIL='ci@example.invalid', GIT_COMMITTER_EMAIL='ci@example.invalid') + subprocess.run(['git', '-C', str(self.root), 'commit', '-qm', 'fixture'], env=env, check=True) + old = self.root / 'go/openai/with tab\tname.go' + old.touch() + git('add', '.') + subprocess.run(['git', '-C', str(self.root), 'commit', '-qm', 'old owner'], env=env, check=True) + old.rename(self.root / 'go/a2a/with tab\tname.go') + git('add', '.') + subprocess.run(['git', '-C', str(self.root), 'commit', '-qm', 'move'], env=env, check=True) + paths = changed_paths('HEAD~1..HEAD', self.root) + self.assertIn('go/openai/with tab\tname.go', paths) + self.assertEqual(self.names(paths), {'openai', 'a2a'}) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..de45da74b --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,215 @@ +name: Integration tests + +# Additive to python-tests.yml and livekit-release-check.yml. Only the seven +# targets listed in integration_plan.py are covered here; absent, not-yet-merged +# packages are ignored so each integration can carry this same CI independently. +on: + pull_request: + branches: [main, dev] + types: [opened, synchronize, reopened, edited] + push: + branches: [main, dev] + workflow_dispatch: + +permissions: + contents: read + +defaults: + run: + shell: bash + +concurrency: + group: integration-tests-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + FI_BASE_URL: http://127.0.0.1:9 + FI_GRPC_URL: http://127.0.0.1:9 + # Contract tests use local stand-ins; this workflow never needs vendor secrets. + CI: 'true' + +jobs: + plan: + name: Integration plan + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + typescript: ${{ steps.plan.outputs.typescript }} + go: ${{ steps.plan.outputs.go }} + java: ${{ steps.plan.outputs.java }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12.3' + - name: Test existing and integration planners and result gates + run: | + python -m unittest discover -s .github/scripts -p 'test_*.py' -v + python -m unittest discover -s .github/integration-ci -p 'test_*.py' -v + - name: Plan changed integrations + id: plan + env: + EVENT_NAME: ${{ github.event_name }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PUSH_BEFORE_SHA: ${{ github.event.before }} + PUSH_AFTER_SHA: ${{ github.event.after }} + run: | + planner=.github/integration-ci/integration_plan.py + plan_file="$RUNNER_TEMP/integration-plan.json" + case "$EVENT_NAME" in + pull_request) + if ! git cat-file -e "$PR_BASE_SHA^{commit}" 2>/dev/null; then + timeout 120 git fetch --no-tags origin "$PR_BASE_SHA" || true + fi + if git cat-file -e "$PR_BASE_SHA^{commit}" 2>/dev/null; then + python "$planner" --diff "$PR_BASE_SHA...$PR_HEAD_SHA" > "$plan_file" + else + python "$planner" --all > "$plan_file" + fi + ;; + push) + if [ "$PUSH_BEFORE_SHA" = 0000000000000000000000000000000000000000 ] || ! git cat-file -e "$PUSH_BEFORE_SHA^{commit}" 2>/dev/null; then + python "$planner" --all > "$plan_file" + else + python "$planner" --diff "$PUSH_BEFORE_SHA..$PUSH_AFTER_SHA" > "$plan_file" + fi + ;; + *) python "$planner" --all > "$plan_file" ;; + esac + jq . "$plan_file" + for language in typescript go java; do + echo "$language=$(jq -c --arg language "$language" '.[$language]' "$plan_file")" >> "$GITHUB_OUTPUT" + done + + typescript: + name: TypeScript (${{ matrix.package.name }}) + needs: plan + if: needs.plan.outputs.typescript != '[]' + runs-on: ubuntu-24.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + package: ${{ fromJSON(needs.plan.outputs.typescript) }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22.23.4' + - name: Install locked target and workspace dependencies + working-directory: typescript + env: + PACKAGE_NAME: ${{ matrix.package.name }} + run: | + # package scripts invoke `pnpm` themselves; setup-node supplies + # corepack, but does not enable its pnpm shim on a clean runner. + mkdir -p "$RUNNER_TEMP/corepack-bin" + corepack enable --install-directory "$RUNNER_TEMP/corepack-bin" pnpm + echo "$RUNNER_TEMP/corepack-bin" >> "$GITHUB_PATH" + # packageManager pins pnpm 10.30.1 with its integrity hash. + corepack pnpm --version + corepack pnpm --filter "@traceai/$PACKAGE_NAME..." install --frozen-lockfile --ignore-scripts + - name: Build workspace dependencies and target, then run unit tests + working-directory: typescript + env: + PACKAGE_NAME: ${{ matrix.package.name }} + run: | + corepack pnpm run build:deps + corepack pnpm --filter "@traceai/$PACKAGE_NAME" run build + corepack pnpm --filter "@traceai/$PACKAGE_NAME" test --runInBand + - name: Codex real SDK contract (local subprocess, no API calls) + if: matrix.package.name == 'codex-sdk' + working-directory: ${{ matrix.package.dir }} + run: node contract/run_real_sdk.mjs + - name: Build installable tarball + working-directory: ${{ matrix.package.dir }} + run: | + mkdir -p "$RUNNER_TEMP/package" + corepack pnpm pack --pack-destination "$RUNNER_TEMP/package" + test -n "$(find "$RUNNER_TEMP/package" -name '*.tgz' -print -quit)" + git diff --exit-code -- ../../pnpm-lock.yaml + + go: + name: Go (${{ matrix.package.name }}) + needs: plan + if: needs.plan.outputs.go != '[]' + runs-on: ubuntu-24.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + package: ${{ fromJSON(needs.plan.outputs.go) }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: '1.26.0' + cache-dependency-path: ${{ matrix.package.dir }}/go.sum + - name: Verify modules, build package and example, vet and race-test + working-directory: ${{ matrix.package.dir }} + env: + GOTOOLCHAIN: local + GOFLAGS: -mod=readonly + run: | + go mod download + go mod verify + go build ./... + go vet ./... + go test -race -count=1 -v ./... + git diff --exit-code -- go.mod go.sum + + java: + name: Java (${{ matrix.package.name }}) + needs: plan + if: needs.plan.outputs.java != '[]' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + package: ${{ fromJSON(needs.plan.outputs.java) }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 + with: + distribution: temurin + java-version: '21.0.8' + - name: Install checksum-pinned Maven 3.9.9 + run: bash .github/integration-ci/install_maven.sh + - name: Build packaged application and run OTLP integration tests + working-directory: ${{ matrix.package.dir }} + run: | + mvn --version + mvn -B -ntp -DfailIfNoTests=true verify + test -n "$(find target -name '*.jar' -print -quit)" + git diff --exit-code -- pom.xml + + result: + name: Integration tests + needs: [plan, typescript, go, java] + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - name: Require success from every planned language matrix + env: + NEEDS_JSON: ${{ toJSON(needs) }} + run: python3 .github/integration-ci/check_results.py diff --git a/java/examples/quarkus-langchain4j-example/.gitignore b/java/examples/quarkus-langchain4j-example/.gitignore new file mode 100644 index 000000000..2f7896d1d --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/java/examples/quarkus-langchain4j-example/README.md b/java/examples/quarkus-langchain4j-example/README.md new file mode 100644 index 000000000..88c6a06a2 --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/README.md @@ -0,0 +1,89 @@ +# Quarkus LangChain4j to Future AGI + +A Quarkus command-line app with one `@RegisterAiService` and one `@Tool`. Quarkus LangChain4j already creates the +spans; `quarkus-opentelemetry` exports them; `application.properties` points the exporter at Future AGI. There is no +traceAI module here and no `traceai-langchain4j` (that one wraps plain LangChain4j, not Quarkus' build-time AI +services). + +Docs page: `docs/integrations/traceai/java/quarkus-langchain4j` in future-agi/docs. + +## Tested + +| | | +| --- | --- | +| `io.quarkiverse.langchain4j:quarkus-langchain4j-bom` | **1.14.2** (latest stable on Maven Central, 2026-10-15) with Quarkus platform **3.33.3.1** (the version 1.14.2 is built against), OpenTelemetry SDK 1.57.0 | +| Also run | 1.8.2 + Quarkus 3.27.2 (the PRD pin), live export only, see "Older versions" | +| JDK / Maven | OpenJDK 21.0.12, Maven 3.10.0 | +| Model | `quarkus-langchain4j-openai` against an OpenAI-compatible endpoint (tests: a scripted local server) | +| Mode | JVM. Native image not built or tested. | + +## Files + +| File | What | +| --- | --- | +| `src/main/resources/application.properties` | The recipe: exporter, headers, resource, privacy defaults | +| `Assistant.java`, `WeatherTools.java`, `AskCommand.java` | The app: one AI service, one tool, a command-mode `main` | +| `FutureAgiTracingDefaults.java` | Application code to copy: `ERROR` status on failed model calls and AI service calls, no error text unless opted in, removes `host.name` | +| `ToolErrorStatus.java` | Application code to copy: `ERROR` status on a failed tool span (needs quarkus-langchain4j 1.12.0+) | +| `src/test/...` | 14 tests against a scripted OpenAI-compatible server and a stand-in OTLP collector | +| `run_e2e.sh` | Tests, negative check, live run against a real model and a real Future AGI | + +## Run + +```bash +export JAVA_HOME= # 21 tested +./run_e2e.sh # tests + "no quarkus-opentelemetry exports nothing", no keys needed + +# live: a real OpenAI-compatible model and a real Future AGI +FI_API_KEY=... FI_SECRET_KEY=... FI_BASE_URL=https://api.futureagi.com FI_PROJECT_NAME=quarkus-demo \ +OPENAI_API_KEY=... OPENAI_BASE_URL=https://api.openai.com/v1/ OPENAI_MODEL=gpt-4o-mini ./run_e2e.sh + +# the app alone +mvn package -DskipTests +java -jar target/quarkus-app/quarkus-run.jar "What is the weather in Paris?" + +# the older pair +QLC4J_VERSION=1.8.2 QUARKUS_VERSION=3.27.2 ./run_e2e.sh +``` + +## What one question exports + +``` +langchain4j.aiservices.Assistant.chat root (AI service span) +├── completion gen_ai.operation.name=chat, gen_ai.provider.name=open_ai, +│ └── POST /chat/completions gen_ai.request/response.model, gen_ai.usage.input/output_tokens +├── langchain4j.tools.currentWeather gen_ai.operation.name=execute_tool, gen_ai.tool.name, gen_ai.tool.call.id +└── completion + └── POST /chat/completions Quarkus REST client span: method, url.full, status code +``` + +In Future AGI the model spans are type `llm` with model and tokens, the tool span is type `tool`, the AI service and +REST client spans are `unknown`. Resource: `project_name`, `project_type=observe`, `service.name`, `service.version`, +`telemetry.sdk.*`, `webengine.*`. + +## Privacy defaults and opt-ins + +| What | Default | Opt in | +| --- | --- | --- | +| Prompt, completion, tool arguments, tool result | off (upstream default, written out) | `quarkus.langchain4j.tracing.include-prompt`, `-completion`, `-tool-arguments`, `-tool-result` = `true` (or `QUARKUS_LANGCHAIN4J_TRACING_INCLUDE_PROMPT=true` ...). Stored as span attributes `gen_ai.prompt`, `gen_ai.completion`, `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result`; Future AGI's Input/Output columns stay empty. | +| Error message in the span status | off | `FI_EXPORT_ERROR_DETAILS=true` (`fi.export-error-details`) | +| Exception events (`exception.message`, full `exception.stacktrace`) | off: `quarkus.otel.span.event.count.limit=0` | raise the limit. Future AGI stores only the event name and time. | +| `host.name` resource attribute | removed by `FutureAgiTracingDefaults.DropHostName` | none | + +## Upstream behaviour the recipe works around (asserted in `UpstreamDefaultsTest`) + +- Quarkus appends `/v1/traces` to `quarkus.otel.exporter.otlp.traces.endpoint`. Set it to `{base}/tracer`; a value + ending in `/tracer/v1/traces` posts to `/tracer/v1/traces/v1/traces`. +- Quarkus always adds `host.name` to the resource, after `quarkus.otel.resource.attributes` and after the + `disabled-keys` filter, so configuration cannot remove it. +- Quarkus LangChain4j calls `recordException` on failures but never sets the span status, so failures arrive as + `UNSET` with the exception message and stack trace in an event. +- `quarkus.otel.span.event.count.limit=0` needs OpenTelemetry SDK 1.51+. Quarkus 3.27.2 (SDK 1.44.1) refuses to start + with it. + +## Older versions + +1.8.2 (the PRD pin) was run live on Quarkus 3.27.2 with `-Dpre112` (leaves `ToolErrorStatus` out, skips the tests): +the trace shape is the same, but the model spans carry no `gen_ai.operation.name` and use +`gen_ai.usage.prompt_tokens`/`completion_tokens`, so Future AGI shows them as `unknown` without tokens. The source shows +the current names from quarkus-langchain4j 1.11.0 and `ToolSpanContributor` from 1.12.0. diff --git a/java/examples/quarkus-langchain4j-example/pom.xml b/java/examples/quarkus-langchain4j-example/pom.xml new file mode 100644 index 000000000..cf4ac511f --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/pom.xml @@ -0,0 +1,171 @@ + + + 4.0.0 + + ai.traceai.examples + quarkus-langchain4j-example + 1.0.0 + TraceAI Quarkus LangChain4j Example + Quarkus LangChain4j exporting its own OpenTelemetry spans to Future AGI. Configuration only, no traceAI module. + + + 21 + UTF-8 + + 1.14.2 + 3.33.3.1 + 1.11.1-alpha + + 4.36.2 + 3.5.4 + 3.14.1 + + + + + + io.quarkus.platform + quarkus-bom + ${quarkus.platform.version} + pom + import + + + io.quarkiverse.langchain4j + quarkus-langchain4j-bom + ${quarkus-langchain4j.version} + pom + import + + + + + + + + io.quarkiverse.langchain4j + quarkus-langchain4j-openai + + + + io.quarkus + quarkus-junit5 + test + + + + io.opentelemetry.proto + opentelemetry-proto + ${opentelemetry-proto.version} + test + + + com.google.protobuf + protobuf-java + ${protobuf-java.version} + test + + + + + + + otel + + + !noOtel + + + + + io.quarkus + quarkus-opentelemetry + + + + + no-otel + + + noOtel + + + + + true + + + + + pre-1.12 + + + pre112 + + + + true + + + + + maven-compiler-plugin + + + **/ToolErrorStatus.java + + + + + + + + + + + + io.quarkus.platform + quarkus-maven-plugin + ${quarkus.platform.version} + true + + + + build + + + + + + maven-compiler-plugin + ${compiler-plugin.version} + + true + + + + maven-surefire-plugin + ${surefire-plugin.version} + + + org.jboss.logmanager.LogManager + + + + + + diff --git a/java/examples/quarkus-langchain4j-example/run_e2e.sh b/java/examples/quarkus-langchain4j-example/run_e2e.sh new file mode 100755 index 000000000..1ebcb9e4d --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/run_e2e.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# End-to-end run of the Quarkus LangChain4j example. +# 1. tests against a scripted model server and a stand-in OTLP collector (no keys needed) +# 2. live: the packaged app, configured only through application.properties + environment, calls a real +# OpenAI-compatible model and exports to a real Future AGI: +# a) one question that makes a tool call -> project $FI_PROJECT_NAME +# b) one failing model call (unknown model) -> project $FI_PROJECT_NAME +# c) the same two with the content opt-ins on -> project $FI_PROJECT_NAME-optin +# 3. flush: a batch-processor build (quarkus.otel.simple=false) exits normally and its spans still arrive +# -> project $FI_PROJECT_NAME-batch +# 4. negative: built without quarkus-opentelemetry, nothing is exported (local stub collector) +# +# Needs JDK 17+ (21 tested) and Maven; JAVA_HOME is respected. +# Live steps need: FI_API_KEY, FI_SECRET_KEY, OPENAI_API_KEY and optionally FI_BASE_URL (default +# https://api.futureagi.com), FI_PROJECT_NAME (default quarkus-langchain4j-example), OPENAI_BASE_URL, OPENAI_MODEL, +# OPENAI_FAIL_MODEL (a model name the endpoint rejects, default no-such-model-quarkus-example), MARKER. +# QLC4J_VERSION + QUARKUS_VERSION select another tested pair, e.g. 1.8.2 + 3.27.2 (tests are skipped on 1.8.x). +set -u +cd "$(dirname "$0")" + +MVN_ARGS=(-B -q) +if [ -n "${QLC4J_VERSION:-}" ]; then + MVN_ARGS+=("-Dquarkus-langchain4j.version=$QLC4J_VERSION" "-Dquarkus.platform.version=${QUARKUS_VERSION:?set QUARKUS_VERSION too}") + case "$QLC4J_VERSION" in 1.8.*|1.9.*|1.10.*|1.11.*) MVN_ARGS+=(-Dpre112) ;; esac + case "$QUARKUS_VERSION" in 3.2[0-9].*|3.1[0-9].*) + # OpenTelemetry SDK < 1.51 refuses quarkus.otel.span.event.count.limit=0 at startup. + export QUARKUS_OTEL_SPAN_EVENT_COUNT_LIMIT=128 + echo "note: Quarkus $QUARKUS_VERSION: span events back on, exception messages and stack traces are exported" ;; + esac +fi +RUN() { java -jar "$1/quarkus-run.jar" "${@:2}"; } +expect_answer() { RUN "$@" || { echo "FAIL: the AI service call did not succeed"; exit 1; }; } +expect_model_failure() { + local out; out=$(RUN "$@" 2>&1) + if echo "$out" | grep -q "The AI service call failed"; then echo "model call failed as expected"; else + echo "FAIL: the app did not reach the model call"; echo "$out" | grep -E "ERROR|Caused by" | head -5; exit 1; fi +} + +echo "== 1. tests" +mvn "${MVN_ARGS[@]}" clean test || { echo "TESTS FAILED"; exit 1; } + +echo "== build: default app, batch-processor app, app without quarkus-opentelemetry" +mvn "${MVN_ARGS[@]}" -DskipTests package || exit 1 +mvn "${MVN_ARGS[@]}" -DskipTests package -Dquarkus.otel.simple=false -Dquarkus.package.output-directory=batch-app || exit 1 +mvn "${MVN_ARGS[@]}" -DskipTests package -DnoOtel -Dquarkus.package.output-directory=no-otel-app || exit 1 + +echo "== 4. negative: no quarkus-opentelemetry, local stub collector" +STUB_PORT="${STUB_PORT:-18270}" +rm -f /tmp/quarkus-lc4j-stub-hits +python3 - "$STUB_PORT" <<'PY' & +import http.server, sys +class H(http.server.BaseHTTPRequestHandler): + def do_POST(s): + s.rfile.read(int(s.headers.get('content-length', 0))) + open('/tmp/quarkus-lc4j-stub-hits', 'a').write(s.path + '\n') + s.send_response(200); s.send_header('content-length', '0'); s.end_headers() + def log_message(*a): pass +http.server.HTTPServer(('127.0.0.1', int(sys.argv[1])), H).serve_forever() +PY +STUB=$! +sleep 1 +# The model call fails fast (closed port): only the export path matters here. +FI_BASE_URL="http://127.0.0.1:$STUB_PORT" FI_API_KEY=k FI_SECRET_KEY=s FI_PROJECT_NAME=negative \ + OPENAI_API_KEY=k OPENAI_BASE_URL=http://127.0.0.1:1/v1/ RUN target/no-otel-app "What is the weather in Paris?" > /tmp/quarkus-lc4j-negative.log 2>&1 +sleep 2; kill "$STUB" 2> /dev/null +if [ -s /tmp/quarkus-lc4j-stub-hits ]; then echo "FAIL exported without quarkus-opentelemetry"; exit 1; fi +echo "PASS nothing exported without quarkus-opentelemetry" + +if [ -z "${FI_API_KEY:-}" ] || [ -z "${FI_SECRET_KEY:-}" ] || [ -z "${OPENAI_API_KEY:-}" ]; then + echo "FI_API_KEY / FI_SECRET_KEY / OPENAI_API_KEY not set: skipping the live steps."; exit 0 +fi +export FI_PROJECT_NAME="${FI_PROJECT_NAME:-quarkus-langchain4j-example}" +MARKER="${MARKER:-QLC4J$(date +%s)}" +FAIL_MODEL="${OPENAI_FAIL_MODEL:-no-such-model-quarkus-example}" +BASE="$FI_PROJECT_NAME" + +echo "== 2a. live tool call -> $BASE (marker $MARKER)" +expect_answer target/quarkus-app "What is the weather in Paris$MARKER right now?" +echo "== 2b. live failing model call -> $BASE" +OPENAI_MODEL="$FAIL_MODEL" expect_model_failure target/quarkus-app "What is the weather in Oslo$MARKER right now?" + +echo "== 2c. content opt-in -> $BASE-optin" +export QUARKUS_LANGCHAIN4J_TRACING_INCLUDE_PROMPT=true QUARKUS_LANGCHAIN4J_TRACING_INCLUDE_COMPLETION=true \ + QUARKUS_LANGCHAIN4J_TRACING_INCLUDE_TOOL_ARGUMENTS=true QUARKUS_LANGCHAIN4J_TRACING_INCLUDE_TOOL_RESULT=true \ + FI_EXPORT_ERROR_DETAILS=true +FI_PROJECT_NAME="$BASE-optin" expect_answer target/quarkus-app "What is the weather in Rome$MARKER right now?" +FI_PROJECT_NAME="$BASE-optin" OPENAI_MODEL="$FAIL_MODEL" expect_model_failure target/quarkus-app "What is the weather in Bern$MARKER right now?" +unset QUARKUS_LANGCHAIN4J_TRACING_INCLUDE_PROMPT QUARKUS_LANGCHAIN4J_TRACING_INCLUDE_COMPLETION \ + QUARKUS_LANGCHAIN4J_TRACING_INCLUDE_TOOL_ARGUMENTS QUARKUS_LANGCHAIN4J_TRACING_INCLUDE_TOOL_RESULT FI_EXPORT_ERROR_DETAILS + +echo "== 3. batch processor, normal exit -> $BASE-batch" +FI_PROJECT_NAME="$BASE-batch" expect_answer target/batch-app "What is the weather in Madrid$MARKER right now?" + +echo "sent. Check the projects in Future AGI: $BASE (tool-call trace + ERROR trace), $BASE-optin, $BASE-batch." +echo "marker: $MARKER" diff --git a/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/AskCommand.java b/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/AskCommand.java new file mode 100644 index 000000000..dc3db897d --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/AskCommand.java @@ -0,0 +1,33 @@ +package ai.traceai.examples.quarkus; + +import io.quarkus.runtime.QuarkusApplication; +import io.quarkus.runtime.annotations.QuarkusMain; +import jakarta.enterprise.context.control.ActivateRequestContext; +import jakarta.inject.Inject; + +/** + * Command-mode entry point: asks one question, prints the answer and exits. Quarkus shuts the OpenTelemetry SDK down + * on exit, which flushes any span still queued. + * + *
java -jar target/quarkus-app/quarkus-run.jar "What is the weather in Paris?"
+ */ +@QuarkusMain +public class AskCommand implements QuarkusApplication { + + @Inject + Assistant assistant; + + @Override + @ActivateRequestContext + public int run(String... args) { + String question = args.length > 0 ? String.join(" ", args) : "What is the weather in Paris right now?"; + try { + System.out.println(assistant.chat(question)); + return 0; + } catch (RuntimeException e) { + // Only the exception type: the message can contain the prompt or the provider's error body. + System.err.println("The AI service call failed: " + e.getClass().getName()); + return 1; + } + } +} diff --git a/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/Assistant.java b/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/Assistant.java new file mode 100644 index 000000000..12472f2ac --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/Assistant.java @@ -0,0 +1,16 @@ +package ai.traceai.examples.quarkus; + +import dev.langchain4j.service.SystemMessage; +import dev.langchain4j.service.UserMessage; +import io.quarkiverse.langchain4j.RegisterAiService; + +/** + * The AI service. Quarkus LangChain4j generates the implementation at build time and, with quarkus-opentelemetry on + * the classpath, opens a span named {@code langchain4j.aiservices.Assistant.chat} for every call. + */ +@RegisterAiService(tools = WeatherTools.class) +public interface Assistant { + + @SystemMessage("You are a weather assistant. Always call the currentWeather tool for the city in the question, then answer in one short sentence.") + String chat(@UserMessage String question); +} diff --git a/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/FutureAgiTracingDefaults.java b/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/FutureAgiTracingDefaults.java new file mode 100644 index 000000000..84f9124e5 --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/FutureAgiTracingDefaults.java @@ -0,0 +1,88 @@ +package ai.traceai.examples.quarkus; + +import dev.langchain4j.model.chat.listener.ChatModelErrorContext; +import dev.langchain4j.observability.api.event.AiServiceErrorEvent; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.quarkiverse.langchain4j.runtime.listeners.ChatModelSpanContributor; +import io.quarkus.arc.Unremovable; +import io.quarkus.opentelemetry.runtime.AutoConfiguredOpenTelemetrySdkBuilderCustomizer; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; +import jakarta.annotation.Priority; +import jakarta.enterprise.event.Observes; +import jakarta.inject.Singleton; +import org.eclipse.microprofile.config.inject.ConfigProperty; + +/** + * Future AGI defaults for the spans Quarkus LangChain4j already creates. Application code: copy this file (and + * {@link ToolErrorStatus}) into your application. Nothing here creates spans. + * + *
    + *
  1. Failures become {@code ERROR}. Quarkus LangChain4j records the exception on the AI service span and on the + * model span but leaves the span status {@code UNSET}, so a failed call would look successful.
  2. + *
  3. The status description stays empty unless {@code fi.export-error-details=true} + * (env {@code FI_EXPORT_ERROR_DETAILS=true}). Exception messages often contain the prompt, a tool argument or the + * provider's error body. The exception events themselves are dropped by {@code quarkus.otel.span.event.count.limit=0} + * in application.properties.
  4. + *
  5. {@link DropHostName} removes {@code host.name}, which Quarkus always adds to the resource.
  6. + *
+ */ +@Singleton +public class FutureAgiTracingDefaults implements ChatModelSpanContributor { + + static final String AI_SERVICE_SPAN_PREFIX = "langchain4j.aiservices."; + + private final boolean exportErrorDetails; + + public FutureAgiTracingDefaults( + @ConfigProperty(name = "fi.export-error-details", defaultValue = "false") boolean exportErrorDetails) { + this.exportErrorDetails = exportErrorDetails; + } + + /** The model span ({@code completion }) of a failed model call. */ + @Override + public void onError(ChatModelErrorContext errorContext, Span currentSpan) { + markError(currentSpan, errorContext.error(), exportErrorDetails); + } + + /** + * Fired by Quarkus LangChain4j for any failure of an AI service method (model error, tool error that propagates, + * output parsing). It runs while the {@code langchain4j.aiservices.*} span is still the current span. + */ + void onAiServiceError(@Observes AiServiceErrorEvent event) { + Span current = Span.current(); + if (current instanceof ReadableSpan readable && readable.getName().startsWith(AI_SERVICE_SPAN_PREFIX)) { + markError(current, event.error(), exportErrorDetails); + } + } + + static void markError(Span span, Throwable error, boolean withDetails) { + String description = withDetails && error != null && error.getMessage() != null ? error.getMessage() : ""; + span.setStatus(StatusCode.ERROR, description); + } + + /** + * Quarkus puts {@code host.name} on the resource after {@code quarkus.otel.resource.attributes} and after the + * resource-key filter, so configuration cannot remove it. Quarkus applies these customizers in descending bean + * priority; the lowest priority makes this one run after Quarkus' own resource customizer. + */ + @Singleton + @Unremovable + @Priority(Integer.MIN_VALUE) + public static class DropHostName implements AutoConfiguredOpenTelemetrySdkBuilderCustomizer { + + static final AttributeKey HOST_NAME = AttributeKey.stringKey("host.name"); + + @Override + public void customize(AutoConfiguredOpenTelemetrySdkBuilder builder) { + builder.addResourceCustomizer((resource, config) -> withoutHostName(resource)); + } + + static Resource withoutHostName(Resource resource) { + return resource.toBuilder().removeIf(HOST_NAME::equals).build(); + } + } +} diff --git a/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/ToolErrorStatus.java b/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/ToolErrorStatus.java new file mode 100644 index 000000000..2af2befe3 --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/ToolErrorStatus.java @@ -0,0 +1,27 @@ +package ai.traceai.examples.quarkus; + +import io.opentelemetry.api.trace.Span; +import io.quarkiverse.langchain4j.runtime.tool.ToolExecutionErrorContext; +import io.quarkiverse.langchain4j.runtime.tool.ToolSpanContributor; +import jakarta.inject.Singleton; +import org.eclipse.microprofile.config.inject.ConfigProperty; + +/** + * Marks the {@code langchain4j.tools.} span of a tool that threw as {@code ERROR}. Quarkus LangChain4j records + * the exception but leaves the status {@code UNSET}. {@code ToolSpanContributor} exists from Quarkus LangChain4j 1.12.0; + * on older lines leave this file out. + */ +@Singleton +public class ToolErrorStatus implements ToolSpanContributor { + + private final boolean exportErrorDetails; + + public ToolErrorStatus(@ConfigProperty(name = "fi.export-error-details", defaultValue = "false") boolean exportErrorDetails) { + this.exportErrorDetails = exportErrorDetails; + } + + @Override + public void onError(ToolExecutionErrorContext context, Span span) { + FutureAgiTracingDefaults.markError(span, context.error(), exportErrorDetails); + } +} diff --git a/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/WeatherTools.java b/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/WeatherTools.java new file mode 100644 index 000000000..f9f30daaf --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/src/main/java/ai/traceai/examples/quarkus/WeatherTools.java @@ -0,0 +1,18 @@ +package ai.traceai.examples.quarkus; + +import dev.langchain4j.agent.tool.Tool; +import jakarta.enterprise.context.ApplicationScoped; + +/** One deterministic tool, so a single question produces a model call, a tool call and a second model call. */ +@ApplicationScoped +public class WeatherTools { + + @Tool("Returns the current weather for a city") + public String currentWeather(String city) { + if (city == null || city.isBlank() || city.startsWith("Atlantis")) { + // The message carries user input on purpose: the tests check it never leaves the JVM by default. + throw new IllegalArgumentException("No weather station for " + city); + } + return "Sunny, 21 C in " + city; + } +} diff --git a/java/examples/quarkus-langchain4j-example/src/main/resources/application.properties b/java/examples/quarkus-langchain4j-example/src/main/resources/application.properties new file mode 100644 index 000000000..af2804b1d --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/src/main/resources/application.properties @@ -0,0 +1,33 @@ +# --- Model provider: any OpenAI-compatible endpoint --------------------------------------------------------------- +quarkus.langchain4j.openai.api-key=${OPENAI_API_KEY} +quarkus.langchain4j.openai.base-url=${OPENAI_BASE_URL:https://api.openai.com/v1/} +quarkus.langchain4j.openai.chat-model.model-name=${OPENAI_MODEL:gpt-4o-mini} +quarkus.langchain4j.timeout=60s + +# --- Future AGI: Quarkus OpenTelemetry exports the spans Quarkus LangChain4j creates -------------------------------- +# HTTP/protobuf, not the gRPC default. Quarkus appends /v1/traces to this URL, so stop at /tracer. +quarkus.otel.exporter.otlp.traces.protocol=http/protobuf +quarkus.otel.exporter.otlp.traces.endpoint=${FI_BASE_URL:https://api.futureagi.com}/tracer +quarkus.otel.exporter.otlp.traces.headers=X-Api-Key=${FI_API_KEY},X-Secret-Key=${FI_SECRET_KEY} +# Already Quarkus' default. Kept explicit: the Future AGI collector does not accept gzip bodies. +quarkus.otel.exporter.otlp.traces.compression=none +quarkus.otel.resource.attributes=project_name=${FI_PROJECT_NAME},project_type=observe + +# --- Privacy defaults ---------------------------------------------------------------------------------------------- +# Prompt, completion, tool arguments and tool results stay off (Quarkus LangChain4j defaults, written out). +# Turn them on with the four properties set to true, or QUARKUS_LANGCHAIN4J_TRACING_INCLUDE_PROMPT=true and so on. +quarkus.langchain4j.tracing.include-prompt=false +quarkus.langchain4j.tracing.include-completion=false +quarkus.langchain4j.tracing.include-tool-arguments=false +quarkus.langchain4j.tracing.include-tool-result=false +# No span events. Quarkus LangChain4j records failures as exception events carrying exception.message and the full +# exception.stacktrace; this drops them. Failed spans still get status ERROR from FutureAgiTracingDefaults. +# 0 needs the OpenTelemetry SDK 1.51+ (Quarkus 3.33.3.1 ships 1.57.0; Quarkus 3.27.2 ships 1.44.1 and refuses to start). +quarkus.otel.span.event.count.limit=0 + +# --- This example is a short command-line run -------------------------------------------------------------------- +# Export each span as it ends instead of waiting for the batch timer (build-time property). A long-running service +# can drop this line: the batch is flushed when Quarkus shuts down normally. +quarkus.otel.simple=true +quarkus.banner.enabled=false +quarkus.log.level=WARN diff --git a/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/ContentOptInTest.java b/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/ContentOptInTest.java new file mode 100644 index 000000000..4f93c3a58 --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/ContentOptInTest.java @@ -0,0 +1,83 @@ +package ai.traceai.examples.quarkus; + +import static ai.traceai.examples.quarkus.DefaultExportTest.COMPLETION_SPAN; +import static ai.traceai.examples.quarkus.DefaultExportTest.TOOL_SPAN; +import static ai.traceai.examples.quarkus.Exported.AI_SERVICE_SPAN; +import static ai.traceai.examples.quarkus.Exported.awaitTrace; +import static ai.traceai.examples.quarkus.Exported.named; +import static ai.traceai.examples.quarkus.Exported.only; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import ai.traceai.examples.quarkus.Exported.ExportedSpan; +import io.opentelemetry.proto.trace.v1.Status; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Conformance profile: the documented opt-ins turned on. Not a production setting. */ +@QuarkusTest +@QuarkusTestResource(StandIns.class) +@TestProfile(ContentOptInTest.OptIn.class) +class ContentOptInTest { + + public static class OptIn implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "quarkus.langchain4j.tracing.include-prompt", "true", + "quarkus.langchain4j.tracing.include-completion", "true", + "quarkus.langchain4j.tracing.include-tool-arguments", "true", + "quarkus.langchain4j.tracing.include-tool-result", "true", + "fi.export-error-details", "true"); + } + } + + @Inject + Assistant assistant; + + String marker; + + @BeforeEach + void reset() { + StandIns.collector.clear(); + marker = "QLC4JOPTIN" + System.nanoTime(); + } + + @Test + void optInExportsPromptCompletionToolArgumentsAndToolResult() throws Exception { + assistant.chat("What is the weather in " + marker + "?"); + List spans = awaitTrace(); + + List completions = named(spans, COMPLETION_SPAN); + assertTrue(completions.stream().allMatch(c -> c.attribute("gen_ai.prompt").orElse("").contains(marker)), + Exported.tree(spans)); + assertTrue(completions.stream().anyMatch(c -> c.attribute("gen_ai.completion").orElse("").contains(marker))); + Map tool = only(spans, TOOL_SPAN).attributes(); + assertEquals("{\"city\":\"" + marker + "\"}", tool.get("gen_ai.tool.call.arguments")); + assertEquals("Sunny, 21 C in " + marker, tool.get("gen_ai.tool.call.result")); + assertFalse(Exported.everythingReceived().contains(StandIns.PROVIDER_KEY)); + } + + @Test + void optInPutsTheErrorMessageInTheSpanStatus() throws Exception { + assertThrows(RuntimeException.class, () -> assistant.chat("What is the weather in MODEL-FAIL-" + marker + "?")); + List spans = awaitTrace(); + + for (ExportedSpan s : List.of(only(spans, AI_SERVICE_SPAN), only(spans, COMPLETION_SPAN))) { + assertEquals(Status.StatusCode.STATUS_CODE_ERROR, s.span().getStatus().getCode()); + assertTrue(s.span().getStatus().getMessage().contains(marker), s.span().getStatus().getMessage()); + } + // Exception events stay off: that is quarkus.otel.span.event.count.limit, a separate switch. + assertTrue(spans.stream().allMatch(s -> s.span().getEventsCount() == 0)); + assertTrue(spans.stream().noneMatch(s -> s.resource().containsKey("host.name"))); + } +} diff --git a/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/DefaultExportTest.java b/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/DefaultExportTest.java new file mode 100644 index 000000000..ef0bb42c1 --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/DefaultExportTest.java @@ -0,0 +1,187 @@ +package ai.traceai.examples.quarkus; + +import static ai.traceai.examples.quarkus.Exported.AI_SERVICE_SPAN; +import static ai.traceai.examples.quarkus.Exported.awaitTrace; +import static ai.traceai.examples.quarkus.Exported.named; +import static ai.traceai.examples.quarkus.Exported.only; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import ai.traceai.examples.quarkus.Exported.ExportedSpan; +import io.opentelemetry.proto.trace.v1.Status; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; +import java.net.InetAddress; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** The shipped application.properties, unchanged: content off, error text off, no host name. */ +@QuarkusTest +@QuarkusTestResource(StandIns.class) +class DefaultExportTest { + + static final String COMPLETION_SPAN = "completion " + StandIns.MODEL; + static final String TOOL_SPAN = "langchain4j.tools.currentWeather"; + static final List CONTENT_KEYS = List.of("gen_ai.prompt", "gen_ai.completion", "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result"); + + @Inject + Assistant assistant; + + String marker; + + @BeforeEach + void reset() { + StandIns.collector.clear(); + StandIns.model.clear(); + marker = "QLC4JMARK" + System.nanoTime(); + } + + @Test + void oneQuestionIsOneTraceWithTheAiServiceSpanAsRoot() throws Exception { + assistant.chat("What is the weather in Lisbon?"); + List spans = awaitTrace(); + + ExportedSpan root = only(spans, AI_SERVICE_SPAN); + assertTrue(root.parentId().isEmpty(), "AI service span is the root in a command-mode app"); + List completions = named(spans, COMPLETION_SPAN); + assertEquals(2, completions.size(), Exported.tree(spans)); + completions.forEach(c -> assertEquals(root.spanId(), c.parentId(), "model span is a child of the AI service span")); + assertEquals(root.spanId(), only(spans, TOOL_SPAN).parentId(), "tool span is a child of the AI service span"); + List http = named(spans, "POST /chat/completions"); + assertEquals(2, http.size(), "one REST client span per model call"); + http.forEach(h -> assertTrue(completions.stream().anyMatch(c -> c.spanId().equals(h.parentId())))); + assertEquals(1, spans.stream().map(ExportedSpan::traceId).distinct().count()); + assertEquals(1, spans.stream().filter(s -> s.parentId().isEmpty()).count(), "one root"); + } + + @Test + void modelAndToolSpansCarryGenAiAttributes() throws Exception { + assistant.chat("What is the weather in Lisbon?"); + List spans = awaitTrace(); + + List completions = named(spans, COMPLETION_SPAN); + for (ExportedSpan c : completions) { + Map a = c.attributes(); + assertEquals("chat", a.get("gen_ai.operation.name")); + assertEquals("open_ai", a.get("gen_ai.provider.name")); + assertEquals(StandIns.MODEL, a.get("gen_ai.request.model")); + assertEquals("gpt-test-2026-10-01", a.get("gen_ai.response.model")); + } + assertEquals(List.of("31/7", "52/11"), completions.stream() + .map(c -> c.attributes().get("gen_ai.usage.input_tokens") + "/" + c.attributes().get("gen_ai.usage.output_tokens")) + .sorted().toList()); + Map tool = only(spans, TOOL_SPAN).attributes(); + assertEquals("execute_tool", tool.get("gen_ai.operation.name")); + assertEquals("currentWeather", tool.get("gen_ai.tool.name")); + } + + @Test + void exportIsHttpProtobufToTheTracerRouteWithBothHeadersAndNoGzip() throws Exception { + assistant.chat("What is the weather in Lisbon?"); + awaitTrace(); + + List posts = StandIns.collector.posts(); + assertFalse(posts.isEmpty()); + for (OtlpRecorder.Post post : posts) { + assertEquals("POST", post.method()); + assertEquals("/tracer/v1/traces", post.path(), "traces only, on the Future AGI route"); + assertEquals("application/x-protobuf", post.headers().get("Content-Type")); + assertFalse(post.headers().containsKey("Content-Encoding"), "no gzip"); + assertEquals(StandIns.FI_API_KEY, post.headers().get("X-Api-Key")); + assertEquals(StandIns.FI_SECRET_KEY, post.headers().get("X-Secret-Key")); + } + } + + @Test + void resourceCarriesTheProjectAndNoHostName() throws Exception { + assistant.chat("What is the weather in Lisbon?"); + for (ExportedSpan s : awaitTrace()) { + Map resource = s.resource(); + assertEquals(StandIns.PROJECT, resource.get("project_name")); + assertEquals("observe", resource.get("project_type")); + assertEquals("quarkus-langchain4j-example", resource.get("service.name")); + assertFalse(resource.keySet().stream().anyMatch(k -> k.startsWith("host.") || k.startsWith("process.")), + resource.toString()); + } + } + + @Test + void contentIsOffByDefault() throws Exception { + assistant.chat("What is the weather in " + marker + "?"); + List spans = awaitTrace(); + + assertTrue(StandIns.model.requests().stream().anyMatch(r -> r.body().contains(marker)), "the model did see it"); + for (ExportedSpan s : spans) { + for (String key : CONTENT_KEYS) { + assertFalse(s.attributes().containsKey(key), key + " on " + s.name()); + } + } + assertNothingPrivateReceived(); + } + + @Test + void failedModelCallIsErrorWithoutTheErrorText() throws Exception { + assertThrows(RuntimeException.class, () -> assistant.chat("What is the weather in MODEL-FAIL-" + marker + "?")); + List spans = awaitTrace(); + + for (ExportedSpan s : List.of(only(spans, AI_SERVICE_SPAN), only(spans, COMPLETION_SPAN))) { + assertEquals(Status.StatusCode.STATUS_CODE_ERROR, s.span().getStatus().getCode(), s.name()); + assertEquals("", s.span().getStatus().getMessage(), s.name()); + } + assertTrue(spans.stream().allMatch(s -> s.span().getEventsCount() == 0), "exception events are dropped"); + assertNothingPrivateReceived(); + } + + @Test + void failedToolIsErrorOnTheToolSpan() throws Exception { + // The tool throws "No weather station for Atlantis-". Quarkus LangChain4j sends that text back to the + // model as the tool result, so the AI service call itself succeeds. + assistant.chat("What is the weather in Atlantis-" + marker + "?"); + List spans = awaitTrace(); + + ExportedSpan tool = only(spans, TOOL_SPAN); + assertEquals(Status.StatusCode.STATUS_CODE_ERROR, tool.span().getStatus().getCode()); + assertEquals("", tool.span().getStatus().getMessage()); + assertEquals(Status.StatusCode.STATUS_CODE_UNSET, only(spans, AI_SERVICE_SPAN).span().getStatus().getCode()); + assertEquals(0, tool.span().getEventsCount()); + assertNothingPrivateReceived(); + } + + @Test + void responsesWithoutUsageGetNoTokenAttributes() throws Exception { + assistant.chat("NO-USAGE What is the weather in Lisbon?"); + for (ExportedSpan c : named(awaitTrace(), COMPLETION_SPAN)) { + assertFalse(c.attributes().keySet().stream().anyMatch(k -> k.startsWith("gen_ai.usage.")), c.attributes().toString()); + } + } + + @Test + void futureAgiKeysOnlyGoToTheCollector() throws Exception { + assistant.chat("What is the weather in Lisbon?"); + awaitTrace(); + for (MockOpenAi.Request r : StandIns.model.requests()) { + String all = r.headers() + r.body(); + assertFalse(all.contains(StandIns.FI_API_KEY) || all.contains(StandIns.FI_SECRET_KEY)); + assertEquals("Bearer " + StandIns.PROVIDER_KEY, r.headers().get("Authorization")); + } + } + + /** Scans every byte the collector received, headers included. */ + private void assertNothingPrivateReceived() throws Exception { + String received = Exported.everythingReceived(); + assertFalse(received.contains(marker), "marker exported"); + assertFalse(received.contains(StandIns.PROVIDER_KEY), "provider key exported"); + assertFalse(received.contains("\tat ") || received.contains("Exception"), "stack trace or exception type exported"); + assertFalse(received.contains("upstream rejected") || received.contains("No weather station"), "error text exported"); + assertFalse(received.contains(InetAddress.getLocalHost().getHostName()), "host name exported"); + assertFalse(received.contains(System.getProperty("user.home")), "home directory exported"); + assertFalse(received.contains(System.getProperty("user.dir")), "working directory exported"); + assertFalse(received.contains("/" + System.getProperty("user.name") + "/"), "user name exported"); + } +} diff --git a/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/Exported.java b/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/Exported.java new file mode 100644 index 000000000..c9d476c7e --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/Exported.java @@ -0,0 +1,140 @@ +package ai.traceai.examples.quarkus; + +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; +import io.opentelemetry.proto.trace.v1.Span; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; + +/** What the stand-in collector received, decoded. */ +final class Exported { + + record ExportedSpan(Span span, Map resource) { + String name() { + return span.getName(); + } + + String spanId() { + return HexFormat.of().formatHex(span.getSpanId().toByteArray()); + } + + String parentId() { + return HexFormat.of().formatHex(span.getParentSpanId().toByteArray()); + } + + String traceId() { + return HexFormat.of().formatHex(span.getTraceId().toByteArray()); + } + + Map attributes() { + return toMap(span.getAttributesList()); + } + + Optional attribute(String key) { + return Optional.ofNullable(attributes().get(key)); + } + } + + private Exported() { + } + + static List spans() { + List out = new ArrayList<>(); + for (OtlpRecorder.Post post : StandIns.collector.posts()) { + for (ResourceSpans rs : post.decode().getResourceSpansList()) { + Map resource = toMap(rs.getResource().getAttributesList()); + for (ScopeSpans ss : rs.getScopeSpansList()) { + for (Span s : ss.getSpansList()) { + out.add(new ExportedSpan(s, resource)); + } + } + } + } + return out; + } + + /** Waits until a span matching {@code done} has been received (the simple processor exports asynchronously). */ + static List awaitSpans(Predicate done, Duration timeout) throws InterruptedException { + long deadline = System.nanoTime() + timeout.toNanos(); + while (System.nanoTime() < deadline) { + List spans = spans(); + if (spans.stream().anyMatch(done)) { + Thread.sleep(200); // let sibling spans of the same trace land + return spans(); + } + Thread.sleep(50); + } + throw new AssertionError("no matching span within " + timeout + ", got " + spans().stream().map(ExportedSpan::name).toList()); + } + + static final String AI_SERVICE_SPAN = "langchain4j.aiservices.Assistant.chat"; + + /** Waits for the AI service span (it ends last) and returns every span of its trace. */ + static List awaitTrace() throws InterruptedException { + List all = awaitSpans(s -> s.name().equals(AI_SERVICE_SPAN), Duration.ofSeconds(10)); + String traceId = all.stream().filter(s -> s.name().equals(AI_SERVICE_SPAN)).reduce((a, b) -> b).orElseThrow().traceId(); + return all.stream().filter(s -> s.traceId().equals(traceId)).toList(); + } + + static ExportedSpan only(List spans, String name) { + List matching = spans.stream().filter(s -> s.name().equals(name)).toList(); + if (matching.size() != 1) { + throw new AssertionError("expected one span " + name + ", got " + matching.size() + "\n" + tree(spans)); + } + return matching.get(0); + } + + static List named(List spans, String name) { + return spans.stream().filter(s -> s.name().equals(name)).toList(); + } + + /** Every byte the collector received, headers included, as one string for leak scans. */ + static String everythingReceived() { + StringBuilder all = new StringBuilder(); + for (OtlpRecorder.Post post : StandIns.collector.posts()) { + all.append(post.path()).append('\n').append(post.headers()).append('\n') + .append(new String(post.body(), StandardCharsets.ISO_8859_1)).append('\n'); + } + return all.toString(); + } + + static String tree(List spans) { + StringBuilder out = new StringBuilder(); + for (ExportedSpan s : spans) { + out.append(s.traceId(), 0, 8).append(' ').append(s.spanId(), 0, 8).append(" <- ") + .append(s.parentId().isEmpty() ? "root " : s.parentId().substring(0, 8)).append(' ') + .append(s.name()).append(" status=").append(s.span().getStatus().getCode()) + .append(" desc='").append(s.span().getStatus().getMessage()).append("' events=") + .append(s.span().getEventsCount()).append(' ').append(s.attributes()).append('\n'); + } + return out.toString(); + } + + static Map toMap(List kvs) { + Map m = new LinkedHashMap<>(); + for (KeyValue kv : kvs) { + m.put(kv.getKey(), value(kv.getValue())); + } + return m; + } + + private static String value(AnyValue v) { + return switch (v.getValueCase()) { + case STRING_VALUE -> v.getStringValue(); + case INT_VALUE -> Long.toString(v.getIntValue()); + case DOUBLE_VALUE -> Double.toString(v.getDoubleValue()); + case BOOL_VALUE -> Boolean.toString(v.getBoolValue()); + default -> v.toString(); + }; + } +} diff --git a/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/MockOpenAi.java b/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/MockOpenAi.java new file mode 100644 index 000000000..7358f0b86 --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/MockOpenAi.java @@ -0,0 +1,137 @@ +package ai.traceai.examples.quarkus; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A scripted OpenAI-compatible {@code /v1/chat/completions} server shaped like the documented Chat Completions API. + *
    + *
  • "... weather in CITY?": first call answers with a {@code currentWeather} tool call, the call after the tool + * result answers with text that repeats the tool result.
  • + *
  • A question containing {@code MODEL-FAIL} gets HTTP 500 whose error body repeats the question.
  • + *
  • A question containing {@code NO-USAGE} gets responses without a {@code usage} object.
  • + *
+ */ +final class MockOpenAi implements AutoCloseable { + + static final int TOOL_CALL_PROMPT_TOKENS = 31; + static final int TOOL_CALL_COMPLETION_TOKENS = 7; + static final int ANSWER_PROMPT_TOKENS = 52; + static final int ANSWER_COMPLETION_TOKENS = 11; + private static final Pattern CITY = Pattern.compile("weather in (\\S+?)\\?"); + + record Request(Map headers, String body) { + } + + private final ObjectMapper json = new ObjectMapper(); + private final HttpServer server; + private final List requests = new CopyOnWriteArrayList<>(); + + MockOpenAi() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/v1/chat/completions", this::handle); + server.start(); + } + + private void handle(HttpExchange exchange) throws IOException { + Map headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + exchange.getRequestHeaders().forEach((k, v) -> headers.put(k, String.join(",", v))); + String body = new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8); + requests.add(new Request(headers, body)); + + JsonNode messages = json.readTree(body).path("messages"); + JsonNode last = messages.get(messages.size() - 1); + String question = ""; + for (JsonNode m : messages) { + if ("user".equals(m.path("role").asText())) { + question = m.path("content").asText(); + } + } + boolean usage = !question.contains("NO-USAGE"); + + if (question.contains("MODEL-FAIL")) { + ObjectNode error = json.createObjectNode(); + error.putObject("error").put("message", "upstream rejected the request: " + question).put("type", "server_error"); + send(exchange, 500, error); + return; + } + + ObjectNode response = json.createObjectNode(); + response.put("id", "chatcmpl-test-" + requests.size()); + response.put("object", "chat.completion"); + response.put("created", 1760000000L); + response.put("model", "gpt-test-2026-10-01"); + ObjectNode choice = response.putArray("choices").addObject(); + choice.put("index", 0); + ObjectNode message = choice.putObject("message"); + message.put("role", "assistant"); + + if ("tool".equals(last.path("role").asText())) { + message.put("content", "Here you go: " + last.path("content").asText()); + choice.put("finish_reason", "stop"); + if (usage) { + usage(response, ANSWER_PROMPT_TOKENS, ANSWER_COMPLETION_TOKENS); + } + } else { + Matcher m = CITY.matcher(question); + String city = m.find() ? m.group(1) : "Paris"; + message.putNull("content"); + ArrayNode calls = message.putArray("tool_calls"); + ObjectNode call = calls.addObject(); + call.put("id", "call_" + requests.size()); + call.put("type", "function"); + call.putObject("function").put("name", "currentWeather") + .put("arguments", json.createObjectNode().put("city", city).toString()); + choice.put("finish_reason", "tool_calls"); + if (usage) { + usage(response, TOOL_CALL_PROMPT_TOKENS, TOOL_CALL_COMPLETION_TOKENS); + } + } + send(exchange, 200, response); + } + + private static void usage(ObjectNode response, int prompt, int completion) { + response.putObject("usage").put("prompt_tokens", prompt).put("completion_tokens", completion) + .put("total_tokens", prompt + completion); + } + + private void send(HttpExchange exchange, int status, JsonNode body) throws IOException { + byte[] bytes = json.writeValueAsBytes(body); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, bytes.length); + exchange.getResponseBody().write(bytes); + exchange.close(); + } + + String baseUrl() { + return "http://127.0.0.1:" + server.getAddress().getPort() + "/v1/"; + } + + List requests() { + return new ArrayList<>(requests); + } + + void clear() { + requests.clear(); + } + + @Override + public void close() { + server.stop(0); + } +} diff --git a/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/OtlpRecorder.java b/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/OtlpRecorder.java new file mode 100644 index 000000000..41e0de2f7 --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/OtlpRecorder.java @@ -0,0 +1,60 @@ +package ai.traceai.examples.quarkus; + +import com.sun.net.httpserver.HttpServer; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.CopyOnWriteArrayList; + +/** A stand-in OTLP/HTTP collector. It keeps the method, path, headers and exact bytes of every request. */ +final class OtlpRecorder implements AutoCloseable { + + record Post(String method, String path, Map headers, byte[] body) { + ExportTraceServiceRequest decode() { + try { + return ExportTraceServiceRequest.parseFrom(body); + } catch (IOException e) { + throw new IllegalStateException("not an OTLP protobuf trace request: " + path, e); + } + } + } + + private final HttpServer server; + private final List posts = new CopyOnWriteArrayList<>(); + + OtlpRecorder() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + Map headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + exchange.getRequestHeaders().forEach((k, v) -> headers.put(k, String.join(",", v))); + posts.add(new Post(exchange.getRequestMethod(), exchange.getRequestURI().getPath(), headers, + exchange.getRequestBody().readAllBytes())); + exchange.getResponseHeaders().add("Content-Type", "application/x-protobuf"); + exchange.sendResponseHeaders(200, -1); + exchange.close(); + }); + server.start(); + } + + String baseUrl() { + return "http://127.0.0.1:" + server.getAddress().getPort(); + } + + List posts() { + return new ArrayList<>(posts); + } + + void clear() { + posts.clear(); + } + + @Override + public void close() { + server.stop(0); + } +} diff --git a/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/StandIns.java b/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/StandIns.java new file mode 100644 index 000000000..f933b6331 --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/StandIns.java @@ -0,0 +1,49 @@ +package ai.traceai.examples.quarkus; + +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; + +import java.util.Map; + +/** + * Starts the scripted model server and the OTLP recorder and points the application at them through the same + * variables a customer sets (OPENAI_*, FI_*). Nothing in the tests talks to a real provider or to Future AGI. + */ +public class StandIns implements QuarkusTestResourceLifecycleManager { + + static final String PROVIDER_KEY = "sk-provider-key-must-not-be-exported-4d1f"; + static final String FI_API_KEY = "fi-api-key-test-9b2e"; + static final String FI_SECRET_KEY = "fi-secret-key-test-51ac"; + static final String PROJECT = "quarkus-langchain4j-test"; + static final String MODEL = "gpt-test"; + + static volatile MockOpenAi model; + static volatile OtlpRecorder collector; + + @Override + public Map start() { + try { + model = new MockOpenAi(); + collector = new OtlpRecorder(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + return Map.of( + "OPENAI_API_KEY", PROVIDER_KEY, + "OPENAI_BASE_URL", model.baseUrl(), + "OPENAI_MODEL", MODEL, + "FI_BASE_URL", collector.baseUrl(), + "FI_API_KEY", FI_API_KEY, + "FI_SECRET_KEY", FI_SECRET_KEY, + "FI_PROJECT_NAME", PROJECT); + } + + @Override + public void stop() { + if (model != null) { + model.close(); + } + if (collector != null) { + collector.close(); + } + } +} diff --git a/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/UpstreamDefaultsTest.java b/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/UpstreamDefaultsTest.java new file mode 100644 index 000000000..d0b513740 --- /dev/null +++ b/java/examples/quarkus-langchain4j-example/src/test/java/ai/traceai/examples/quarkus/UpstreamDefaultsTest.java @@ -0,0 +1,81 @@ +package ai.traceai.examples.quarkus; + +import static ai.traceai.examples.quarkus.DefaultExportTest.COMPLETION_SPAN; +import static ai.traceai.examples.quarkus.Exported.AI_SERVICE_SPAN; +import static ai.traceai.examples.quarkus.Exported.awaitTrace; +import static ai.traceai.examples.quarkus.Exported.only; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import ai.traceai.examples.quarkus.Exported.ExportedSpan; +import io.opentelemetry.proto.trace.v1.Span; +import io.opentelemetry.proto.trace.v1.Status; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import java.net.InetAddress; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * What Quarkus and Quarkus LangChain4j do without this recipe's helpers and limits. These are the reasons the + * helpers exist; a failure here after an upgrade means upstream changed and the docs need a re-check. + */ +@QuarkusTest +@QuarkusTestResource(StandIns.class) +@TestProfile(UpstreamDefaultsTest.Upstream.class) +class UpstreamDefaultsTest { + + public static class Upstream implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "quarkus.arc.exclude-types", "FutureAgiTracingDefaults,DropHostName,ToolErrorStatus", + "quarkus.otel.span.event.count.limit", "128", + // The full traces URL, as other OTLP SDKs take it. Quarkus appends /v1/traces again. + "quarkus.otel.exporter.otlp.traces.endpoint", "${FI_BASE_URL}/tracer/v1/traces"); + } + } + + @Inject + Assistant assistant; + + @BeforeEach + void reset() { + StandIns.collector.clear(); + } + + @Test + void quarkusAddsTheHostNameToTheResource() throws Exception { + assistant.chat("What is the weather in Lisbon?"); + String hostName = InetAddress.getLocalHost().getHostName(); + assertTrue(awaitTrace().stream().allMatch(s -> hostName.equals(s.resource().get("host.name")))); + } + + @Test + void upstreamLeavesFailuresUnsetAndExportsTheErrorTextAndStackTrace() throws Exception { + String marker = "QLC4JUP" + System.nanoTime(); + assertThrows(RuntimeException.class, () -> assistant.chat("What is the weather in MODEL-FAIL-" + marker + "?")); + List spans = awaitTrace(); + + for (ExportedSpan s : List.of(only(spans, AI_SERVICE_SPAN), only(spans, COMPLETION_SPAN))) { + assertEquals(Status.StatusCode.STATUS_CODE_UNSET, s.span().getStatus().getCode(), s.name()); + Span.Event exception = s.span().getEventsList().stream().filter(e -> e.getName().equals("exception")).findFirst().orElseThrow(); + Map attributes = Exported.toMap(exception.getAttributesList()); + assertTrue(attributes.get("exception.message").contains(marker), s.name()); + assertTrue(attributes.get("exception.stacktrace").contains("\tat "), s.name()); + } + } + + @Test + void anEndpointEndingInV1TracesIsPostedToADoubledPath() throws Exception { + assistant.chat("What is the weather in Lisbon?"); + awaitTrace(); + assertTrue(StandIns.collector.posts().stream().allMatch(p -> p.path().equals("/tracer/v1/traces/v1/traces"))); + } +}