Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions python/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,27 @@
## [1.2.2] - 2026-10-30
### Fixed
- Apply content privacy to creation attributes and links before OpenTelemetry SDK
sampling, and to sampler-produced attributes before span processors run. Preserve
the sampler's decision, trace state, attribute selection and ordinary retention
limits; keep FI's priority only for a caller-declared span kind.
- Extend input/output hiding to known tool, image, retrieval, reranking, audio,
voice and computer-use content fields and their descendants, including event,
exception and link attributes. Keep non-content metadata available. Mixed-direction
content is hidden when either input or output hiding is enabled. These controls
cover known semantic keys, not arbitrary custom attributes or free-text secrets.
- `FI_HIDE_EMBEDDING_VECTORS` now also hides `gen_ai.embeddings.vectors` and its
descendants without evaluating hidden lazy values.
- Restore nested and concurrent `suppress_tracing` contexts correctly, including
asynchronous use and exception exits.
- Honor explicit positional and keyword HTTP/gRPC exporter endpoints and headers
instead of overwriting them with environment defaults.

### Changed
- Provider initialization leaves application SIGINT and SIGTERM handlers untouched.
Normal-exit cleanup is still registered; applications must call `provider.shutdown()`
through their own shutdown lifecycle to export pending spans on SIGTERM or SIGINT.
See [application-owned shutdown](README.md#application-owned-shutdown) for an example.

## [1.2.1] - 2026-10-26
### Fixed
- `register()` no longer marks its own exporter as a default that the next
Expand Down
19 changes: 19 additions & 0 deletions python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,25 @@ finally:

Open the `traceai-quickstart` project in **Observe** in [Future AGI](https://app.futureagi.com). `provider.shutdown()` flushes pending spans before exit.

## Shutdown on termination

From `fi-instrumentation-otel` 1.2.2, `register()` leaves your SIGINT and SIGTERM handlers unchanged. Default SIGTERM does not run Python exit hooks, so call `provider.shutdown()` from your application's shutdown path to export pending spans on container stop, and allow enough termination grace time. SIGKILL cannot flush spans.

A standalone script that owns its signal policy can opt in:

```python
import signal

def stop(signum, frame):
provider.shutdown()
raise SystemExit(0) # Your application chooses the exit status.

signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
```

In a web server, worker or async service, use the framework's shutdown hook instead of replacing its handlers: stop accepting work, finish or cancel active requests, then call `provider.shutdown()`. `force_flush()` exports ended spans without shutting down the provider; it does not replace shutdown for operations still running.

## Context and privacy

Use `using_attributes(session_id=..., user_id=..., metadata=...)` to attach application context. The core also provides `using_session`, `using_user`, `using_tags`, and `suppress_tracing`.
Expand Down
29 changes: 24 additions & 5 deletions python/examples/anannas-ai/tests/test_anannas_ai_recipe.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Recipe contracts: mocked vendor traffic and guarded loopback subprocesses."""

import json
from importlib.metadata import PackageNotFoundError, distribution
import os
import re
import socket
Expand Down Expand Up @@ -41,6 +42,18 @@
# versions record the requested model. Accept both so this recipe passes on either version.
PROVIDER_NAMES = ("openai", "anannas")

# Use distribution metadata: the 1.1.0 wheel's runtime version says 0.1.19.
# Match the imported file too, so a stale installed wheel cannot relax source tests.
try:
CORE_DISTRIBUTION = distribution("fi-instrumentation-otel")
except PackageNotFoundError:
LEGACY_CORE = False
else:
LEGACY_CORE = CORE_DISTRIBUTION.version == "1.1.0" and (
Path(fi_instrumentation.__file__).resolve()
== Path(CORE_DISTRIBUTION.locate_file("fi_instrumentation/__init__.py")).resolve()
)

RECIPE = Path(__file__).resolve().parents[1]
VENDOR_KEY = "placeholder-anannas-key"
FI_KEY = "placeholder-fi-api-key"
Expand Down Expand Up @@ -145,10 +158,16 @@ def assert_export(receiver, project):
return span, attrs


def assert_parameters(attrs, *, stream=False):
params = json.loads(attrs["gen_ai.request.parameters"])
assert params["model"] == REQUEST_MODEL
assert params.get("stream", False) is stream
def assert_parameters(attrs, *, stream=False, hide_inputs=False):
# The CI compatibility cell pins core 1.1.0, before hide_inputs also hid
# invocation parameters. Require the sentinel on the repaired core; only
# that historical release retains the JSON contract in masked cases.
if hide_inputs and not LEGACY_CORE:
assert attrs["gen_ai.request.parameters"] == "__REDACTED__"
else:
params = json.loads(attrs["gen_ai.request.parameters"])
assert params["model"] == REQUEST_MODEL
assert params.get("stream", False) is stream


def assert_usage(attrs, expected):
Expand Down Expand Up @@ -410,7 +429,7 @@ def handler(request):
else:
assert answer in exported
assert attrs["output.value"] == answer
assert_parameters(attrs, stream=stream)
assert_parameters(attrs, stream=stream, hide_inputs=hide_inputs)


def allowed_urls():
Expand Down
34 changes: 17 additions & 17 deletions python/examples/gradio/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,10 +99,10 @@ if __name__ == "__main__":
headers, and stamps every batch with the resource attributes
`project_name` and `project_type=observe`. fi-collector rejects a batch
whose resource has no `project_name` (source reading). Spans are batched;
`register()` flushes the batch on normal exit (tested: with a 10-minute
batch delay every span still arrived) and on SIGINT/SIGTERM, so Ctrl+C on
`python src/app.py` flushes too (checked in a live run: every turn was
exported after SIGINT).
`register()` registers a flush on normal Python exit. With core 1.2.2,
it leaves signal handlers unchanged: arrange `provider.shutdown()` in your
application's shutdown lifecycle for SIGTERM or SIGINT. See
[application-owned shutdown](../../README.md#application-owned-shutdown).
- **Content off.** traceAI records prompts and answers by default
(`TraceConfig`'s `hide_inputs` and `hide_outputs` default to `False`). The
recipe passes `hide_inputs=True, hide_outputs=True`. See [Privacy](#privacy).
Expand Down Expand Up @@ -254,11 +254,10 @@ again after each save. So `init_tracing()` runs twice per launch even if you
never save, and once more per save:

- Each of those runs' `register()` builds another provider with its own
batch queue and logs `Failed to register signal handlers: ... This is
expected when running in non-main threads.` Its
`OpenAIInstrumentor().instrument()` is ignored with the warning
`Attempting to instrument while already instrumented`. So both warnings
appear once at startup and once per save. Spans keep going to the first
batch queue. Its `OpenAIInstrumentor().instrument()` is ignored with the
warning `Attempting to instrument while already instrumented`. The warning
can appear at startup and once per save. Core 1.2.2 no longer attempts to
install signal handlers from the watcher thread. Spans keep going to the first
provider, with the first `TraceConfig`; the other providers export
nothing.
- **Edits to the tracing setup do not take effect until you restart.** In
Expand All @@ -269,10 +268,11 @@ never save, and once more per save:
- The session key survives the re-runs, so an open session keeps its id
(tested).

A batch is lost only if the process ends without the exit flush.
`register()` flushes on normal exit, SIGINT and SIGTERM. SIGKILL, or the
SIGHUP a closed terminal sends, skips it (source reading of
`fi_instrumentation/otel.py`, `setup_signal_handlers`; not tested). The
Pending spans can be lost if the process ends without cleanup.
`register()` registers normal Python exit cleanup, not signal handlers.
Default SIGTERM, SIGHUP and SIGKILL skip exit hooks. Use your application's
shutdown lifecycle to call `provider.shutdown()` before exiting; SIGKILL
cannot be handled. The
startup run leaves one idle provider from the start, and each save one
more, until the process exits (that they stay until exit is source
reading).
Expand All @@ -294,10 +294,10 @@ the tests open no server port.
so no session id was set.
- **`session.id` is not Gradio's session hash.** By design: it is a keyed
digest of the hash (see Session above).
- **Startup warnings in reload mode.** `Failed to register signal handlers`
and `Attempting to instrument while already instrumented` come from
Gradio's startup re-run of the file. In the tested replay the spans still
went to the first provider; see [Gradio reload mode](#gradio-reload-mode).
- **Startup warnings in reload mode.** `Attempting to instrument while
already instrumented` comes from Gradio's startup re-run of the file.
Spans keep going to the first provider; see
[Gradio reload mode](#gradio-reload-mode).
- **A tracing change did nothing after a save.** Reload mode keeps the
first instrumentation. Restart the process.

Expand Down
3 changes: 2 additions & 1 deletion python/examples/gradio/src/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,5 +93,6 @@ def predict(message: str, history: list, request: gr.Request = None) -> str:

if __name__ == "__main__":
init_tracing()
# register() flushes the batch on normal exit and on SIGINT/SIGTERM.
# register() cleans up on normal exit; signal cleanup belongs to the app.
# See README.md for shutdown lifecycle guidance.
demo.launch()
34 changes: 28 additions & 6 deletions python/examples/perplexity/tests/test_perplexity_recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import ast
from contextlib import contextmanager
import importlib.util
from importlib.metadata import PackageNotFoundError, distribution
import json
import os
from pathlib import Path
Expand All @@ -24,6 +25,18 @@
# Accept both so this recipe passes on either version.
PROVIDER_NAMES = ("openai", "perplexity")

# Use distribution metadata: the 1.1.0 wheel's runtime version says 0.1.19.
# Match the imported file too, so a stale installed wheel cannot relax source tests.
try:
CORE_DISTRIBUTION = distribution("fi-instrumentation-otel")
except PackageNotFoundError:
LEGACY_CORE = False
else:
LEGACY_CORE = CORE_DISTRIBUTION.version == "1.1.0" and (
Path(fi_instrumentation.__file__).resolve()
== Path(CORE_DISTRIBUTION.locate_file("fi_instrumentation/__init__.py")).resolve()
)

TEST_DIR = Path(__file__).resolve().parent
RECIPE_DIR = TEST_DIR.parent
VENDOR_KEY = "placeholder-perplexity-key"
Expand Down Expand Up @@ -88,7 +101,10 @@ def _attributes(span):
}


def _assert_export(receiver, provider, project, error=False, requested_model=fake.REQUEST_MODEL):
def _assert_export(
receiver, provider, project, error=False, requested_model=fake.REQUEST_MODEL,
hide_inputs=False,
):
assert provider.force_flush()
[span] = receiver.spans()
[export] = receiver.requests()
Expand All @@ -98,11 +114,17 @@ def _assert_export(receiver, provider, project, error=False, requested_model=fak
assert attrs["gen_ai.provider.name"] in PROVIDER_NAMES
# A successful response overrides the request model, including on streams.
assert attrs["gen_ai.request.model"] == (requested_model if error else fake.RESPONSE_MODEL)
parameters = json.loads(attrs["gen_ai.request.parameters"])
if requested_model is None:
assert "model" not in parameters
# The CI compatibility cell pins core 1.1.0, before hide_inputs also hid
# invocation parameters. Require the sentinel on the repaired core; only
# that historical release retains the JSON contract in masked cases.
if hide_inputs and not LEGACY_CORE:
assert attrs["gen_ai.request.parameters"] == "__REDACTED__"
else:
assert parameters["model"] == requested_model
parameters = json.loads(attrs["gen_ai.request.parameters"])
if requested_model is None:
assert "model" not in parameters
else:
assert parameters["model"] == requested_model
assert span["status"]["code"] == ("STATUS_CODE_ERROR" if error else "STATUS_CODE_OK")
assert export["path"] == "/tracer/v1/traces"
assert export["headers"]["x-api-key"] == FI_KEY
Expand Down Expand Up @@ -325,7 +347,7 @@ def handler(request):

with tracing() as (receiver, provider, project):
assert _call(handler, prompt=marker) == fake.ANSWER
span, _ = _assert_export(receiver, provider, project)
span, _ = _assert_export(receiver, provider, project, hide_inputs=hidden)
assert (marker in json.dumps(span)) is (not hidden)
# Output echoes remain visible even when the input itself is hidden.
for output_marker in (fake.QUERY_MARKER, fake.INSTRUCTIONS_MARKER):
Expand Down
39 changes: 34 additions & 5 deletions python/fi_instrumentation/instrumentation/_spans.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import wrapt # type: ignore[import-untyped]
from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes
from opentelemetry.trace import Span, Status, StatusCode
from opentelemetry.trace import Span, SpanContext, Status, StatusCode
from opentelemetry.util.types import Attributes, AttributeValue

from ._attributes import (
Expand Down Expand Up @@ -54,9 +54,14 @@ class FiSpan(wrapt.ObjectProxy): # type: ignore[misc]
exception events keep only ``exception.type`` / ``exception.escaped``.
"""

def __init__(self, wrapped: Span, config: TraceConfig) -> None:
def __init__(
self, wrapped: Span, config: TraceConfig, span_kind: Optional[AttributeValue] = None
) -> None:
super().__init__(wrapped)
self._self_config = config
# Logical kind permits the helper methods even if a sampler omitted the
# kind. It is not written back unless the sampler or a setter chose it.
self._self_span_kind = span_kind
self._self_important_attributes: Dict[str, AttributeValue] = {}
# Only summaries (class names, message) of recorded exceptions are kept,
# never the exceptions, so their traceback frames and locals are freed.
Expand Down Expand Up @@ -111,6 +116,7 @@ def record_exception(
span = cast(Span, self.__wrapped__)
if not span.is_recording():
return
attributes = self._mask_event_attributes(attributes)
error_type = get_error_type(exception)
self._remember_recorded(error_type, summarize_exception(exception))
if escaped:
Expand All @@ -131,6 +137,26 @@ def record_exception(
name=EXCEPTION_EVENT_NAME, attributes=event_attributes, timestamp=timestamp
)

def _mask_event_attributes(self, attributes: Attributes) -> Attributes:
if attributes is None:
return None
result: Dict[str, AttributeValue] = {}
for key, value in attributes.items():
masked = self._self_config.mask(key, value)
if masked is not None:
result[key] = masked
return result

def add_link(self, context: SpanContext, attributes: Attributes = None) -> None:
# Available on newer OTel SDKs; preserve the underlying API's behavior
# on versions that do not support adding links after start.
span = cast(Span, self.__wrapped__)
span.add_link(context, attributes=self._mask_event_attributes(attributes))

def _remember_sampled_kind(self, value: AttributeValue) -> None:
# Exported selection and the caller's logical helper kind are distinct.
self._self_important_attributes[GEN_AI_SPAN_KIND] = value

def add_event(
self,
name: str,
Expand All @@ -151,6 +177,7 @@ def add_event(
)
if str(escaped).lower() == "true":
self._self_escaped_error_type = exception_type
attributes = self._mask_event_attributes(attributes)
if self._self_config.hide_error_details:
attributes = strip_error_details(attributes)
span.add_event(name, attributes=attributes, timestamp=timestamp)
Expand Down Expand Up @@ -244,6 +271,8 @@ def set_attribute(
if masked_value is not None:
if key in _IMPORTANT_ATTRIBUTES:
self._self_important_attributes[key] = masked_value
if key == GEN_AI_SPAN_KIND:
self._self_span_kind = masked_value
else:
span = cast(Span, self.__wrapped__)
span.set_attribute(key, masked_value)
Expand All @@ -264,7 +293,7 @@ def set_input(
*,
mime_type: Optional[FiMimeType] = None,
) -> None:
if GEN_AI_SPAN_KIND not in self._self_important_attributes:
if self._self_span_kind is None:
raise ValueError("Cannot set input attributes on a non-Fi span")
self.set_attributes(get_input_attributes(value, mime_type=mime_type))

Expand All @@ -274,7 +303,7 @@ def set_output(
*,
mime_type: Optional[FiMimeType] = None,
) -> None:
if GEN_AI_SPAN_KIND not in self._self_important_attributes:
if self._self_span_kind is None:
raise ValueError("Cannot set output attributes on a non-Fi span")
self.set_attributes(get_output_attributes(value, mime_type=mime_type))

Expand All @@ -285,7 +314,7 @@ def set_tool(
description: Optional[str] = None,
parameters: Union[str, Dict[str, Any]],
) -> None:
if self._self_important_attributes.get(GEN_AI_SPAN_KIND) != TOOL:
if self._self_span_kind != TOOL:
raise ValueError("Cannot set tool attributes on a non-tool span")
self.set_attributes(
get_tool_attributes(
Expand Down
Loading
Loading