diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5c129b8..0201d65 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,3 +29,8 @@ jobs: - name: Pytest run: PYTHONPATH=src uv run pytest -q + + - name: Pytest with the oldest supported sentry-sdk + run: | + uv pip install "sentry-sdk==2.15.0" + PYTHONPATH=src uv run --no-sync pytest -q diff --git a/.gitignore b/.gitignore index e92fd54..0df0569 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,14 @@ # template so it stays tracked. .env.* !.env.example + +# Python build and environment artifacts. +__pycache__/ +*.py[cod] +.venv/ +.pytest_cache/ +.ruff_cache/ +.coverage +dist/ +build/ +*.egg-info/ diff --git a/CLAUDE.md b/CLAUDE.md index 6c573d8..3e5b60e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,7 @@ pytest tests/ -v pytest tests/ --cov=openadapt_telemetry # Run linting -ruff check src/ +ruff check . ``` ## Package Structure @@ -31,13 +31,16 @@ openadapt-telemetry/ │ ├── client.py # TelemetryClient (Sentry wrapper) │ ├── privacy.py # PII filtering and scrubbing │ ├── decorators.py # Convenience decorators -│ └── events.py # Event types and helpers +│ ├── events.py # Event types and helpers +│ ├── posthog.py # PostHog usage events +│ └── failure_signals.py # Closed-schema automation failure signals ├── tests/ -│ ├── conftest.py # Pytest fixtures -│ ├── test_client.py # Client tests -│ ├── test_config.py # Configuration tests -│ ├── test_decorators.py # Decorator tests -│ └── test_privacy.py # Privacy filtering tests +│ ├── conftest.py # Fixtures, including an in-memory Sentry transport +│ ├── test_sentry_payload.py # What a real Sentry client sends +│ ├── test_opt_out.py # The shared opt-out rule +│ ├── test_readme.py # Runs the README and docs examples +│ └── ... +├── docs/COMPLIANCE.md # Data-handling reference for reviewers ├── pyproject.toml # Package metadata ├── README.md # Documentation └── LICENSE # MIT License @@ -121,12 +124,12 @@ Config file location: `~/.config/openadapt/telemetry.json` ## Internal User Detection -The package automatically detects internal/developer usage via: +The package detects internal/developer usage via: 1. `OPENADAPT_INTERNAL=true` environment variable 2. `OPENADAPT_DEV=true` environment variable -3. Running from source (not frozen executable) -4. Git repository present in working directory -5. CI environment detected (GitHub Actions, GitLab CI, etc.) +3. CI environment detected (GitHub Actions, GitLab CI, etc.) +4. `OPENADAPT_INTERNAL_FROM_GIT=true` and a `.git` directory in the working directory or its parent +5. `"internal": true` in the config file Internal users are tagged with `internal: true` for filtering in GlitchTip. @@ -164,7 +167,7 @@ except ImportError: ## Dependencies Core: -- `sentry-sdk>=2.0.0` - Sentry/GlitchTip SDK +- `sentry-sdk>=2.15.0` - Sentry/GlitchTip SDK (CI also tests against 2.15.0) Dev: - `pytest>=8.0.0` - Testing diff --git a/README.md b/README.md index 6610ef3..4d339b7 100644 --- a/README.md +++ b/README.md @@ -5,249 +5,177 @@ [![PyPI version](https://img.shields.io/pypi/v/openadapt-telemetry.svg)](https://pypi.org/project/openadapt-telemetry/) [![Downloads](https://img.shields.io/pypi/dm/openadapt-telemetry.svg)](https://pypi.org/project/openadapt-telemetry/) -Unified telemetry and error tracking for OpenAdapt packages. +Error reports and usage events for [OpenAdapt](https://github.com/OpenAdaptAI/OpenAdapt) +packages. Data is filtered on the user's machine before it is sent, and +`DO_NOT_TRACK=1` turns everything off. -## Features +- **If an OpenAdapt package you use depends on this one**, see + [what is sent](#what-is-sent) and [how to turn it off](#turn-it-off). + `pip show openadapt-telemetry` lists those packages under `Required-by`. +- **If you build an OpenAdapt package**, the quick start below sends error + reports to your GlitchTip or Sentry project. Other Python projects can use it + for error reports the same way. Usage events go to OpenAdapt's PostHog project + unless `OPENADAPT_POSTHOG_PROJECT_API_KEY` names another. -- **Unified Error Tracking**: Consistent error reporting across all OpenAdapt packages -- **Usage Counters (PostHog)**: Lightweight product usage events for adoption metrics -- **Privacy-First Design**: Automatic PII scrubbing and path sanitization -- **Configurable Opt-Out**: Respects `DO_NOT_TRACK` and custom environment variables -- **Internal Usage Tagging**: Explicit flags + CI detection with optional git heuristic -- **GlitchTip/Sentry Compatible**: Uses the Sentry SDK for maximum compatibility -- **Failure recurrence without customer evidence**: Closed categorical failure - signatures omit application, tenant, workflow, step, text, input, screenshot, - origin, and exception details +## Quick start -## Installation +Install the package and set the DSN, the URL with a key that GlitchTip or Sentry +shows in your project's settings: ```bash pip install openadapt-telemetry +export OPENADAPT_TELEMETRY_DSN="https://@app.glitchtip.com/" ``` -Or with development dependencies: +```python +from openadapt_telemetry import get_telemetry -```bash -pip install openadapt-telemetry[dev] +telemetry = get_telemetry() +telemetry.initialize(package_name="mypackage", package_version="0.1.0") + +try: + open("/Users/alice/notes.txt") +except OSError as exc: + telemetry.capture_exception(exc) ``` -## Quick Start +GlitchTip receives +`FileNotFoundError: [Errno 2] No such file or directory: '/Users//notes.txt'` +with the file, function and line of each stack frame. Without a DSN, or after +an opt-out, `initialize()` returns `False` and no error reports are sent. -### Initialize Telemetry +To see the filter without a DSN: ```python -from openadapt_telemetry import get_telemetry - -# Initialize once at package startup -get_telemetry().initialize( - dsn="https://xxx@app.glitchtip.com/XXXX", - package_name="openadapt-mypackage", - package_version="0.1.0", -) +>>> from openadapt_telemetry import scrub_string +>>> scrub_string("Saved /Users/alice/notes.txt for alice@example.com") +'Saved /Users//notes.txt for [REDACTED]' ``` -### Capture Exceptions +## Turn it off -```python -from openadapt_telemetry import get_telemetry +Any one of these turns off both error reports and usage events: -try: - risky_operation() -except Exception as e: - get_telemetry().capture_exception(e) - raise -``` +| Where | Setting | +|-------|---------| +| Environment | `DO_NOT_TRACK=1` (or `true`, `yes`, `on`) | +| Environment | `OPENADAPT_TELEMETRY_ENABLED=false` | +| The nearest `pyproject.toml` at or above the current directory, for everyone who works in that repository | `telemetry = false` under `[tool.openadapt]` | +| `~/.config/openadapt/telemetry.json` | `{"enabled": false}` | -### Capture Usage Events (PostHog) +`DO_NOT_TRACK` overrides everything else. `OPENADAPT_TELEMETRY_ENABLED=true` +overrides the two files. Code can check the result with +`openadapt_telemetry.is_telemetry_allowed()`. Usage events are also off in CI +unless `OPENADAPT_TELEMETRY_IN_CI=true` or `OPENADAPT_TELEMETRY_ENABLED=true`. -```python -from openadapt_telemetry import capture_usage_event +## What is sent -capture_usage_event( - "agent_run", - properties={"entrypoint": "oa evals run", "mode": "live"}, - package_name="openadapt-evals", -) -``` +| | Error reports | Usage events | +|-|---------------|--------------| +| Destination | The GlitchTip or Sentry project named by the DSN. With no DSN, nothing is sent. | OpenAdapt's PostHog project, unless `OPENADAPT_POSTHOG_PROJECT_API_KEY` names another one | +| When | After `initialize()`: on `capture_exception` or `capture_message`, an uncaught exception, an `ERROR` log record, or a call to a function wrapped by a `track_*` decorator | When a package calls `capture_usage_event` | +| Contents | Exception type and message; file, function and line of each stack frame; recent log lines and HTTP request URLs; timings; package, Python and OS versions; names and versions of installed Python packages; whether the run looks internal or CI | Event name; package and Python versions; OS name; a timestamp; a random install ID kept in `~/.openadapt/telemetry_distinct_id`; the properties the package passes | +| Removed before sending | User names in home-directory paths; emails, phone, card and social security numbers, bearer tokens and long key-like strings; values under keys such as `password`, `token` or `email`; the host name; the command line; local variable values; source code lines; URL query strings | Properties under sensitive keys; the same patterns in property values. Strings are cut to 256 characters. | -### Capture an automation failure safely +The exact filter rules are in +[docs/COMPLIANCE.md](https://github.com/OpenAdaptAI/openadapt-telemetry/blob/main/docs/COMPLIANCE.md), +along with `capture_automation_failure`, which reports an automation failure as +fixed categories, a version, a count and an hour, with no free text. -Use the closed-schema failure API rather than sending an exception message or -run report as analytics. Its grouping key is derived only from categorical -runtime facts; it is not an installation or tenant identifier. +## Usage events ```python -from openadapt_telemetry import ( - ActionKind, AutomationFailureSignal, DeliveryState, ExecutionOutcome, - FailureKind, RiskClass, Substrate, capture_automation_failure, -) - -capture_automation_failure(AutomationFailureSignal( - failure_kind=FailureKind.DELIVERY_UNCERTAIN, - substrate=Substrate.CITRIX, - action_kind=ActionKind.CLICK, - risk_class=RiskClass.CONSEQUENTIAL, - delivery_state=DeliveryState.UNCERTAIN, - outcome=ExecutionOutcome.HALTED, -)) +from openadapt_telemetry import capture_usage_event + +capture_usage_event("demo_recorded", properties={"steps": 12}, package_name="mypackage") ``` -This signal supports aggregate discovery only. Full evidence remains local, -and no signal can authorize or promote a repair. +A background thread sends the event with a 1-second timeout, so a slow or +unreachable server doesn't block the caller. Events still queued when the +process exits are dropped, so a script that exits right after the call may send +nothing. The function returns `False` when telemetry is off or the queue is +full. -### Using Decorators +## Decorators and spans ```python -from openadapt_telemetry import track_errors, track_performance, track_feature - -@track_errors() -def process_data(data): - """Exceptions are automatically captured.""" - return transform(data) - -@track_performance("indexing.build_faiss") -def build_index(vectors): - """Execution time is automatically tracked.""" - return create_index(vectors) - -@track_feature("retrieval.add_demo") -def add_demo(demo_id, task): - """Feature usage is tracked for analytics.""" - save_demo(demo_id, task) -``` +from openadapt_telemetry import TelemetrySpan, track_errors, track_feature, track_performance -### Span Context Manager +@track_errors() # report exceptions, then re-raise +def process(data): ... -```python -from openadapt_telemetry import TelemetrySpan +@track_performance("indexing.build") # send a timed transaction (1% by default) +def build_index(vectors): ... + +@track_feature("retrieval.add_demo") # send a message event on each call +def add_demo(demo_id): ... -with TelemetrySpan("indexing", "build_faiss_index") as span: - span.set_tag("num_vectors", 1000) - # ... indexing operations ... +with TelemetrySpan("indexing", "load_vectors") as span: # a timed step + span.set_data("count", 1000) ``` +These send to the same GlitchTip or Sentry project as error reports, and do +nothing until `initialize()` succeeds. A `TelemetrySpan` is sent as a step of +the `track_performance` transaction it runs in. Outside one, it sends nothing. + ## Configuration -### Environment Variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `DO_NOT_TRACK` | - | Universal opt-out (1 = disabled) | -| `OPENADAPT_TELEMETRY_ENABLED` | `true` | Enable/disable telemetry | -| `OPENADAPT_INTERNAL` | `false` | Tag as internal usage | -| `OPENADAPT_DEV` | `false` | Development mode | -| `OPENADAPT_INTERNAL_FROM_GIT` | `false` | Optional: tag as internal when running from a git checkout | -| `OPENADAPT_TELEMETRY_DSN` | - | GlitchTip/Sentry DSN | -| `OPENADAPT_POSTHOG_PROJECT_API_KEY` | embedded default | PostHog ingestion project token (`phc_...`) | +Environment variables take precedence over `~/.config/openadapt/telemetry.json`, +which takes precedence over the defaults. + +| Variable | Default | Effect | +|----------|---------|--------| +| `DO_NOT_TRACK` | unset | `1`, `true`, `yes` or `on` turns off all telemetry | +| `OPENADAPT_TELEMETRY_ENABLED` | unset (on) | `false` turns telemetry off; `true` overrides the opt-out files | +| `OPENADAPT_TELEMETRY_DSN` | none | GlitchTip or Sentry DSN for error reports | +| `OPENADAPT_TELEMETRY_ENVIRONMENT` | `production` | Environment name on error reports | +| `OPENADAPT_TELEMETRY_SAMPLE_RATE` | `1.0` | Share of errors sent (0.0 to 1.0) | +| `OPENADAPT_TELEMETRY_TRACES_SAMPLE_RATE` | `0.01` | Share of performance transactions sent | +| `OPENADAPT_TELEMETRY_IN_CI` | `false` | Send usage events from CI | +| `OPENADAPT_POSTHOG_PROJECT_API_KEY` | OpenAdapt's project | PostHog project token (`phc_...`) for usage events | | `OPENADAPT_POSTHOG_HOST` | `https://us.i.posthog.com` | PostHog ingestion host | -| `OPENADAPT_TELEMETRY_DISTINCT_ID` | generated UUID | Stable anonymous identifier override | -| `OPENADAPT_TELEMETRY_TIMEOUT_SECONDS` | `1.0` | PostHog network timeout | -| `OPENADAPT_TELEMETRY_IN_CI` | `false` | Enable usage events in CI pipelines | -| `OPENADAPT_TELEMETRY_ENVIRONMENT` | `production` | Environment name | -| `OPENADAPT_TELEMETRY_SAMPLE_RATE` | `1.0` | Error sampling rate (0.0-1.0) | -| `OPENADAPT_TELEMETRY_TRACES_SAMPLE_RATE` | `0.01` | Performance sampling rate | -| `OPENADAPT_TELEMETRY_ANON_SALT` | generated | Optional anonymization salt override (advanced use only) | +| `OPENADAPT_TELEMETRY_DISTINCT_ID` | random UUID | Install ID for usage events | +| `OPENADAPT_TELEMETRY_TIMEOUT_SECONDS` | `1.0` | Network timeout for usage events | +| `OPENADAPT_INTERNAL`, `OPENADAPT_DEV` | `false` | Tag error reports as internal | +| `OPENADAPT_INTERNAL_FROM_GIT` | `false` | Tag error reports as internal when run from a git checkout | +| `OPENADAPT_TELEMETRY_ANON_SALT` | random, saved in the config file | Key for hashing user IDs passed to `set_user` (32 characters or more) | -### Configuration File - -Create `~/.config/openadapt/telemetry.json`: +The config file takes these keys: ```json { "enabled": true, "internal": false, - "dsn": "https://xxx@app.glitchtip.com/XXXX", + "dsn": "https://@app.glitchtip.com/", "environment": "production", "sample_rate": 1.0, "traces_sample_rate": 0.01 } ``` -### Priority Order - -1. Environment variables (highest priority) -2. Configuration file -3. Package defaults (lowest priority) - -## Opt-Out - -To disable telemetry, set either: - -```bash -# Universal standard -export DO_NOT_TRACK=1 - -# Or package-specific -export OPENADAPT_TELEMETRY_ENABLED=false -``` - -## Privacy - -### What We Collect - -| Category | Data | Purpose | -|----------|------|---------| -| Errors | Exception type, stack trace | Bug fixing | -| Performance | Function timing | Optimization | -| Feature Usage | Feature names, counts | Prioritization | -| Environment | OS, Python version | Compatibility | - -### What We Never Collect - -- Screenshots or images -- Text content or file contents -- Personal information (names, emails, IPs) -- API keys or passwords -- Full file paths with usernames +## Limits -### Automatic Scrubbing - -- File paths have usernames replaced with `` -- Sensitive fields (password, token, api_key, etc.) are redacted -- Email addresses and phone numbers are scrubbed from messages -- Top-level event messages/logentry strings are scrubbed -- Tag keys are validated, sensitive/invalid keys are dropped, and values are scrubbed before upload -- User IDs are HMAC-anonymized before upload (`anon:v2:`) -- `send_default_pii` is enforced to `false` by the client -- PostHog events set `$geoip_disable: true` to suppress IP geolocation enrichment - -## Internal Usage Tagging - -Internal/developer usage is automatically detected via: - -1. `OPENADAPT_INTERNAL=true` environment variable -2. `OPENADAPT_DEV=true` environment variable -3. CI environment detected (GitHub Actions, GitLab CI, etc.) -4. Optional git repository heuristic when `OPENADAPT_INTERNAL_FROM_GIT=true` - -Filter in GlitchTip: -``` -tag:internal IS false # External users only -tag:internal IS true # Internal users only -``` +- The string filter matches patterns. It doesn't recognize names, addresses or + other free text, so an exception message such as `no record for Jane Doe` is + sent as written. +- Paths are kept apart from the user name. Stack frames carry each file's full + path and HTTP breadcrumbs carry each URL's path, so a name or identifier in a + path (for example `/Users//clients/acme/app.py` or `/patients/123`) is + sent. +- GlitchTip, Sentry and PostHog see the IP address of each connection, as any + web server does. The library doesn't put IP addresses in events and asks + PostHog not to look up location. ## Development ```bash -# Clone and install git clone https://github.com/OpenAdaptAI/openadapt-telemetry cd openadapt-telemetry pip install -e ".[dev]" - -# Run tests -pytest tests/ -v - -# Run with coverage -pytest tests/ --cov=openadapt_telemetry +pytest +ruff check . ``` ## License -MIT License - see [LICENSE](LICENSE) for details. - -## Links - -- [Documentation](https://github.com/OpenAdaptAI/openadapt-telemetry#readme) -- [Issues](https://github.com/OpenAdaptAI/openadapt-telemetry/issues) -- [GlitchTip](https://glitchtip.com) -- [Sentry SDK](https://docs.sentry.io/platforms/python/) +MIT. See [LICENSE](https://github.com/OpenAdaptAI/openadapt-telemetry/blob/main/LICENSE). diff --git a/docs/COMPLIANCE.md b/docs/COMPLIANCE.md new file mode 100644 index 0000000..4a2e327 --- /dev/null +++ b/docs/COMPLIANCE.md @@ -0,0 +1,163 @@ +# Data handling reference + +This page is for operators, security reviewers and maintainers. The +[README](../README.md) lists what is sent and how to turn it off. This page +gives the rules the code applies, so a reviewer can check them against +`src/openadapt_telemetry/`. + +## Opt-out rule + +`config.is_telemetry_allowed()` decides whether anything is sent. Error +reports, usage events and automation failure signals all call it. It applies +these checks in order and stops at the first that decides: + +1. `DO_NOT_TRACK` set to `1`, `true`, `yes` or `on` (any case): off. +2. `OPENADAPT_TELEMETRY_ENABLED` set to a non-empty value: on for `1`, `true`, + `yes` or `on`, off for anything else. +3. `telemetry = false` under `[tool.openadapt]` in the nearest `pyproject.toml`, + searching from the current directory upward: off. Only the nearest file is + read. +4. `"enabled": false` (or the string `"false"`) in + `~/.config/openadapt/telemetry.json`: off. +5. Otherwise: on. + +Usage events and failure signals have one more rule: in CI (detected from +`CI`, `GITHUB_ACTIONS`, `GITLAB_CI` and other common variables) they stay off +unless `OPENADAPT_TELEMETRY_IN_CI` or `OPENADAPT_TELEMETRY_ENABLED` is true. +Error reports in CI are sent, tagged `ci` and `internal`. + +## Error reports + +Error reports use the Sentry SDK (2.15 or later) and go to the DSN in +`OPENADAPT_TELEMETRY_DSN`, the config file, or the `dsn` argument of +`initialize()`. There is no default DSN. + +`TelemetryClient.initialize()` sets these SDK options: + +| Option | Value | +|--------|-------| +| `send_default_pii` | SDK default (off). Passing it is refused with a warning. | +| `release` | `@`. A caller can pass another value. Left unset, the SDK would send the commit hash of the git checkout the process runs in. | +| `data_collection` | Not set. Passing it is refused with a warning, because newer SDKs let it override `send_default_pii`. | +| `include_local_variables` | `False`. A caller can pass `True`, but the filter still removes local variables. | +| `include_source_context` | `False`. A caller can pass `True`, but the filter still removes source lines. | +| `before_send`, `before_send_transaction` | The privacy filter. A caller's own hook runs first and the filter runs last. | + +If the filter raises, the event is dropped. + +### What the filter does + +The filter (`privacy.create_before_send_filter()`) runs on every error, +message and transaction event: + +| Event field | Treatment | +|-------------|-----------| +| `server_name` | Removed | +| `message`, `logentry` message, formatted text and arguments | String scrub | +| Exception messages | String scrub | +| Stack frames (exceptions and threads) | Local variables (`vars`) and source lines (`pre_context`, `context_line`, `post_context`) removed. `filename` and `abs_path` lose the user name. File, function and line stay. | +| Breadcrumbs | Message: string scrub. Data: `http.query` and `http.fragment` removed, then key and string scrub. | +| `extra` | `sys.argv` removed, then key and string scrub | +| `contexts` | Key and string scrub, except `trace_id`, `span_id`, `parent_span_id` and the sampling context in `contexts.trace` | +| Transaction spans | Description: string scrub. Data: key and string scrub. Tags: tag rules. | +| `tags` | Tag rules | +| `request` | `query_string` and `cookies` removed. Headers and data: key and string scrub. | +| `user` | Only `id` is kept, replaced by `anon:v2:` and the first 16 hex characters of an HMAC-SHA256 | + +**String scrub** (`privacy.scrub_string`) replaces the user name in +`/Users//`, `/home//` and `C:\Users\\` paths with ``, +then replaces these with `[REDACTED]`: email addresses, phone numbers, card +numbers, social security numbers, `Bearer` tokens, alphanumeric strings of 32 +or more characters, and base64-like strings of 40 or more characters. + +**Key scrub** (`privacy.scrub_dict`) replaces the value with `[REDACTED]` +when the key contains any entry of `privacy.PII_DENYLIST`, for example +`password`, `token`, `secret`, `auth`, `cookie`, `session`, `email`, `phone`, +`address`, `ssn` or `credit_card`. + +**Tag rules** keep the default tags (`package`, `package_version`, +`python_version`, `os`, `os_version`, `ci`, `internal`), drop keys that are +sensitive or don't match `^[a-zA-Z0-9._:-]{1,64}$`, keep at most 64 tags, and +string-scrub each value and cut it to 256 characters. + +## Usage events + +`capture_usage_event()` posts this JSON to `/capture/`: + +| Field | Value | +|-------|-------| +| `api_key` | PostHog project token. OpenAdapt's by default, or `OPENADAPT_POSTHOG_PROJECT_API_KEY`. | +| `event` | Event name given by the caller | +| `distinct_id` | Random UUID saved in `~/.openadapt/telemetry_distinct_id`, or `OPENADAPT_TELEMETRY_DISTINCT_ID` | +| `properties` | `package`, `version`, `python_version`, `platform`, `timestamp` (Unix seconds), the caller's properties, and `$geoip_disable: true` | + +Caller properties keep `None`, numbers and booleans as they are. Other values +become strings cut to 256 characters and get the string scrub. Properties +under sensitive keys, and values that are fully redacted, are dropped. + +Events go into an in-memory queue of up to 2,048 events and a daemon thread +sends them with a 1-second timeout (`OPENADAPT_TELEMETRY_TIMEOUT_SECONDS`). +Network errors are ignored. Events still queued when the process exits are +not sent. + +## Automation failure signals + +`capture_automation_failure()` sends a closed-schema description of an +automation failure, for counting how often the same kind of failure recurs +across installs. + +```python +from openadapt_telemetry import ( + ActionKind, AutomationFailureSignal, DeliveryState, ExecutionOutcome, + FailureKind, RiskClass, Substrate, capture_automation_failure, +) + +capture_automation_failure(AutomationFailureSignal( + failure_kind=FailureKind.DELIVERY_UNCERTAIN, + substrate=Substrate.CITRIX, + action_kind=ActionKind.CLICK, + risk_class=RiskClass.CONSEQUENTIAL, + delivery_state=DeliveryState.UNCERTAIN, + outcome=ExecutionOutcome.HALTED, +)) +``` + +The event is `automation_failure_observed` with schema +`openadapt.automation-failure-signal/v1`. It carries the base properties of a +usage event (package, versions, platform and send time) and the signal's +fields. Every signal field is an enum, except: + +- `runtime_version`: a release identifier such as `1.34.0`, or `unknown` +- `model_calls`: an integer from 0 to 1,000,000 +- `external_network_calls`: `none`, `observed` or `unknown` +- `occurred_at`: rounded down to the hour, in UTC + +The constructor rejects any other type or value, so application names, window +titles, workflow or step IDs, typed text, screenshots, exception messages and +URLs can't be attached. The `distinct_id` is `failure:` followed by a SHA-256 +of the categorical fields, not the install ID, so the event shows that a +failure shape recurred but not which install reported it. + +The signal carries none of the run report, screenshots or other evidence; +those stay wherever the caller keeps them. This package only sends the signal +and has no way to act on one, so a signal can't approve or apply a repair. + +## Internal usage + +Error reports carry an `internal` tag (`True` or `False`) so dashboards can +separate OpenAdapt's own use from outside use. A run counts as internal when +any of these holds: + +- `OPENADAPT_INTERNAL` or `OPENADAPT_DEV` is true +- a CI variable is set +- `OPENADAPT_INTERNAL_FROM_GIT` is true and `.git` exists in the current or + parent directory +- `"internal": true` is in the config file + +## Identifiers + +| Identifier | Where it comes from | Sent with | +|------------|---------------------|-----------| +| Install ID | Random UUID in `~/.openadapt/telemetry_distinct_id` | Usage events | +| Anonymous user ID | HMAC-SHA256 of the value passed to `set_user()`, keyed with `OPENADAPT_TELEMETRY_ANON_SALT` or a random salt saved as `anon_salt` in the config file | Error reports, only when a package calls `set_user()` | +| Failure signature | SHA-256 of a failure signal's categorical fields | Failure signals | diff --git a/pyproject.toml b/pyproject.toml index 99bb782..22b6dbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "openadapt-telemetry" version = "0.3.1" -description = "Unified telemetry and error tracking for OpenAdapt packages" +description = "Error reports and usage events for OpenAdapt packages, filtered before sending, with one opt-out" readme = "README.md" requires-python = ">=3.10" license = "MIT" @@ -37,7 +37,9 @@ classifiers = [ ] dependencies = [ - "sentry-sdk>=2.0.0", + "sentry-sdk>=2.15.0", + # Reads [tool.openadapt] in pyproject.toml; Python 3.11+ has tomllib. + "tomli>=1.1.0; python_version < '3.11'", ] [project.optional-dependencies] diff --git a/src/openadapt_telemetry/__init__.py b/src/openadapt_telemetry/__init__.py index 5f24daa..8b7801c 100644 --- a/src/openadapt_telemetry/__init__.py +++ b/src/openadapt_telemetry/__init__.py @@ -4,11 +4,12 @@ using GlitchTip/Sentry SDK for error tracking and performance monitoring. Features: -- Unified error tracking across all packages -- Privacy-first design with automatic PII scrubbing -- Configurable opt-out (DO_NOT_TRACK, OPENADAPT_TELEMETRY_ENABLED) +- Error reports to GlitchTip or Sentry, filtered before sending +- Usage events to PostHog +- One opt-out rule for both (DO_NOT_TRACK, OPENADAPT_TELEMETRY_ENABLED, + [tool.openadapt] in pyproject.toml, or the config file) - CI/dev mode detection for filtering internal usage -- Convenient decorators for tracking errors, performance, and features +- Decorators for tracking errors, performance, and features Quick Start: from openadapt_telemetry import get_telemetry @@ -61,6 +62,7 @@ def add_demo(demo_id, task): CONFIG_FILE, DEFAULTS, TelemetryConfig, + is_telemetry_allowed, load_config, save_config, ) @@ -126,6 +128,7 @@ def add_demo(demo_id, task): "is_running_from_executable", # Config "TelemetryConfig", + "is_telemetry_allowed", "load_config", "save_config", "CONFIG_DIR", diff --git a/src/openadapt_telemetry/__pycache__/__init__.cpython-312.pyc b/src/openadapt_telemetry/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 6f216df..0000000 Binary files a/src/openadapt_telemetry/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/src/openadapt_telemetry/__pycache__/client.cpython-312.pyc b/src/openadapt_telemetry/__pycache__/client.cpython-312.pyc deleted file mode 100644 index 733129d..0000000 Binary files a/src/openadapt_telemetry/__pycache__/client.cpython-312.pyc and /dev/null differ diff --git a/src/openadapt_telemetry/__pycache__/config.cpython-312.pyc b/src/openadapt_telemetry/__pycache__/config.cpython-312.pyc deleted file mode 100644 index 3b2d7f2..0000000 Binary files a/src/openadapt_telemetry/__pycache__/config.cpython-312.pyc and /dev/null differ diff --git a/src/openadapt_telemetry/__pycache__/decorators.cpython-312.pyc b/src/openadapt_telemetry/__pycache__/decorators.cpython-312.pyc deleted file mode 100644 index 5920ec9..0000000 Binary files a/src/openadapt_telemetry/__pycache__/decorators.cpython-312.pyc and /dev/null differ diff --git a/src/openadapt_telemetry/__pycache__/events.cpython-312.pyc b/src/openadapt_telemetry/__pycache__/events.cpython-312.pyc deleted file mode 100644 index 6bc608a..0000000 Binary files a/src/openadapt_telemetry/__pycache__/events.cpython-312.pyc and /dev/null differ diff --git a/src/openadapt_telemetry/__pycache__/privacy.cpython-312.pyc b/src/openadapt_telemetry/__pycache__/privacy.cpython-312.pyc deleted file mode 100644 index 88a538b..0000000 Binary files a/src/openadapt_telemetry/__pycache__/privacy.cpython-312.pyc and /dev/null differ diff --git a/src/openadapt_telemetry/client.py b/src/openadapt_telemetry/client.py index d25da41..e1c7ce8 100644 --- a/src/openadapt_telemetry/client.py +++ b/src/openadapt_telemetry/client.py @@ -89,6 +89,13 @@ def is_internal_user() -> bool: return False +# Sentry options a caller may not override, with the reason given in the warning. +_ENFORCED_SENTRY_OPTIONS = { + "send_default_pii": "OpenAdapt telemetry never sends default PII.", + "data_collection": "OpenAdapt telemetry does not collect personal data.", +} + + def _compose_before_send(base: BeforeSendFn, extra: BeforeSendFn) -> BeforeSendFn: """Compose custom before_send before final privacy filtering.""" @@ -119,7 +126,7 @@ def __init__(self) -> None: self._initialized = False self._config: Optional[TelemetryConfig] = None self._enabled = self._check_enabled() - self._internal = is_internal_user() + self._internal = self._check_internal() @classmethod def get_instance(cls) -> "TelemetryClient": @@ -148,6 +155,10 @@ def _check_enabled(self) -> bool: self._config = load_config() return bool(self._config.enabled) + def _check_internal(self) -> bool: + """Return True for internal usage: env flags, CI, or ``"internal": true`` in the config file.""" + return is_internal_user() or bool(self._config is not None and self._config.internal) + @property def enabled(self) -> bool: """Whether telemetry is enabled.""" @@ -205,6 +216,7 @@ def initialize( if environment: self._config.environment = environment self._enabled = bool(self._config.enabled) + self._internal = self._check_internal() if not self._enabled: return False @@ -213,26 +225,30 @@ def initialize( if not self._config.dsn: return False - # Always enforce privacy scrubber first; optional custom filter can run afterward. + # The privacy filter always runs last, on errors and on transactions. + # A caller's own filter may run first. base_before_send = create_before_send_filter() - custom_before_send = kwargs.pop("before_send", None) - if custom_before_send is not None: - if not callable(custom_before_send): - raise TypeError("before_send must be callable") + hooks = {} + for hook in ("before_send", "before_send_transaction"): + custom = kwargs.pop(hook, None) + if custom is None: + hooks[hook] = base_before_send + continue + if not callable(custom): + raise TypeError(f"{hook} must be callable") warnings.warn( - "Custom before_send runs before OpenAdapt privacy filtering; final payload is always scrubbed.", + f"Custom {hook} runs before OpenAdapt privacy filtering; final payload is always scrubbed.", stacklevel=2, ) - before_send = _compose_before_send(base_before_send, custom_before_send) - else: - before_send = base_before_send + hooks[hook] = _compose_before_send(base_before_send, custom) - if "send_default_pii" in kwargs: - kwargs.pop("send_default_pii") - warnings.warn( - "Ignoring sentry init override for send_default_pii; OpenAdapt telemetry enforces send_default_pii=False.", - stacklevel=2, - ) + for option, reason in _ENFORCED_SENTRY_OPTIONS.items(): + if option in kwargs: + kwargs.pop(option) + warnings.warn( + f"Ignoring sentry init override for {option}; {reason}", + stacklevel=2, + ) # Initialize Sentry SDK sentry_kwargs = { @@ -240,9 +256,19 @@ def initialize( "environment": self._config.environment, "sample_rate": self._config.sample_rate, "traces_sample_rate": self._config.traces_sample_rate, - # Enforced for privacy safety across all callers/configs. - "send_default_pii": False, - "before_send": before_send, + # Without a release, the SDK sends the commit hash of whatever git + # checkout the process runs in. The package version names the code + # that raised instead; a caller can still pass release=. + "release": f"{package_name}@{package_version}", + # send_default_pii is left at the SDK default (off) rather than + # passed, because newer SDKs deprecate the option. Callers cannot + # override it (see _ENFORCED_SENTRY_OPTIONS). + # Local variable values and source lines can hold application data. + # The privacy filter drops them too; these options stop the SDK + # from collecting them in the first place. + "include_local_variables": False, + "include_source_context": False, + **hooks, } # Merge in any additional kwargs @@ -320,7 +346,7 @@ def capture_event( properties = properties or {} - with sentry_sdk.push_scope() as scope: + with sentry_sdk.new_scope() as scope: for key, value in properties.items(): scope.set_extra(key, value) return sentry_sdk.capture_message( diff --git a/src/openadapt_telemetry/config.py b/src/openadapt_telemetry/config.py index 8129f79..700c2b1 100644 --- a/src/openadapt_telemetry/config.py +++ b/src/openadapt_telemetry/config.py @@ -5,6 +5,9 @@ 1. Environment variables (highest priority) 2. Configuration file (~/.config/openadapt/telemetry.json) 3. Package defaults (lowest priority) + +Whether telemetry runs at all is decided by :func:`is_telemetry_allowed`, which +also honors ``[tool.openadapt] telemetry = false`` in the nearest pyproject.toml. """ from __future__ import annotations @@ -68,7 +71,78 @@ def __post_init__(self) -> None: def _parse_bool(value: str) -> bool: """Parse a boolean from a string value.""" - return value.lower() in ("true", "1", "yes", "on") + return value.strip().lower() in ("true", "1", "yes", "on") + + +def _pyproject_opts_out() -> bool: + """Return True when the nearest pyproject.toml turns telemetry off. + + Walks up from the current directory to the first ``pyproject.toml`` and + reads:: + + [tool.openadapt] + telemetry = false + + This lets a team commit one file to turn telemetry off for everyone who + works in the repository. + """ + try: + import tomllib + except ImportError: + try: + import tomli as tomllib # type: ignore[no-redef] + except ImportError: + return False + + try: + cwd = Path.cwd() + except OSError: + return False + + for parent in [cwd, *cwd.parents]: + pyproject = parent / "pyproject.toml" + if pyproject.is_file(): + try: + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + except Exception: + return False + value = data.get("tool", {}).get("openadapt", {}).get("telemetry") + if value is None: + return False + return not _parse_bool(str(value)) + return False + + +def _file_opts_out(file_config: dict[str, Any]) -> bool: + """Return True when the config file sets ``"enabled"`` to a false value.""" + value = file_config.get("enabled") + if isinstance(value, bool): + return not value + if isinstance(value, str): + return not _parse_bool(value) + return False + + +def is_telemetry_allowed() -> bool: + """Apply the opt-out rule shared by error reports and usage events. + + In order of precedence: + + 1. ``DO_NOT_TRACK`` set to ``1``, ``true``, ``yes`` or ``on`` turns telemetry off. + 2. ``OPENADAPT_TELEMETRY_ENABLED``, when set, decides. + 3. ``[tool.openadapt] telemetry = false`` in the nearest pyproject.toml turns it off. + 4. ``"enabled": false`` in ~/.config/openadapt/telemetry.json turns it off. + + Otherwise telemetry is allowed. + """ + if _parse_bool(os.getenv("DO_NOT_TRACK", "")): + return False + explicit = os.getenv("OPENADAPT_TELEMETRY_ENABLED", "").strip() + if explicit: + return _parse_bool(explicit) + if _pyproject_opts_out(): + return False + return not _file_opts_out(_load_config_file()) def _load_config_file() -> dict[str, Any]: @@ -94,7 +168,7 @@ def _get_env_config() -> dict[str, Any]: config["enabled"] = _parse_bool(enabled_env) # Universal opt-out (DO_NOT_TRACK standard) always wins. - if os.getenv("DO_NOT_TRACK", "").lower() in ("1", "true"): + if _parse_bool(os.getenv("DO_NOT_TRACK", "")): config["enabled"] = False # Internal/developer flags @@ -216,6 +290,9 @@ def load_config() -> TelemetryConfig: env_config = _get_env_config() merged.update(env_config) + # One opt-out rule for every telemetry path (see is_telemetry_allowed). + merged["enabled"] = is_telemetry_allowed() + # Remove None values for fields that should use defaults config_dict = {k: v for k, v in merged.items() if v is not None or k in {"dsn", "anon_salt"}} diff --git a/src/openadapt_telemetry/decorators.py b/src/openadapt_telemetry/decorators.py index 1e6d714..5004732 100644 --- a/src/openadapt_telemetry/decorators.py +++ b/src/openadapt_telemetry/decorators.py @@ -62,7 +62,7 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: raise finally: duration = time.perf_counter() - start - sentry_sdk.set_measurement("duration_ms", duration * 1000, "millisecond") + transaction.set_data("duration_ms", duration * 1000) return wrapper # type: ignore @@ -234,7 +234,7 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: else: self._span.set_status("ok") - self._span.set_measurement("duration_ms", duration * 1000, "millisecond") + self._span.set_data("duration_ms", duration * 1000) self._span.__exit__(exc_type, exc_val, exc_tb) def set_tag(self, key: str, value: str) -> None: diff --git a/src/openadapt_telemetry/events.py b/src/openadapt_telemetry/events.py index 9f46d4b..c93ef75 100644 --- a/src/openadapt_telemetry/events.py +++ b/src/openadapt_telemetry/events.py @@ -7,7 +7,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from datetime import datetime +from datetime import datetime, timezone from enum import Enum from typing import Any, Dict, Optional @@ -60,7 +60,7 @@ class TelemetryEvent: severity: EventSeverity = EventSeverity.INFO properties: Dict[str, Any] = field(default_factory=dict) tags: Dict[str, str] = field(default_factory=dict) - timestamp: datetime = field(default_factory=datetime.utcnow) + timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) def send(self) -> Optional[str]: """Send this event to the telemetry backend. diff --git a/src/openadapt_telemetry/posthog.py b/src/openadapt_telemetry/posthog.py index b7af03c..1269941 100644 --- a/src/openadapt_telemetry/posthog.py +++ b/src/openadapt_telemetry/posthog.py @@ -20,6 +20,7 @@ from typing import Any from .client import is_ci_environment +from .config import is_telemetry_allowed from .privacy import scrub_dict DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com" @@ -37,57 +38,22 @@ def _is_truthy(raw: str | None) -> bool: return str(raw or "").strip().lower() in {"1", "true", "yes", "on"} -def _pyproject_telemetry_disabled() -> bool: - """Check if telemetry is disabled via pyproject.toml [tool.openadapt]. - - Walks up from cwd looking for pyproject.toml with:: - - [tool.openadapt] - telemetry = false - - This lets enterprises commit one file to their repo to disable - telemetry for all developers — no per-user .env needed. - """ - try: - import tomllib - except ImportError: - try: - import tomli as tomllib # type: ignore[no-redef] - except ImportError: - return False - - from pathlib import Path - - for parent in [Path.cwd(), *Path.cwd().parents]: - pyproject = parent / "pyproject.toml" - if pyproject.is_file(): - try: - data = tomllib.loads(pyproject.read_text()) - val = data.get("tool", {}).get("openadapt", {}).get("telemetry") - if val is not None: - return not _is_truthy(str(val)) - except Exception: - pass - break # only check the nearest pyproject.toml - return False - - def _usage_enabled() -> bool: - # 1. Standard DO_NOT_TRACK env var (consoledonottrack.com) - if _is_truthy(os.getenv("DO_NOT_TRACK")): - return False + """Return True when usage events may be sent. - # 2. Explicit env var - explicit = os.getenv("OPENADAPT_TELEMETRY_ENABLED") - if explicit is not None: - return _is_truthy(explicit) - - # 3. pyproject.toml [tool.openadapt] telemetry = false - if _pyproject_telemetry_disabled(): + Uses the same opt-out rule as error reports (see + :func:`openadapt_telemetry.config.is_telemetry_allowed`). In addition, usage + events stay off in CI unless ``OPENADAPT_TELEMETRY_IN_CI`` or + ``OPENADAPT_TELEMETRY_ENABLED`` turns them on. + """ + if not is_telemetry_allowed(): return False - # 4. CI environments default to off - if is_ci_environment() and not _is_truthy(os.getenv("OPENADAPT_TELEMETRY_IN_CI")): + if ( + is_ci_environment() + and not _is_truthy(os.getenv("OPENADAPT_TELEMETRY_IN_CI")) + and not _is_truthy(os.getenv("OPENADAPT_TELEMETRY_ENABLED")) + ): return False return True @@ -126,10 +92,11 @@ def _normalize_value(value: Any) -> Any: def _sanitize_properties(properties: dict[str, Any] | None) -> dict[str, Any]: + """Drop sensitive keys and scrub string values (emails, tokens, home paths).""" if not properties: return {} normalized = {str(k): _normalize_value(v) for k, v in properties.items() if str(k).strip()} - redacted = scrub_dict(normalized, deep=True, scrub_values=False) + redacted = scrub_dict(normalized, deep=True, scrub_values=True) return {k: v for k, v in redacted.items() if v != "[REDACTED]"} diff --git a/src/openadapt_telemetry/privacy.py b/src/openadapt_telemetry/privacy.py index 027d10b..95bda36 100644 --- a/src/openadapt_telemetry/privacy.py +++ b/src/openadapt_telemetry/privacy.py @@ -144,6 +144,12 @@ def _scrub_top_level_messages(event: Dict[str, Any]) -> None: logentry["message"] = scrub_string(logentry["message"]) if isinstance(logentry.get("formatted"), str): logentry["formatted"] = scrub_string(logentry["formatted"]) + # Arguments of a log call such as logging.error("failed for %s", path). + params = logentry.get("params") + if isinstance(params, (list, tuple)): + logentry["params"] = scrub_list(list(params), scrub_values=True) + elif isinstance(params, dict): + logentry["params"] = scrub_dict(params, deep=True, scrub_values=True) def _is_safe_tag_key(key: Any) -> bool: @@ -242,12 +248,17 @@ def is_sensitive_key(key: str) -> bool: def scrub_string(value: str) -> str: """Scrub sensitive patterns from a string value. + Removes the user name from home-directory paths, then replaces emails, + phone numbers, card numbers, SSNs, bearer tokens and long key-like strings + with ``[REDACTED]``. + Args: value: The string to scrub. Returns: The scrubbed string with sensitive patterns replaced. """ + value = sanitize_path(value) for pattern in SENSITIVE_PATTERNS: value = pattern.sub("[REDACTED]", value) return value @@ -334,12 +345,33 @@ def scrub_list(data: List[Any], scrub_values: bool = False) -> List[Any]: return result +# Stack-frame fields that can carry application data rather than code location: +# local variable values and the source lines around each frame. +FRAME_CONTENT_KEYS = ("vars", "pre_context", "context_line", "post_context") + + +def _scrub_stacktrace(stacktrace: Any) -> None: + """Keep only code locations in stack frames, with user names removed.""" + if not isinstance(stacktrace, dict) or not isinstance(stacktrace.get("frames"), list): + return + for frame in stacktrace["frames"]: + if not isinstance(frame, dict): + continue + for key in FRAME_CONTENT_KEYS: + frame.pop(key, None) + if isinstance(frame.get("filename"), str): + frame["filename"] = sanitize_path(frame["filename"]) + if isinstance(frame.get("abs_path"), str): + frame["abs_path"] = sanitize_path(frame["abs_path"]) + + def scrub_exception_data(exception_data: Dict[str, Any]) -> None: """Scrub PII from Sentry exception data in place. - This modifies the exception data dictionary to sanitize: - - File paths in stack traces - - Variable values that may contain PII + This modifies the exception data dictionary to: + - Remove user names from file paths in stack traces + - Drop local variable values and source lines from stack frames + - Scrub sensitive patterns from exception messages Args: exception_data: The Sentry exception data dictionary. @@ -348,26 +380,51 @@ def scrub_exception_data(exception_data: Dict[str, Any]) -> None: return for value in exception_data["values"]: - # Sanitize stack trace file paths - if "stacktrace" in value and "frames" in value["stacktrace"]: - for frame in value["stacktrace"]["frames"]: - if "filename" in frame: - frame["filename"] = sanitize_path(frame["filename"]) - if "abs_path" in frame: - frame["abs_path"] = sanitize_path(frame["abs_path"]) - - # Scrub local variables if present - if "vars" in frame and isinstance(frame["vars"], dict): - frame["vars"] = scrub_dict(frame["vars"], deep=True, scrub_values=True) + _scrub_stacktrace(value.get("stacktrace")) # Scrub exception message for sensitive patterns if "value" in value and isinstance(value["value"], str): value["value"] = scrub_string(value["value"]) +# HTTP breadcrumb fields that hold the query string or fragment of a URL. +URL_DETAIL_KEYS = ("http.query", "http.fragment") + + +# Trace identifiers are random hex. The key-like pattern in SENSITIVE_PATTERNS +# would replace them, which breaks the link between errors and traces. +TRACE_CONTEXT_KEYS = ("trace_id", "span_id", "parent_span_id", "dynamic_sampling_context") + + +def _scrub_contexts(contexts: Dict[str, Any]) -> Dict[str, Any]: + """Scrub event contexts but keep trace identifiers intact.""" + scrubbed = scrub_dict(contexts, deep=True, scrub_values=True) + trace = contexts.get("trace") + if isinstance(trace, dict) and isinstance(scrubbed.get("trace"), dict): + for key in TRACE_CONTEXT_KEYS: + if key in trace: + scrubbed["trace"][key] = trace[key] + return scrubbed + + +def _scrub_spans(spans: List[Any]) -> None: + """Scrub span descriptions, data and tags in a transaction in place.""" + for span in spans: + if not isinstance(span, dict): + continue + if isinstance(span.get("description"), str): + span["description"] = scrub_string(span["description"]) + if isinstance(span.get("data"), dict): + span["data"] = scrub_dict(span["data"], deep=True, scrub_values=True) + if isinstance(span.get("tags"), dict): + span["tags"] = _scrub_tags(span["tags"]) + + def create_before_send_filter(): """Create a before_send filter function for Sentry. + The same function also serves as ``before_send_transaction``. + Returns: A function suitable for use as Sentry's before_send callback. """ @@ -376,29 +433,50 @@ def create_before_send_filter(): def before_send(event: Event, hint: Hint) -> Optional[Event]: """Filter and sanitize events before sending to Sentry/GlitchTip.""" try: + # The host name often contains the user's name (for example + # "alices-macbook"), so it is never sent. + event.pop("server_name", None) + _scrub_top_level_messages(event) # Scrub exception data if "exception" in event: scrub_exception_data(event["exception"]) - # Scrub breadcrumbs + # Thread stack traces (attach_stacktrace) get the same treatment. + threads = event.get("threads") + if isinstance(threads, dict) and isinstance(threads.get("values"), list): + for thread in threads["values"]: + if isinstance(thread, dict): + _scrub_stacktrace(thread.get("stacktrace")) + + # Scrub breadcrumbs. Query strings and fragments of HTTP requests + # often carry tokens or record IDs, so they are dropped. if "breadcrumbs" in event and "values" in event["breadcrumbs"]: for breadcrumb in event["breadcrumbs"]["values"]: if "message" in breadcrumb and isinstance(breadcrumb["message"], str): breadcrumb["message"] = scrub_string(breadcrumb["message"]) if "data" in breadcrumb and isinstance(breadcrumb["data"], dict): - breadcrumb["data"] = scrub_dict( - breadcrumb["data"], deep=True, scrub_values=True - ) - - # Scrub extra data + data = { + k: v + for k, v in breadcrumb["data"].items() + if k not in URL_DETAIL_KEYS + } + breadcrumb["data"] = scrub_dict(data, deep=True, scrub_values=True) + + # Scrub extra data. Command-line arguments can name files or carry + # user text, so the SDK's sys.argv entry is dropped. if "extra" in event and isinstance(event["extra"], dict): - event["extra"] = scrub_dict(event["extra"], deep=True, scrub_values=True) + extra = {k: v for k, v in event["extra"].items() if k != "sys.argv"} + event["extra"] = scrub_dict(extra, deep=True, scrub_values=True) # Scrub contexts if "contexts" in event and isinstance(event["contexts"], dict): - event["contexts"] = scrub_dict(event["contexts"], deep=True, scrub_values=True) + event["contexts"] = _scrub_contexts(event["contexts"]) + + # Scrub transaction spans + if isinstance(event.get("spans"), list): + _scrub_spans(event["spans"]) # Scrub tags (allowlist + scrub values) if "tags" in event and isinstance(event["tags"], dict): @@ -407,6 +485,8 @@ def before_send(event: Event, hint: Hint) -> Optional[Event]: # Scrub request data (if present) if "request" in event and isinstance(event["request"], dict): request = event["request"] + request.pop("query_string", None) + request.pop("cookies", None) if "headers" in request: if isinstance(request["headers"], dict): request["headers"] = scrub_dict( diff --git a/tests/__pycache__/__init__.cpython-312.pyc b/tests/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 02e054c..0000000 Binary files a/tests/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/tests/__pycache__/conftest.cpython-312-pytest-8.3.5.pyc b/tests/__pycache__/conftest.cpython-312-pytest-8.3.5.pyc deleted file mode 100644 index dd41f82..0000000 Binary files a/tests/__pycache__/conftest.cpython-312-pytest-8.3.5.pyc and /dev/null differ diff --git a/tests/__pycache__/test_client.cpython-312-pytest-8.3.5.pyc b/tests/__pycache__/test_client.cpython-312-pytest-8.3.5.pyc deleted file mode 100644 index ca35ac6..0000000 Binary files a/tests/__pycache__/test_client.cpython-312-pytest-8.3.5.pyc and /dev/null differ diff --git a/tests/__pycache__/test_config.cpython-312-pytest-8.3.5.pyc b/tests/__pycache__/test_config.cpython-312-pytest-8.3.5.pyc deleted file mode 100644 index 2248bdd..0000000 Binary files a/tests/__pycache__/test_config.cpython-312-pytest-8.3.5.pyc and /dev/null differ diff --git a/tests/__pycache__/test_decorators.cpython-312-pytest-8.3.5.pyc b/tests/__pycache__/test_decorators.cpython-312-pytest-8.3.5.pyc deleted file mode 100644 index b7e153c..0000000 Binary files a/tests/__pycache__/test_decorators.cpython-312-pytest-8.3.5.pyc and /dev/null differ diff --git a/tests/__pycache__/test_privacy.cpython-312-pytest-8.3.5.pyc b/tests/__pycache__/test_privacy.cpython-312-pytest-8.3.5.pyc deleted file mode 100644 index 53953c5..0000000 Binary files a/tests/__pycache__/test_privacy.cpython-312-pytest-8.3.5.pyc and /dev/null differ diff --git a/tests/conftest.py b/tests/conftest.py index 05c3861..348ddda 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,10 +4,64 @@ from unittest.mock import patch import pytest +import sentry_sdk +from sentry_sdk.transport import Transport from openadapt_telemetry.client import TelemetryClient +class CapturingTransport(Transport): + """Keep every Sentry envelope in memory instead of sending it.""" + + envelopes: list = [] + + def capture_envelope(self, envelope): + CapturingTransport.envelopes.append(envelope) + + +@pytest.fixture +def sentry_events(): + """Route real Sentry clients to memory; return a reader for captured events. + + ``sentry_events("event")`` returns error and message events and + ``sentry_events("transaction")`` returns transactions, as the server + would receive them. + """ + real_init = sentry_sdk.init + + def init(*args, **kwargs): + kwargs["transport"] = CapturingTransport + return real_init(*args, **kwargs) + + def read(kind: str) -> list[dict]: + sentry_sdk.flush() + return [ + item.payload.json + for envelope in CapturingTransport.envelopes + for item in envelope.items + if item.headers.get("type") == kind + ] + + CapturingTransport.envelopes = [] + with patch.object(sentry_sdk, "init", init): + yield read + sentry_sdk.flush() + real_init() # replace the client with one that sends nothing + TelemetryClient.reset_instance() + + +@pytest.fixture(autouse=True) +def isolated_home(tmp_path, monkeypatch): + """Keep tests away from the real ~/.config/openadapt and ~/.openadapt files.""" + config_dir = tmp_path / "config" / "openadapt" + monkeypatch.setattr("openadapt_telemetry.config.CONFIG_DIR", config_dir) + monkeypatch.setattr("openadapt_telemetry.config.CONFIG_FILE", config_dir / "telemetry.json") + monkeypatch.setattr( + "openadapt_telemetry.posthog.DISTINCT_ID_FILE", + tmp_path / "openadapt" / "telemetry_distinct_id", + ) + + @pytest.fixture(autouse=True) def reset_telemetry(): """Reset telemetry singleton before and after each test.""" diff --git a/tests/test_client.py b/tests/test_client.py index cc2606f..3f22dde 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -217,7 +217,7 @@ def test_initialize_with_dsn(self, mock_sentry): assert client.initialized is True mock_sentry.init.assert_called_once() init_kwargs = mock_sentry.init.call_args.kwargs - assert init_kwargs["send_default_pii"] is False + assert "send_default_pii" not in init_kwargs # SDK default is off @patch("openadapt_telemetry.client.sentry_sdk") def test_capture_exception_when_enabled(self, mock_sentry): @@ -325,7 +325,58 @@ def test_send_default_pii_override_is_ignored(self, mock_sentry): send_default_pii=True, ) - assert mock_sentry.init.call_args.kwargs["send_default_pii"] is False + assert "send_default_pii" not in mock_sentry.init.call_args.kwargs + + @patch("openadapt_telemetry.client.sentry_sdk") + def test_data_collection_override_is_ignored(self, mock_sentry): + """data_collection would supersede send_default_pii, so it is refused.""" + with patch.dict(os.environ, {"DO_NOT_TRACK": ""}, clear=False): + TelemetryClient.reset_instance() + client = TelemetryClient.get_instance() + with pytest.warns(UserWarning, match="Ignoring sentry init override for data_collection"): + client.initialize( + dsn="https://test@example.com/1", + data_collection={"user_info": True}, + ) + + assert "data_collection" not in mock_sentry.init.call_args.kwargs + + @patch("openadapt_telemetry.client.sentry_sdk") + def test_privacy_filter_covers_transactions(self, mock_sentry): + """Transactions get the same filter as errors, and frames carry no values.""" + with patch.dict(os.environ, {"DO_NOT_TRACK": ""}, clear=False): + TelemetryClient.reset_instance() + client = TelemetryClient.get_instance() + client.initialize(dsn="https://test@example.com/1") + + kwargs = mock_sentry.init.call_args.kwargs + assert kwargs["include_local_variables"] is False + assert kwargs["include_source_context"] is False + transaction = {"server_name": "alices-laptop", "tags": {"password": "x"}} + output = kwargs["before_send_transaction"](transaction, {}) + assert output is not None + assert "server_name" not in output + assert "password" not in output["tags"] + + @patch("openadapt_telemetry.client.sentry_sdk") + def test_custom_before_send_transaction_runs_before_privacy_filter(self, mock_sentry): + """A caller's before_send_transaction cannot bypass the privacy filter.""" + + def custom(event, hint): + event["server_name"] = "alices-laptop" + return event + + with patch.dict(os.environ, {"DO_NOT_TRACK": ""}, clear=False): + TelemetryClient.reset_instance() + client = TelemetryClient.get_instance() + with pytest.warns(UserWarning, match="Custom before_send_transaction runs before"): + client.initialize( + dsn="https://test@example.com/1", + before_send_transaction=custom, + ) + + hook = mock_sentry.init.call_args.kwargs["before_send_transaction"] + assert "server_name" not in hook({}, {}) def test_initialize_rejects_non_callable_before_send(self): """Non-callable before_send should raise TypeError.""" diff --git a/tests/test_opt_out.py b/tests/test_opt_out.py new file mode 100644 index 0000000..661bbca --- /dev/null +++ b/tests/test_opt_out.py @@ -0,0 +1,154 @@ +"""One opt-out rule covers error reports and usage events.""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import patch + +import pytest + +import openadapt_telemetry.config as config +import openadapt_telemetry.posthog as posthog +from openadapt_telemetry import TelemetryClient, is_telemetry_allowed + +TELEMETRY_ENV = ( + "DO_NOT_TRACK", + "OPENADAPT_TELEMETRY_ENABLED", + "OPENADAPT_TELEMETRY_IN_CI", + "CI", + "CONTINUOUS_INTEGRATION", + "GITHUB_ACTIONS", + "GITLAB_CI", + "JENKINS_URL", + "TRAVIS", + "CIRCLECI", + "BUILDKITE", + "AZURE_PIPELINES", + "TF_BUILD", + "CODEBUILD_BUILD_ID", + "TEAMCITY_VERSION", + "BITBUCKET_BUILD_NUMBER", +) + + +@pytest.fixture +def project(tmp_path, monkeypatch): + """Run from an empty project directory with no telemetry variables set.""" + for name in TELEMETRY_ENV: + monkeypatch.delenv(name, raising=False) + root = tmp_path / "project" + root.mkdir() + monkeypatch.chdir(root) + return root + + +def _write_pyproject(root, telemetry): + value = "true" if telemetry else "false" + (root / "pyproject.toml").write_text(f"[tool.openadapt]\ntelemetry = {value}\n") + + +def _write_config_file(enabled): + config.CONFIG_DIR.mkdir(parents=True, exist_ok=True) + config.CONFIG_FILE.write_text(json.dumps({"enabled": enabled})) + + +def _paths_enabled() -> tuple[bool, bool]: + """Return (error reports enabled, usage events enabled).""" + TelemetryClient.reset_instance() + return TelemetryClient.get_instance().enabled, posthog._usage_enabled() + + +def test_enabled_by_default(project): + assert is_telemetry_allowed() is True + assert _paths_enabled() == (True, True) + + +@pytest.mark.parametrize("value", ["1", "true", "yes", "on", "TRUE"]) +def test_do_not_track_turns_off_both_paths(project, monkeypatch, value): + monkeypatch.setenv("DO_NOT_TRACK", value) + assert _paths_enabled() == (False, False) + + +def test_do_not_track_beats_explicit_enable(project, monkeypatch): + monkeypatch.setenv("DO_NOT_TRACK", "1") + monkeypatch.setenv("OPENADAPT_TELEMETRY_ENABLED", "true") + assert _paths_enabled() == (False, False) + + +def test_pyproject_opt_out_turns_off_both_paths(project): + _write_pyproject(project, telemetry=False) + assert _paths_enabled() == (False, False) + + +def test_pyproject_opt_out_found_from_subdirectory(project, monkeypatch): + _write_pyproject(project, telemetry=False) + sub = project / "src" / "pkg" + sub.mkdir(parents=True) + monkeypatch.chdir(sub) + assert _paths_enabled() == (False, False) + + +def test_pyproject_telemetry_true_keeps_defaults(project): + _write_pyproject(project, telemetry=True) + assert _paths_enabled() == (True, True) + + +def test_explicit_env_enable_overrides_pyproject(project, monkeypatch): + _write_pyproject(project, telemetry=False) + monkeypatch.setenv("OPENADAPT_TELEMETRY_ENABLED", "true") + assert _paths_enabled() == (True, True) + + +def test_config_file_opt_out_turns_off_both_paths(project): + _write_config_file(False) + assert _paths_enabled() == (False, False) + + +def test_config_file_string_false_is_an_opt_out(project): + _write_config_file("false") + assert _paths_enabled() == (False, False) + + +def test_usage_events_stay_off_in_ci_by_default(project, monkeypatch): + monkeypatch.setenv("CI", "true") + assert _paths_enabled() == (True, False) + monkeypatch.setenv("OPENADAPT_TELEMETRY_IN_CI", "true") + assert posthog._usage_enabled() is True + + +def test_empty_explicit_variable_means_unset(project, monkeypatch): + monkeypatch.setenv("OPENADAPT_TELEMETRY_ENABLED", "") + assert _paths_enabled() == (True, True) + + +def test_opted_out_usage_event_is_not_queued(project): + _write_pyproject(project, telemetry=False) + with patch("openadapt_telemetry.posthog._ensure_worker") as worker: + assert posthog.capture_usage_event("agent_run") is False + worker.assert_not_called() + + +def test_config_file_internal_flag_tags_reports(project): + config.CONFIG_DIR.mkdir(parents=True, exist_ok=True) + config.CONFIG_FILE.write_text(json.dumps({"internal": True})) + TelemetryClient.reset_instance() + assert TelemetryClient.get_instance().internal is True + + +def test_python_310_installs_a_toml_parser(): + """Python 3.10 has no tomllib, so the package must depend on tomli there. + + Without it the pyproject.toml opt-out is silently ignored. pytest pulls in + tomli on 3.10, so this checks the declared dependency instead of importing. + """ + try: + import tomllib + except ImportError: # Python 3.10, where pytest provides tomli + import tomli as tomllib + + text = (Path(__file__).resolve().parent.parent / "pyproject.toml").read_text() + dependencies = tomllib.loads(text)["project"]["dependencies"] + assert any( + d.startswith("tomli") and "python_version < '3.11'" in d for d in dependencies + ) diff --git a/tests/test_posthog.py b/tests/test_posthog.py index b854ca5..ab42c2b 100644 --- a/tests/test_posthog.py +++ b/tests/test_posthog.py @@ -73,3 +73,28 @@ def test_capture_event_scrubs_sensitive_properties() -> None: assert props["entrypoint"] == "oa evals run" assert "api_key" not in props assert "password" not in props + + +def test_capture_event_scrubs_property_values() -> None: + queue = _CaptureQueue() + with patch.dict( + os.environ, + { + "OPENADAPT_TELEMETRY_ENABLED": "true", + "OPENADAPT_TELEMETRY_DISTINCT_ID": "test-id", + }, + clear=False, + ): + with patch("openadapt_telemetry.posthog._ensure_worker", return_value=queue): + assert posthog.capture_event( + "demo_recorded", + { + "steps": 12, + "owner": "alice@example.com", + "source": "/Users/alice/demos/intake.json", + }, + ) + props = queue.payload["properties"] + assert props["steps"] == 12 + assert "owner" not in props + assert props["source"] == "/Users//demos/intake.json" diff --git a/tests/test_privacy.py b/tests/test_privacy.py index 67f4f31..4395360 100644 --- a/tests/test_privacy.py +++ b/tests/test_privacy.py @@ -254,8 +254,8 @@ def test_exception_message_scrubbing(self): assert "user@example.com" not in exception_data["values"][0]["value"] - def test_local_variables_scrubbing(self): - """Local variables in stack frames should be scrubbed.""" + def test_local_variables_and_source_lines_dropped(self): + """Stack frames keep the code location only, not values or source lines.""" exception_data = { "values": [ { @@ -263,10 +263,15 @@ def test_local_variables_scrubbing(self): "frames": [ { "filename": "app.py", + "function": "save", + "lineno": 12, "vars": { "password": "secret123", - "username": "john", + "note": "patient text", }, + "pre_context": ["note = read()"], + "context_line": "save(note)", + "post_context": ["return"], }, ], }, @@ -276,8 +281,23 @@ def test_local_variables_scrubbing(self): scrub_exception_data(exception_data) frame = exception_data["values"][0]["stacktrace"]["frames"][0] - assert frame["vars"]["password"] == "[REDACTED]" - assert frame["vars"]["username"] == "john" + assert frame == {"filename": "app.py", "function": "save", "lineno": 12} + + def test_home_path_in_exception_message_sanitized(self): + """A path in an exception message loses the user name.""" + exception_data = { + "values": [ + { + "type": "FileNotFoundError", + "value": "No such file: '/Users/john/notes/visit.txt'", + }, + ], + } + scrub_exception_data(exception_data) + + assert exception_data["values"][0]["value"] == ( + "No such file: '/Users//notes/visit.txt'" + ) class TestAnonymizeIdentifier: diff --git a/tests/test_readme.py b/tests/test_readme.py index 15bf09b..fcbfdd9 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -1,20 +1,39 @@ -"""Checks that the README matches the published state of the package.""" +"""Checks that the README matches the published package and the code's behavior.""" +import doctest import re from pathlib import Path +from unittest.mock import patch -README = Path(__file__).resolve().parent.parent / "README.md" +import pytest + +from openadapt_telemetry import TelemetryClient, get_telemetry + +ROOT = Path(__file__).resolve().parent.parent +README = ROOT / "README.md" HTML_COMMENT = re.compile(r"", re.DOTALL) +PYTHON_BLOCK = re.compile(r"```python\n(.*?)```", re.DOTALL) + + +def _readme() -> str: + return README.read_text(encoding="utf-8") def _visible_readme() -> str: """Return the README text with HTML comments removed.""" - return HTML_COMMENT.sub("", README.read_text(encoding="utf-8")) + return HTML_COMMENT.sub("", _readme()) + + +def _python_block(containing: str) -> str: + """Return the one README python block that contains the given text.""" + blocks = [b for b in PYTHON_BLOCK.findall(_readme()) if containing in b] + assert len(blocks) == 1, f"expected one python block containing {containing!r}" + return blocks[0] def test_readme_does_not_say_package_is_unpublished(): """The package is on PyPI, so no note may say it isn't yet.""" - assert "uncomment once package is published" not in README.read_text(encoding="utf-8") + assert "uncomment once package is published" not in _readme() def test_readme_shows_pypi_badges(): @@ -22,3 +41,127 @@ def test_readme_shows_pypi_badges(): visible = _visible_readme() assert "img.shields.io/pypi/v/openadapt-telemetry" in visible assert "img.shields.io/pypi/dm/openadapt-telemetry" in visible + + +def test_readme_doctest_examples_pass(): + """Every >>> example in the README produces the output it shows.""" + parser = doctest.DocTestParser() + runner = doctest.DocTestRunner(optionflags=doctest.ELLIPSIS) + examples = [b for b in PYTHON_BLOCK.findall(_readme()) if ">>>" in b] + assert examples + for i, block in enumerate(examples): + test = parser.get_doctest(block, {}, f"README example {i}", str(README), 0) + runner.run(test) + assert runner.summarize(verbose=False).failed == 0 + + +def test_quick_start_sends_the_report_the_readme_describes(clean_env, monkeypatch, sentry_events): + """Run the quick start against an in-memory transport and compare with the README.""" + monkeypatch.setenv("OPENADAPT_TELEMETRY_DSN", "https://public@example.invalid/1") + TelemetryClient.reset_instance() + exec(_python_block("telemetry.initialize("), {}) + + (event,) = sentry_events("event") + (exception,) = event["exception"]["values"] + received = f"{exception['type']}: {exception['value']}" + claimed = re.search(r"GlitchTip receives\s+`([^`]+)`", _readme()).group(1) + assert received == claimed + assert "server_name" not in event + for frame in exception["stacktrace"]["frames"]: + assert {"filename", "function", "lineno"} <= set(frame) + assert not {"vars", "pre_context", "context_line", "post_context"} & set(frame) + + +def test_quick_start_sends_nothing_without_a_dsn(clean_env, sentry_events): + """Without a DSN, initialize() returns False and nothing is sent.""" + TelemetryClient.reset_instance() + namespace: dict = {} + exec(_python_block("telemetry.initialize("), namespace) + assert namespace["telemetry"].initialized is False + assert sentry_events("event") == [] + + +def test_usage_event_example_queues_an_event(monkeypatch): + """The usage-event example queues one event with the properties shown.""" + for name in ("DO_NOT_TRACK", "CI", "GITHUB_ACTIONS"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("OPENADAPT_TELEMETRY_ENABLED", "true") + queued = [] + + class Queue: + def put_nowait(self, payload): + queued.append(payload) + + with patch("openadapt_telemetry.posthog._ensure_worker", return_value=Queue()): + exec(_python_block("capture_usage_event("), {}) + + (payload,) = queued + assert payload["event"] == "demo_recorded" + assert payload["properties"]["steps"] == 12 + + +def test_decorator_example_runs_before_initialize(): + """The decorator example runs as written and does nothing until initialize().""" + exec(_python_block("TelemetrySpan("), {}) + + +def test_decorator_example_after_initialize(clean_env, sentry_events): + """track_feature sends a message per call; a span outside a transaction sends nothing.""" + TelemetryClient.reset_instance() + get_telemetry().initialize(dsn="https://public@example.invalid/1", traces_sample_rate=1.0) + namespace: dict = {} + exec(_python_block("TelemetrySpan("), namespace) + assert sentry_events("transaction") == [] + + namespace["add_demo"]("demo-1") + namespace["add_demo"]("demo-2") + assert len(sentry_events("event")) == 2 + namespace["build_index"]([1, 2, 3]) + (transaction,) = sentry_events("transaction") + assert transaction["transaction"] == "indexing.build" + + +@pytest.mark.parametrize( + "term", + [ + "admission", + "admitted", + "signed ledger", + "evidence authority", + "product target", + "lifecycle state", + "governed", + ], +) +def test_readme_has_no_governance_language(term): + """Process and compliance terms belong in docs/COMPLIANCE.md, not the README.""" + assert term not in _readme().lower() + + +def test_readme_links_compliance_doc_once(): + """The data-handling reference is linked once, and the link target exists.""" + links = re.findall(r"\]\(([^)]*docs/COMPLIANCE\.md)\)", _readme()) + assert len(links) == 1 + assert (ROOT / "docs" / "COMPLIANCE.md").is_file() + + +def test_compliance_failure_signal_example_queues_one_event(monkeypatch): + """The failure-signal example in docs/COMPLIANCE.md runs and sends only the closed schema.""" + for name in ("DO_NOT_TRACK", "CI", "GITHUB_ACTIONS"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("OPENADAPT_TELEMETRY_ENABLED", "true") + doc = (ROOT / "docs" / "COMPLIANCE.md").read_text(encoding="utf-8") + (block,) = PYTHON_BLOCK.findall(doc) + queued = [] + + class Queue: + def put_nowait(self, payload): + queued.append(payload) + + with patch("openadapt_telemetry.posthog._ensure_worker", return_value=Queue()): + exec(block, {}) + + (payload,) = queued + assert payload["event"] == "automation_failure_observed" + assert payload["distinct_id"].startswith("failure:") + assert payload["properties"]["occurred_at"].endswith(":00:00Z") diff --git a/tests/test_sentry_payload.py b/tests/test_sentry_payload.py new file mode 100644 index 0000000..b6b9cef --- /dev/null +++ b/tests/test_sentry_payload.py @@ -0,0 +1,109 @@ +"""End-to-end checks on what the Sentry SDK would send, using a capturing transport.""" + +from __future__ import annotations + +import logging +import re + +import pytest +import sentry_sdk + +from openadapt_telemetry import TelemetryClient, TelemetrySpan, get_telemetry, track_performance + +HEX32 = re.compile(r"^[0-9a-f]{32}$") + + +@pytest.fixture +def live_client(clean_env, sentry_events): + """A real Sentry client whose transport keeps events in memory.""" + TelemetryClient.reset_instance() + client = get_telemetry() + assert client.initialize( + dsn="https://public@example.invalid/1", + package_name="openadapt-test", + package_version="1.0.0", + traces_sample_rate=1.0, + ) + return client + + +def _fail(note: str) -> None: + path = "/Users/alice/records/" + note + raise FileNotFoundError(f"No such file: {path!r}") + + +def test_sdk_default_pii_is_off(live_client): + assert sentry_sdk.get_client().should_send_default_pii() is False + + +def test_error_event_carries_no_personal_or_application_data(live_client, sentry_events): + try: + _fail("visit-notes.txt") + except FileNotFoundError as exc: + assert live_client.capture_exception(exc) + + (event,) = sentry_events("event") + assert "server_name" not in event + assert "sys.argv" not in event.get("extra", {}) + (exception,) = event["exception"]["values"] + assert exception["value"] == "No such file: '/Users//records/visit-notes.txt'" + for frame in exception["stacktrace"]["frames"]: + assert not {"vars", "pre_context", "context_line", "post_context"} & set(frame) + assert "alice" not in frame.get("abs_path", "") + assert HEX32.match(event["contexts"]["trace"]["trace_id"]) + assert event["tags"]["package"] == "openadapt-test" + + +def test_release_is_the_package_version_not_a_git_commit(live_client, sentry_events): + """The tests run inside a git checkout, where the SDK would send its commit hash.""" + live_client.capture_message("hello") + (event,) = sentry_events("event") + assert event["release"] == "openadapt-test@1.0.0" + + +def test_caller_can_set_the_release(clean_env, sentry_events): + TelemetryClient.reset_instance() + get_telemetry().initialize( + dsn="https://public@example.invalid/1", package_name="pkg", release="pkg@abc" + ) + get_telemetry().capture_message("hello") + (event,) = sentry_events("event") + assert event["release"] == "pkg@abc" + + +def test_transactions_go_through_the_privacy_filter(live_client, sentry_events): + @track_performance("indexing.build") + def build(): + with TelemetrySpan("indexing", "load") as span: + span.set_data("source", "/home/alice/data.csv") + span.set_data("api_key", "abc") + return 1 + + build() + + (transaction,) = sentry_events("transaction") + assert transaction["transaction"] == "indexing.build" + assert "server_name" not in transaction + assert "sys.argv" not in transaction.get("extra", {}) + assert HEX32.match(transaction["contexts"]["trace"]["trace_id"]) + (span,) = transaction["spans"] + assert span["data"]["source"] == "/home//data.csv" + assert span["data"]["api_key"] == "[REDACTED]" + + +def test_log_arguments_and_url_queries_are_scrubbed(live_client, sentry_events): + logging.getLogger("openadapt-test").error( + "save failed for %s (%s)", "/Users/alice/out.json", "alice@example.com" + ) + sentry_sdk.add_breadcrumb( + type="http", + category="httplib", + data={"url": "https://emr.example/api/visits", "http.query": "token=abc123"}, + ) + live_client.capture_message("after request") + + log_event, message_event = sentry_events("event") + assert log_event["logentry"]["params"] == ["/Users//out.json", "[REDACTED]"] + assert "alice" not in log_event["logentry"].get("formatted", "") + (http,) = [b for b in message_event["breadcrumbs"]["values"] if b.get("type") == "http"] + assert http["data"] == {"url": "https://emr.example/api/visits"}