diff --git a/.agents/dd-apm-sdk-review-overrides/repo-context.md b/.agents/dd-apm-sdk-review-overrides/repo-context.md new file mode 100644 index 0000000000..cda085ee8a --- /dev/null +++ b/.agents/dd-apm-sdk-review-overrides/repo-context.md @@ -0,0 +1,13 @@ +# Repo context — dd-trace-php + +Read only by the orchestrator (Step 0 of `SKILL.md`), not by individual reviewers. Repo-specific; not part of the shared core. + +## Related skills in this repo + +Existing skills live under `.claude/skills/`. Cite them as authoritative for their area. Do not invoke them, and they must not invoke this skill: + +- `check-ci` — GitLab CI / GitHub Actions watch and failure investigation +- `crash-analysis` — wild crash reports (`event.json`) for this tracer +- `release-notes` — `CHANGELOG.md` for a minor/major release + +`.claude/skills/dd-apm-sdk-review` is a symlink to `.agents/skills/dd-apm-sdk-review`. No name clash. diff --git a/.agents/dd-apm-sdk-review-overrides/reviewers/conventions.md b/.agents/dd-apm-sdk-review-overrides/reviewers/conventions.md new file mode 100644 index 0000000000..b1070e3461 --- /dev/null +++ b/.agents/dd-apm-sdk-review-overrides/reviewers/conventions.md @@ -0,0 +1,13 @@ +Override for `reviewers/conventions.md` (in the core skill folder) — read that file first, then this. + +# Conventions — dd-trace-php specifics + +This file starts with one confirmed pattern and should grow. Do not treat it as exhaustive. + +The source of truth for *how* style is checked is [`phpcs.xml`](../../../phpcs.xml) via `composer lint` / `composer fix-lint`. [`CONTRIBUTING.md`](../../../CONTRIBUTING.md) § "PHP linting" still names PSR-2; the ruleset that command actually runs is [PSR-12](https://www.php-fig.org/psr/psr-12/). When those disagree, `phpcs.xml` wins. + +## PHP userland follows the phpcs ruleset (PSR-12) + +A new or edited `.php` file that fails `composer lint` is a finding (Allman braces and tab indent fail that check). Do not invent a different house style than `phpcs.xml`. + +CI does **not** run `composer lint`. It runs `composer ci-lint`, a separate nearly-empty gate (same CONTRIBUTING section). That is not "this repo has no PHP style standard" — the standard is still `composer lint` / `phpcs.xml`. diff --git a/.gitignore b/.gitignore index 6539a75e64..65d9f5fa34 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,8 @@ tests/tested_versions .gitlab/*-gen.yml .gitlab-ci-local/ /.claude/settings.local.json + +# LLM Validation local/CI artifacts +.llm-validation/results.json +.llm-validation/report.md +.llm-validation/details.json diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6ef68e5ce..a95f506c2b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,6 +17,26 @@ variables: description: "Run a specific datadog-reliability-env branch downstream" SYSTEM_TESTS_LIBRARY: php +include: + - project: "ddoghq/llm-validation-platform" + ref: &llmval_platform_sha "c331696647a78672101002ba48ccebbd41b0fae8" + file: "/ci/llm-validation.gitlab-ci.yml" + +"llm validation": + variables: + LLMVAL_PLATFORM_PROJECT: "ddoghq/llm-validation-platform" + LLMVAL_PLATFORM_REF: *llmval_platform_sha + LLMVAL_BASE_REF: "master" + # Repo-wide default is recursive (libdatadog, etc.). JS/Java skip submodules + # on this job; inherit that so the gate does not clone GitHub before run.sh. + GIT_SUBMODULE_STRATEGY: none + # dd-trace-php's fabric-proxy 403s NodeSource/npm/dot.net/NuGet; JS/Java SAs allow + # them. Bypass the proxy for the hosts run.sh needs (CIEXE-714 annotation). + # Temporary: drop once ddoghq/dd-source#93032 (ACL) is deployed. + # dot.net 301s to builds.dotnet.microsoft.com; the install script also uses aka.ms. + # After the SDK lands, `dotnet build` restores from api.nuget.org. + KUBERNETES_POD_ANNOTATIONS_NODESOURCE: "beta.fabric.datadoghq.com/no-proxy-additions=deb.nodesource.com,registry.npmjs.org,dot.net,builds.dotnet.microsoft.com,aka.ms,api.nuget.org,nuget.org" + generate-templates: stage: build image: registry.ddbuild.io/images/mirror/php:8.2-cli diff --git a/.llm-validation/README.md b/.llm-validation/README.md new file mode 100644 index 0000000000..7938b33cb2 --- /dev/null +++ b/.llm-validation/README.md @@ -0,0 +1,72 @@ +# LLM Validation — `dd-apm-sdk-review` + +This folder is how we test the review skill. It is **not** a PHPUnit run. +The cases live here; the runner lives in [`ddoghq/llm-validation-platform`](https://github.com/ddoghq/llm-validation-platform). + +Same gate as [`dd-trace-js#10137`](https://github.com/DataDog/dd-trace-js/pull/10137) and +[`dd-trace-java#12409`](https://github.com/DataDog/dd-trace-java/pull/12409). + +It answers: *did an edit to a review rule make the agent better or worse?* + +## Add a rule (this is the whole contribution) + +Overrides are owned by this repo. The shared core is not — never edit `.agents/skills/dd-apm-sdk-review/`. + +1. Create or extend a file under [`.agents/dd-apm-sdk-review-overrides/reviewers/`](../.agents/dd-apm-sdk-review-overrides/reviewers/). + Copy the shape of `conventions.md`: one pattern, why it matters, the fix. +2. Add the new override path to `instruction_files` in [`config.yaml`](./config.yaml) so CI + watches it. +3. Add a case in [`suites/dd-apm-sdk-review.yaml`](./suites/dd-apm-sdk-review.yaml). Copy the starter case. + A good case is a 10-line snippet plus 2–3 `expected_criteria` that would fail if the rule disappeared. +4. List the new case id under `presets.gate.cases` in [`config.yaml`](./config.yaml) if you want CI to run it. +5. Open a PR. That is it. + +The starter case in this folder is the example. Keep new ones that short. + +## Layout + +| Path | Role | +|---|---| +| [`config.yaml`](./config.yaml) | Monitored instruction files, model, `--level` presets | +| [`suites/dd-apm-sdk-review.yaml`](./suites/dd-apm-sdk-review.yaml) | Cases (one file only — the CLI errors if `suites/` has more than one YAML) | + +## Run locally (Docker) + +From the **`dd-trace-php` repo root**: + +```bash +export LLMVAL_IMAGE=registry.ddbuild.io/ci/llm-validation-platform/llmval:latest +docker pull "$LLMVAL_IMAGE" + +# Offline smoke — no gateway, no Claude (1 case) +docker run --rm -v "$PWD:/repo" "$LLMVAL_IMAGE" \ + --repo /repo --base-sha master --level minimum --fake + +# Cheap real smoke — still 1 case +export LLMVAL_AUTH_HEADER="$(ddtool auth token rapid-ai-platform --datacenter us1.staging.dog --http-header)" +docker run --rm -e LLMVAL_AUTH_HEADER -v "$PWD:/repo" "$LLMVAL_IMAGE" \ + --repo /repo --base-sha master --level minimum --runs 1 + +# The gate preset (starter case; add more ids in config.yaml to grow it) +docker run --rm -e LLMVAL_AUTH_HEADER -v "$PWD:/repo" "$LLMVAL_IMAGE" \ + --repo /repo --base-sha master --level gate --runs 1 + +# One named case +docker run --rm -e LLMVAL_AUTH_HEADER -v "$PWD:/repo" "$LLMVAL_IMAGE" \ + --repo /repo --base-sha master --case php-conventions-psr2 --runs 1 +``` + +`--level` picks **which cases** run (`minimum` / `gate` = the starter case until you add more, `full` = every case). +`--runs` only repeats those cases. Needs `ddtool` on the host for a real (non-`--fake`) run. + +CI includes the reusable `"llm validation"` job from the platform repo (see `.gitlab-ci.yml`). +This repo's default branch is `master`, so the job sets `LLMVAL_BASE_REF: "master"`. + +## What a pass means + +This is an A/B comparison, not an absolute score: + +- **Candidate** = the working tree. Uncommitted edits count. +- **Baseline** = `git show :`. A file that is not on `master` yet is treated as added. + +The gate fails only on a **confident regression**. Noisy changes WARN and do not block. diff --git a/.llm-validation/config.yaml b/.llm-validation/config.yaml new file mode 100644 index 0000000000..c137c31a68 --- /dev/null +++ b/.llm-validation/config.yaml @@ -0,0 +1,40 @@ +# LLM Validation Platform — dd-trace-php config +# +# Starter suite for the dd-apm-sdk-review skill. One case on purpose. +model: claude-sonnet-5 +runs: 2 + +instruction_files: + - AGENTS.md + - .agents/skills/dd-apm-sdk-review/SKILL.md + - .agents/skills/dd-apm-sdk-review/reviewers/_common.md + - .agents/skills/dd-apm-sdk-review/reviewers/coherence.md + - .agents/skills/dd-apm-sdk-review/reviewers/correctness.md + - .agents/skills/dd-apm-sdk-review/reviewers/design.md + - .agents/skills/dd-apm-sdk-review/reviewers/performance.md + - .agents/skills/dd-apm-sdk-review/reviewers/maintainability.md + - .agents/skills/dd-apm-sdk-review/reviewers/conventions.md + - .agents/skills/dd-apm-sdk-review/reviewers/cross-sdk.md + - .agents/skills/dd-apm-sdk-review/reviewers/report-template.md + - .agents/skills/dd-apm-sdk-review/review-without-harness.md + - .agents/dd-apm-sdk-review-overrides/repo-context.md + - .agents/dd-apm-sdk-review-overrides/reviewers/conventions.md + +default_level: gate +presets: + gate: + cases: + - php-conventions-psr2 + runs: 2 + minimum: + cases: + - php-conventions-psr2 + runs: 2 + full: + runs: 2 + +policy: + noise_threshold: 1.0 + pairwise_win_floor: 0.45 + blocking_fail_floor: 0.45 + blocking_fail_ci_upper: 0.55 diff --git a/.llm-validation/suites/dd-apm-sdk-review.yaml b/.llm-validation/suites/dd-apm-sdk-review.yaml new file mode 100644 index 0000000000..fb2039985c --- /dev/null +++ b/.llm-validation/suites/dd-apm-sdk-review.yaml @@ -0,0 +1,39 @@ +name: dd-apm-sdk-review +version: "0.1" + +# One file on purpose: the platform CLI errors if suites/ has more than one YAML. +# One starter case — a template. Copy this block to add the next one. +cases: + - id: php-conventions-psr2 + files: + - CONTRIBUTING.md + - phpcs.xml + - .agents/skills/dd-apm-sdk-review/reviewers/conventions.md + - .agents/skills/dd-apm-sdk-review/reviewers/_common.md + - .agents/dd-apm-sdk-review-overrides/reviewers/conventions.md + input: | + Apply this repo's conventions-review criteria (reviewers/conventions.md, its + .agents/dd-apm-sdk-review-overrides/reviewers/conventions.md override, _common.md, + CONTRIBUTING.md § "PHP linting", and phpcs.xml) to the following change. No git + checkout available — this snippet is the entire change to review. + + ```php +