diff --git a/.claude/ci/check-ci b/.claude/ci/check-ci index a1ae5e91e5e..f0ec59e7322 100755 --- a/.claude/ci/check-ci +++ b/.claude/ci/check-ci @@ -35,6 +35,41 @@ class GitLabError(Exception): """Raised on fatal GitLab API errors (auth failure, network error, timeout).""" +class GitLabClient: + """GitLab API session that refreshes a rejected OAuth token once.""" + + def __init__(self, token: str): + self.session = aiohttp.ClientSession( + headers={"Authorization": f"Bearer {token}"}, + connector=aiohttp.TCPConnector(limit=20), + ) + self._refresh_lock = asyncio.Lock() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.session.close() + + async def refresh_token(self, rejected_authorization: str | None) -> bool: + """Refresh the token unless another request already refreshed it.""" + async with self._refresh_lock: + current_authorization = self.session.headers.get("Authorization") + if current_authorization != rejected_authorization: + return True + + token = await asyncio.to_thread(get_gitlab_token) + if not token: + return False + + self.session.headers["Authorization"] = f"Bearer {token}" + print( + "GitLab token expired or was revoked; refreshed it and retrying.", + file=sys.stderr, + ) + return True + + def parse_args(): parser = argparse.ArgumentParser(description="Monitor a GitLab CI pipeline and GitHub Actions workflows.") group = parser.add_mutually_exclusive_group() @@ -107,27 +142,38 @@ def get_gitlab_token() -> str | None: # GitLab API helpers # --------------------------------------------------------------------------- -async def api_get(session: aiohttp.ClientSession, path: str, params: dict | None = None) -> tuple[int, Any, dict]: +async def api_get(client: GitLabClient, path: str, params: dict | None = None) -> tuple[int, Any, dict]: """Make a GET request. Returns (status_code, json_or_text, headers).""" url = f"{API_BASE}{path}" - try: - async with session.get(url, params=params) as resp: - if resp.status in (401, 403): - text = await resp.text() - msg = f"Error: authentication failed ({resp.status}): {text}" - print(msg, file=sys.stderr) - raise GitLabError(msg) - headers = dict(resp.headers) - if resp.content_type and "json" in resp.content_type: - return resp.status, await resp.json(), headers - return resp.status, await resp.text(), headers - except aiohttp.ClientError as e: - return 0, str(e), {} - - -async def api_get_json(session: aiohttp.ClientSession, path: str, params: dict | None = None) -> Any: + for attempt in range(2): + rejected_authorization = client.session.headers.get("Authorization") + try: + async with client.session.get(url, params=params) as resp: + if resp.status == 401: + text = await resp.text() + if attempt == 0 and await client.refresh_token(rejected_authorization): + continue + msg = f"Error: authentication failed ({resp.status}): {text}" + print(msg, file=sys.stderr) + raise GitLabError(msg) + if resp.status == 403: + text = await resp.text() + msg = f"Error: authentication failed ({resp.status}): {text}" + print(msg, file=sys.stderr) + raise GitLabError(msg) + headers = dict(resp.headers) + if resp.content_type and "json" in resp.content_type: + return resp.status, await resp.json(), headers + return resp.status, await resp.text(), headers + except aiohttp.ClientError as e: + return 0, str(e), {} + + raise AssertionError("unreachable") + + +async def api_get_json(client: GitLabClient, path: str, params: dict | None = None) -> Any: """GET request expecting JSON. Returns None on network error.""" - status, data, _ = await api_get(session, path, params) + status, data, _ = await api_get(client, path, params) if status == 0: print(f"Warning: network error fetching {path}: {data}", file=sys.stderr) return None @@ -137,15 +183,15 @@ async def api_get_json(session: aiohttp.ClientSession, path: str, params: dict | return data -async def api_get_text(session: aiohttp.ClientSession, path: str) -> str | None: +async def api_get_text(client: GitLabClient, path: str) -> str | None: """GET request expecting text. Returns None on error.""" - status, data, _ = await api_get(session, path) + status, data, _ = await api_get(client, path) if status == 0 or status >= 400: return None return data -async def paginated_get(session: aiohttp.ClientSession, path: str, params: dict | None = None) -> list: +async def paginated_get(client: GitLabClient, path: str, params: dict | None = None) -> list: """Fetch all pages of a paginated endpoint. Reads X-Total-Pages from the first response, then fetches all remaining pages in parallel. @@ -154,7 +200,7 @@ async def paginated_get(session: aiohttp.ClientSession, path: str, params: dict p["per_page"] = 100 p["page"] = 1 - status, first_page, headers = await api_get(session, path, p) + status, first_page, headers = await api_get(client, path, p) if status == 0: print(f"Warning: network error fetching {path}: {first_page}", file=sys.stderr) return [] @@ -171,19 +217,19 @@ async def paginated_get(session: aiohttp.ClientSession, path: str, params: dict async def fetch_page(n: int) -> list: pp = {**p, "page": n} - data = await api_get_json(session, path, pp) + data = await api_get_json(client, path, pp) return data if isinstance(data, list) else [] rest = await asyncio.gather(*[fetch_page(n) for n in range(2, total_pages + 1)]) return first_page + [item for page in rest for item in page] -async def discover_pipeline(session: aiohttp.ClientSession, sha: str, timeout: int) -> int: +async def discover_pipeline(client: GitLabClient, sha: str, timeout: int) -> int: """Poll GitLab until a pipeline is found for the given SHA.""" deadline = time.monotonic() + timeout interval = 5 while True: - status, data, _ = await api_get(session, f"/projects/{PROJECT_ID}/pipelines", {"sha": sha, "per_page": 20}) + status, data, _ = await api_get(client, f"/projects/{PROJECT_ID}/pipelines", {"sha": sha, "per_page": 20}) if status == 0: msg = f"Error: network error during pipeline discovery: {data}" print(msg, file=sys.stderr) @@ -209,7 +255,7 @@ async def discover_pipeline(session: aiohttp.ClientSession, sha: str, timeout: i async def discover_pipelines_and_jobs( - session: aiohttp.ClientSession, root_id: int + client: GitLabClient, root_id: int ) -> tuple[list[int], dict[int, str], dict[int, list[dict]], dict[int, dict]]: """Discover all pipelines and fetch all jobs in a single pass (no double bridge fetches). @@ -230,9 +276,9 @@ async def discover_pipelines_and_jobs( async def visit(pid: int, depth: int): pipeline_ids.append(pid) jobs, bridges, details = await asyncio.gather( - paginated_get(session, f"/projects/{PROJECT_ID}/pipelines/{pid}/jobs"), - paginated_get(session, f"/projects/{PROJECT_ID}/pipelines/{pid}/bridges"), - api_get_json(session, f"/projects/{PROJECT_ID}/pipelines/{pid}"), + paginated_get(client, f"/projects/{PROJECT_ID}/pipelines/{pid}/jobs"), + paginated_get(client, f"/projects/{PROJECT_ID}/pipelines/{pid}/bridges"), + api_get_json(client, f"/projects/{PROJECT_ID}/pipelines/{pid}"), ) jobs_by_pipeline[pid] = list(jobs or []) + list(bridges or []) if details and isinstance(details, dict): @@ -276,10 +322,10 @@ def compute_duration(job: dict) -> int | None: return None -async def download_job_log(session: aiohttp.ClientSession, job_id: int, log_dir: Path): +async def download_job_log(client: GitLabClient, job_id: int, log_dir: Path): """Download a job's trace log to a file.""" log_path = log_dir / f"{job_id}.log" - text = await api_get_text(session, f"/projects/{PROJECT_ID}/jobs/{job_id}/trace") + text = await api_get_text(client, f"/projects/{PROJECT_ID}/jobs/{job_id}/trace") if text is None: msg = f"Failed to download log for job {job_id}" print(f"Warning: {msg}", file=sys.stderr) @@ -414,10 +460,10 @@ async def gh_download_job_log(session: aiohttp.ClientSession, job_id: int, log_d # list-jobs mode # --------------------------------------------------------------------------- -async def list_jobs(session: aiohttp.ClientSession, root_id: int, patterns: list[str] | None = None): +async def list_jobs(client: GitLabClient, root_id: int, patterns: list[str] | None = None): """List all GitLab jobs grouped by pipeline, then exit. Filtered to matched jobs when patterns given.""" patterns = patterns or [] - pipeline_ids, pipeline_names, jobs_by_pipeline, pipeline_statuses = await discover_pipelines_and_jobs(session, root_id) + pipeline_ids, pipeline_names, jobs_by_pipeline, pipeline_statuses = await discover_pipelines_and_jobs(client, root_id) for pid in pipeline_ids: jobs_by_pipeline[pid] = sorted(jobs_by_pipeline.get(pid, []), key=lambda j: j.get("name", "")) @@ -504,13 +550,10 @@ async def _run(args): if sha and not github_token: print("Warning: could not get GitHub token via 'ddtool auth github token' — skipping GitHub Actions monitoring", file=sys.stderr) - gl_connector = aiohttp.TCPConnector(limit=20) - gl_headers = {"Authorization": f"Bearer {token}"} - gh_session: aiohttp.ClientSession | None = None gh_connector: aiohttp.TCPConnector | None = None - async with aiohttp.ClientSession(headers=gl_headers, connector=gl_connector) as gl_session: + async with GitLabClient(token) as gl_client: # Open GitHub session if we have a token and SHA if sha and github_token: gh_connector = aiohttp.TCPConnector(limit=10) @@ -522,7 +565,7 @@ async def _run(args): gh_session = aiohttp.ClientSession(headers=gh_headers, connector=gh_connector) try: - await _monitor(args, sha, gl_session, gh_session) + await _monitor(args, sha, gl_client, gh_session) finally: if gh_session: await gh_session.close() @@ -530,20 +573,20 @@ async def _run(args): await gh_connector.close() -async def _monitor(args, sha: str | None, gl_session: aiohttp.ClientSession, gh_session: aiohttp.ClientSession | None): +async def _monitor(args, sha: str | None, gl_client: GitLabClient, gh_session: aiohttp.ClientSession | None): # Determine root GitLab pipeline ID if args.pipeline is not None: root_id = args.pipeline print(f"Using pipeline {root_id}") else: - root_id = await discover_pipeline(gl_session, sha, args.discovery_timeout) + root_id = await discover_pipeline(gl_client, sha, args.discovery_timeout) patterns = parse_job_patterns(args.jobs) if patterns: print(f"Filtering to jobs matching (any of): {patterns}") if args.list_jobs: - await list_jobs(gl_session, root_id, patterns) + await list_jobs(gl_client, root_id, patterns) if gh_session and sha: await list_github_jobs(gh_session, sha, patterns) return @@ -591,7 +634,7 @@ async def _monitor(args, sha: str | None, gl_session: aiohttp.ClientSession, gh_ # Phase 1: fetch GitLab pipeline tree and GitHub run list in parallel. if gh_monitoring: gl_result, gh_runs_raw = await asyncio.gather( - discover_pipelines_and_jobs(gl_session, root_id), + discover_pipelines_and_jobs(gl_client, root_id), _gh_get_runs_safe(), ) if gh_runs_raw is None: @@ -601,7 +644,7 @@ async def _monitor(args, sha: str | None, gl_session: aiohttp.ClientSession, gh_ else: all_runs = gh_runs_raw else: - gl_result = await discover_pipelines_and_jobs(gl_session, root_id) + gl_result = await discover_pipelines_and_jobs(gl_client, root_id) all_runs = [] pipeline_ids, _, jobs_by_pipeline, pipeline_statuses = gl_result @@ -689,7 +732,7 @@ async def _monitor(args, sha: str | None, gl_session: aiohttp.ClientSession, gh_ # Phase 4: download GL and GH failure logs in parallel. download_coros = [ - download_job_log(gl_session, job["id"], fail_log_dir) + download_job_log(gl_client, job["id"], fail_log_dir) for job in new_failure_jobs if "downstream_pipeline" not in job # bridge/trigger jobs have no trace log ] diff --git a/.github/workflows/prof_correctness.yml b/.github/workflows/prof_correctness.yml index 20c92f71290..1bbf333fa85 100644 --- a/.github/workflows/prof_correctness.yml +++ b/.github/workflows/prof_correctness.yml @@ -12,7 +12,9 @@ jobs: prof-correctness: runs-on: ubuntu-24.04 env: - LLVM_VERSION: "20" + # LLVM's own release build from GitHub, pinned by the SHA256 GitHub publishes for the asset. + LLVM_RELEASE: "20.1.8" + LLVM_SHA256: "1ead36b3dfcb774b57be530df42bec70ab2d239fbce9889447c7a29a4ddc1ae6" PROFILER_SO: ${{ github.workspace }}/tmp/build_profiler/modules/datadog-profiling.so strategy: fail-fast: false @@ -49,30 +51,50 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ tmp/build_profiler/target-profiling/ - key: ${{ runner.os }}-cargo-llvm-${{ env.LLVM_VERSION }}-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }}-${{ matrix.php-version }}-${{ matrix.phpts }} + key: ${{ runner.os }}-cargo-llvm-${{ env.LLVM_RELEASE }}-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }}-${{ matrix.php-version }}-${{ matrix.phpts }} + + # All matrix jobs share one LLVM, so it is cached separately from the + # per-job build cache. + - name: Restore LLVM + id: llvm-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/llvm + key: ${{ runner.os }}-llvm-${{ env.LLVM_RELEASE }}-${{ env.LLVM_SHA256 }} + + - name: Download LLVM + if: steps.llvm-cache.outputs.cache-hit != 'true' + run: | + # Only the parts the build uses: the full toolchain unpacks to over 9 GB. + llvm_archive="LLVM-${LLVM_RELEASE}-Linux-X64.tar.xz" + llvm_top="${llvm_archive%.tar.xz}" + curl -fsSL --retry 3 -o "/tmp/${llvm_archive}" \ + "https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_RELEASE}/${llvm_archive}" + echo "${LLVM_SHA256} /tmp/${llvm_archive}" | sha256sum -c - + mkdir -p ~/llvm + tar -xJf "/tmp/${llvm_archive}" -C ~/llvm --strip-components=1 --wildcards \ + "${llvm_top}/bin/clang" "${llvm_top}/bin/clang++" "${llvm_top}/bin/clang-[0-9]*" \ + "${llvm_top}/bin/ld.lld" "${llvm_top}/bin/lld" "${llvm_top}/bin/llvm-config" \ + "${llvm_top}/lib/libclang.so*" "${llvm_top}/lib/clang/*" + rm -f "/tmp/${llvm_archive}" + + - name: Cache LLVM + if: steps.llvm-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/llvm + key: ${{ runner.os }}-llvm-${{ env.LLVM_RELEASE }}-${{ env.LLVM_SHA256 }} - name: Build profiler run: | - codename="$(lsb_release -cs)" - curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/llvm-archive-keyring.gpg - echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-${LLVM_VERSION} main" | sudo tee /etc/apt/sources.list.d/llvm.list - sudo apt-get update - llvm18_packages="$(dpkg-query -W -f='${binary:Package}\n' \ - '*clang*18*' '*llvm*18*' '*lld*18*' '*libomp*18*' \ - '*libc++*18*' '*libc++abi*18*' '*mlir*18*' '*flang*18*' \ - '*bolt*18*' '*polly*18*' 2>/dev/null || true)" - if [ -n "$llvm18_packages" ]; then - sudo apt-get purge -y $llvm18_packages - sudo apt-get autoremove -y - fi - sudo apt-get install -y clang-${LLVM_VERSION} lld-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev libclang-${LLVM_VERSION}-dev libclang-rt-${LLVM_VERSION}-dev - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 100 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${LLVM_VERSION} 100 - sudo update-alternatives --install /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-${LLVM_VERSION} 100 - export CC=clang-${LLVM_VERSION} - export CXX=clang++-${LLVM_VERSION} - export LLVM_CONFIG_PATH=/usr/bin/llvm-config-${LLVM_VERSION} - export LIBCLANG_PATH=/usr/lib/llvm-${LLVM_VERSION}/lib + llvm_root="${HOME}/llvm" + # lld links against the system libxml2. + dpkg -s libxml2 >/dev/null 2>&1 || { sudo apt-get update && sudo apt-get install -y libxml2; } + export PATH="${llvm_root}/bin:${PATH}" + export CC=clang + export CXX=clang++ + export LLVM_CONFIG_PATH="${llvm_root}/bin/llvm-config" + export LIBCLANG_PATH="${llvm_root}/lib" export LD_LIBRARY_PATH="${LIBCLANG_PATH}:${LD_LIBRARY_PATH:-}" clang --version ld.lld --version @@ -91,7 +113,7 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ tmp/build_profiler/target-profiling/ - key: ${{ runner.os }}-cargo-llvm-${{ env.LLVM_VERSION }}-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }}-${{ matrix.php-version }}-${{ matrix.phpts }} + key: ${{ runner.os }}-cargo-llvm-${{ env.LLVM_RELEASE }}-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }}-${{ matrix.php-version }}-${{ matrix.phpts }} - name: Run no profile test run: | diff --git a/.gitlab/compile_extension.sh b/.gitlab/compile_extension.sh index 5c266d5c3d7..1202c121c4b 100755 --- a/.gitlab/compile_extension.sh +++ b/.gitlab/compile_extension.sh @@ -20,9 +20,7 @@ if [ "${WITH_ASAN}" -eq "1" ]; then export COMPILE_ASAN=1 fi # Compile Rust and PHP in parallel -rust_build_log=$(mktemp) -trap 'rm -f "$rust_build_log"' EXIT -SHARED=1 ./compile_rust.sh 2>&1 | tee "$rust_build_log" & +SHARED=1 ./compile_rust.sh & rust_build_pid=$! make -j static & c_build_pid=$! @@ -36,12 +34,6 @@ if [ "$c_build_status" -ne 0 ]; then exit "$c_build_status" fi if [ "$rust_build_status" -ne 0 ]; then - # Retry the job with a clean target directory: libddwaf may have left a - # partially extracted archive, which cannot safely be reused locally. - if grep -Eq 'Failed to (download archive|write archive entry contents to file):.*reqwest::Error.*(ConnectionReset|ConnectionAborted|TimedOut|IncompleteMessage|UnexpectedEof)' "$rust_build_log"; then - echo "Transient libddwaf download failure; exiting 75 for GitLab retry." >&2 - exit 75 - fi exit "$rust_build_status" fi diff --git a/.gitlab/generate-ci-images.php b/.gitlab/generate-ci-images.php index ee9db882d3c..8d8805aebc9 100644 --- a/.gitlab/generate-ci-images.php +++ b/.gitlab/generate-ci-images.php @@ -113,6 +113,7 @@ function parse_compose(string $path, array $env): array fwrite(STDERR, "ERROR: no services parsed for Windows\n"); exit(1); } + ?> # CI image build + publish child pipeline, generated by # .gitlab/generate-ci-images.php from the docker-compose.yml + .env files. @@ -245,7 +246,7 @@ function parse_compose(string $path, array $env): array if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } Write-Host "Initializing submodules..." - git submodule update --init --recursive + git submodule update --init --recursive -- libdatadog tests/FeatureFlags/ffe-system-test-data appsec/third_party/libddwaf-rust if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } Write-Host "Git setup complete." diff --git a/.gitlab/generate-common.php b/.gitlab/generate-common.php index 830b762c8f8..e33fdd183cc 100644 --- a/.gitlab/generate-common.php +++ b/.gitlab/generate-common.php @@ -82,7 +82,7 @@ function windows_git_setup() { # Initialize submodules Write-Host "Initializing submodules..." - git submodule update --init --recursive + git submodule update --init --recursive -- libdatadog tests/FeatureFlags/ffe-system-test-data appsec/third_party/libddwaf-rust if ($LASTEXITCODE -ne 0) { exit 75 } # transient network (submodule fetch); 75 triggers default retry Write-Host "Git setup complete." " - mkdir -p tmp/build_zai && cd tmp/build_zai - CMAKE_PREFIX_PATH=/opt/catch2 Tea_ROOT=../../tmp/tea/ cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_ZAI_TESTING=ON -DPhpConfig_ROOT=$(php-config --prefix) ../../zend_abstract_interface - - ../../.gitlab/run-with-retryable-download.sh make -j all + - make -j all - mkdir -p "${CI_PROJECT_DIR}/artifacts" - make test ARGS="--output-junit ${CI_PROJECT_DIR}/artifacts/zai---results.xml --output-on-failure" - grep -e "=== Total [0-9]+ memory leaks detected ===" Testing/Temporary/LastTest.log && exit 1 || true @@ -213,7 +213,7 @@ - echo "extension=curl.so" | sudo tee $(php -i | awk -F"=> " '/Scan this dir for additional .ini files/ {print $2}')/curl.ini - mkdir -p tmp/build_zai && cd tmp/build_zai - CMAKE_PREFIX_PATH=/opt/catch2 Tea_ROOT=../../tmp/tea/nts cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_ZAI_TESTING=ON -DRUN_SHARED_EXTS_TESTS=1 -DPhpConfig_ROOT=$(php-config --prefix) ../../zend_abstract_interface - - ../../.gitlab/run-with-retryable-download.sh make -j all + - make -j all - TEA_INI_IGNORE=0 make test - grep -e "=== Total [0-9]+ memory leaks detected ===" Testing/Temporary/LastTest.log && exit 1 || true "] @@ -18,7 +20,8 @@ authors = ["Datadog Inc. "] [package] name = "datadog-php" version.workspace = true -edition.workspace = true +# Not yet migrated to edition 2024; pinned independently of the workspace. +edition = "2021" rust-version.workspace = true license.workspace = true authors.workspace = true diff --git a/appsec/tests/integration/build.gradle b/appsec/tests/integration/build.gradle index 998f707aa49..afa7ddda71c 100644 --- a/appsec/tests/integration/build.gradle +++ b/appsec/tests/integration/build.gradle @@ -295,6 +295,9 @@ def buildRunInDockerTask = { Map options -> if (!options.get('runAsRoot', false)) { commandLine.addAll(['--user', uuid, '-e', 'HOME=/tmp']) } + if (System.getenv('GITLAB_CI') != null) { + commandLine.addAll(['-e', 'GITLAB_CI']) + } binds.each { source, dest -> commandLine.addAll(['--mount', "type=bind,src=${source},dst=${dest}"]) } @@ -1284,7 +1287,7 @@ buildRunInDockerTask( cargo fmt -p helper-rust --check echo '=== Running Clippy ===' - cargo clippy --release -p helper-rust + cargo clippy --release -p helper-rust --no-deps echo '=== Building helper-rust ===' cargo build --release -p helper-rust diff --git a/appsec/third_party/CMakeLists.txt b/appsec/third_party/CMakeLists.txt index ae485834569..526d45a82ed 100644 --- a/appsec/third_party/CMakeLists.txt +++ b/appsec/third_party/CMakeLists.txt @@ -47,36 +47,35 @@ if(DD_APPSEC_BUILD_EXTENSION) endif() endif() -# spdlog -FetchContent_Declare( - spdlog - GIT_REPOSITORY https://github.com/gabime/spdlog.git - GIT_TAG 79524ddd08a4ec981b7fea76afd08ee05f83755d) -FetchContent_MakeAvailable(spdlog) -set_target_properties(spdlog PROPERTIES POSITION_INDEPENDENT_CODE 1) - -# zlib -set(ZLIB_VERSION v1.3.1) -FetchContent_Declare( - zlib - GIT_REPOSITORY https://github.com/madler/zlib.git - GIT_TAG ${ZLIB_VERSION} -) -FetchContent_MakeAvailable(zlib) -if(NOT(MSVC)) - set_property(TARGET zlibstatic PROPERTY POSITION_INDEPENDENT_CODE ON) +if(DD_APPSEC_TESTING) + set(SPDLOG_COMMIT 79524ddd08a4ec981b7fea76afd08ee05f83755d) + if(DEFINED ENV{GITLAB_CI}) + set(SPDLOG_URL + "https://depot-read-api-bzl.us1.ddbuild.io/github.com/gabime/spdlog/archive/${SPDLOG_COMMIT}.tar.gz") + else() + set(SPDLOG_URL + "https://github.com/gabime/spdlog/archive/${SPDLOG_COMMIT}.tar.gz") + endif() + FetchContent_Declare( + spdlog + URL "${SPDLOG_URL}" + URL_HASH SHA256=c6dee844e35a54cc80e3ab76aae2310763e0b49c7397549dfa4d4657d9737138) + FetchContent_MakeAvailable(spdlog) + set_target_properties(spdlog PROPERTIES POSITION_INDEPENDENT_CODE 1) endif() -target_compile_definitions(zlibstatic PUBLIC ZLIB_CONST=1) -target_include_directories(zlibstatic INTERFACE ${zlib_SOURCE_DIR} ${zlib_BINARY_DIR}) - -# rapidJson +set(RAPIDJSON_COMMIT 24b5e7a8b27f42fa16b96fc70aade9106cf7102f) +if(DEFINED ENV{GITLAB_CI}) + set(RAPIDJSON_URL + "https://depot-read-api-bzl.us1.ddbuild.io/github.com/Tencent/rapidjson/archive/${RAPIDJSON_COMMIT}.tar.gz") +else() + set(RAPIDJSON_URL + "https://github.com/Tencent/rapidjson/archive/${RAPIDJSON_COMMIT}.tar.gz") +endif() FetchContent_Declare( rapidjson - GIT_REPOSITORY https://github.com/Tencent/rapidjson.git - GIT_TAG 24b5e7a8b27f42fa16b96fc70aade9106cf7102f - GIT_SHALLOW TRUE -) + URL "${RAPIDJSON_URL}" + URL_HASH SHA256=2d2601a82d2d3b7e143a3c8d43ef616671391034bc46891a9816b79cf2d3e7a8) FetchContent_Populate(rapidjson) # don't read its CMakeLists.txt add_library(rapidjson_appsec INTERFACE) target_include_directories(rapidjson_appsec INTERFACE ${rapidjson_SOURCE_DIR}/include) diff --git a/appsec/third_party/libddwaf-rust b/appsec/third_party/libddwaf-rust index b7569bea223..4ffe0d8297e 160000 --- a/appsec/third_party/libddwaf-rust +++ b/appsec/third_party/libddwaf-rust @@ -1 +1 @@ -Subproject commit b7569bea223b1eb329944154d95bf804fa9fae67 +Subproject commit 4ffe0d8297ec8a3aa31593fa9913bc5b4e4311ce diff --git a/cbindgen.toml b/cbindgen.toml index dd4223ffb7e..e2e54e2f762 100644 --- a/cbindgen.toml +++ b/cbindgen.toml @@ -11,6 +11,8 @@ no_includes = true sys_includes = ["stdbool.h", "stddef.h", "stdint.h"] includes = ["common.h", "telemetry.h", "sidecar.h"] +after_includes = "#define ddog_DYANMIC_CONFIG_UPDATE_UNMODIFIED (_zend_string*)1" + [defines] "feature = profiling" = "DDTRACE_PROFILING" "target_os = linux" = "__linux__" diff --git a/components-rs/common.h b/components-rs/common.h index d9f89bd27a5..099371d5730 100644 --- a/components-rs/common.h +++ b/components-rs/common.h @@ -52,6 +52,15 @@ typedef struct ddog_Endpoint ddog_Endpoint; +/** + * A shared handle to a [`MutableMetadata`]. + * + * This is a cheap-to-clone `Arc>`: all clones observe the same + * underlying value. See the + * [module documentation](self) for the write/read protocol. + */ +typedef struct ddog_MutableMetadataHandle ddog_MutableMetadataHandle; + /** * Holds the raw parts of a Rust Vec; it should only be created from Rust, * never from C. @@ -208,6 +217,24 @@ typedef struct ddog_ArrayQueue_UsizeResult { }; } ddog_ArrayQueue_UsizeResult; +/** + * A generic result type for when an operation may fail, + * but there's nothing to return in the case of success. + */ +typedef enum ddog_VoidResult_Tag { + DDOG_VOID_RESULT_OK, + DDOG_VOID_RESULT_ERR, +} ddog_VoidResult_Tag; + +typedef struct ddog_VoidResult { + ddog_VoidResult_Tag tag; + union { + struct { + struct ddog_Error err; + }; + }; +} ddog_VoidResult; + typedef enum ddog_Option_U32_Tag { DDOG_OPTION_U32_SOME_U32, DDOG_OPTION_U32_NONE_U32, @@ -264,6 +291,8 @@ typedef struct ddog_Vec_Tag_ParseResult { typedef struct _zend_string _zend_string; +#define ddog_DYANMIC_CONFIG_UPDATE_UNMODIFIED (_zend_string*)1 + #define ddog_LOG_ONCE (1 << 3) /** @@ -477,6 +506,12 @@ typedef struct ddog_SidecarActionsBuffer ddog_SidecarActionsBuffer; */ typedef struct ddog_SidecarTransport ddog_SidecarTransport; +/** + * One pre-encoded flush. The duplicate fd preserves the template connection until drop; it + * shares packet ordering with normal sends and never reads the normal client's replies. + * Construction and destruction require ordinary thread context. + */ +typedef struct ddog_SignalFlush ddog_SignalFlush; /** * Opaque shared-memory span stats concentrator exposed to C. * @@ -725,24 +760,6 @@ typedef struct ddog_Vec_DebuggerPayload { */ typedef uint64_t ddog_QueueId; -/** - * A generic result type for when an operation may fail, - * but there's nothing to return in the case of success. - */ -typedef enum ddog_VoidResult_Tag { - DDOG_VOID_RESULT_OK, - DDOG_VOID_RESULT_ERR, -} ddog_VoidResult_Tag; - -typedef struct ddog_VoidResult { - ddog_VoidResult_Tag tag; - union { - struct { - struct ddog_Error err; - }; - }; -} ddog_VoidResult; - /** * A (key, value) pair for peer-service tags, borrowed from PHP/concentrator memory. */ @@ -949,8 +966,6 @@ typedef uint8_t ddog_Bytes[16]; */ typedef ddog_Bytes ddog_Uuid; -#define ddog_DYANMIC_CONFIG_UPDATE_UNMODIFIED (_zend_string*)1 - typedef struct ddog_DebuggerCapture ddog_DebuggerCapture; typedef struct ddog_DebuggerValue ddog_DebuggerValue; @@ -1217,6 +1232,15 @@ typedef struct ddog_MappedMem_ShmHandle ddog_MappedMem_ShmHandle; */ typedef struct ddog_PlatformHandle_File ddog_PlatformHandle_File; +/** + * Opaque registration for a Windows remote configuration callback. + * + * Create it with `ddog_sidecar_remote_config_notification_new`, pass it to + * `ddog_sidecar_session_set_config`, and release it with + * `ddog_sidecar_remote_config_notification_drop`. + */ +typedef struct ddog_RemoteConfigNotification ddog_RemoteConfigNotification; + typedef struct ddog_RemoteConfigReader ddog_RemoteConfigReader; /** @@ -1281,7 +1305,6 @@ typedef struct ddog_Slice_FfeExposure { */ uintptr_t len; } ddog_Slice_FfeExposure; - typedef struct ddog_FfeFlagEvaluation { int64_t timestamp_ms; ddog_CharSlice flag_key; @@ -1315,7 +1338,6 @@ typedef struct ddog_Slice_FfeFlagEvaluation { */ uintptr_t len; } ddog_Slice_FfeFlagEvaluation; - typedef struct ddog_FfeEvaluationMetric { ddog_CharSlice flag_key; ddog_CharSlice variant; @@ -1337,7 +1359,6 @@ typedef struct ddog_Slice_FfeEvaluationMetric { */ uintptr_t len; } ddog_Slice_FfeEvaluationMetric; - /** * Holds the raw parts of a Rust Vec; it should only be created from Rust, * never from C. @@ -1347,7 +1368,6 @@ typedef struct ddog_Vec_SpanBytes { uintptr_t len; uintptr_t capacity; } ddog_Vec_SpanBytes; - typedef struct ddog_Vec_SpanBytes ddog_TraceBytes; /** @@ -1359,7 +1379,6 @@ typedef struct ddog_Vec_TraceBytes { uintptr_t len; uintptr_t capacity; } ddog_Vec_TraceBytes; - typedef struct ddog_Vec_TraceBytes ddog_TracesBytes; typedef struct ddog_SenderParameters { @@ -2173,6 +2192,52 @@ void ddog_endpoint_set_use_system_resolver(struct ddog_Endpoint *endpoint, void ddog_endpoint_drop(struct ddog_Endpoint*); +/** + * Creates a shared, updatable metadata handle initialized with default values. + * + * Use the metadata setters to configure or update its values. + * + * # Safety + * + * `out_handle` must point to valid, writable (uninitialized) memory for a + * `ddog_MutableMetadataHandle *`. + */ +void ddog_mutable_metadata_new(struct ddog_MutableMetadataHandle **out_handle); + +/** + * Frees a `ddog_MutableMetadataHandle` handle. + * + * Call once this handle is no longer needed. It must not be used concurrently + * with this call or accessed afterward. Other cloned handles remain valid. + * + * # Safety + * + * `handle` must be a valid mutable metadata handle obtained through [`ddog_mutable_metadata_new`] + */ +void ddog_mutable_metadata_free(struct ddog_MutableMetadataHandle *handle); + +/** + * Replaces the `runtime_id` held by the shared mutable metadata handle. + * + * # Safety + * + * `handle` must be a valid mutable metadata handle obtained through [`ddog_mutable_metadata_new`] + */ +DDOG_CHECK_RETURN +struct ddog_VoidResult ddog_mutable_metadata_set_runtime_id(const struct ddog_MutableMetadataHandle *handle, + ddog_CharSlice runtime_id); + +/** + * Replaces the `process_tags` held by the shared mutable metadata handle. + * + * # Safety + * + * `handle` must be a valid mutable metadata handle obtained through [`ddog_mutable_metadata_new`] + */ +DDOG_CHECK_RETURN +struct ddog_VoidResult ddog_mutable_metadata_set_process_tags(const struct ddog_MutableMetadataHandle *handle, + ddog_CharSlice process_tags); + struct ddog_Option_U32 ddog_Option_U32_some(uint32_t v); struct ddog_Option_U32 ddog_Option_U32_none(void); diff --git a/components-rs/datadog.h b/components-rs/datadog.h index 446d6fd51be..90a3805eaba 100644 --- a/components-rs/datadog.h +++ b/components-rs/datadog.h @@ -273,6 +273,12 @@ void datadog_sidecar_set_reconnect_fn(struct ddog_SidecarTransport **transport, void datadog_sidecar_clear_reconnect_fn(struct ddog_SidecarTransport **transport); +bool ddog_shm_limiter_inc(const struct ddog_MaybeShmLimiter *limiter, uint32_t limit); + +bool ddog_exception_hash_limiter_inc(struct ddog_SidecarTransport *connection, + uint64_t hash, + uint32_t granularity_seconds); + #if defined(__linux__) /** * Execute the prepared flush. For a default signal disposition, terminate the process afterward. @@ -284,12 +290,6 @@ int32_t datadog_sidecar_signal_flush_run(const struct ddog_SignalFlush *flush, bool terminate_process); #endif -bool ddog_shm_limiter_inc(const struct ddog_MaybeShmLimiter *limiter, uint32_t limit); - -bool ddog_exception_hash_limiter_inc(struct ddog_SidecarTransport *connection, - uint64_t hash, - uint32_t granularity_seconds); - /** * Returns true once the agent /info has been received and applied. * Used by the PHP extension to skip stats computation until the concentrator diff --git a/components-rs/sidecar.h b/components-rs/sidecar.h index 6ba2fc8511a..cd5a2c62e5a 100644 --- a/components-rs/sidecar.h +++ b/components-rs/sidecar.h @@ -208,7 +208,7 @@ ddog_MaybeError ddog_sidecar_session_set_config(struct ddog_SidecarTransport **t uintptr_t force_drop_size, ddog_CharSlice log_level, ddog_CharSlice log_path, - void *_remote_config_notify_function, + const struct ddog_RemoteConfigNotification *win_remote_config_notification, const enum ddog_RemoteConfigProduct *remote_config_products, uintptr_t remote_config_products_count, const enum ddog_RemoteConfigCapabilities *remote_config_capabilities, @@ -533,6 +533,94 @@ struct ddog_AppsecCResponse datadog_sidecar_send_appsec_message_without_reconnec */ void ddog_sidecar_appsec_response_drop(struct ddog_AppsecCResponse response); +#if defined(_WIN32) +/** + * Create a Windows notification that invokes `callback(context)` when remote configuration may + * have changed. + * + * On success, `*out` receives a newly allocated notification. Pass that pointer to + * `ddog_sidecar_session_set_config` to associate it with a session, and eventually release it + * with `ddog_sidecar_remote_config_notification_drop`. Session configuration does not take + * ownership of the notification. + * + * The callback runs asynchronously on a Windows thread-pool thread. Invocations of the + * caller-provided callback for the same notification do not overlap, but several remote + * configuration updates may be coalesced into one callback invocation. Treat the callback as a + * prompt to read the latest configuration rather than as a count of updates. + * + * If the function returns an error, a valid `out` parameter is set to NULL. + * + * # Safety + * + * - `out` must point to writable storage for one notification pointer. + * - `callback` must be non-NULL and safe to call with `context` from a Windows thread-pool thread. + * - If creation succeeds, the callback code and any data reached through `context` must remain + * valid until `ddog_sidecar_remote_config_notification_drop` returns. + * - The callback must not drop its own notification. + */ +ddog_MaybeError ddog_sidecar_remote_config_notification_new(void (*callback)(void*), + void *context, + struct ddog_RemoteConfigNotification **out); +#endif + +#if defined(_WIN32) +/** + * Disable a remote configuration notification and release it. + * + * Passing NULL has no effect. If its callback is currently running, this function waits for the + * callback to return. Once this function returns, no callback for this notification is running or + * can start, so the caller may safely release the callback context or unload the callback code. + * A sidecar that still has the session configuration may continue sending signals, but those + * signals can no longer invoke the callback. + * + * # Safety + * + * - `notification` must be NULL or a live pointer returned by + * `ddog_sidecar_remote_config_notification_new`. + * - A non-NULL pointer may be passed to this function only once and must not be used concurrently + * by another call, including `ddog_sidecar_session_set_config`. + * - This function must not be called from the notification's callback. + */ +void ddog_sidecar_remote_config_notification_drop(struct ddog_RemoteConfigNotification *notification); +#endif + +#if defined(__linux__) +/** + * Prepare a flush on this transport with a private completion pipe. + * Normal thread context only. The returned object owns a duplicate of the transport fd; + * refresh it after reconnect and drop it before normal connection shutdown. + * + * # Safety + * `transport` must be exclusively borrowed and `output` must be writable for this call. + */ +ddog_MaybeError ddog_sidecar_prepare_signal_flush(struct ddog_SidecarTransport *transport, + struct ddog_SidecarFlushOptions options, + struct ddog_SignalFlush **output); +#endif + +#if defined(__linux__) +/** + * Destroy a prepared flush in ordinary thread context. + * + * # Safety + * `flush` must be null or an owned pointer returned by prepare. Any raw worker must have exited. + */ +void ddog_sidecar_signal_flush_drop(struct ddog_SignalFlush *flush); +#endif + +#if defined(__linux__) +/** + * Run one bounded flush without TLS access, allocation, unwinding, or process termination. + * Returns zero when the sidecar closes the pipe (completion or exit), or a negative Linux errno. + * + * # Safety + * The object must remain alive through the call, with exclusive one-shot use of this object. + * The normal transport may continue sending and receiving concurrently. + * All worker signals must be blocked. Do not use an inherited object after fork. + */ +int32_t ddog_sidecar_signal_flush_run(const struct ddog_SignalFlush *flush); +#endif + ddog_TracesBytes *ddog_get_traces(void); void ddog_free_traces(ddog_TracesBytes *_traces); @@ -692,50 +780,4 @@ void ddog_add_event_attributes_float(ddog_SpanEventBytes *event, ddog_CharSlice */ ddog_CharSlice ddog_serialize_trace_into_charslice(ddog_TraceBytes *trace); -#if defined(__linux__) -/** - * One pre-encoded flush. The duplicate fd preserves the template connection until drop; it - * shares packet ordering with normal sends and never reads the normal client's replies. - * Construction and destruction require ordinary thread context. - */ -typedef struct ddog_SignalFlush ddog_SignalFlush; -#endif - -#if defined(__linux__) -/** - * Prepare a flush on this transport with a private completion pipe. - * Normal thread context only. The returned object owns a duplicate of the transport fd; - * refresh it after reconnect and drop it before normal connection shutdown. - * - * # Safety - * `transport` must be exclusively borrowed and `output` must be writable for this call. - */ -ddog_MaybeError ddog_sidecar_prepare_signal_flush(struct ddog_SidecarTransport *transport, - struct ddog_SidecarFlushOptions options, - struct ddog_SignalFlush **output); -#endif - -#if defined(__linux__) -/** - * Destroy a prepared flush in ordinary thread context. - * - * # Safety - * `flush` must be null or an owned pointer returned by prepare. Any raw worker must have exited. - */ -void ddog_sidecar_signal_flush_drop(struct ddog_SignalFlush *flush); -#endif - -#if defined(__linux__) -/** - * Run one bounded flush without TLS access, allocation, unwinding, or process termination. - * Returns zero when the sidecar closes the pipe (completion or exit), or a negative Linux errno. - * - * # Safety - * The object must remain alive through the call, with exclusive one-shot use of this object. - * The normal transport may continue sending and receiving concurrently. - * All worker signals must be blocked. Do not use an inherited object after fork. - */ -int32_t ddog_sidecar_signal_flush_run(const struct ddog_SignalFlush *flush); -#endif - #endif /* DDOG_SIDECAR_H */ diff --git a/ext/remote_config.c b/ext/remote_config.c index 17ef51109ef..07d83c55823 100644 --- a/ext/remote_config.c +++ b/ext/remote_config.c @@ -5,6 +5,7 @@ #include #include #include "threads.h" +#include #include #include @@ -16,6 +17,12 @@ ZEND_EXTERN_MODULE_GLOBALS(datadog); +#ifdef _WIN32 +static struct ddog_RemoteConfigNotification *remote_config_notification; + +static void datadog_remote_config_notify(void *context); +#endif + static void (*dd_prev_interrupt_function)(zend_execute_data *execute_data); static void dd_vm_interrupt(zend_execute_data *execute_data) { if (dd_prev_interrupt_function) { @@ -30,7 +37,7 @@ static void dd_vm_interrupt(zend_execute_data *execute_data) { } } -// We need this exported to call it via CreateRemoteThread on Windows +// The Windows remote configuration notification invokes this asynchronously. DATADOG_PUBLIC void datadog_set_all_thread_vm_interrupt(void) { // broadcast interrupt to all threads on ZTS #if ZTS @@ -131,7 +138,11 @@ void datadog_minit_remote_config(void) { dd_prev_interrupt_function = zend_interrupt_function; zend_interrupt_function = dd_vm_interrupt; -#ifndef _WIN32 +#ifdef _WIN32 + datadog_ffi_try("Failed to initialize remote config notification", + ddog_sidecar_remote_config_notification_new(datadog_remote_config_notify, NULL, + &remote_config_notification)); +#else struct sigaction act = {0}; act.sa_flags = SA_SIGINFO | SA_RESTART; act.sa_sigaction = dd_sigvtalarm_handler; @@ -140,13 +151,27 @@ void datadog_minit_remote_config(void) { } void datadog_mshutdown_remote_config(void) { -#ifndef _WIN32 +#ifdef _WIN32 + ddog_sidecar_remote_config_notification_drop(remote_config_notification); + remote_config_notification = NULL; +#else struct sigaction act = {0}; act.sa_handler = SIG_IGN; sigaction(SIGVTALRM, &act, NULL); #endif } +#ifdef _WIN32 +const struct ddog_RemoteConfigNotification *datadog_remote_config_notification_get(void) { + return remote_config_notification; +} + +static void datadog_remote_config_notify(void *context) { + UNUSED(context); + datadog_set_all_thread_vm_interrupt(); +} +#endif + void datadog_rinit_remote_config(void) { DATADOG_G(reread_remote_configuration) = 0; } diff --git a/ext/remote_config.h b/ext/remote_config.h index e615515b833..fafce0f7a21 100644 --- a/ext/remote_config.h +++ b/ext/remote_config.h @@ -9,6 +9,10 @@ void datadog_rinit_remote_config(void); void datadog_rshutdown_remote_config(void); void datadog_check_for_new_config_now(void); +#ifdef _WIN32 +struct ddog_RemoteConfigNotification; +const struct ddog_RemoteConfigNotification *datadog_remote_config_notification_get(void); +#endif DATADOG_PUBLIC void datadog_set_all_thread_vm_interrupt(void); diff --git a/ext/sidecar.c b/ext/sidecar.c index 37dc5e9398d..3a5ae36a0df 100644 --- a/ext/sidecar.c +++ b/ext/sidecar.c @@ -133,6 +133,11 @@ static void dd_sidecar_post_connect(ddog_SidecarTransport **transport, bool is_f ddog_CharSlice parent_session_id = datadog_is_empty_session_id(datadog_formatted_parent_session_id) ? DDOG_CHARSLICE_C("") : (ddog_CharSlice) {.ptr = (char *) datadog_formatted_parent_session_id, .len = sizeof(datadog_formatted_parent_session_id)}; const ddog_Vec_Tag *process_tags = datadog_process_tags_get_vec(); ddog_Endpoint *otlp_metrics_endpoint = datadog_otel_metrics_endpoint(); +#ifdef _WIN32 + const struct ddog_RemoteConfigNotification *remote_config_notification = datadog_remote_config_notification_get(); +#else + const struct ddog_RemoteConfigNotification *remote_config_notification = NULL; +#endif ddog_sidecar_session_set_config(transport, session_id, datadog_endpoint, dogstatsd_endpoint, otlp_metrics_endpoint, DDOG_CHARSLICE_C("php"), php_version_rt, @@ -148,7 +153,7 @@ static void dd_sidecar_post_connect(ddog_SidecarTransport **transport, bool is_f get_global_DD_TRACE_AGENT_STACK_BACKLOG() * get_global_DD_TRACE_AGENT_MAX_PAYLOAD_SIZE(), get_global_DD_TRACE_DEBUG() ? DDOG_CHARSLICE_C("debug") : dd_zend_string_to_CharSlice(get_global_DD_TRACE_LOG_LEVEL()), (ddog_CharSlice){ .ptr = logpath, .len = strlen(logpath) }, - datadog_set_all_thread_vm_interrupt, + remote_config_notification, DATADOG_REMOTE_CONFIG_PRODUCTS.ptr, DATADOG_REMOTE_CONFIG_PRODUCTS.len, DATADOG_REMOTE_CONFIG_CAPABILITIES.ptr, diff --git a/libdatadog b/libdatadog index 89ee2a8b042..f93a1535465 160000 --- a/libdatadog +++ b/libdatadog @@ -1 +1 @@ -Subproject commit 89ee2a8b042db71e99017d5156facf5dd897c36b +Subproject commit f93a1535465785b7dac3858dbe114091d13fc876 diff --git a/tests/Benchmarks/API/TraceSerializationBench.php b/tests/Benchmarks/API/TraceSerializationBench.php index 964c76ac23b..2e5ad353146 100644 --- a/tests/Benchmarks/API/TraceSerializationBench.php +++ b/tests/Benchmarks/API/TraceSerializationBench.php @@ -11,13 +11,46 @@ class TraceSerializationBench * @Iterations(20) * @OutputTimeUnit("microseconds") * @RetryThreshold(10.0) - * @BeforeMethods("setUp") + * @BeforeMethods({"warmUpSampling", "setUp"}) */ public function benchSerializeTrace() { \dd_trace_serialize_closed_spans(); } + public function warmUpSampling() + { + if (!\dd_trace_env_config('DD_TRACE_SIDECAR_TRACE_SENDER')) { + return; + } + + // PHPBench runs BeforeMethods in each worker, outside the subject timer. + // A connected sidecar may not have published /info or agent sampling rates yet. + if (!\dd_trace_internal_fn('await_agent_info', 5000)) { + throw new \RuntimeException('Trace serialization benchmark requires a ready agent /info response'); + } + + // Sampling rates are published after a trace response. Exercise automatic sampling + // and the real sender, without changing the priority of the measured trace. + $span = \DDTrace\start_trace_span(); + $span->name = 'bench.trace_serialization.warmup'; + \DDTrace\close_span(); + \DDTrace\flush(); + + $deadline = microtime(true) + 5; + do { + // Trace enqueueing is asynchronous, so the first flush can precede it. + \dd_trace_synchronous_flush(5000); + $config = \dd_trace_internal_fn('get_agent_sampling_config'); + if (isset($config['rate_by_service']) && is_array($config['rate_by_service'])) { + return; + } + usleep(10000); + } while (microtime(true) < $deadline); + + throw new \RuntimeException('Trace serialization benchmark requires agent sampling rates'); + } + public function setUp() { for ($i = 0; $i < 100; $i++) { diff --git a/tests/Benchmarks/README.md b/tests/Benchmarks/README.md index c1fee4e1856..4d673dde9b7 100644 --- a/tests/Benchmarks/README.md +++ b/tests/Benchmarks/README.md @@ -16,6 +16,11 @@ or if you want to run the benchmarks with OPcache enabled: make benchmarks_opcache ``` +`TraceSerializationBench` measures steady-state serialization. With the sidecar +trace sender enabled, each worker sends a warmup trace and waits for agent info +and sampling rates before timing the subject. It requires a reachable agent and +fails setup if readiness times out. + ## How to add a new benchmark The benchmarks are located in the [tests/Benchmarks](.) folder and are written using [PHPBench](https://github.com/phpbench/phpbench). diff --git a/tests/ext/inferred_proxy/serialization-sampling-chunk.phpt b/tests/ext/inferred_proxy/serialization-sampling-chunk.phpt new file mode 100644 index 00000000000..23d77530908 --- /dev/null +++ b/tests/ext/inferred_proxy/serialization-sampling-chunk.phpt @@ -0,0 +1,46 @@ +--TEST-- +Inferred spans use the sampling snapshot of their serialized trace chunk +--ENV-- +DD_TRACE_GENERATE_ROOT_SPAN=0 +DD_TRACE_AUTO_FLUSH_ENABLED=0 +DD_TRACE_SAMPLE_RATE=0 +DD_SPAN_SAMPLING_RULES=[{"name":"keep.root","sample_rate":1}] +DD_TRACE_STATS_COMPUTATION_ENABLED=0 +DD_TRACE_INFERRED_PROXY_SERVICES_ENABLED=1 +HTTP_X_DD_PROXY=aws-apigateway +HTTP_X_DD_PROXY_REQUEST_TIME_MS=100 +HTTP_X_DD_PROXY_PATH=/test +HTTP_X_DD_PROXY_HTTPMETHOD=GET +HTTP_X_DD_PROXY_DOMAIN_NAME=example.com +--GET-- +test=1 +--FILE-- +name = 'keep.root'; +$root->metrics['http.status_code'] = new ChangeSamplingDuringSerialization(); +DDTrace\close_span(); + +foreach (dd_trace_serialize_closed_spans() as $span) { + echo $span['name'], "\n"; + if (isset($span['metrics']['_dd.span_sampling.mechanism'])) { + echo 'single span: ', $span['metrics']['_dd.span_sampling.mechanism'], "\n"; + } + if (isset($span['metrics']['_sampling_priority_v1'])) { + echo 'trace priority: ', $span['metrics']['_sampling_priority_v1'], "\n"; + } +} + +?> +--EXPECT-- +keep.root +single span: 8 +aws.apigateway +trace priority: -1 diff --git a/tests/ext/priority_sampling/serialization-chunk.phpt b/tests/ext/priority_sampling/serialization-chunk.phpt new file mode 100644 index 00000000000..11192c42235 --- /dev/null +++ b/tests/ext/priority_sampling/serialization-chunk.phpt @@ -0,0 +1,54 @@ +--TEST-- +Serialization snapshots trace sampling across sibling spans and attached stacks +--ENV-- +DD_TRACE_GENERATE_ROOT_SPAN=0 +DD_TRACE_AUTO_FLUSH_ENABLED=0 +DD_TRACE_SAMPLE_RATE=0 +DD_SPAN_SAMPLING_RULES=[{"name":"keep*","sample_rate":1}] +DD_TRACE_STATS_COMPUTATION_ENABLED=0 +--FILE-- +name = 'root'; +// Serialization converts this metric to a string tag and releases the old value. +$root->metrics['http.status_code'] = new ChangeSamplingDuringSerialization(); + +DDTrace\start_span()->name = 'keep.child'; +DDTrace\close_span(); +DDTrace\create_stack(); +DDTrace\start_span()->name = 'keep.attached'; +DDTrace\close_span(); +DDTrace\switch_stack($root); +DDTrace\start_span()->name = 'unmatched'; +DDTrace\close_span(); +DDTrace\close_span(); + +foreach (dd_trace_serialize_closed_spans() as $span) { + echo $span['name'], ': ', isset($span['metrics']['_dd.span_sampling.mechanism']) ? 'single span' : 'trace', "\n"; +} + +// Subsequent chunks must observe the new configuration. +DDTrace\start_span()->name = 'keep.next'; +DDTrace\close_span(); +$span = dd_trace_serialize_closed_spans()[0]; +echo $span['name'], ': ', isset($span['metrics']['_dd.span_sampling.mechanism']) ? 'single span' : 'trace', "\n"; +var_dump($span['metrics']['_sampling_priority_v1']); +var_dump(dd_trace_serialize_closed_spans()); + +?> +--EXPECT-- +root: trace +unmatched: trace +keep.child: single span +keep.attached: single span +keep.next: trace +float(2) +array(0) { +} diff --git a/tests/ext/priority_sampling/serialization-later-chunk.phpt b/tests/ext/priority_sampling/serialization-later-chunk.phpt new file mode 100644 index 00000000000..45893525f25 --- /dev/null +++ b/tests/ext/priority_sampling/serialization-later-chunk.phpt @@ -0,0 +1,60 @@ +--TEST-- +Serialization reevaluates sampling for later chunks of the same root +--DESCRIPTION-- +This documents existing behavior rather than a spec requirement. The automatic +(non-manual) sampling decision is not locked: each serialized chunk of a root +reevaluates it, so a later chunk may carry a different priority than an earlier +one. This appears deliberate, part of the extended sampling design (5f7f4a4d1) +where the decision is recomputed as spans close and rules start matching. + +It departs from the Priority Sampling RFC, which requires the priority to be +locked once any span finishes or the trace propagates, and from the other +tracers (Python, Java, Go, Node.js, .NET, Ruby), none of which rerun the +automatic sampler for later chunks of a trace. Ruby does reconsider rule +sampling on resource changes, but stops once the first chunk is flushed. +--ENV-- +DD_TRACE_GENERATE_ROOT_SPAN=0 +DD_TRACE_AUTO_FLUSH_ENABLED=0 +DD_AUTOFINISH_SPANS=0 +DD_TRACE_SAMPLE_RATE=0 +DD_TRACE_STATS_COMPUTATION_ENABLED=0 +--FILE-- +name = 'root'; + +// Flush a completed attached stack while the root stack has no closed spans. +DDTrace\create_stack(); +DDTrace\start_span()->name = 'first chunk'; +DDTrace\close_span(); +DDTrace\flush(); +var_dump($root->samplingPriority); + +ini_set('datadog.trace.sample_rate', '1'); +DDTrace\switch_stack($root); +DDTrace\create_stack(); +DDTrace\start_span()->name = 'second chunk'; +DDTrace\close_span(); +DDTrace\flush(); +var_dump($root->samplingPriority); + +// The root can be emitted before an orphan stack; neither decision is permanent. +DDTrace\switch_stack($root); +$orphan = DDTrace\create_stack(); +DDTrace\start_span()->name = 'orphan'; +DDTrace\switch_stack($root); +DDTrace\close_span(); +DDTrace\flush(); + +ini_set('datadog.trace.sample_rate', '0'); +DDTrace\switch_stack($orphan); +DDTrace\close_span(); +DDTrace\flush(); +var_dump($root->samplingPriority); + +?> +--EXPECT-- +int(-1) +int(2) +int(-1) diff --git a/tracer/serializer.c b/tracer/serializer.c index e1fc3e71721..783609124c1 100644 --- a/tracer/serializer.c +++ b/tracer/serializer.c @@ -1329,7 +1329,7 @@ void transfer_metrics_data(ddog_SpanBytes *source, ddog_SpanBytes *destination, } } -ddog_SpanBytes *ddtrace_serialize_span_to_rust_span(ddtrace_span_data *span, ddog_TraceBytes *trace) { +ddog_SpanBytes *ddtrace_serialize_span_to_rust_span(ddtrace_span_data *span, ddog_TraceBytes *trace, bool p0_trace) { zend_array *meta = ddtrace_property_array(&span->property_meta); zend_array *metrics = ddtrace_property_array(&span->property_metrics); @@ -1399,8 +1399,7 @@ ddog_SpanBytes *ddtrace_serialize_span_to_rust_span(ddtrace_span_data *span, ddo profiling_notify_trace_finished(span->span_id, type, resource); } - // Determine sampling before allocating the rust span to avoid unnecessary work. - bool p0_trace = ddtrace_fetch_priority_sampling_from_span(span->root) <= 0; + // Apply per-span sampling to the trace sampling decision snapshotted for this chunk. bool span_sampling_applied = false; double span_sampling_rate = 1.0; double span_sampling_max_per_second = 0.0; @@ -1867,7 +1866,7 @@ ddog_SpanBytes *ddtrace_serialize_span_to_rust_span(ddtrace_span_data *span, ddo } if (inferred_span) { - ddog_SpanBytes *serialized_inferred_span = ddtrace_serialize_span_to_rust_span(inferred_span, trace); + ddog_SpanBytes *serialized_inferred_span = ddtrace_serialize_span_to_rust_span(inferred_span, trace, p0_trace); rust_span = ddog_get_span(trace, rust_span_index); transfer_metrics_data(rust_span, serialized_inferred_span, "_dd.agent_psr", true); diff --git a/tracer/serializer.h b/tracer/serializer.h index 4be37e80184..0bb42c31d5a 100644 --- a/tracer/serializer.h +++ b/tracer/serializer.h @@ -6,7 +6,7 @@ int ddtrace_serialize_simple_array(zval *trace, zval *retval); int ddtrace_serialize_simple_array_into_c_string(zval *trace, char **data_p, size_t *size_p); -ddog_SpanBytes *ddtrace_serialize_span_to_rust_span(ddtrace_span_data *span, ddog_TraceBytes *trace); +ddog_SpanBytes *ddtrace_serialize_span_to_rust_span(ddtrace_span_data *span, ddog_TraceBytes *trace, bool p0_trace); zval dd_serialize_rust_traces_to_zval(ddog_TracesBytes *traces); void ddtrace_save_active_error_to_metadata(void); diff --git a/tracer/span.c b/tracer/span.c index ff4653eea53..9a7f4efd364 100644 --- a/tracer/span.c +++ b/tracer/span.c @@ -1164,15 +1164,24 @@ void ddtrace_serialize_closed_spans(ddog_TracesBytes *traces, bool fast_shutdown next_stack = stack->next; } ddog_TraceBytes *trace = ddog_traces_new_trace(traces); + bool sampling_decided = false; + bool p0_trace = false; do { // Note this ->next: We always splice in new spans at next, so start at next to mostly preserve order ddtrace_span_data *span = stack->closed_ring_flush->next, *end = span; stack->closed_ring_flush = NULL; + if (!sampling_decided) { + // Detach the ring before sampling, which can trigger destructors or GC. + // Snapshot once for this chunk, including attached stacks and inferred spans. + // Later chunks reevaluate automatic sampling (existing behavior, unlike other tracers). + p0_trace = ddtrace_fetch_priority_sampling_from_span(span->root) <= 0; + sampling_decided = true; + } do { ddtrace_span_data *tmp = span; span = tmp->next; - ddtrace_serialize_span_to_rust_span(tmp, trace); + ddtrace_serialize_span_to_rust_span(tmp, trace, p0_trace); #if PHP_VERSION_ID < 70400 // remove the artificially increased RC while closing again GC_SET_REFCOUNT(&tmp->std, GC_REFCOUNT(&tmp->std) - DD_RC_CLOSED_MARKER);