From 9e13e690d39d34aeb503044df98ea7fb690d3a12 Mon Sep 17 00:00:00 2001 From: Jules Wiriath Date: Wed, 15 Jul 2026 14:05:55 +0200 Subject: [PATCH 1/5] feat: added file capability --- Cargo.lock | 28 ++-- crates/capabilities/Cargo.toml | 4 +- crates/capabilities/src/file.rs | 155 ++++++++++++++++++++++ crates/capabilities/src/file_transport.js | 41 ++++++ crates/capabilities/src/lib.rs | 48 +++++-- crates/pipeline/Cargo.toml | 16 +-- eslint.config.js | 6 +- test/file-transport.js | 51 +++++++ yarn.lock | 6 +- 9 files changed, 316 insertions(+), 39 deletions(-) create mode 100644 crates/capabilities/src/file.rs create mode 100644 crates/capabilities/src/file_transport.js create mode 100644 test/file-transport.js diff --git a/Cargo.lock b/Cargo.lock index 22d8213..6cbceb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -917,7 +917,7 @@ dependencies = [ [[package]] name = "libdd-capabilities" version = "2.1.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=8cd68ab922fb7aade0f089ccfc91291d874673af#8cd68ab922fb7aade0f089ccfc91291d874673af" +source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" dependencies = [ "anyhow", "bytes", @@ -941,8 +941,9 @@ dependencies = [ [[package]] name = "libdd-capabilities-impl" version = "3.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=8cd68ab922fb7aade0f089ccfc91291d874673af#8cd68ab922fb7aade0f089ccfc91291d874673af" +source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" dependencies = [ + "anyhow", "bytes", "http", "http-body-util", @@ -989,7 +990,7 @@ dependencies = [ [[package]] name = "libdd-common" version = "5.1.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=8cd68ab922fb7aade0f089ccfc91291d874673af#8cd68ab922fb7aade0f089ccfc91291d874673af" +source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" dependencies = [ "anyhow", "bytes", @@ -1052,7 +1053,7 @@ dependencies = [ [[package]] name = "libdd-data-pipeline" version = "7.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=8cd68ab922fb7aade0f089ccfc91291d874673af#8cd68ab922fb7aade0f089ccfc91291d874673af" +source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" dependencies = [ "anyhow", "arc-swap", @@ -1094,7 +1095,7 @@ dependencies = [ [[package]] name = "libdd-ddsketch" version = "1.1.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=8cd68ab922fb7aade0f089ccfc91291d874673af#8cd68ab922fb7aade0f089ccfc91291d874673af" +source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" dependencies = [ "prost", ] @@ -1102,7 +1103,7 @@ dependencies = [ [[package]] name = "libdd-dogstatsd-client" version = "4.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=8cd68ab922fb7aade0f089ccfc91291d874673af#8cd68ab922fb7aade0f089ccfc91291d874673af" +source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" dependencies = [ "anyhow", "cadence", @@ -1174,7 +1175,7 @@ dependencies = [ [[package]] name = "libdd-shared-runtime" version = "2.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=8cd68ab922fb7aade0f089ccfc91291d874673af#8cd68ab922fb7aade0f089ccfc91291d874673af" +source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" dependencies = [ "async-trait", "futures", @@ -1218,7 +1219,7 @@ dependencies = [ [[package]] name = "libdd-tinybytes" version = "1.1.1" -source = "git+https://github.com/DataDog/libdatadog.git?rev=8cd68ab922fb7aade0f089ccfc91291d874673af#8cd68ab922fb7aade0f089ccfc91291d874673af" +source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" dependencies = [ "serde", ] @@ -1226,7 +1227,7 @@ dependencies = [ [[package]] name = "libdd-trace-normalization" version = "3.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=8cd68ab922fb7aade0f089ccfc91291d874673af#8cd68ab922fb7aade0f089ccfc91291d874673af" +source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" dependencies = [ "anyhow", "libdd-trace-protobuf 4.0.0", @@ -1235,7 +1236,7 @@ dependencies = [ [[package]] name = "libdd-trace-obfuscation" version = "5.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=8cd68ab922fb7aade0f089ccfc91291d874673af#8cd68ab922fb7aade0f089ccfc91291d874673af" +source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" dependencies = [ "anyhow", "fluent-uri", @@ -1261,7 +1262,7 @@ dependencies = [ [[package]] name = "libdd-trace-protobuf" version = "4.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=8cd68ab922fb7aade0f089ccfc91291d874673af#8cd68ab922fb7aade0f089ccfc91291d874673af" +source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" dependencies = [ "prost", "serde", @@ -1271,7 +1272,7 @@ dependencies = [ [[package]] name = "libdd-trace-stats" version = "6.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=8cd68ab922fb7aade0f089ccfc91291d874673af#8cd68ab922fb7aade0f089ccfc91291d874673af" +source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" dependencies = [ "anyhow", "arc-swap", @@ -1298,7 +1299,7 @@ dependencies = [ [[package]] name = "libdd-trace-utils" version = "9.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?rev=8cd68ab922fb7aade0f089ccfc91291d874673af#8cd68ab922fb7aade0f089ccfc91291d874673af" +source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" dependencies = [ "anyhow", "base64", @@ -1310,6 +1311,7 @@ dependencies = [ "http-body", "http-body-util", "indexmap", + "itoa", "libdd-capabilities 2.1.0", "libdd-capabilities-impl 3.0.0", "libdd-common 5.1.0", diff --git a/crates/capabilities/Cargo.toml b/crates/capabilities/Cargo.toml index cbb376e..d3a587a 100644 --- a/crates/capabilities/Cargo.toml +++ b/crates/capabilities/Cargo.toml @@ -15,9 +15,7 @@ http = "1" bytes = "1.4" futures-core = "0.3" anyhow = "1" -# TODO: Replace this temporary libdatadog PR rev with the official release/tag -# that contains DataDog/libdatadog#2235, then regenerate Cargo.lock. -libdd-capabilities = { git = "https://github.com/DataDog/libdatadog.git", rev = "8cd68ab922fb7aade0f089ccfc91291d874673af" } +libdd-capabilities = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps" } [dev-dependencies] wasm-bindgen-test = "0.3" diff --git a/crates/capabilities/src/file.rs b/crates/capabilities/src/file.rs new file mode 100644 index 0000000..fc2644c --- /dev/null +++ b/crates/capabilities/src/file.rs @@ -0,0 +1,155 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Wasm implementation of [`FileCapability`] backed by Node.js `fs`. +//! +//! The JS transport is imported via `wasm_bindgen(module = ...)` from +//! `file_transport.js`, which ships alongside the wasm output. + +use std::future::Future; + +use bytes::Bytes; +use js_sys::{self, Reflect, Uint8Array}; +use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; + +use libdd_capabilities::file::{FileCapability, FileError, FileMetadata}; +use libdd_capabilities::maybe_send::MaybeSend; + +#[wasm_bindgen(module = "/src/file_transport.js")] +extern "C" { + #[wasm_bindgen(js_name = "readFile", catch)] + fn js_read_file(path: &str) -> Result; + + #[wasm_bindgen(js_name = "writeFile", catch)] + fn js_write_file(path: &str, data: &[u8]) -> Result; + + #[wasm_bindgen(js_name = "metadata", catch)] + fn js_metadata(path: &str) -> Result; + + #[wasm_bindgen(js_name = "exists", catch)] + fn js_exists(path: &str) -> Result; +} + +#[derive(Debug, Clone)] +pub struct WasmFileCapability; + +impl FileCapability for WasmFileCapability { + fn new() -> Self { + Self + } + + #[allow(clippy::manual_async_fn)] + fn read(&self, path: &str) -> impl Future> + MaybeSend { + let path = path.to_owned(); + async move { + let promise = + js_read_file(&path).map_err(|e| map_js_error(&e, &path))?; + let value = JsFuture::from(promise) + .await + .map_err(|e| map_js_error(&e, &path))?; + let array = Uint8Array::new(&value); + Ok(Bytes::from(array.to_vec())) + } + } + + #[allow(clippy::manual_async_fn)] + fn write( + &self, + path: &str, + contents: Bytes, + ) -> impl Future> + MaybeSend { + let path = path.to_owned(); + async move { + let promise = js_write_file(&path, &contents) + .map_err(|e| map_js_error(&e, &path))?; + JsFuture::from(promise) + .await + .map_err(|e| map_js_error(&e, &path))?; + Ok(()) + } + } + + #[allow(clippy::manual_async_fn)] + fn metadata( + &self, + path: &str, + ) -> impl Future> + MaybeSend { + let path = path.to_owned(); + async move { + let promise = + js_metadata(&path).map_err(|e| map_js_error(&e, &path))?; + let value = JsFuture::from(promise) + .await + .map_err(|e| map_js_error(&e, &path))?; + parse_metadata(&value, &path) + } + } + + #[allow(clippy::manual_async_fn)] + fn exists(&self, path: &str) -> impl Future> + MaybeSend { + let path = path.to_owned(); + async move { + let promise = js_exists(&path).map_err(|e| map_js_error(&e, &path))?; + let value = JsFuture::from(promise) + .await + .map_err(|e| map_js_error(&e, &path))?; + value + .as_bool() + .ok_or_else(|| FileError::Io(anyhow::anyhow!("exists({path}) did not return a boolean"))) + } + } +} + +fn map_js_error(err: &JsValue, path: &str) -> FileError { + let code = Reflect::get(err, &JsValue::from_str("code")) + .ok() + .and_then(|v| v.as_string()); + match code.as_deref() { + Some("ENOENT") => FileError::NotFound(path.to_owned()), + Some("EACCES") | Some("EPERM") => FileError::PermissionDenied(path.to_owned()), + _ => { + let message = Reflect::get(err, &JsValue::from_str("message")) + .ok() + .and_then(|v| v.as_string()) + .unwrap_or_else(|| format!("{err:?}")); + FileError::Io(anyhow::anyhow!("{message} (path: {path})")) + } + } +} + +fn parse_metadata(value: &JsValue, path: &str) -> Result { + let size = read_number(value, "size", path)?; + // Node populates `stat().ino` on every platform, so `inode` is always Some. + let inode = Some(read_number(value, "inode", path)? as u64); + let is_file = read_bool(value, "is_file", path)?; + let is_dir = read_bool(value, "is_dir", path)?; + Ok(FileMetadata { + size: size as u64, + inode, + is_file, + is_dir, + }) +} + +fn read_number(value: &JsValue, key: &str, path: &str) -> Result { + Reflect::get(value, &JsValue::from_str(key)) + .ok() + .and_then(|v| v.as_f64()) + .ok_or_else(|| { + FileError::Io(anyhow::anyhow!( + "metadata({path}) is missing numeric field `{key}`" + )) + }) +} + +fn read_bool(value: &JsValue, key: &str, path: &str) -> Result { + Reflect::get(value, &JsValue::from_str(key)) + .ok() + .and_then(|v| v.as_bool()) + .ok_or_else(|| { + FileError::Io(anyhow::anyhow!( + "metadata({path}) is missing boolean field `{key}`" + )) + }) +} diff --git a/crates/capabilities/src/file_transport.js b/crates/capabilities/src/file_transport.js new file mode 100644 index 0000000..046cd8a --- /dev/null +++ b/crates/capabilities/src/file_transport.js @@ -0,0 +1,41 @@ +// Lazy `require('node:fs')` — see http_transport.js. + +'use strict' + +module.exports.readFile = function (path) { + const fs = require('node:fs') + return fs.promises.readFile(path) +} + +module.exports.writeFile = function (path, data) { + const fs = require('node:fs') + // Copy off the wasm-memory view before the async write; a memory grow would + // otherwise detach the underlying ArrayBuffer mid-write. + return fs.promises.writeFile(path, Buffer.from(data)) +} + +module.exports.metadata = function (path) { + const fs = require('node:fs') + try { + const s = fs.statSync(path) + return Promise.resolve({ + size: Number(s.size), + inode: Number(s.ino), + is_file: s.isFile(), + is_dir: s.isDirectory(), + }) + } catch (error) { + return Promise.reject(error) + } +} + +module.exports.exists = function (path) { + const fs = require('node:fs') + return fs.promises.stat(path).then( + () => true, + (error) => { + if (error && error.code === 'ENOENT') return false + throw error + }, + ) +} diff --git a/crates/capabilities/src/lib.rs b/crates/capabilities/src/lib.rs index f7cb4b0..15bc833 100644 --- a/crates/capabilities/src/lib.rs +++ b/crates/capabilities/src/lib.rs @@ -12,12 +12,15 @@ use std::future::Future; use std::time::Duration; +use libdd_capabilities::file::{FileCapability, FileError, FileMetadata}; use libdd_capabilities::http::HttpError; use libdd_capabilities::{HttpClientCapability, LogWriterCapability, MaybeSend, SleepCapability}; +pub mod file; pub mod http; pub mod sleep; +pub use file::WasmFileCapability; pub use http::WasmHttpClient; pub use sleep::WasmSleepCapability; @@ -33,6 +36,7 @@ pub use sleep::WasmSleepCapability; pub struct WasmCapabilities { http: WasmHttpClient, sleep: WasmSleepCapability, + file: WasmFileCapability, } impl Default for WasmCapabilities { @@ -46,16 +50,14 @@ impl WasmCapabilities { Self { http: WasmHttpClient::new_client(), sleep: WasmSleepCapability, + file: WasmFileCapability, } } } impl HttpClientCapability for WasmCapabilities { fn new_client() -> Self { - Self { - http: WasmHttpClient::new_client(), - sleep: WasmSleepCapability, - } + Self::new() } fn request( @@ -68,10 +70,7 @@ impl HttpClientCapability for WasmCapabilities { impl SleepCapability for WasmCapabilities { fn new() -> Self { - Self { - http: WasmHttpClient::new_client(), - sleep: WasmSleepCapability, - } + Self::new() } fn sleep(&self, duration: Duration) -> impl Future + MaybeSend { @@ -89,3 +88,36 @@ impl LogWriterCapability for WasmCapabilities { Ok(()) } } + +impl FileCapability for WasmCapabilities { + fn new() -> Self { + Self::new() + } + + fn read( + &self, + path: &str, + ) -> impl Future> + MaybeSend { + self.file.read(path) + } + + fn write( + &self, + path: &str, + contents: ::bytes::Bytes, + ) -> impl Future> + MaybeSend { + self.file.write(path, contents) + } + + fn metadata( + &self, + path: &str, + ) -> impl Future> + MaybeSend { + self.file.metadata(path) + } + + fn exists(&self, path: &str) -> impl Future> + MaybeSend { + self.file.exists(path) + } +} + diff --git a/crates/pipeline/Cargo.toml b/crates/pipeline/Cargo.toml index f8e318a..a2af410 100644 --- a/crates/pipeline/Cargo.toml +++ b/crates/pipeline/Cargo.toml @@ -14,15 +14,13 @@ js-sys = "0.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1" libdatadog-nodejs-capabilities = { path = "../capabilities" } -# TODO: Replace these temporary libdatadog PR revs with the official release/tag -# that contains DataDog/libdatadog#2235, then regenerate Cargo.lock. -libdd-capabilities = { git = "https://github.com/DataDog/libdatadog.git", rev = "8cd68ab922fb7aade0f089ccfc91291d874673af" } -libdd-common = { git = "https://github.com/DataDog/libdatadog.git", rev = "8cd68ab922fb7aade0f089ccfc91291d874673af", default-features = false } -libdd-data-pipeline = { git = "https://github.com/DataDog/libdatadog.git", rev = "8cd68ab922fb7aade0f089ccfc91291d874673af", default-features = false } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog.git", rev = "8cd68ab922fb7aade0f089ccfc91291d874673af", default-features = false, features = ["change-buffer"] } -libdd-trace-stats = { git = "https://github.com/DataDog/libdatadog.git", rev = "8cd68ab922fb7aade0f089ccfc91291d874673af", default-features = false } -libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog.git", rev = "8cd68ab922fb7aade0f089ccfc91291d874673af", default-features = false } -libdd-shared-runtime = { git = "https://github.com/DataDog/libdatadog.git", rev = "8cd68ab922fb7aade0f089ccfc91291d874673af", default-features = false } +libdd-capabilities = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps" } +libdd-common = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps", default-features = false } +libdd-data-pipeline = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps", default-features = false } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps", default-features = false, features = ["change-buffer"] } +libdd-trace-stats = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps", default-features = false } +libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps", default-features = false } +libdd-shared-runtime = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps", default-features = false } rmp-serde = "1" bytes = "1" http = "1" diff --git a/eslint.config.js b/eslint.config.js index e7b6736..690debf 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -80,9 +80,9 @@ module.exports = [ }, }, { - // Loaded by Rust via `wasm_bindgen(module = ".../http_transport.js")`, so the - // snake_case filename must match the Rust module path. - files: ['**/http_transport.js'], + // Loaded by Rust via `wasm_bindgen(module = ".../_transport.js")`, so + // the snake_case filename must match the Rust module path. + files: ['**/*_transport.js'], rules: { 'unicorn/filename-case': 'off', }, diff --git a/test/file-transport.js b/test/file-transport.js new file mode 100644 index 0000000..0e28433 --- /dev/null +++ b/test/file-transport.js @@ -0,0 +1,51 @@ +'use strict' + +// The transport shim is plain CommonJS, so drive it directly. + +const { describe, it, before, after } = require('node:test') +const assert = require('node:assert') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +const fileTransport = require('../crates/capabilities/src/file_transport') + +describe('file_transport', () => { + let tmp + before(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'libdd-file-')) + }) + after(() => { + fs.rmSync(tmp, { recursive: true, force: true }) + }) + + it('writes and reads a file round-trip', async () => { + const p = path.join(tmp, 'hello.bin') + await fileTransport.writeFile(p, Buffer.from('hello')) + const got = await fileTransport.readFile(p) + assert.strictEqual(Buffer.from(got).toString('utf8'), 'hello') + }) + + it('readFile on a missing path rejects with ENOENT', async () => { + const p = path.join(tmp, 'does-not-exist') + await assert.rejects(fileTransport.readFile(p), error => error && error.code === 'ENOENT') + }) + + it('metadata reports size, kind, and a positive inode', async () => { + const p = path.join(tmp, 'meta.bin') + fs.writeFileSync(p, '0123456789') + const m = await fileTransport.metadata(p) + assert.strictEqual(m.size, 10) + assert.strictEqual(m.is_file, true) + assert.strictEqual(m.is_dir, false) + assert.ok(m.inode > 0, `expected positive inode, got ${m.inode}`) + }) + + it('exists returns true for a present path and false for a missing one', async () => { + const present = path.join(tmp, 'here') + fs.writeFileSync(present, '') + const absent = path.join(tmp, 'gone') + assert.strictEqual(await fileTransport.exists(present), true) + assert.strictEqual(await fileTransport.exists(absent), false) + }) +}) diff --git a/yarn.lock b/yarn.lock index 14dca93..85f5c9f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -280,7 +280,7 @@ baseline-browser-mapping@^2.9.0: resolved "https://registry.yarnpkg.com/baseline-browser-mapping/-/baseline-browser-mapping-2.10.0.tgz#5b09935025bf8a80e29130251e337c6a7fc8cbb9" integrity sha512-lIyg0szRfYbiy67j9KN8IyeD7q7hcmqnJ1ddWmNt19ItGpNN64mnllmxUNFIOdOm6by97jlL6wfpTTJrmnjWAA== -brace-expansion@^5.0.2, brace-expansion@^5.0.6: +brace-expansion@^5.0.2: version "5.0.6" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-5.0.6.tgz#ec68fe0a641a29d8711579caf641d05bae1f2285" integrity sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g== @@ -608,7 +608,7 @@ flat-cache@^4.0.0: flatted "^3.2.9" keyv "^4.5.4" -flatted@^3.2.9, flatted@^3.4.2: +flatted@^3.2.9: version "3.4.2" resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.4.2.tgz#f5c23c107f0f37de8dbdf24f13722b3b98d52726" integrity sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA== @@ -801,7 +801,7 @@ picocolors@^1.1.1: resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== -picomatch@^4.0.2, picomatch@^4.0.3, picomatch@^4.0.4: +picomatch@^4.0.2, picomatch@^4.0.3: version "4.0.4" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-4.0.4.tgz#fd6f5e00a143086e074dffe4c924b8fb293b0589" integrity sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A== From ba4247dda9ecc72e5834ec28ad834fbf7b65c11d Mon Sep 17 00:00:00 2001 From: Jules Wiriath Date: Thu, 16 Jul 2026 13:33:31 +0200 Subject: [PATCH 2/5] fix: JS tomfoolery --- crates/capabilities/src/file.rs | 22 ++++++++++------------ crates/capabilities/src/file_transport.js | 6 +++--- test/file-transport.js | 4 ++-- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/crates/capabilities/src/file.rs b/crates/capabilities/src/file.rs index fc2644c..aeb1d0c 100644 --- a/crates/capabilities/src/file.rs +++ b/crates/capabilities/src/file.rs @@ -119,28 +119,26 @@ fn map_js_error(err: &JsValue, path: &str) -> FileError { } fn parse_metadata(value: &JsValue, path: &str) -> Result { - let size = read_number(value, "size", path)?; + let size = read_bigint_u64(value, "size", path)?; // Node populates `stat().ino` on every platform, so `inode` is always Some. - let inode = Some(read_number(value, "inode", path)? as u64); + let inode = Some(read_bigint_u64(value, "inode", path)?); let is_file = read_bool(value, "is_file", path)?; let is_dir = read_bool(value, "is_dir", path)?; Ok(FileMetadata { - size: size as u64, + size, inode, is_file, is_dir, }) } -fn read_number(value: &JsValue, key: &str, path: &str) -> Result { - Reflect::get(value, &JsValue::from_str(key)) - .ok() - .and_then(|v| v.as_f64()) - .ok_or_else(|| { - FileError::Io(anyhow::anyhow!( - "metadata({path}) is missing numeric field `{key}`" - )) - }) +fn read_bigint_u64(value: &JsValue, key: &str, path: &str) -> Result { + let v = Reflect::get(value, &JsValue::from_str(key)) + .map_err(|_| FileError::Io(anyhow::anyhow!("metadata({path}) missing field `{key}`")))?; + let bigint = js_sys::BigInt::try_from(v) + .map_err(|_| FileError::Io(anyhow::anyhow!("metadata({path}) field `{key}` is not a BigInt")))?; + u64::try_from(bigint) + .map_err(|_| FileError::Io(anyhow::anyhow!("metadata({path}) field `{key}` overflows u64"))) } fn read_bool(value: &JsValue, key: &str, path: &str) -> Result { diff --git a/crates/capabilities/src/file_transport.js b/crates/capabilities/src/file_transport.js index 046cd8a..bdcd3f6 100644 --- a/crates/capabilities/src/file_transport.js +++ b/crates/capabilities/src/file_transport.js @@ -17,10 +17,10 @@ module.exports.writeFile = function (path, data) { module.exports.metadata = function (path) { const fs = require('node:fs') try { - const s = fs.statSync(path) + const s = fs.statSync(path, { bigint: true }) return Promise.resolve({ - size: Number(s.size), - inode: Number(s.ino), + size: s.size, + inode: s.ino, is_file: s.isFile(), is_dir: s.isDirectory(), }) diff --git a/test/file-transport.js b/test/file-transport.js index 0e28433..e2151dd 100644 --- a/test/file-transport.js +++ b/test/file-transport.js @@ -35,10 +35,10 @@ describe('file_transport', () => { const p = path.join(tmp, 'meta.bin') fs.writeFileSync(p, '0123456789') const m = await fileTransport.metadata(p) - assert.strictEqual(m.size, 10) + assert.strictEqual(m.size, 10n) assert.strictEqual(m.is_file, true) assert.strictEqual(m.is_dir, false) - assert.ok(m.inode > 0, `expected positive inode, got ${m.inode}`) + assert.ok(m.inode > 0n, `expected positive inode, got ${m.inode}`) }) it('exists returns true for a present path and false for a missing one', async () => { From a1b469928fa35caca8e56818a38d95d54f8abc8a Mon Sep 17 00:00:00 2001 From: Jules Wiriath Date: Wed, 22 Jul 2026 13:48:16 +0200 Subject: [PATCH 3/5] chore: change input to main now that libdatadog's side was merged --- Cargo.lock | 39 ++++++++++++++++++++++------------ crates/capabilities/Cargo.toml | 2 +- crates/pipeline/Cargo.toml | 14 ++++++------ 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6cbceb8..e0f6000 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -917,7 +917,7 @@ dependencies = [ [[package]] name = "libdd-capabilities" version = "2.1.0" -source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "bytes", @@ -941,7 +941,7 @@ dependencies = [ [[package]] name = "libdd-capabilities-impl" version = "3.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "bytes", @@ -990,7 +990,7 @@ dependencies = [ [[package]] name = "libdd-common" version = "5.1.0" -source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "bytes", @@ -1053,13 +1053,14 @@ dependencies = [ [[package]] name = "libdd-data-pipeline" version = "7.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "arc-swap", "async-trait", "bytes", "either", + "futures", "getrandom 0.2.17", "http", "http-body-util", @@ -1082,6 +1083,7 @@ dependencies = [ "tokio-util", "tracing", "uuid", + "web-time", ] [[package]] @@ -1095,7 +1097,7 @@ dependencies = [ [[package]] name = "libdd-ddsketch" version = "1.1.0" -source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "prost", ] @@ -1103,7 +1105,7 @@ dependencies = [ [[package]] name = "libdd-dogstatsd-client" version = "4.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "cadence", @@ -1175,7 +1177,7 @@ dependencies = [ [[package]] name = "libdd-shared-runtime" version = "2.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "async-trait", "futures", @@ -1219,7 +1221,7 @@ dependencies = [ [[package]] name = "libdd-tinybytes" version = "1.1.1" -source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "serde", ] @@ -1227,7 +1229,7 @@ dependencies = [ [[package]] name = "libdd-trace-normalization" version = "3.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "libdd-trace-protobuf 4.0.0", @@ -1236,7 +1238,7 @@ dependencies = [ [[package]] name = "libdd-trace-obfuscation" version = "5.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "fluent-uri", @@ -1262,7 +1264,7 @@ dependencies = [ [[package]] name = "libdd-trace-protobuf" version = "4.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "prost", "serde", @@ -1272,7 +1274,7 @@ dependencies = [ [[package]] name = "libdd-trace-stats" version = "6.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "arc-swap", @@ -1294,12 +1296,13 @@ dependencies = [ "tokio", "tokio-util", "tracing", + "web-time", ] [[package]] name = "libdd-trace-utils" version = "9.0.0" -source = "git+https://github.com/DataDog/libdatadog.git?branch=jwiriath%2Ffile-caps#cc69e41b12bbaaaba3d35c5f5b525ba84721b1be" +source = "git+https://github.com/DataDog/libdatadog.git?rev=3081603d3c74f209be4e3be951f78a1a7469397f#3081603d3c74f209be4e3be951f78a1a7469397f" dependencies = [ "anyhow", "base64", @@ -2759,6 +2762,16 @@ dependencies = [ "semver", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-root-certs" version = "1.0.8" diff --git a/crates/capabilities/Cargo.toml b/crates/capabilities/Cargo.toml index d3a587a..edb40ad 100644 --- a/crates/capabilities/Cargo.toml +++ b/crates/capabilities/Cargo.toml @@ -15,7 +15,7 @@ http = "1" bytes = "1.4" futures-core = "0.3" anyhow = "1" -libdd-capabilities = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps" } +libdd-capabilities = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f" } [dev-dependencies] wasm-bindgen-test = "0.3" diff --git a/crates/pipeline/Cargo.toml b/crates/pipeline/Cargo.toml index a2af410..b3e9ef3 100644 --- a/crates/pipeline/Cargo.toml +++ b/crates/pipeline/Cargo.toml @@ -14,13 +14,13 @@ js-sys = "0.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1" libdatadog-nodejs-capabilities = { path = "../capabilities" } -libdd-capabilities = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps" } -libdd-common = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps", default-features = false } -libdd-data-pipeline = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps", default-features = false } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps", default-features = false, features = ["change-buffer"] } -libdd-trace-stats = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps", default-features = false } -libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps", default-features = false } -libdd-shared-runtime = { git = "https://github.com/DataDog/libdatadog.git", branch = "jwiriath/file-caps", default-features = false } +libdd-capabilities = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f" } +libdd-common = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false } +libdd-data-pipeline = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false, features = ["change-buffer"] } +libdd-trace-stats = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false } +libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false } +libdd-shared-runtime = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false } rmp-serde = "1" bytes = "1" http = "1" From 1723eb3f02a7be13f9facfd66ecefcf7d4d0bfad Mon Sep 17 00:00:00 2001 From: Jules Wiriath Date: Wed, 22 Jul 2026 13:59:26 +0200 Subject: [PATCH 4/5] fix: post bump fixes --- Cargo.lock | 1 + crates/pipeline/Cargo.toml | 1 + crates/pipeline/src/stats.rs | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index e0f6000..d45e097 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1821,6 +1821,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test", + "web-time", ] [[package]] diff --git a/crates/pipeline/Cargo.toml b/crates/pipeline/Cargo.toml index b3e9ef3..29fc272 100644 --- a/crates/pipeline/Cargo.toml +++ b/crates/pipeline/Cargo.toml @@ -21,6 +21,7 @@ libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog.git", rev = " libdd-trace-stats = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false } libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false } libdd-shared-runtime = { git = "https://github.com/DataDog/libdatadog.git", rev = "3081603d3c74f209be4e3be951f78a1a7469397f", default-features = false } +web-time = "1" rmp-serde = "1" bytes = "1" http = "1" diff --git a/crates/pipeline/src/stats.rs b/crates/pipeline/src/stats.rs index 127a21e..a42aa2f 100644 --- a/crates/pipeline/src/stats.rs +++ b/crates/pipeline/src/stats.rs @@ -7,7 +7,8 @@ //! HTTP transport for flushing stats to the Datadog agent's `/v0.6/stats` //! endpoint. -use std::time::{Duration, SystemTime}; +use std::time::Duration; +use web_time::SystemTime; /// Wall-clock now() for wasm. `std::time::SystemTime::now()` is unimplemented on /// `wasm32-unknown-unknown` (it panics/traps), so derive the time from JS @@ -67,6 +68,7 @@ impl StatsCollector { ], Vec::new(), None, + Vec::new(), ), meta, agent_url, From 676357af4e2ceccaa0300dae79ea23ab24615c48 Mon Sep 17 00:00:00 2001 From: Jules Wiriath Date: Wed, 22 Jul 2026 15:09:26 +0200 Subject: [PATCH 5/5] fix: review --- crates/capabilities/src/file.rs | 5 ++++- crates/capabilities/src/file_transport.js | 17 ++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/crates/capabilities/src/file.rs b/crates/capabilities/src/file.rs index aeb1d0c..b633d70 100644 --- a/crates/capabilities/src/file.rs +++ b/crates/capabilities/src/file.rs @@ -134,7 +134,10 @@ fn parse_metadata(value: &JsValue, path: &str) -> Result Result { let v = Reflect::get(value, &JsValue::from_str(key)) - .map_err(|_| FileError::Io(anyhow::anyhow!("metadata({path}) missing field `{key}`")))?; + .map_err(|_| FileError::Io(anyhow::anyhow!("metadata({path}) could not read field `{key}`")))?; + if v.is_undefined() || v.is_null() { + return Err(FileError::Io(anyhow::anyhow!("metadata({path}) missing field `{key}`"))); + } let bigint = js_sys::BigInt::try_from(v) .map_err(|_| FileError::Io(anyhow::anyhow!("metadata({path}) field `{key}` is not a BigInt")))?; u64::try_from(bigint) diff --git a/crates/capabilities/src/file_transport.js b/crates/capabilities/src/file_transport.js index bdcd3f6..036003c 100644 --- a/crates/capabilities/src/file_transport.js +++ b/crates/capabilities/src/file_transport.js @@ -16,17 +16,12 @@ module.exports.writeFile = function (path, data) { module.exports.metadata = function (path) { const fs = require('node:fs') - try { - const s = fs.statSync(path, { bigint: true }) - return Promise.resolve({ - size: s.size, - inode: s.ino, - is_file: s.isFile(), - is_dir: s.isDirectory(), - }) - } catch (error) { - return Promise.reject(error) - } + return fs.promises.stat(path, { bigint: true }).then(s => ({ + size: s.size, + inode: s.ino, + is_file: s.isFile(), + is_dir: s.isDirectory(), + })) } module.exports.exists = function (path) {