diff --git a/.changeset/typed-networks-flow.md b/.changeset/typed-networks-flow.md new file mode 100644 index 0000000000..12caf268da --- /dev/null +++ b/.changeset/typed-networks-flow.md @@ -0,0 +1,9 @@ +--- +"@browserbasehq/stagehand-protocol": patch +"@browserbasehq/stagehand-python": patch +"@browserbasehq/stagehand-extension": patch +"@browserbasehq/stagehand-go": patch +"@browserbasehq/stagehand": patch +--- + +expose page-scoped network capture events with response bodies diff --git a/packages/docs/tests/sdk-reference.test.ts b/packages/docs/tests/sdk-reference.test.ts index b61d517fa4..687d8afb15 100644 --- a/packages/docs/tests/sdk-reference.test.ts +++ b/packages/docs/tests/sdk-reference.test.ts @@ -308,6 +308,7 @@ describe("SDK reference surface", () => { "Awaited", "Buffer", "Error", + "EventName", "EvaluateResult", "Input", "Map", diff --git a/packages/docs/v4/reference/page.mdx b/packages/docs/v4/reference/page.mdx index fff4074143..464411ef7f 100644 --- a/packages/docs/v4/reference/page.mdx +++ b/packages/docs/v4/reference/page.mdx @@ -336,8 +336,9 @@ const title = await page.evaluate(() => document.title); ## on() -Subscribe to console messages from this page and its page-owned sessions. Stagehand -delivers each message using the underlying `"Runtime.consoleAPICalled"` event envelope. +Subscribe to console or network events from this page and its page-owned sessions. The +listener type is narrowed from the event name: `"console"` delivers `PageConsoleEvent`, +while `"network"` delivers `PageNetworkEvent`. ```typescript const subscription = await page.on("console", (event) => { @@ -348,13 +349,13 @@ await page.evaluate(() => console.log("ready")); await subscription.unsubscribe(); ``` - - The console event name. Currently, the only supported value is `"console"`. + + The event name: `"console"` or `"network"`. - - A callback that receives the event method, raw parameters, session ID, target ID, - and page ID. Async callbacks may overlap and are not awaited by later page calls. + + A callback that receives the typed event payload. Async callbacks may overlap and + are not awaited by later page calls. @@ -1016,8 +1017,9 @@ title = await page.evaluate("document.title", result_type=str) ## on() -Subscribe to console messages from this page and its page-owned sessions. Stagehand -delivers each message using the underlying `"Runtime.consoleAPICalled"` event envelope. +Subscribe to console or network events from this page and its page-owned sessions. +Both values deliver a `PageCDPEvent` envelope with the corresponding CDP method and +parameters. ```python async def handle_console(event: PageCDPEvent) -> None: @@ -1029,7 +1031,7 @@ await subscription.unsubscribe() ``` - The console event name. Currently, the only supported value is `"console"`. + The event name: `"console"` or `"network"`. @@ -1722,8 +1724,9 @@ fmt.Println(title) ## On() -Subscribe to console messages from this page and its page-owned sessions. Stagehand -delivers each message using the underlying `"Runtime.consoleAPICalled"` event envelope. +Subscribe to console or network events from this page and its page-owned sessions. +Both values deliver a `PageCDPEvent` envelope with the corresponding CDP method and +parameters. ```go subscription, err := page.On(ctx, stagehand.PageEventNameConsole, func(event stagehand.PageCDPEvent) { @@ -1742,7 +1745,7 @@ if err := subscription.Close(ctx); err != nil { ``` - The console event name. Currently, the only supported value is `PageEventNameConsole`. + The event name: `PageEventNameConsole` or `PageEventNameNetwork`. diff --git a/packages/extension/runtime.ts b/packages/extension/runtime.ts index 403cd7fd33..3a9ca5f192 100644 --- a/packages/extension/runtime.ts +++ b/packages/extension/runtime.ts @@ -177,7 +177,10 @@ export type UnderstudyRuntimePage = { close(): Promise | void; captureSnapshot(options?: SnapshotOptions): Promise; deepLocator(selector: string): UnderstudyRuntimeLocator; - subscribeCDPEvent(listener: (event: PageCDPEvent) => void): () => void; + subscribeCDPEvent( + eventName: PageOnParams["event"], + listener: (event: PageCDPEvent) => void, + ): () => void; }; export type UnderstudyRuntimeScreenshotOptions = Omit & { @@ -743,7 +746,7 @@ export class StagehandRuntime { if (this.pageEventSubscriptions.has(params.subscriptionId)) { throw new DuplicatePageEventSubscriptionError(); } - const dispose = this.resolvePage(params.pageId).subscribeCDPEvent((event) => { + const dispose = this.resolvePage(params.pageId).subscribeCDPEvent(params.event, (event) => { this.adapters.emitPageCDPEvent({ subscriptionId: params.subscriptionId, event }); }); this.pageEventSubscriptions.set(params.subscriptionId, { pageId: params.pageId, dispose }); diff --git a/packages/extension/tests/page-cdp-events.test.ts b/packages/extension/tests/page-cdp-events.test.ts index bd67b7bd5d..973db79c43 100644 --- a/packages/extension/tests/page-cdp-events.test.ts +++ b/packages/extension/tests/page-cdp-events.test.ts @@ -6,10 +6,17 @@ import { Page } from "../understudy/page.js"; class FakeCDPSession implements CDPSessionLike { readonly handlers = new Map void>>(); + readonly sendCalls: Array<{ method: string; params?: object }> = []; + readonly responseBodies = new Map(); constructor(readonly id: string) {} - async send(): Promise { + async send(method: string, params?: object): Promise { + this.sendCalls.push({ method, params }); + if (method === "Network.getResponseBody") { + const requestId = (params as { requestId?: string } | undefined)?.requestId ?? ""; + return (this.responseBodies.get(requestId) ?? {}) as Result; + } return {} as Result; } @@ -45,13 +52,13 @@ function createPage( } describe("Page CDP event subscriptions", () => { - it("covers the main session plus current and future OOPIF sessions", () => { + it("covers the main session plus current and future OOPIF sessions", async () => { const main = new FakeCDPSession("main"); const child = new FakeCDPSession("child"); const page = createPage(main); const events: unknown[] = []; - const unsubscribe = page.subscribeCDPEvent((event) => { + const unsubscribe = await page.subscribeCDPEvent("console", (event) => { events.push(event); }); main.emit("Runtime.consoleAPICalled", { type: "log", args: [] }); @@ -81,29 +88,29 @@ describe("Page CDP event subscriptions", () => { expect(main.listenerCount("Runtime.consoleAPICalled")).toBe(0); }); - it("removes every raw listener when the page is disposed", () => { + it("removes every raw listener when the page is disposed", async () => { const main = new FakeCDPSession("main"); const child = new FakeCDPSession("child"); const page = createPage(main); page.adoptOopifSession(child, "frame-child"); - page.subscribeCDPEvent(() => {}); + await page.subscribeCDPEvent("console", () => {}); page.dispose(); expect(main.listenerCount("Runtime.consoleAPICalled")).toBe(0); expect(child.listenerCount("Runtime.consoleAPICalled")).toBe(0); }); - it("isolates listener failures so other subscriptions still receive the event", () => { + it("isolates listener failures so other subscriptions still receive the event", async () => { const main = new FakeCDPSession("main"); const logError = vi.fn(); const page = createPage(main, { error: logError } as unknown as StagehandLogger); const events: PageCDPEvent[] = []; - page.subscribeCDPEvent(() => { + await page.subscribeCDPEvent("console", () => { throw new Error("listener failed"); }); - page.subscribeCDPEvent((event) => events.push(event)); + await page.subscribeCDPEvent("console", (event) => events.push(event)); expect(() => main.emit("Runtime.consoleAPICalled", { type: "log", args: [] })).not.toThrow(); expect(events).toHaveLength(1); @@ -117,4 +124,83 @@ describe("Page CDP event subscriptions", () => { }), ); }); + + it("emits typed network captures with response bodies across page sessions", async () => { + const main = new FakeCDPSession("main"); + const child = new FakeCDPSession("child"); + main.responseBodies.set("request-1", { body: '{"ok":true}', base64Encoded: false }); + const page = createPage(main); + page.adoptOopifSession(child, "frame-child"); + const events: PageCDPEvent[] = []; + + const unsubscribe = await page.subscribeCDPEvent("network", (event) => events.push(event)); + main.emit("Network.requestWillBeSent", { + requestId: "request-1", + request: { + url: "https://example.test/api", + method: "POST", + headers: { "Content-Type": "application/json", attempts: 2 }, + postData: '{"ready":true}', + }, + type: "Fetch", + }); + main.emit("Network.responseReceived", { + requestId: "request-1", + response: { + url: "https://example.test/api", + status: 200, + statusText: "OK", + headers: { "Content-Type": "application/json" }, + mimeType: "application/json", + }, + }); + main.emit("Network.loadingFinished", { requestId: "request-1" }); + child.emit("Network.requestWillBeSent", { + requestId: "request-1", + request: { url: "https://child.example.test/", method: "GET", headers: {} }, + type: "Document", + }); + child.emit("Network.loadingFailed", { + requestId: "request-1", + errorText: "net::ERR_FAILED", + }); + + await vi.waitFor(() => expect(events).toHaveLength(4)); + expect(events[0]).toMatchObject({ + method: "Network.requestWillBeSent", + sessionId: "main", + targetId: "target-main", + params: { + requestKey: "main:request-1", + requestId: "request-1", + httpMethod: "POST", + headers: { "Content-Type": "application/json", attempts: "2" }, + body: '{"ready":true}', + }, + }); + expect(events[1]).toMatchObject({ + method: "Network.requestWillBeSent", + sessionId: "child", + targetId: "target-child", + params: { requestKey: "child:request-1" }, + }); + expect(events[2]).toMatchObject({ + method: "Network.loadingFailed", + sessionId: "child", + params: { requestKey: "child:request-1", errorText: "net::ERR_FAILED" }, + }); + expect(events[3]).toMatchObject({ + method: "Network.loadingFinished", + sessionId: "main", + params: { + requestKey: "main:request-1", + status: 200, + body: '{"ok":true}', + base64Encoded: false, + }, + }); + + unsubscribe(); + expect(main.sendCalls.some((call) => call.method === "Network.disable")).toBe(false); + }); }); diff --git a/packages/extension/tests/stagehand-clients.test.ts b/packages/extension/tests/stagehand-clients.test.ts index 3c1c78d37b..52f27fa391 100644 --- a/packages/extension/tests/stagehand-clients.test.ts +++ b/packages/extension/tests/stagehand-clients.test.ts @@ -44,6 +44,7 @@ import type { PageEvaluateParams, PageKeyPressParams, PageNavigationOptions, + PageOnParams, PageReloadParams, PageSnapshotOptions, PageSetExtraHTTPHeadersParams, @@ -457,14 +458,25 @@ class FakeUnderstudyRuntimePage implements UnderstudyRuntimePage { return locator; } - subscribeCDPEvent(listener: (event: PageCDPEvent) => void): () => void { - const method = "Runtime.consoleAPICalled"; - const listeners = this.cdpEventListeners.get(method) ?? new Set(); - listeners.add(listener); - this.cdpEventListeners.set(method, listeners); + subscribeCDPEvent( + eventName: PageOnParams["event"], + listener: (event: PageCDPEvent) => void, + ): () => void { + const methods: PageCDPEvent["method"][] = + eventName === "console" + ? ["Runtime.consoleAPICalled"] + : ["Network.requestWillBeSent", "Network.loadingFinished", "Network.loadingFailed"]; + for (const method of methods) { + const listeners = this.cdpEventListeners.get(method) ?? new Set(); + listeners.add(listener); + this.cdpEventListeners.set(method, listeners); + } return () => { - listeners.delete(listener); - if (listeners.size === 0) this.cdpEventListeners.delete(method); + for (const method of methods) { + const listeners = this.cdpEventListeners.get(method); + listeners?.delete(listener); + if (listeners?.size === 0) this.cdpEventListeners.delete(method); + } }; } diff --git a/packages/extension/types/private/network.ts b/packages/extension/types/private/network.ts index 4f864a4354..cd7d5601ac 100644 --- a/packages/extension/types/private/network.ts +++ b/packages/extension/types/private/network.ts @@ -1,4 +1,5 @@ import { Protocol } from "devtools-protocol"; +import type { PageNetworkEvent } from "../../../protocol/types.js"; /** Metadata tracked for each network request currently in-flight. */ export type NetworkRequestInfo = { @@ -20,6 +21,14 @@ export interface NetworkObserver { onRequestFailed(info: NetworkRequestInfo): void; } +export type NetworkCaptureEvent = PageNetworkEvent extends infer Event + ? Event extends PageNetworkEvent + ? Omit + : never + : never; + +export type NetworkCaptureObserver = (event: NetworkCaptureEvent) => void; + /** Options for the idle waiter helper. */ export type WaitForIdleOptions = { startTime?: number; diff --git a/packages/extension/understudy/networkManager.ts b/packages/extension/understudy/networkManager.ts index 67dd2deca4..baf60d49f7 100644 --- a/packages/extension/understudy/networkManager.ts +++ b/packages/extension/understudy/networkManager.ts @@ -3,12 +3,32 @@ import type { CDPSessionLike } from "./cdp.js"; import { DEFAULT_IDLE_WAIT, IGNORED_RESOURCE_TYPES, + NetworkCaptureEvent, + NetworkCaptureObserver, NetworkObserver, NetworkRequestInfo, WaitForIdleHandle, WaitForIdleOptions, } from "../types/private/network.js"; +const NETWORK_CAPTURE_BUFFER_LIMITS = { + maxResourceBufferSize: 5_000_000, + maxTotalBufferSize: 10_000_000, +} as const; + +type NetworkCaptureRequest = { + requestKey: string; + requestId: string; + startedAt: number; +}; + +type NetworkCaptureResponse = { + status: number; + statusText: string; + headers: Record; + mimeType: string; +}; + /** * Cross-session network tracker. * @@ -33,6 +53,12 @@ export class NetworkManager { readonly observers = new Set(); + readonly captureObservers = new Set(); + + readonly captureRequests = new Map(); + + readonly captureResponses = new Map(); + readonly requests = new Map(); readonly documentRequestsByFrame = new Map(); @@ -66,6 +92,31 @@ export class NetworkManager { } this.emitStart(info); + + if (this.captureObservers.size > 0) { + if (evt.redirectResponse) { + this.emitRedirectCapture(sid, info.requestKey, evt.requestId, evt.redirectResponse); + } + this.captureRequests.set(info.requestKey, { + requestKey: info.requestKey, + requestId: evt.requestId, + startedAt: Date.now(), + }); + this.emitCapture({ + method: "Network.requestWillBeSent", + params: { + requestKey: info.requestKey, + requestId: evt.requestId, + url: evt.request.url, + httpMethod: evt.request.method, + headers: normalizeHeaders(evt.request.headers), + body: evt.request.postData ?? null, + resourceType: evt.type ?? "Other", + timestamp: new Date().toISOString(), + }, + sessionId: sid, + }); + } }; const finish = (reqId: string) => { @@ -105,15 +156,29 @@ export class NetworkManager { const onFinished = (evt: { requestId: string }) => { if (!evt?.requestId) return; finish(evt.requestId); + if (this.captureObservers.size > 0) { + void this.emitFinishedCapture(session, sid, evt.requestId); + } }; const onFailed = (evt: Protocol.Network.LoadingFailedEvent) => { if (!evt?.requestId) return; fail(evt.requestId); + if (this.captureObservers.size > 0) { + this.emitFailedCapture(sid, evt); + } }; const onResponse = (evt: Protocol.Network.ResponseReceivedEvent) => { if (!evt?.requestId) return; + if (this.captureObservers.size > 0 && evt.response) { + this.captureResponses.set(this.requestKey(sid, evt.requestId), { + status: evt.response.status, + statusText: evt.response.statusText, + headers: normalizeHeaders(evt.response.headers), + mimeType: evt.response.mimeType, + }); + } const url = evt.response?.url ?? ""; if (url.startsWith("data:")) finish(evt.requestId); }; @@ -139,7 +204,12 @@ export class NetworkManager { session.on("Network.responseReceived", onResponse); session.on("Page.frameStoppedLoading", onFrameStopped); - void session.send("Network.enable").catch(() => {}); + void session + .send( + "Network.enable", + this.captureObservers.size > 0 ? NETWORK_CAPTURE_BUFFER_LIMITS : undefined, + ) + .catch(() => {}); void session.send("Page.enable").catch(() => {}); this.sessions.set(sid, { @@ -168,6 +238,12 @@ export class NetworkManager { for (const key of this.requests.keys()) { if (key.startsWith(`${sid}:`)) this.requests.delete(key); } + for (const key of this.captureRequests.keys()) { + if (key.startsWith(`${sid}:`)) this.captureRequests.delete(key); + } + for (const key of this.captureResponses.keys()) { + if (key.startsWith(`${sid}:`)) this.captureResponses.delete(key); + } for (const [frameId, key] of this.documentRequestsByFrame.entries()) { if (key.startsWith(`${sid}:`)) { @@ -187,6 +263,28 @@ export class NetworkManager { }; } + /** + * Register an observer for typed request/response capture records. + * Network tracking remains enabled after disposal because navigation and idle + * detection share the same CDP domain. + */ + public addCaptureObserver(observer: NetworkCaptureObserver): () => void { + const firstObserver = this.captureObservers.size === 0; + this.captureObservers.add(observer); + if (firstObserver) { + for (const { session } of this.sessions.values()) { + void session.send("Network.enable", NETWORK_CAPTURE_BUFFER_LIMITS).catch(() => undefined); + } + } + return () => { + this.captureObservers.delete(observer); + if (this.captureObservers.size === 0) { + this.captureRequests.clear(); + this.captureResponses.clear(); + } + }; + } + /** * Resolve once no (filtered) requests are in flight for the given quiet window. * The waiter automatically unregisters itself on completion or timeout. @@ -304,7 +402,10 @@ export class NetworkManager { } this.sessions.clear(); this.observers.clear(); + this.captureObservers.clear(); this.requests.clear(); + this.captureRequests.clear(); + this.captureResponses.clear(); this.documentRequestsByFrame.clear(); } @@ -329,6 +430,101 @@ export class NetworkManager { } } + private emitCapture(event: NetworkCaptureEvent): void { + for (const observer of this.captureObservers) { + try { + observer(event); + } catch { + // Capture observers are passive and must not disrupt CDP dispatch. + } + } + } + + private emitRedirectCapture( + sessionId: string, + requestKey: string, + requestId: string, + response: Protocol.Network.Response, + ): void { + const request = this.captureRequests.get(requestKey); + this.emitCapture({ + method: "Network.loadingFinished", + params: { + requestKey, + requestId, + status: response.status, + statusText: response.statusText, + headers: normalizeHeaders(response.headers), + mimeType: response.mimeType, + body: null, + base64Encoded: false, + durationMs: request ? Date.now() - request.startedAt : 0, + }, + sessionId, + }); + this.captureRequests.delete(requestKey); + this.captureResponses.delete(requestKey); + } + + private async emitFinishedCapture( + session: CDPSessionLike, + sessionId: string, + requestId: string, + ): Promise { + const requestKey = this.requestKey(sessionId, requestId); + const request = this.captureRequests.get(requestKey); + const response = this.captureResponses.get(requestKey); + let body: string | null = null; + let base64Encoded = false; + try { + const result = await session.send<{ body?: string; base64Encoded?: boolean }>( + "Network.getResponseBody", + { requestId }, + ); + body = result.body ?? null; + base64Encoded = result.base64Encoded ?? false; + } catch { + // Some response types do not expose a body through CDP. + } + + this.emitCapture({ + method: "Network.loadingFinished", + params: { + requestKey, + requestId, + status: response?.status ?? 0, + statusText: response?.statusText ?? "", + headers: response?.headers ?? {}, + mimeType: response?.mimeType ?? "", + body, + base64Encoded, + durationMs: request ? Date.now() - request.startedAt : 0, + }, + sessionId, + }); + if (this.captureRequests.get(requestKey) === request) { + this.captureRequests.delete(requestKey); + this.captureResponses.delete(requestKey); + } + } + + private emitFailedCapture(sessionId: string, event: Protocol.Network.LoadingFailedEvent): void { + const requestKey = this.requestKey(sessionId, event.requestId); + const request = this.captureRequests.get(requestKey); + this.emitCapture({ + method: "Network.loadingFailed", + params: { + requestKey, + requestId: event.requestId, + errorText: event.errorText, + durationMs: request ? Date.now() - request.startedAt : 0, + }, + sessionId, + }); + this.captureRequests.delete(requestKey); + this.captureResponses.delete(requestKey); + } + /** Compute a stable key for a session (falls back to synthetic root id). */ sessionKey(session: CDPSessionLike): string { return session.id ?? "__main__"; @@ -339,3 +535,9 @@ export class NetworkManager { return `${sessionId}:${requestId}`; } } + +function normalizeHeaders(headers: Protocol.Network.Headers | undefined): Record { + return Object.fromEntries( + Object.entries(headers ?? {}).map(([name, value]) => [name, String(value)]), + ); +} diff --git a/packages/extension/understudy/page.ts b/packages/extension/understudy/page.ts index 936aebc833..ac519de6e9 100644 --- a/packages/extension/understudy/page.ts +++ b/packages/extension/understudy/page.ts @@ -21,6 +21,7 @@ import type { LoadState, LocalBrowserLaunchOptions, PageCDPEvent, + PageEventName, PageSnapshotOptions, SnapshotResult, WebMCPAnnotation, @@ -91,8 +92,10 @@ type WebMCPInvocationRecord = { }; type CDPEventSubscription = { + eventName: PageEventName; listener: (event: PageCDPEvent) => void; sessionHandlers: Map void }>; + networkDispose?: () => void; }; function createDeferred(): Deferred { @@ -404,7 +407,9 @@ export class Page { if (childSession.id) this.sessions.set(childSession.id, childSession); for (const subscription of this.cdpEventSubscriptions) { - this.attachCDPEventSubscription(subscription, childSession); + if (subscription.eventName === "console") { + this.attachCDPEventSubscription(subscription, childSession); + } } this.networkManager.trackSession(childSession); @@ -507,15 +512,36 @@ export class Page { return this.mainSession.send(method, params); } - /** Subscribe to console events on every session owned by this page. */ - public subscribeCDPEvent(listener: (event: PageCDPEvent) => void): () => void { + /** Subscribe to a typed event stream owned by this page. */ + public subscribeCDPEvent( + eventName: PageEventName, + listener: (event: PageCDPEvent) => void, + ): () => void { const subscription: CDPEventSubscription = { + eventName, listener, sessionHandlers: new Map(), }; this.cdpEventSubscriptions.add(subscription); - for (const session of this.sessions.values()) { - this.attachCDPEventSubscription(subscription, session); + if (eventName === "console") { + for (const session of this.sessions.values()) { + this.attachCDPEventSubscription(subscription, session); + } + } else { + subscription.networkDispose = this.networkManager.addCaptureObserver((networkEvent) => { + const session = + networkEvent.sessionId === "__main__" + ? this.mainSession + : this.sessions.get(networkEvent.sessionId); + this.deliverCDPEvent( + subscription, + PageCDPEventSchema.parse({ + ...networkEvent, + pageId: this.pageId, + targetId: this.conn.targetIdForSession(session?.id ?? null) ?? this._targetId, + }), + ); + }); } let active = true; @@ -523,6 +549,7 @@ export class Page { if (!active) return; active = false; this.cdpEventSubscriptions.delete(subscription); + subscription.networkDispose?.(); for (const sessionId of subscription.sessionHandlers.keys()) { this.detachCDPEventSubscription(subscription, sessionId); } @@ -547,17 +574,7 @@ export class Page { sessionId, targetId: this.conn.targetIdForSession(session.id) ?? this._targetId, }); - try { - subscription.listener(event); - } catch (error) { - this.logger.error("Page CDP event listener failed", { - category: "page", - pageId: this.pageId, - method: "Runtime.consoleAPICalled", - sessionId, - error: error instanceof Error ? error.message : String(error), - }); - } + this.deliverCDPEvent(subscription, event); }; subscription.sessionHandlers.set(sessionId, { session, handler }); session.on("Runtime.consoleAPICalled", handler); @@ -570,6 +587,20 @@ export class Page { subscription.sessionHandlers.delete(sessionId); } + private deliverCDPEvent(subscription: CDPEventSubscription, event: PageCDPEvent): void { + try { + subscription.listener(event); + } catch (error) { + this.logger.error("Page CDP event listener failed", { + category: "page", + pageId: this.pageId, + method: event.method, + sessionId: event.sessionId, + error: error instanceof Error ? error.message : String(error), + }); + } + } + /** * Return a fresh snapshot of the WebMCP tools registered by the current page. * @@ -805,6 +836,7 @@ export class Page { public dispose(): void { for (const subscription of this.cdpEventSubscriptions) { this.cdpEventSubscriptions.delete(subscription); + subscription.networkDispose?.(); for (const sessionId of subscription.sessionHandlers.keys()) { this.detachCDPEventSubscription(subscription, sessionId); } diff --git a/packages/protocol/schemas.ts b/packages/protocol/schemas.ts index a1931f7352..05bfc9906e 100644 --- a/packages/protocol/schemas.ts +++ b/packages/protocol/schemas.ts @@ -1441,19 +1441,92 @@ export const ResponseFinishedResultSchema = z }) .meta({ id: "ResponseFinishedResult" }); -export const PageEventNameSchema = z.enum(["console"]).meta({ id: "PageEventName" }); +export const PageEventNameSchema = z.enum(["console", "network"]).meta({ id: "PageEventName" }); export const PageCDPEventParamsSchema = z .record(z.string(), z.json()) .meta({ id: "PageCDPEventParams" }); +const PageEventEnvelopeSchema = z.strictObject({ + pageId: z.string().min(1), + sessionId: z.string().min(1), + targetId: z.string().min(1), +}); + +export const PageConsoleEventSchema = PageEventEnvelopeSchema.extend({ + method: z.literal("Runtime.consoleAPICalled"), + params: PageCDPEventParamsSchema, +}).meta({ id: "PageConsoleEvent" }); + +export const PageNetworkRequestEventParamsSchema = z + .strictObject({ + requestKey: z.string().min(1), + requestId: z.string().min(1), + url: z.string(), + httpMethod: z.string().min(1), + headers: z.record(z.string(), z.string()), + body: z.string().nullable(), + resourceType: z.string(), + timestamp: z.iso.datetime(), + }) + .meta({ id: "PageNetworkRequestEventParams" }); + +export const PageNetworkRequestEventSchema = PageEventEnvelopeSchema.extend({ + method: z.literal("Network.requestWillBeSent"), + params: PageNetworkRequestEventParamsSchema, +}).meta({ id: "PageNetworkRequestEvent" }); + +export const PageNetworkLoadingFinishedEventParamsSchema = z + .strictObject({ + requestKey: z.string().min(1), + requestId: z.string().min(1), + status: z.number(), + statusText: z.string(), + headers: z.record(z.string(), z.string()), + mimeType: z.string(), + body: z.string().nullable(), + base64Encoded: z.boolean(), + durationMs: z.number().nonnegative(), + }) + .meta({ id: "PageNetworkLoadingFinishedEventParams" }); + +export const PageNetworkLoadingFinishedEventSchema = PageEventEnvelopeSchema.extend({ + method: z.literal("Network.loadingFinished"), + params: PageNetworkLoadingFinishedEventParamsSchema, +}).meta({ id: "PageNetworkLoadingFinishedEvent" }); + +export const PageNetworkLoadingFailedEventParamsSchema = z + .strictObject({ + requestKey: z.string().min(1), + requestId: z.string().min(1), + errorText: z.string(), + durationMs: z.number().nonnegative(), + }) + .meta({ id: "PageNetworkLoadingFailedEventParams" }); + +export const PageNetworkLoadingFailedEventSchema = PageEventEnvelopeSchema.extend({ + method: z.literal("Network.loadingFailed"), + params: PageNetworkLoadingFailedEventParamsSchema, +}).meta({ id: "PageNetworkLoadingFailedEvent" }); + +export const PageNetworkEventSchema = z + .discriminatedUnion("method", [ + PageNetworkRequestEventSchema, + PageNetworkLoadingFinishedEventSchema, + PageNetworkLoadingFailedEventSchema, + ]) + .meta({ id: "PageNetworkEvent" }); + export const PageCDPEventSchema = z .strictObject({ - pageId: z.string().min(1), - method: z.literal("Runtime.consoleAPICalled"), + ...PageEventEnvelopeSchema.shape, + method: z.enum([ + "Runtime.consoleAPICalled", + "Network.requestWillBeSent", + "Network.loadingFinished", + "Network.loadingFailed", + ]), params: PageCDPEventParamsSchema, - sessionId: z.string().min(1), - targetId: z.string().min(1), }) .meta({ id: "PageCDPEvent" }); diff --git a/packages/protocol/stagehand.v4.json b/packages/protocol/stagehand.v4.json index 35ab624400..5a61d34f38 100644 --- a/packages/protocol/stagehand.v4.json +++ b/packages/protocol/stagehand.v4.json @@ -3634,7 +3634,7 @@ }, "PageEventName": { "type": "string", - "enum": ["console"] + "enum": ["console", "network"] }, "PageOffParams": { "type": "object", @@ -4880,13 +4880,6 @@ "type": "string", "minLength": 1 }, - "method": { - "type": "string", - "const": "Runtime.consoleAPICalled" - }, - "params": { - "$ref": "#/$defs/PageCDPEventParams" - }, "session_id": { "type": "string", "minLength": 1 @@ -4894,9 +4887,21 @@ "target_id": { "type": "string", "minLength": 1 + }, + "method": { + "type": "string", + "enum": [ + "Runtime.consoleAPICalled", + "Network.requestWillBeSent", + "Network.loadingFinished", + "Network.loadingFailed" + ] + }, + "params": { + "$ref": "#/$defs/PageCDPEventParams" } }, - "required": ["page_id", "method", "params", "session_id", "target_id"], + "required": ["page_id", "session_id", "target_id", "method", "params"], "additionalProperties": false }, "PageCDPEventParams": { diff --git a/packages/protocol/tests/protocol/page-events.test.ts b/packages/protocol/tests/protocol/page-events.test.ts index 55cc78d5fa..9b44dd9755 100644 --- a/packages/protocol/tests/protocol/page-events.test.ts +++ b/packages/protocol/tests/protocol/page-events.test.ts @@ -4,12 +4,14 @@ import { PageCDPEventNotificationSchema, PageCDPEventSchema, PageEventNameSchema, + PageNetworkEventSchema, } from "../../schemas.js"; import { StagehandMethods, StagehandNotifications } from "../../schema-registry.js"; -describe("console page events", () => { - it("only accepts the public console event name", () => { +describe("page events", () => { + it("accepts the public console and network event names", () => { expect(PageEventNameSchema.parse("console")).toBe("console"); + expect(PageEventNameSchema.parse("network")).toBe("network"); expect(() => PageEventNameSchema.parse("Runtime.consoleAPICalled")).toThrow(); expect(() => PageEventNameSchema.parse("Network.responseReceived")).toThrow(); }); @@ -30,13 +32,64 @@ describe("console page events", () => { sessionId: "session-1", targetId: "target-1", }); + }); + + it("validates typed network request, completion, and failure events", () => { + const envelope = { + pageId: "page-1", + sessionId: "session-1", + targetId: "target-1", + }; + expect( + PageNetworkEventSchema.parse({ + ...envelope, + method: "Network.requestWillBeSent", + params: { + requestKey: "session-1:request-1", + requestId: "request-1", + url: "https://example.com/api", + httpMethod: "POST", + headers: { "content-type": "application/json" }, + body: '{"ready":true}', + resourceType: "Fetch", + timestamp: "2026-08-27T12:00:00.000Z", + }, + }).method, + ).toBe("Network.requestWillBeSent"); + expect( + PageNetworkEventSchema.parse({ + ...envelope, + method: "Network.loadingFinished", + params: { + requestKey: "session-1:request-1", + requestId: "request-1", + status: 200, + statusText: "OK", + headers: { "content-type": "application/json" }, + mimeType: "application/json", + body: '{"ok":true}', + base64Encoded: false, + durationMs: 12, + }, + }).method, + ).toBe("Network.loadingFinished"); + expect( + PageNetworkEventSchema.parse({ + ...envelope, + method: "Network.loadingFailed", + params: { + requestKey: "session-1:request-2", + requestId: "request-2", + errorText: "net::ERR_FAILED", + durationMs: 2, + }, + }).method, + ).toBe("Network.loadingFailed"); expect(() => PageCDPEventSchema.parse({ - pageId: "page-1", + ...envelope, method: "Network.responseReceived", params: {}, - sessionId: "session-1", - targetId: "target-1", }), ).toThrow(); }); diff --git a/packages/protocol/types.ts b/packages/protocol/types.ts index 751aac123d..fbee0cea73 100644 --- a/packages/protocol/types.ts +++ b/packages/protocol/types.ts @@ -148,6 +148,7 @@ import type { PageCDPEventNotificationSchema, PageCDPEventParamsSchema, PageCDPEventSchema, + PageConsoleEventSchema, PageClickParamsSchema, PageCloseResultSchema, PageDragAndDropParamsSchema, @@ -164,6 +165,13 @@ import type { PageLocatorSchema, PageNavigationResultSchema, PageNavigationOptionsSchema, + PageNetworkEventSchema, + PageNetworkLoadingFailedEventParamsSchema, + PageNetworkLoadingFailedEventSchema, + PageNetworkLoadingFinishedEventParamsSchema, + PageNetworkLoadingFinishedEventSchema, + PageNetworkRequestEventParamsSchema, + PageNetworkRequestEventSchema, PageOffParamsSchema, PageOnParamsSchema, PageRefSchema, @@ -301,6 +309,18 @@ export type ContextVoidResult = z.infer; export type PageRef = z.infer; export type PageEventName = z.infer; export type PageCDPEventParams = z.infer; +export type PageConsoleEvent = z.infer; +export type PageNetworkRequestEventParams = z.infer; +export type PageNetworkRequestEvent = z.infer; +export type PageNetworkLoadingFinishedEventParams = z.infer< + typeof PageNetworkLoadingFinishedEventParamsSchema +>; +export type PageNetworkLoadingFinishedEvent = z.infer; +export type PageNetworkLoadingFailedEventParams = z.infer< + typeof PageNetworkLoadingFailedEventParamsSchema +>; +export type PageNetworkLoadingFailedEvent = z.infer; +export type PageNetworkEvent = z.infer; export type PageCDPEvent = z.infer; export type PageCDPEventNotification = z.infer; export type PageNavigationOptions = z.infer; diff --git a/packages/sdk-go/internal/extensionassets/stagehand-extension.zip b/packages/sdk-go/internal/extensionassets/stagehand-extension.zip index dbac537eee..c3bb6f4c2d 100644 Binary files a/packages/sdk-go/internal/extensionassets/stagehand-extension.zip and b/packages/sdk-go/internal/extensionassets/stagehand-extension.zip differ diff --git a/packages/sdk-go/models.gen.go b/packages/sdk-go/models.gen.go index e3131bcad8..a2464b5ba2 100644 --- a/packages/sdk-go/models.gen.go +++ b/packages/sdk-go/models.gen.go @@ -1201,7 +1201,7 @@ type PageAddInitScriptParams struct { type PageCDPEvent struct { // Method corresponds to the JSON schema field "method". - Method string `json:"method"` + Method PageCDPEventMethod `json:"method"` // PageID corresponds to the JSON schema field "page_id". PageID string `json:"page_id"` @@ -1216,6 +1216,13 @@ type PageCDPEvent struct { TargetID string `json:"target_id"` } +type PageCDPEventMethod string + +const PageCDPEventMethodNetworkLoadingFailed PageCDPEventMethod = "Network.loadingFailed" +const PageCDPEventMethodNetworkLoadingFinished PageCDPEventMethod = "Network.loadingFinished" +const PageCDPEventMethodNetworkRequestWillBeSent PageCDPEventMethod = "Network.requestWillBeSent" +const PageCDPEventMethodRuntimeConsoleAPICalled PageCDPEventMethod = "Runtime.consoleAPICalled" + type PageCDPEventNotification struct { // Event corresponds to the JSON schema field "event". Event PageCDPEvent `json:"event"` @@ -1311,6 +1318,7 @@ type PageEvaluateResult struct { type PageEventName string const PageEventNameConsole PageEventName = "console" +const PageEventNameNetwork PageEventName = "network" type PageGoBackParams struct { // Options corresponds to the JSON schema field "options". diff --git a/packages/sdk-go/page.go b/packages/sdk-go/page.go index 6a98cbc7c1..d5e80cf25a 100644 --- a/packages/sdk-go/page.go +++ b/packages/sdk-go/page.go @@ -20,7 +20,7 @@ type Page struct { reportEventListenerPanic func(any) } -// CDPSubscription is a page-scoped console event listener registration. +// CDPSubscription is a page-scoped event listener registration. type CDPSubscription struct { rpc protocolClient page *Page @@ -243,7 +243,7 @@ func (p *Page) AddInitScript(ctx context.Context, source string) error { return p.rpc.call(ctx, "page.add_init_script", params, &result) } -// On subscribes to console events for this page and its OOPIF sessions. +// On subscribes to console or network events for this page and its OOPIF sessions. func (p *Page) On( ctx context.Context, event PageEventName, diff --git a/packages/sdk-python/src/stagehand/_generated/input_types.py b/packages/sdk-python/src/stagehand/_generated/input_types.py index a271ac6fe1..82f0ad5335 100644 --- a/packages/sdk-python/src/stagehand/_generated/input_types.py +++ b/packages/sdk-python/src/stagehand/_generated/input_types.py @@ -676,10 +676,10 @@ class PageAddInitScriptParams(TypedDict): class PageCDPEvent(TypedDict): page_id: str - method: Literal["Runtime.consoleAPICalled"] - params: PageCDPEventParams session_id: str target_id: str + method: Literal["Runtime.consoleAPICalled", "Network.requestWillBeSent", "Network.loadingFinished", "Network.loadingFailed"] + params: PageCDPEventParams class PageCDPEventNotification(TypedDict): @@ -733,7 +733,7 @@ class PageEvaluateResult(TypedDict): value: FieldSchema9 -PageEventName: TypeAlias = Literal["console"] +PageEventName: TypeAlias = Literal["console", "network"] class PageHoverParams(TypedDict): diff --git a/packages/sdk-python/src/stagehand/_generated/models.py b/packages/sdk-python/src/stagehand/_generated/models.py index f9ecf9cda2..63583f7c62 100644 --- a/packages/sdk-python/src/stagehand/_generated/models.py +++ b/packages/sdk-python/src/stagehand/_generated/models.py @@ -1297,6 +1297,13 @@ class LogLevel(StrEnum): debug = "debug" +class Method(StrEnum): + runtime_console_api_called = "Runtime.consoleAPICalled" + network_request_will_be_sent = "Network.requestWillBeSent" + network_loading_finished = "Network.loadingFinished" + network_loading_failed = "Network.loadingFailed" + + class Mode(StrEnum): auto = "auto" required = "required" @@ -1476,10 +1483,10 @@ class PageCDPEvent(WireModel): validate_by_name=True, ) page_id: Annotated[StrictStr, Field(min_length=1)] - method: Literal["Runtime.consoleAPICalled"] - params: PageCDPEventParams session_id: Annotated[StrictStr, Field(min_length=1)] target_id: Annotated[StrictStr, Field(min_length=1)] + method: Method + params: PageCDPEventParams class PageCDPEventNotification(WireModel): @@ -1573,8 +1580,9 @@ class PageEvaluateResult(WireModel): value: Optional[FieldSchema9] -class PageEventName(RootModel[Literal["console"]]): - root: Literal["console"] +class PageEventName(StrEnum): + console = "console" + network = "network" class PageGoBackParams(WireModel): diff --git a/packages/sdk-ts/src/index.ts b/packages/sdk-ts/src/index.ts index 04f950a6d1..ce60f07754 100644 --- a/packages/sdk-ts/src/index.ts +++ b/packages/sdk-ts/src/index.ts @@ -23,6 +23,7 @@ export { Page, type PageClickOptions, type PageDragAndDropOptions, + type PageEventForName, type PageEventListener, type PageKeyPressOptions, type PageReloadOptions, @@ -74,9 +75,14 @@ export type { MouseButton, ObserveResult, PageCDPEvent, + PageConsoleEvent, PageDragAndDropRoutePoint, PageEventName, PageNavigationOptions, + PageNetworkEvent, + PageNetworkLoadingFailedEvent, + PageNetworkLoadingFinishedEvent, + PageNetworkRequestEvent, PageScreenshotClip, PageSnapshotOptions, RgbaColor, diff --git a/packages/sdk-ts/src/page.ts b/packages/sdk-ts/src/page.ts index 8a0fc8f766..27803456e5 100644 --- a/packages/sdk-ts/src/page.ts +++ b/packages/sdk-ts/src/page.ts @@ -1,7 +1,8 @@ import type { LoadState, - PageCDPEvent, + PageConsoleEvent, PageEventName, + PageNetworkEvent, PageClickParams, PageDragAndDropParams, PageKeyPressParams, @@ -43,8 +44,12 @@ export type PageSetViewportSizeOptions = NonNullable; export type PageWaitForSelectorOptions = NonNullable; -export interface PageEventListener { - (event: PageCDPEvent): unknown; +export type PageEventForName = EventName extends "console" + ? PageConsoleEvent + : PageNetworkEvent; + +export interface PageEventListener { + (event: PageEventForName): unknown; } export class CDPSubscription { @@ -209,7 +214,10 @@ export class Page { }); } - async on(event: PageEventName, listener: PageEventListener): Promise { + async on( + event: EventName, + listener: PageEventListener, + ): Promise { const subscriptionId = crypto.randomUUID(); const removeNotificationListener = this.rpcClient.onNotification((notification) => { if ( @@ -219,7 +227,7 @@ export class Page { return; } try { - const result = listener(notification.params.event); + const result = listener(notification.params.event as PageEventForName); if (result && typeof result === "object" && "then" in result) { void Promise.resolve(result).catch(reportPageEventListenerError); } diff --git a/packages/sdk-ts/tests/objectWrapper.test.ts b/packages/sdk-ts/tests/objectWrapper.test.ts index 15ac59d4f7..db859955f0 100644 --- a/packages/sdk-ts/tests/objectWrapper.test.ts +++ b/packages/sdk-ts/tests/objectWrapper.test.ts @@ -579,6 +579,54 @@ describe("Stagehand TS object wrapper", () => { expect(client.listeners).toHaveLength(0); }); + it("narrows and delivers typed network events", async () => { + const client = new FakeProtocolClient(); + client.queueResponse(StagehandMethods.pageOn, { ok: true }); + client.queueResponse(StagehandMethods.pageOff, { ok: true }); + const page = new Page(client, { pageId: "page-1" }); + const requestKeys: string[] = []; + + const subscription = await page.on("network", (event) => { + if (event.method === "Network.requestWillBeSent") { + requestKeys.push(event.params.requestKey); + } + }); + const subscriptionId = (client.calls[0]!.params as { subscriptionId: string }).subscriptionId; + client.emitNotification({ + jsonrpc: "2.0", + method: "page.cdp_event", + params: { + subscriptionId, + event: { + pageId: "page-1", + method: "Network.requestWillBeSent", + params: { + requestKey: "session-1:request-1", + requestId: "request-1", + url: "https://example.test/api", + httpMethod: "GET", + headers: {}, + body: null, + resourceType: "Fetch", + timestamp: "2026-08-27T12:00:00.000Z", + }, + sessionId: "session-1", + targetId: "target-1", + }, + }, + }); + + expect(requestKeys).toStrictEqual(["session-1:request-1"]); + expect(client.calls[0]).toStrictEqual( + requestCall(StagehandMethods.pageOn, { + pageId: "page-1", + subscriptionId, + event: "network", + }), + ); + await subscription.unsubscribe(); + }); + it("cleans up page.on state when remote registration fails", async () => { const client = new FakeProtocolClient(); client.queueResponse(StagehandMethods.pageOn, new Error("registration failed"));