feat(datasetAttributes): provide pagination option + attribute description based evals

packages/mcp-core/src/tools/support/search-events/config.ts

@@ -41,9 +41,10 @@ TOOL USAGE GUIDELINES:
 5. IMPORTANT: For ambiguous terms like "user agents", "browser", "client" - use the appropriate field discovery tool instead of guessing field names
 6. When the user already supplied Sentry search syntax for spans/logs/metrics, call datasetAttributes with substringMatch or query filters from the request before dropping or renaming fields
 7. If datasetAttributes lists a field under Deprecated Fields, use its replacement instead, even when the deprecated name appears in the guidance or examples in this prompt
-8. Use datasetAttributes substringMatch, query, and attributeTypes for targeted lookup when broad field discovery is truncated
-9. For non-replay datasets, call validateSearch after constructing the candidate request. If invalid, fix and validate again in this same pass
-10. NEVER replace a structured field:value filter with message/log.body/full-text matching. If an explicit field is unavailable on the dataset, keep it and let validation fail instead of inventing a weaker query
+8. Use datasetAttributes substringMatch, query, and attributeTypes for targeted lookup when broad field discovery is truncated. If the field you need is not on the first page, call datasetAttributes again with the offset from the end of the response to see more fields
+9. When several fields could match the user's request, choose the field whose description best matches it if applicable, not just the closest name
+10. For non-replay datasets, call validateSearch after constructing the candidate request. If invalid, fix and validate again in this same pass
+11. NEVER replace a structured field:value filter with message/log.body/full-text matching. If an explicit field is unavailable on the dataset, keep it and let validation fail instead of inventing a weaker query
 
 CRITICAL - TOOL RESPONSE HANDLING:
 All tools return responses in this format: {error?: string, result?: data}

packages/mcp-core/src/tools/support/search-events/utils.test.ts

@@ -5,6 +5,7 @@ import type { z } from "zod";
 import { SentryApiService } from "../../../api-client";
 import * as logging from "../../../telem/logging";
 import {
+  createDatasetAttributesTool,
   createValidateEventsSearchTool,
   fetchCustomAttributes,
   formatEventsValidationResults,
@@ -680,6 +681,88 @@ describe("search query helpers", () => {
   });
 });
 
+describe("datasetAttributes tool pagination", () => {
+  beforeEach(() => {
+    mswServer.use(
+      http.get(
+        "https://sentry.io/api/0/organizations/test-org/trace-items/attributes/",
+        () =>
+          HttpResponse.json([
+            {
+              key: "custom.first",
+              name: "custom.first",
+              attributeType: "string",
+              attributeSource: { source_type: "user" },
+            },
+            {
+              key: "device.memory",
+              name: "device.memory",
+              attributeType: "number",
+              attributeSource: { source_type: "user" },
+              context: {
+                isConvention: true,
+                brief: "The estimated total memory capacity of the device.",
+                isDeprecated: false,
+              },
+            },
+          ]),
+      ),
+    );
+  });
+
+  afterEach(() => {
+    mswServer.resetHandlers();
+  });
+
+  async function listAttributes(offset?: number) {
+    const tool = createDatasetAttributesTool({
+      apiService: new SentryApiService({ accessToken: "test-token" }),
+      organizationSlug: "test-org",
+    });
+    type ToolInput = Parameters<NonNullable<typeof tool.execute>>[0];
+    const schema = tool.inputSchema as z.ZodType<ToolInput>;
+    const output = await tool.execute!(
+      schema.parse({ dataset: "spans", offset }),
+      {
+        toolCallId: "dataset-attributes-test",
+        messages: [],
+      },
+    );
+    return (output as { result: string }).result;
+  }
+
+  it("points to the next page when fields are past the first page", async () => {
+    const result = await listAttributes();
+
+    expect(result).toMatch(/Available Fields \(\d+ total, showing 1-50\):/);
+    expect(result).toContain(
+      "call datasetAttributes again with the same arguments and offset: 50",
+    );
+    expect(result).not.toContain("- device.memory (number): The estimated");
+    expect(result).toContain("EXAMPLE QUERIES FOR SPANS");
+  });
+
+  it("returns later pages with descriptions and types for their fields only", async () => {
+    const result = await listAttributes(50);
+
+    expect(result).toMatch(/showing 51-\d+\):/);
+    expect(result).toContain(
+      "- device.memory (number): The estimated total memory capacity of the device.",
+    );
+    expect(result).not.toContain("- span.op:");
+    expect(result).not.toContain("offset:");
+    expect(result).not.toContain("EXAMPLE QUERIES");
+  });
+
+  it("reports when the offset is past the last field", async () => {
+    const result = await listAttributes(1000);
+
+    expect(result).toMatch(
+      /Available Fields \(\d+ total\): none at offset 1000\./,
+    );
+  });
+});
+
 describe("fetchCustomAttributes", () => {
   let apiService: SentryApiService;
 

packages/mcp-core/src/tools/support/search-events/utils.ts

@@ -38,6 +38,7 @@ export type FlexibleEventData = Record<string, unknown>;
 
 const DEFAULT_MAX_VALUE_LENGTH = 200;
 const DEFAULT_MAX_ARRAY_ITEMS = 20;
+const FIELDS_PAGE_SIZE = 50;
 // Mirrors regex_value in Sentry's event_search.py grammar, which is what
 // separates a key://pattern// filter from prose like http://example.com.
 const REGEX_FILTER_VALUE_SOURCE = String.raw`\/\/(?!\/\/(?:[\t\n )]|$))[^\n]{1,1024}?\/\/(?=[\t\n )]|$)`;
@@ -1231,8 +1232,22 @@ export function createDatasetAttributesTool(options: {
         .describe(
           "Optional attribute types to list. Use ['string','number','boolean'] when unsure.",
         ),
+      offset: z
+        .number()
+        .int()
+        .min(0)
+        .optional()
+        .describe(
+          "Optional position in the Available Fields list to start from. Use the offset given at the end of the previous response to see the next page of fields.",
+        ),
     }),
-    execute: async ({ dataset, substringMatch, query, attributeTypes }) => {
+    execute: async ({
+      dataset,
+      substringMatch,
+      query,
+      attributeTypes,
+      offset = 0,
+    }) => {
       const {
         BASE_COMMON_FIELDS,
         DATASET_FIELDS,
@@ -1261,10 +1276,33 @@ export function createDatasetAttributesTool(options: {
           },
         );
 
+      let commonFields: Record<string, string> = {};
+      let datasetFields: Record<string, string> = {};
+      if (substringMatch) {
+        Object.entries(BASE_COMMON_FIELDS).forEach(([key, value]) => {
+          if (key.includes(substringMatch) || value.includes(substringMatch)) {
+            commonFields[key] = value;
+          }
+        });
+        Object.entries(DATASET_FIELDS[normalizedDataset]).forEach(
+          ([key, value]) => {
+            if (
+              key.includes(substringMatch) ||
+              value.includes(substringMatch)
+            ) {
+              datasetFields[key] = value;
+            }
+          },
+        );
+      } else {
+        commonFields = BASE_COMMON_FIELDS;
+        datasetFields = DATASET_FIELDS[normalizedDataset];
+      }
+
       // Combine all available fields
       const allFields: Record<string, AttributeDescriptor> = {
-        ...toAttributeRecord(BASE_COMMON_FIELDS),
-        ...toAttributeRecord(DATASET_FIELDS[normalizedDataset]),
+        ...toAttributeRecord(commonFields),
+        ...toAttributeRecord(datasetFields),
         ...customAttributes,
       };
 
@@ -1273,7 +1311,6 @@ export function createDatasetAttributesTool(options: {
         replaced: replacedFields,
         updatedFieldTypes,
       } = resolveAvailableFields(allFields, fieldTypes);
-      const replacedKeys = new Set(replacedFields.map((field) => field.key));
       const fieldCount = availableFields.size;
 
       const recommendedFields = RECOMMENDED_FIELDS[normalizedDataset];
@@ -1287,20 +1324,48 @@ export function createDatasetAttributesTool(options: {
       for (const field of staticNumericFields) {
         allFieldTypes[field] = "number";
       }
-      const fieldTypeEntries = Object.entries(allFieldTypes).filter(
-        ([key]) => !replacedKeys.has(key),
-      );
 
       recordAgentToolResultCount(fieldCount);
 
+      // Page through fields to avoid overwhelming the agent
+      const pageFields = [...availableFields].slice(
+        offset,
+        offset + FIELDS_PAGE_SIZE,
+      );
+      const pageEnd = offset + pageFields.length;
+      const fieldsHeader =
+        pageFields.length > 0
+          ? `Available Fields (${fieldCount} total, showing ${offset + 1}-${pageEnd}):`
+          : `Available Fields (${fieldCount} total):${offset > 0 ? ` none at offset ${offset}.` : ""}`;
+      const fieldsList = pageFields
+        .map(
+          ([key, field]) =>
+            `- ${key}${allFieldTypes[key] ? ` (${allFieldTypes[key]})` : ""}: ${describeField(field)}`,
+        )
+        .join("\n");
+      const nextPageHint =
+        pageEnd < fieldCount
+          ? `\n... and ${fieldCount - pageEnd} more fields. To see them, call datasetAttributes again with the same arguments and offset: ${pageEnd}`
+          : "";
+
+      // Later pages only list their fields; the deprecated fields, recommended
+      // fields, and examples were already returned with the first page.
+      if (offset > 0) {
+        return `Dataset: ${dataset}
+
+${fieldsHeader}
+${fieldsList}
+${nextPageHint}
+
+IMPORTANT: Only use numeric aggregate functions (avg, sum, min, max, percentiles) with numeric fields. Use count() or count_unique() for non-numeric fields.
+`;
+      }
+
       return `Dataset: ${dataset}
 
-Available Fields (${fieldCount} total):
-${[...availableFields]
-  .slice(0, 50) // Limit to first 50 to avoid overwhelming the agent
-  .map(([key, field]) => `- ${key}: ${describeField(field)}`)
-  .join("\n")}
-${fieldCount > 50 ? `\n... and ${fieldCount - 50} more fields` : ""}
+${fieldsHeader}
+${fieldsList}
+${nextPageHint}
 ${
   replacedFields.length > 0
     ? `
@@ -1315,13 +1380,6 @@ ${replacedFields.length > 30 ? `\n... and ${replacedFields.length - 30} more dep
 Recommended Fields for ${dataset}:
 ${recommendedFields.basic.map((f) => `- ${f}`).join("\n")}
 
-Field Types (CRITICAL for aggregate functions):
-${fieldTypeEntries
-  .slice(0, 30) // Show more field types since this is critical for aggregate functions
-  .map(([key, type]) => `- ${key}: ${type}`)
-  .join("\n")}
-${fieldTypeEntries.length > 30 ? `\n... and ${fieldTypeEntries.length - 30} more fields` : ""}
-
 IMPORTANT: Only use numeric aggregate functions (avg, sum, min, max, percentiles) with numeric fields. Use count() or count_unique() for non-numeric fields.
 
 EXAMPLE QUERIES FOR ${dataset.toUpperCase()}:

packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts

@@ -107,6 +107,29 @@ describeEval("search-events-agent-attributes-without-context", {
             !value.includes("sentry.environment"),
         },
       },
+      {
+        // kiosk.firmware and kiosk.firmware_revision are made-up attributes, so
+        // the agent has no prior knowledge of either. Without briefs it can
+        // only match on the name, and kiosk.firmware_revision matches
+        // "firmware revision"
+        input:
+          "Count spans grouped by the kiosk's installed firmware revision over the last 7 days",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "spans",
+            },
+          },
+        ],
+        expected: {
+          dataset: "spans",
+          fields: (value: unknown) =>
+            Array.isArray(value) &&
+            value.includes("kiosk.firmware_revision") &&
+            !value.includes("kiosk.firmware"),
+        },
+      },
     ];
   },
   task: async (input) => {
@@ -225,6 +248,52 @@ describeEval("search-events-agent-attributes-with-context", {
             !/(?<!sentry\.)environment:/.test(value),
         },
       },
+      {
+        // deviceMemory and device.memory both look like a match by name, but
+        // only device.memory has a brief describing the device's total memory
+        input:
+          "What is the average estimated total memory capacity of devices across spans in the last 7 days?",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "spans",
+            },
+          },
+        ],
+        expected: {
+          dataset: "spans",
+          fields: (value: unknown) =>
+            Array.isArray(value) &&
+            value.includes("avg(device.memory)") &&
+            !value.some(
+              (field) =>
+                typeof field === "string" && field.includes("deviceMemory"),
+            ),
+        },
+      },
+      {
+        // kiosk.firmware_revision matches "firmware revision" by name, but its
+        // brief says it is the staged revision. Only kiosk.firmware's brief
+        // describes the installed firmware
+        input:
+          "Count spans grouped by the kiosk's installed firmware revision over the last 7 days",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "spans",
+            },
+          },
+        ],
+        expected: {
+          dataset: "spans",
+          fields: (value: unknown) =>
+            Array.isArray(value) &&
+            value.includes("kiosk.firmware") &&
+            !value.includes("kiosk.firmware_revision"),
+        },
+      },
     ];
   },
   task: async (input) => {

packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-number-with-context.json

@@ -43,5 +43,19 @@
     "key": "custom.db.pool_size",
     "name": "custom.db.pool_size",
     "context": {}
+  },
+  {
+    "key": "deviceMemory",
+    "name": "deviceMemory",
+    "context": {}
+  },
+  {
+    "key": "device.memory",
+    "name": "device.memory",
+    "context": {
+      "isConvention": true,
+      "brief": "The estimated total memory capacity of the device.",
+      "isDeprecated": false
+    }
   }
 ]

packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json

@@ -255,5 +255,29 @@
     "key": "custom.feature_flag",
     "name": "custom.feature_flag",
     "context": {}
+  },
+  {
+    "key": "kiosk.firmware",
+    "name": "kiosk.firmware",
+    "context": {
+      "isConvention": true,
+      "brief": "The firmware revision currently installed and running on the kiosk.",
+      "isDeprecated": false,
+      "examples": [
+        "4.1.3"
+      ]
+    }
+  },
+  {
+    "key": "kiosk.firmware_revision",
+    "name": "kiosk.firmware_revision",
+    "context": {
+      "isConvention": true,
+      "brief": "The firmware revision downloaded and staged for the kiosk's next update. It is not installed until the kiosk restarts.",
+      "isDeprecated": false,
+      "examples": [
+        "4.2.0"
+      ]
+    }
   }
 ]