Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 95 additions & 0 deletions src/agent/lexical-rank.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,3 +99,98 @@ describe("scoreLexical", () => {
}
});
});

describe("scoreLexical tool names across a trailing plural s", () => {
// No fixture name contains this raw query whole, so only the per-token
// tiers differ between the scores compared.
const rawQuery = "linear issue";

function nameScore(name: string, query: string): number {
return scoreLexical(
lexicalFields(name, "", NO_LEXICAL_PARAMS),
tokenizeLexical(query),
rawQuery,
);
}

// `substring` holds the query word only inside a longer name token.
function expectFoldedBetween(
query: string,
names: { substring: string; folded: string; exact: string },
) {
const folded = nameScore(names.folded, query);
expect(folded).toBeGreaterThan(nameScore(names.substring, query));
expect(folded).toBeLessThan(nameScore(names.exact, query));
}

test("a singular query word matches its plural in a name", () => {
expectFoldedBetween("issue", {
substring: "mcp__acme__issuetracker",
folded: "mcp__linear__list_issues",
exact: "mcp__linear__list_issue",
});
});

test("a plural query word matches its singular in a name", () => {
expectFoldedBetween("issues", {
substring: "mcp__acme__issuestracker",
folded: "mcp__linear__get_issue",
exact: "mcp__linear__get_issues",
});
});

test("a word given in both forms counts once, as its better spelling", () => {
const fields = lexicalFields(
"mcp__linear__list_issues",
"List issues in Linear",
NO_LEXICAL_PARAMS,
);
expect(
scoreLexical(fields, tokenizeLexical("linear issue issues"), rawQuery),
).toBe(scoreLexical(fields, tokenizeLexical("linear issues"), rawQuery));
});

test("a word given in both forms scores the same in either order", () => {
// "issue" sits only inside "issuetracker", so only a substring tier
// matches, and it matches the singular but not the plural.
const fields = lexicalFields(
"mcp__acme__issuetracker",
"",
NO_LEXICAL_PARAMS,
);
const pluralFirst = scoreLexical(
fields,
tokenizeLexical("issues issue"),
rawQuery,
);
expect(pluralFirst).toBeGreaterThan(0);
expect(pluralFirst).toBe(
scoreLexical(fields, tokenizeLexical("issue issues"), rawQuery),
);
});

test("a word of three letters keeps its trailing s", () => {
const fields = lexicalFields("mcp__acme__get_it", "", NO_LEXICAL_PARAMS);
expect(scoreLexical(fields, tokenizeLexical("its"), "its")).toBe(0);
});

test("a plural query word earns no substring credit for its singular", () => {
// "users" folds to "user", which sits inside "username"; the substring
// tier still looks for "users".
const fields = lexicalFields(
"mcp__acme__username_lookup",
"",
NO_LEXICAL_PARAMS,
);
expect(scoreLexical(fields, tokenizeLexical("users"), "users")).toBe(0);
});

test("a description word does not match a query word that folds to it", () => {
const fields = lexicalFields(
"mcp__linear__save_issue",
"Create a new issue",
NO_LEXICAL_PARAMS,
);
expect(scoreLexical(fields, tokenizeLexical("news"), "news")).toBe(0);
});
});
52 changes: 47 additions & 5 deletions src/agent/lexical-rank.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
// One shared dependency-free lexical ranker behind tool_search,
// search_agents, and skill_search. Exact name-token hits weigh most, then
// name tokens that match across a trailing plural "s" (so "issue" finds
// list_issues while the spelling the query uses still wins ties), then
// blob-token hits, then a raw name substring (so "linear" finds
// mcp__linear__* even though it is not a whole token there), then a raw blob
// substring. Parameter name and description tokens (tool_search only) weigh
Expand All @@ -11,10 +13,21 @@ export function tokenizeLexical(text: string): string[] {
return text.toLowerCase().match(/[a-z0-9]+/g) ?? [];
}

// Folding also merges real word pairs such as "news" and "new", so it stays
// on names: in descriptions, a common word like "new" would turn a miss into
// a false hit. Three-letter words keep their "s", so "its" never becomes "it".
function foldPlural(token: string): string {
return token.length > 3 && token.endsWith("s") ? token.slice(0, -1) : token;
}

// Under exact (3); 3 + this clears TOOL_SEARCH_SCORE_RATIO * 6 ("issue list").
const FOLDED_NAME_TOKEN_SCORE = 2.5;

export interface LexicalFields {
readonly name: string;
readonly blob: string;
readonly nameTokens: readonly string[];
readonly foldedNameTokens: readonly string[];
readonly blobTokens: ReadonlySet<string>;
readonly paramNameTokens: ReadonlySet<string>;
readonly paramBlobTokens: ReadonlySet<string>;
Expand All @@ -40,10 +53,12 @@ export function lexicalFields(
blob: string,
params: LexicalParams,
): LexicalFields {
const nameTokens = tokenizeLexical(name);
return {
name,
blob,
nameTokens: tokenizeLexical(name),
nameTokens,
foldedNameTokens: nameTokens.map(foldPlural),
blobTokens: new Set(tokenizeLexical(blob)),
paramNameTokens: new Set(
tokenizeLexical(params.names.map(splitCamelCase).join(" ")),
Expand All @@ -59,6 +74,9 @@ function directTokenScore(
token: string,
): number {
if (fields.nameTokens.includes(token)) return 3;
if (fields.foldedNameTokens.includes(foldPlural(token))) {
return FOLDED_NAME_TOKEN_SCORE;
}
if (fields.blobTokens.has(token)) return 1;
if (name.includes(token)) return 0.75;
if (blob.includes(token)) return 0.25;
Expand All @@ -78,15 +96,39 @@ export function scoreLexical(
): number {
const name = fields.name.toLowerCase();
const blob = fields.blob.toLowerCase();
let matched = name.includes(rawQuery) ? 1 : 0;
// The whole-query bonus lifts a one-word query buried inside a longer name
// token ("granola" in mygranolahoard) above a description hit. A word that
// is already a name token, exact or folded, was scored by the token pass;
// a second point for it would cut its folded matches at the score ratio.
const single = new Set(queryTokens).size === 1 ? queryTokens[0] : undefined;
const nameHit =
single !== undefined &&
(fields.nameTokens.includes(single) ||
fields.foldedNameTokens.includes(foldPlural(single)));
let matched = !nameHit && name.includes(rawQuery) ? 1 : 0;
let total = matched;
// A model repeats a word because its sentence needs it ("release ID or
// release slug"), not to weight it; counting each copy multiplies the
// bonus for every tool that matches that one word.
// bonus for every tool that matches that one word. "issue" and "issues"
// are one word, scored as whichever spelling matches better. The merge
// applies on every tier, so an unrelated pair like "new" and "news" also
// counts once against a description; that is accepted.
const words = new Map<string, string[]>();
for (const token of new Set(queryTokens)) {
const direct = directTokenScore(fields, name, blob, token);
const folded = foldPlural(token);
const spellings = words.get(folded);
if (spellings === undefined) words.set(folded, [token]);
else spellings.push(token);
}
for (const spellings of words.values()) {
const direct = Math.max(
...spellings.map((token) => directTokenScore(fields, name, blob, token)),
);
const param = Math.max(
...spellings.map((token) => paramTokenScore(fields, token)),
);
matched += direct;
total += Math.max(direct, paramTokenScore(fields, token));
total += Math.max(direct, param);
}
return matched > 0 ? total : 0;
}
Expand Down
37 changes: 37 additions & 0 deletions src/agent/tool-search-linear-replay.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { describe, expect, test } from "bun:test";

import { liveLinearCatalog } from "../../testkit/linear-mcp-catalog.js";
import { createToolIndex } from "./tool-search.js";

const index = createToolIndex(() => liveLinearCatalog);

// Queries models sent while looking for a release's issues, at the limit
// each one used; every one should shortlist list_issues.
describe("tool_search on logged Linear issue queries", () => {
test.each([
[
"Get a Linear release by URL or identifier and list its issues including titles, status, and descriptions",
10,
],
["Linear retrieve pipeline release issue list and issue details", 5],
["Linear list issues filtered by release ID or release slug", 5],
["issue list", 5],
] as const)("%p (limit %p) shortlists list_issues", (query, limit) => {
expect(index.search(query, limit)).toContain("mcp__linear__list_issues");
});

// Shortlisted only while "issue" and "issues" counted as two words; the
// query splits between release and issue tools, and release tools match more
// of its words.
test.failing(
'"Linear read pipeline release issues and issue details" (limit 5) shortlists list_issues',
() => {
expect(
index.search(
"Linear read pipeline release issues and issue details",
5,
),
).toContain("mcp__linear__list_issues");
},
);
});
74 changes: 74 additions & 0 deletions src/agent/tool-search.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,80 @@ describe("createToolIndex", () => {
expect(teamIndex.search("linear team")[0]).toBe("mcp__linear__list_teams");
});

// Trimmed from the live Linear MCP catalog and sorted by name, so five
// tools that carry the singular "issue" come ahead of list_issues.
const linearIssueCatalog: ToolDefinition[] = [
mcpDef(
"mcp__linear__create_issue_label",
"[linear] Create a new Linear issue label. Deprecated: use `save_issue_label`, which can also update labels.",
),
mcpDef(
"mcp__linear__get_issue",
"[linear] Retrieve detailed information about an issue by ID, including attachments, git branch name, and active Triage Intelligence suggestions when the issue is in triage.",
),
mcpDef(
"mcp__linear__get_issue_status",
"[linear] Retrieve detailed information about an issue status in Linear by name or ID",
),
mcpDef(
"mcp__linear__list_comments",
"[linear] List comments on a Linear issue, project, initiative, document, project milestone, or project/initiative status update.",
),
mcpDef(
"mcp__linear__list_issue_labels",
"[linear] List available issue labels in a Linear workspace or team. A retired label carries `retiredAt` and cannot be applied to new issues.",
),
mcpDef(
"mcp__linear__list_issue_statuses",
"[linear] List available issue statuses in a Linear team",
),
mcpDef(
"mcp__linear__list_issues",
"[linear] List issues in the user's Linear workspace, including active Triage Intelligence suggestions for issues in triage.",
),
mcpDef(
"mcp__linear__list_projects",
"[linear] List projects in the user's Linear workspace",
),
mcpDef(
"mcp__linear__list_teams",
"[linear] List teams in the user's Linear workspace, including their parent IDs and hierarchy names",
),
mcpDef(
"mcp__linear__save_issue",
"[linear] Create or update a Linear issue. If `id` is provided, updates the existing issue; otherwise creates a new one.",
),
];
const linearIssueIndexes = [
createToolIndex(() => linearIssueCatalog),
createToolIndex(() => [...linearIssueCatalog].reverse()),
];

test("a singular query word finds the tool named for its plural", () => {
for (const index of linearIssueIndexes) {
expect(index.search("issue list")).toContain("mcp__linear__list_issues");
}
});

test("a plural query word ranks the tool named for it first", () => {
for (const index of linearIssueIndexes) {
expect(index.search("list issues")[0]).toBe("mcp__linear__list_issues");
expect(index.search("linear issues")[0]).toBe("mcp__linear__list_issues");
}
});

test("a one-word plural query keeps the tools named for its singular", () => {
const nameOnlyIndex = createToolIndex(() => [
mcpDef("mcp__linear__list_issues", ""),
mcpDef("mcp__linear__get_issue", ""),
mcpDef("mcp__linear__save_issue", ""),
]);
const ranked = nameOnlyIndex.search("issues", 20);
expect(ranked[0]).toBe("mcp__linear__list_issues");
expect(ranked).toContain("mcp__linear__get_issue");
expect(ranked).toContain("mcp__linear__save_issue");
});

test("a malformed input schema still scores on name and description", () => {
const malformedIndex = createToolIndex(() => [
{
Expand Down
Loading
Loading