diff --git a/content/docs/get-started/agent-frameworks/crewai/use-arcade-tools.mdx b/content/docs/get-started/agent-frameworks/crewai/use-arcade-tools.mdx index d79ec0d05..9106bc1ba 100644 --- a/content/docs/get-started/agent-frameworks/crewai/use-arcade-tools.mdx +++ b/content/docs/get-started/agent-frameworks/crewai/use-arcade-tools.mdx @@ -356,7 +356,7 @@ You should see the agent responding to your prompts like any model, as well as h Now that you have integrated Arcade tools into your CrewAI agent team, you can: -- Experiment with different toolkits, such as "Math" or "Search." +- Add other MCP servers to `MCP_SERVERS`, such as `"GoogleCalendar"` or `"Linear"`. - Customize the agent's prompts for specific tasks. - Customize the tool authorization and execution flow to meet your application's requirements. diff --git a/lib/integration-catalog.ts b/lib/integration-catalog.ts index f133d2160..233f8698d 100644 --- a/lib/integration-catalog.ts +++ b/lib/integration-catalog.ts @@ -1,65 +1,46 @@ -import type { Toolkit } from "@arcadeai/design-system"; -import { TOOLKITS } from "@arcadeai/design-system/metadata/toolkits"; +import type { ToolkitData } from "@/components/toolkit-docs/types"; import { PARTNER_TOOLKITS } from "@/data/partner-toolkits"; -import { normalizeToolkitId } from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; -import { readToolkitData } from "./toolkit-data"; +import { resolveToolkitDataDir } from "@/toolkit-docs-generator/src/shared/toolkit-data-dir"; +import { loadAllToolkitData } from "./toolkit-data"; import type { ToolkitWithDocsLink } from "./toolkit-slug"; -const getToolkitDocsLink = (toolkit: Toolkit): string | undefined => { - if ("docsLink" in toolkit) { - const value = (toolkit as ToolkitWithDocsLink).docsLink; - return value ?? undefined; - } - return; -}; +const toCatalogToolkit = ({ + id, + label, + metadata, +}: ToolkitData): ToolkitWithDocsLink => ({ + id, + label, + isBYOC: metadata.isBYOC, + isPro: metadata.isPro, + publicIconUrl: metadata.iconUrl, + docsLink: metadata.docsLink, + relativeDocsLink: new URL(metadata.docsLink).pathname, + type: metadata.type, + category: metadata.category, + isComingSoon: metadata.isComingSoon, + isHidden: metadata.isHidden, +}); /** - * Toolkits that have launched but are still flagged `isComingSoon` in the - * design-system catalog. Remove an entry once DS ships the flag as `false`. + * The integrations index: every toolkit the public catalog publishes, plus + * docs-local partner toolkits. + * + * `generated` is the nightly output of toolkit-docs-generator, which reads the + * same public catalog arcade.dev/tools is built from, so both list the same + * toolkits with the same badges. */ -const LAUNCHED_TOOLKIT_IDS = new Set(["workday"]); - -const clearLaunchedComingSoon = (toolkit: Toolkit): Toolkit => - LAUNCHED_TOOLKIT_IDS.has(normalizeToolkitId(toolkit.id)) - ? { ...toolkit, isComingSoon: false } - : toolkit; +export const toIntegrationCatalog = ( + generated: Iterable, + partners: readonly ToolkitWithDocsLink[] +): ToolkitWithDocsLink[] => [ + ...[...generated].map(toCatalogToolkit), + ...partners, +]; -/** - * The full integrations catalog the index renders: design-system toolkits - * (enriched with a `docsLink` from their data file when the catalog entry - * lacks one, so the card's slug matches the generated page) plus docs-local - * partner toolkits. - */ export const getToolkitsWithDocsLinks = async (): Promise< ToolkitWithDocsLink[] > => { - const docsLinkById = new Map(); - - await Promise.all( - TOOLKITS.map(async (toolkit) => { - const existing = getToolkitDocsLink(toolkit); - if (existing) { - return; - } - - const data = await readToolkitData(toolkit.id); - if (data?.metadata?.docsLink) { - docsLinkById.set( - normalizeToolkitId(toolkit.id), - data.metadata.docsLink - ); - } - }) - ); - - const dsToolkits: ToolkitWithDocsLink[] = TOOLKITS.map((dsToolkit) => { - const toolkit = clearLaunchedComingSoon(dsToolkit); - const existing = getToolkitDocsLink(toolkit); - const docsLink = - existing ?? docsLinkById.get(normalizeToolkitId(toolkit.id)); - - return docsLink ? { ...toolkit, docsLink } : toolkit; - }); - - return [...dsToolkits, ...PARTNER_TOOLKITS]; + const { byNormalizedId } = await loadAllToolkitData(resolveToolkitDataDir()); + return toIntegrationCatalog(byNormalizedId.values(), PARTNER_TOOLKITS); }; diff --git a/tests/integration-catalog.test.ts b/tests/integration-catalog.test.ts new file mode 100644 index 000000000..a5ffd35ef --- /dev/null +++ b/tests/integration-catalog.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, test } from "vitest"; +import type { ToolkitData } from "@/components/toolkit-docs/types"; +import type { PartnerToolkit } from "@/data/partner-toolkits"; +import { toIntegrationCatalog } from "@/lib/integration-catalog"; + +const INTEGRATIONS = "https://docs.arcade.dev/en/resources/integrations"; + +// Minimal generated toolkit file. toIntegrationCatalog only reads `id`, +// `label`, and `metadata`. +const makeGenerated = ( + id: string, + metadata: Partial = {} +): ToolkitData => + ({ + id, + label: `${id} Label`, + metadata: { + category: "social", + iconUrl: `https://design-system.arcade.dev/icons/${id.toLowerCase()}.svg`, + isBYOC: false, + isPro: false, + type: "arcade", + docsLink: `${INTEGRATIONS}/social/${id.toLowerCase()}`, + isComingSoon: false, + isHidden: false, + ...metadata, + }, + }) as unknown as ToolkitData; + +const partner = { + id: "Partner", + label: "Partner", + category: "search", + publicIconUrl: "/images/partners/partner.svg", + isBYOC: true, + isPro: false, + isPartner: true, + type: "verified", + mcpUrl: "https://mcp.example.com/mcp", + docsLink: `${INTEGRATIONS}/search/partner`, + relativeDocsLink: "/en/resources/integrations/search/partner", + isComingSoon: false, + isHidden: false, +} satisfies PartnerToolkit; + +describe("toIntegrationCatalog", () => { + test("lists a published toolkit with the card fields from its generated file", () => { + const [toolkit] = toIntegrationCatalog( + [makeGenerated("Twilio", { isBYOC: true, type: "verified" })], + [] + ); + + expect(toolkit).toEqual({ + id: "Twilio", + label: "Twilio Label", + isBYOC: true, + isPro: false, + publicIconUrl: "https://design-system.arcade.dev/icons/twilio.svg", + docsLink: `${INTEGRATIONS}/social/twilio`, + relativeDocsLink: "/en/resources/integrations/social/twilio", + type: "verified", + category: "social", + isComingSoon: false, + isHidden: false, + }); + }); + + test("lists only toolkits that have a generated file", () => { + const ids = toIntegrationCatalog( + [makeGenerated("Slack"), makeGenerated("SlackApi")], + [] + ).map((toolkit) => toolkit.id); + + expect(ids).toEqual(["Slack", "SlackApi"]); + }); + + test("appends partner toolkits after the published ones", () => { + const ids = toIntegrationCatalog([makeGenerated("Slack")], [partner]).map( + (toolkit) => toolkit.id + ); + + expect(ids).toEqual(["Slack", "Partner"]); + }); +}); diff --git a/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts b/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts index 079396069..0f93d0a23 100644 --- a/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts +++ b/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts @@ -31,7 +31,6 @@ import { } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { TOOLKITS as DESIGN_SYSTEM_TOOLKITS } from "@arcadeai/design-system/metadata/toolkits"; import { PARTNER_TOOLKITS, type PartnerToolkit, @@ -50,20 +49,6 @@ import type { const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); -// Import design system toolkits -type DesignSystemToolkit = { - id: string; - label: string; - category?: string; - isHidden?: boolean; -}; -let TOOLKITS: DesignSystemToolkit[] = - DESIGN_SYSTEM_TOOLKITS as DesignSystemToolkit[]; - -export function setToolkitsForTesting(toolkits: DesignSystemToolkit[]): void { - TOOLKITS = toolkits; -} - // Get project root (two levels up from toolkit-docs-generator/scripts) const PROJECT_ROOT = resolve(__dirname, "..", ".."); @@ -187,33 +172,9 @@ export function getToolkitFiles(dataDir: string): string[] { } /** - * Get category for a toolkit ID from design system - */ -export function getToolkitCategory(toolkitId: string): string | null { - const toolkit = TOOLKITS.find( - (t) => t.id.toLowerCase() === toolkitId.toLowerCase() - ); - - if (toolkit && !toolkit.isHidden) { - return toolkit.category || null; - } - - return null; -} - -/** - * Get label for a toolkit ID from design system + * A readable label from a toolkit ID, for a toolkit file with no label. */ export function getToolkitLabel(toolkitId: string): string { - const toolkit = TOOLKITS.find( - (t) => t.id.toLowerCase() === toolkitId.toLowerCase() - ); - - if (toolkit) { - return toolkit.label; - } - - // Fallback: Convert ID to readable label return toolkitId .replace(CAPITAL_LETTER_REGEX, " $1") .replace(FIRST_CHARACTER_REGEX, (str) => str.toUpperCase()) @@ -269,35 +230,21 @@ export function inferNavGroup( } /** - * Build toolkit info from available JSON files + * Build a sidebar entry from a toolkit's generated JSON file, or `null` when + * the toolkit is hidden or has no category. */ -type ToolkitInfoEntry = { - info: ToolkitInfo; - priority: number; -}; - -function resolveToolkitInfo( - dataDir: string, - slug: string -): ToolkitInfoEntry | null { +function resolveToolkitInfo(dataDir: string, slug: string): ToolkitInfo | null { const jsonData = readToolkitJson(dataDir, slug); const toolkitId = jsonData?.id ?? slug; const docsSlug = getToolkitSlug({ docsLink: jsonData?.metadata?.docsLink ?? null, id: toolkitId, }); - const designSystemToolkit = TOOLKITS.find( - (t) => t.id.toLowerCase() === toolkitId.toLowerCase() - ); - const isHidden = jsonData?.metadata?.isHidden ?? false; - if (designSystemToolkit?.isHidden || isHidden) { + if (jsonData?.metadata?.isHidden) { return null; } - // Keep sidebar routes aligned with static params: toolkit JSON is source of - // truth for category, with design system as fallback when JSON is missing. - const category = - jsonData?.metadata?.category ?? designSystemToolkit?.category; + const category = jsonData?.metadata?.category; if (!category) { return null; } @@ -306,41 +253,30 @@ function resolveToolkitInfo( `Unrecognized integration category "${category}" for toolkit "${toolkitId}".` ); } - const labelFromDesignSystem = designSystemToolkit?.label ?? null; const labelFromJson = jsonData?.label ?? jsonData?.name ?? null; const typeFromJson = jsonData?.metadata?.type ?? null; - const info: ToolkitInfo = { + return { category, id: toolkitId, - // Prefer the Design System label (has correct spacing/casing), but fall back - // to JSON label for toolkits not present in the design system. - label: labelFromDesignSystem ?? labelFromJson ?? getToolkitLabel(toolkitId), + label: labelFromJson ?? getToolkitLabel(toolkitId), navGroup: inferNavGroup(toolkitId, typeFromJson), slug: docsSlug, }; - - const priority = designSystemToolkit ? 2 : 0; - return { info, priority }; } export function buildToolkitInfoList(dataDir: string): ToolkitInfo[] { const files = getToolkitFiles(dataDir); - const toolkitsBySlug = new Map(); + const toolkitsBySlug = new Map(); for (const slug of files) { - const entry = resolveToolkitInfo(dataDir, slug); - if (!entry) { - continue; - } - - const existing = toolkitsBySlug.get(entry.info.slug); - if (!existing || entry.priority > existing.priority) { - toolkitsBySlug.set(entry.info.slug, entry); + const info = resolveToolkitInfo(dataDir, slug); + if (info && !toolkitsBySlug.has(info.slug)) { + toolkitsBySlug.set(info.slug, info); } } - return Array.from(toolkitsBySlug.values()).map((entry) => entry.info); + return Array.from(toolkitsBySlug.values()); } /** diff --git a/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts b/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts index e28bc3847..de273632f 100644 --- a/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts +++ b/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts @@ -16,7 +16,6 @@ import { type CategoryMeta, findMissingMainMetaCategories, generateCategoryMeta, - getToolkitCategory, getToolkitFiles, getToolkitLabel, getToolkitLabelFromJson, @@ -25,31 +24,11 @@ import { type PartnerNavSource, parseBooleanCliFlag, resolveRemoveEmptySections, - setToolkitsForTesting, syncToolkitSidebar, type ToolkitInfo, } from "../../scripts/sync-toolkit-sidebar"; import { INTEGRATION_CATEGORIES } from "../../src/shared/toolkit-primitives"; -setToolkitsForTesting([ - { id: "Gmail", label: "Gmail", category: "productivity" }, - { id: "Slack", label: "Slack", category: "social" }, - { id: "Github", label: "GitHub", category: "development" }, - { id: "Stripe", label: "Stripe", category: "payments" }, - { id: "Zendesk", label: "Zendesk", category: "customer-support" }, - { id: "GoogleSearch", label: "Google Search", category: "search" }, - { id: "Hubspot", label: "HubSpot", category: "sales" }, - { id: "Spotify", label: "Spotify", category: "entertainment" }, - { id: "Postgres", label: "Postgres", category: "databases" }, - { id: "WeaviateApi", label: "Weaviate API", category: "development" }, - { - id: "HiddenToolkit", - label: "Hidden", - category: "productivity", - isHidden: true, - }, -]); - // Test directory setup const TEST_DIR = join(process.cwd(), ".test-sync-sidebar"); const TEST_DATA_DIR = join(TEST_DIR, "data/toolkits"); @@ -114,41 +93,11 @@ describe("getToolkitFiles", () => { }); }); -// ============================================================================ -// Unit Tests: getToolkitCategory -// ============================================================================ - -describe("getToolkitCategory", () => { - it("should return category for known toolkit", () => { - expect(getToolkitCategory("Gmail")).toBe("productivity"); - expect(getToolkitCategory("Slack")).toBe("social"); - }); - - it("should return null for unknown toolkit", () => { - expect(getToolkitCategory("Unknown")).toBeNull(); - }); - - it("should return null for hidden toolkit", () => { - expect(getToolkitCategory("HiddenToolkit")).toBeNull(); - }); -}); - // ============================================================================ // Unit Tests: getToolkitLabel // ============================================================================ describe("getToolkitLabel", () => { - it("should return label for known toolkit", () => { - expect(getToolkitLabel("Gmail")).toBe("Gmail"); - expect(getToolkitLabel("Github")).toBe("GitHub"); - expect(getToolkitLabel("GoogleSearch")).toBe("Google Search"); - }); - - it("should return label for known toolkit regardless of case", () => { - expect(getToolkitLabel("gmail")).toBe("Gmail"); - expect(getToolkitLabel("GITHUB")).toBe("GitHub"); - }); - it("should generate fallback label for unknown toolkit", () => { expect(getToolkitLabel("MyCustomToolkit")).toBe("My Custom Toolkit"); expect(getToolkitLabel("APIHelper")).toBe("A P I Helper"); @@ -234,8 +183,14 @@ describe("buildToolkitInfoList", () => { }); it("should build list of toolkits with correct labels and categories", () => { - createToolkitJson("gmail", { label: "Gmail" }); - createToolkitJson("slack", { label: "Slack" }); + createToolkitJson("gmail", { + label: "Gmail", + metadata: { category: "productivity" }, + }); + createToolkitJson("slack", { + label: "Slack", + metadata: { category: "social" }, + }); createToolkitJson("unknowntoolkit", { label: "Unknown Toolkit" }); const result = buildToolkitInfoList(TEST_DATA_DIR); @@ -258,27 +213,42 @@ describe("buildToolkitInfoList", () => { }); it("should skip hidden toolkits", () => { - createToolkitJson("HiddenToolkit", { label: "Hidden" }); + createToolkitJson("hiddentoolkit", { + id: "HiddenToolkit", + label: "Hidden", + metadata: { category: "productivity", isHidden: true }, + }); + createToolkitJson("gmail", { + label: "Gmail", + metadata: { category: "productivity" }, + }); const result = buildToolkitInfoList(TEST_DATA_DIR); - expect(result).toHaveLength(0); + expect(result.map((item) => item.label)).toEqual(["Gmail"]); }); - it("should prefer design system label over JSON", () => { - createToolkitJson("gmail", { label: "Custom Gmail Label" }); + it("uses the label from the toolkit file", () => { + createToolkitJson("gmail", { + label: "Custom Gmail Label", + metadata: { category: "productivity" }, + }); const result = buildToolkitInfoList(TEST_DATA_DIR); - expect(result[0].label).toBe("Gmail"); + expect(result[0].label).toBe("Custom Gmail Label"); }); - it("should use design system label as fallback", () => { - createToolkitJson("gmail", { label: null }); + it("falls back to a readable name when the file has no label", () => { + createToolkitJson("googlesearch", { + id: "GoogleSearch", + label: null, + metadata: { category: "search" }, + }); const result = buildToolkitInfoList(TEST_DATA_DIR); - expect(result[0].label).toBe("Gmail"); + expect(result[0].label).toBe("Google Search"); }); it("should use docsLink slug when available", () => {