diff --git a/api/src/lib/oss-index.ts b/api/src/lib/oss-index.ts index de95050b5..b4afd8091 100644 --- a/api/src/lib/oss-index.ts +++ b/api/src/lib/oss-index.ts @@ -155,3 +155,65 @@ export const sortQuery = Type.Object({ ), ), }); + +// The collections pipe inserts the project slug before status, so its tuples cannot reuse the +// positional checks of the 9 slot shape. +export type CollectionProjectTuple = [ + id: string, + count: number, + name: string, + logo: string, + softwareValue: number, + avgScore: number, + healthScore: number, + description: string, + slug: string, + status: string, +]; + +export const isCollectionProjectTuple = (value: unknown): value is CollectionProjectTuple => + Array.isArray(value) && + value.length === 10 && + isString(value[0]) && + isCount(value[1]) && + isString(value[2]) && + isString(value[3]) && + isCount(value[4]) && + isNumber(value[5]) && + isNumber(value[6]) && + isString(value[7]) && + isString(value[8]) && + isString(value[9]); + +export const CollectionProject = Type.Object( + { + ...TopProject.properties, + slug: Type.String({ description: 'URL slug of the project.' }), + }, + { title: 'CollectionProject' }, +); +export type CollectionProject = Static; + +export const toCollectionProject = ([ + id, + count, + name, + logo, + softwareValue, + avgScore, + healthScore, + description, + slug, + status, +]: CollectionProjectTuple): CollectionProject => ({ + id, + count, + name, + slug, + logoUrl: logo, + description, + softwareValue, + avgScore, + healthScore, + status, +}); diff --git a/api/src/versions/v1-alpha/oss-index/collections.ts b/api/src/versions/v1-alpha/oss-index/collections.ts new file mode 100644 index 000000000..fe3628cf5 --- /dev/null +++ b/api/src/versions/v1-alpha/oss-index/collections.ts @@ -0,0 +1,192 @@ +// Copyright (c) 2025 The Linux Foundation and each contributor. +// SPDX-License-Identifier: MIT +import type { FastifyPluginAsyncTypebox } from '@fastify/type-provider-typebox'; +import { Type } from '@sinclair/typebox'; + +import { fetchPipe } from '../../../clients/tinybird.js'; +import { NotFoundError } from '../../../lib/errors.js'; +import { + CollectionProject, + isCollectionProjectTuple, + sortQuery, + toCollectionProject, + type CollectionProjectTuple, +} from '../../../lib/oss-index.js'; +import { pipePages, requestedPage, toCountedPage } from '../../../lib/pagination.js'; +import { isCount, isString } from '../../../lib/security.js'; +import { PaginationQuery, paginated } from '../../../schemas/common.js'; + +const pipePath = '/v0/pipes/collections_oss_index.json'; +const categoryPipePath = '/v0/pipes/category_list.json'; + +interface Row { + id: string; + name: string; + slug: string; + totalContributors: number; + softwareValue: number; + avgScore: number; + projectCount: number; + topProjects: CollectionProjectTuple[]; +} + +interface CategoryRow { + id: string; + name: string; + slug: string; + categoryGroupId: string; + categoryGroupName: string; + categoryGroupSlug: string; + categoryGroupType: string; +} + +const isNumber = (value: unknown) => typeof value === 'number' && Number.isFinite(value); + +const isRow = (row: Row) => + isString(row.id) && + isString(row.name) && + isString(row.slug) && + isCount(row.totalContributors) && + isCount(row.softwareValue) && + isNumber(row.avgScore) && + isCount(row.projectCount) && + Array.isArray(row.topProjects) && + row.topProjects.every(isCollectionProjectTuple); + +const isCategoryRow = (row: CategoryRow) => + isString(row.id) && + isString(row.name) && + isString(row.slug) && + isString(row.categoryGroupId) && + isString(row.categoryGroupName) && + isString(row.categoryGroupSlug) && + isString(row.categoryGroupType); + +const Query = Type.Object({ + categorySlug: Type.Optional( + Type.String({ + description: + 'Keeps the collections of this category. An unknown slug answers 404, and the response then includes the category.', + }), + ), + categoryGroupId: Type.Optional( + Type.String({ description: 'Keeps the collections of this category group.' }), + ), + ...sortQuery.properties, + ...PaginationQuery.properties, +}); + +const OssIndexCollection = Type.Object( + { + id: Type.String({ description: 'Identifier of the collection.' }), + name: Type.String({ description: 'Display name of the collection.' }), + slug: Type.String({ description: 'URL slug of the collection.' }), + totalContributors: Type.Integer({ + minimum: 0, + description: 'Contributors across the collection.', + }), + softwareValue: Type.Integer({ minimum: 0, description: 'Software value of the collection.' }), + avgScore: Type.Number({ + description: 'Average health score of the collection, from 0 to 1.', + }), + projectCount: Type.Integer({ minimum: 0, description: 'Projects in the collection.' }), + topProjects: Type.Array(CollectionProject, { + description: 'Leading projects of the collection, ordered by contributors.', + }), + }, + { title: 'OssIndexCollection' }, +); + +const OssIndexCollectionCategory = Type.Object( + { + id: Type.String({ description: 'Identifier of the category.' }), + name: Type.String({ description: 'Display name of the category.' }), + slug: Type.String({ description: 'URL slug of the category.' }), + categoryGroupId: Type.String({ description: 'Identifier of the category group.' }), + categoryGroupName: Type.String({ description: 'Display name of the category group.' }), + categoryGroupSlug: Type.String({ description: 'URL slug of the category group.' }), + categoryGroupType: Type.String({ + description: 'Type of the category group, for example `vertical` or `horizontal`.', + }), + }, + { + title: 'OssIndexCollectionCategory', + description: 'The category named by `categorySlug`. Left out when `categorySlug` is not sent.', + }, +); + +const Collections = Type.Object({ + ...paginated(OssIndexCollection, { + data: 'Collections in the requested order. Empty when none match the filters.', + }).properties, + category: Type.Optional(OssIndexCollectionCategory), +}); + +const collectionRoutes: FastifyPluginAsyncTypebox = async (scope) => { + scope.get( + '/oss-index/collections', + { + schema: { + tags: ['OSS Index'], + summary: 'List the OSS Index collections', + description: + 'Returns collections with their totals and leading projects. `categorySlug` and `categoryGroupId` narrow the list, ' + + 'and `sort` orders it by contributors or by software value. With `categorySlug`, the response also includes that category. ' + + 'Pages follow position, as the Pagination guide describes. A final page that holds exactly `pageSize` collections still returns a `nextCursor`, and the page it points to is empty.', + querystring: Query, + response: { 200: Collections }, + }, + }, + async (request) => { + const { categorySlug, categoryGroupId, sort } = request.query; + const page = requestedPage(request.query); + const { pages, skip } = pipePages(page); + + let category: CategoryRow | undefined; + if (categorySlug) { + [category] = await fetchPipe( + request, + categoryPipePath, + { slug: categorySlug }, + isCategoryRow, + ); + if (!category) { + throw new NotFoundError('Category not found'); + } + } + + const chunks = await Promise.all( + pages.map((number) => + fetchPipe( + request, + pipePath, + { + categorySlug: categorySlug || undefined, + categoryGroupId: categoryGroupId || undefined, + orderBy: sort ?? 'totalContributors', + page: number, + pageSize: page.pageSize, + }, + isRow, + ), + ), + ); + const fetched = chunks.flat(); + const rows = fetched.slice(skip, skip + page.pageSize); + // The pipe's row total ignores the category filter, so only a full last pipe page signals + // that more rows may follow. + const last = chunks[chunks.length - 1]!; + const total = + pages[0]! * page.pageSize + fetched.length + (last.length === page.pageSize ? 1 : 0); + + const result = toCountedPage( + rows.map((row) => ({ ...row, topProjects: row.topProjects.map(toCollectionProject) })), + page, + total, + ); + return category ? { ...result, category } : result; + }, + ); +}; + +export default collectionRoutes; diff --git a/api/tests/oss-index-collections.test.ts b/api/tests/oss-index-collections.test.ts new file mode 100644 index 000000000..6c71fd9c7 --- /dev/null +++ b/api/tests/oss-index-collections.test.ts @@ -0,0 +1,291 @@ +// Copyright (c) 2025 The Linux Foundation and each contributor. +// SPDX-License-Identifier: MIT +import { beforeEach, describe, expect, it } from 'vitest'; + +import { + callsTo, + mockFetch, + pipeCalls, + tinybirdStub, + useApp, + type OpenApiDoc, +} from './helpers/tinybird.js'; + +const route = '/v1-alpha/oss-index/collections'; +const pipePath = '/v0/pipes/collections_oss_index.json'; +const categoryPath = '/v0/pipes/category_list.json'; + +const category = { + id: 'cat1', + name: 'API Clients', + slug: 'api-clients', + categoryGroupId: 'g1', + categoryGroupName: 'Developer Tools', + categoryGroupSlug: 'developer-tools-horizontal', + categoryGroupType: 'horizontal', +}; + +const row = (overrides: Record = {}) => ({ + id: 'c1', + name: 'REST API & HTTP Client Libraries', + slug: 'rest-api-client-libraries', + totalContributors: 133041, + softwareValue: 962602659, + avgScore: 0.41, + projectCount: 171, + topProjects: [ + ['p1', 13759, 'Istio', 'https://logo.test/i.png', 475232855, 0, 83, 'Mesh', 'istio', 'active'], + ], + ...overrides, +}); + +const numbered = (count: number) => Array.from({ length: count }, (_, i) => row({ id: `c${i}` })); + +let rows: unknown[] = []; +let categories: unknown[] = []; +const { get } = useApp(); + +beforeEach(() => { + rows = [row()]; + categories = [category]; + mockFetch.mockImplementation( + tinybirdStub((url) => (url.pathname === categoryPath ? categories : rows)), + ); +}); + +const cursorOf = (offset: number) => Buffer.from(String(offset)).toString('base64url'); + +describe('GET /v1-alpha/oss-index/collections', () => { + it('maps collection fields and converts project tuples including the slug', async () => { + const res = await get(route); + expect(res.statusCode).toBe(200); + expect(res.json()).toEqual({ + data: [ + { + id: 'c1', + name: 'REST API & HTTP Client Libraries', + slug: 'rest-api-client-libraries', + totalContributors: 133041, + softwareValue: 962602659, + avgScore: 0.41, + projectCount: 171, + topProjects: [ + { + id: 'p1', + count: 13759, + name: 'Istio', + slug: 'istio', + logoUrl: 'https://logo.test/i.png', + description: 'Mesh', + softwareValue: 475232855, + avgScore: 0, + healthScore: 83, + status: 'active', + }, + ], + }, + ], + pageSize: 50, + nextCursor: null, + }); + }); + + it('answers an empty list', async () => { + rows = []; + const res = await get(route); + expect(res.statusCode).toBe(200); + expect(res.json()).toEqual({ data: [], pageSize: 50, nextCursor: null }); + }); + + it('sends the default sort, first page and page size, and omits the filters', async () => { + await get(route); + expect(pipeCalls()).toHaveLength(1); + const [call] = callsTo(pipePath); + expect(call?.searchParams.get('orderBy')).toBe('totalContributors'); + expect(call?.searchParams.get('page')).toBe('0'); + expect(call?.searchParams.get('pageSize')).toBe('50'); + expect(call?.searchParams.has('categorySlug')).toBe(false); + expect(call?.searchParams.has('categoryGroupId')).toBe(false); + }); + + it('forwards an explicit sort as orderBy', async () => { + await get(`${route}?sort=softwareValue`); + expect(callsTo(pipePath)[0]?.searchParams.get('orderBy')).toBe('softwareValue'); + }); + + it('rejects an unknown sort with 400 before calling a pipe', async () => { + const res = await get(`${route}?sort=bogus`); + expect(res.statusCode).toBe(400); + expect(pipeCalls()).toHaveLength(0); + }); + + it('forwards categoryGroupId when given and omits it when empty', async () => { + await get(`${route}?categoryGroupId=g1`); + await get(`${route}?categoryGroupId=`); + const calls = callsTo(pipePath); + expect(calls[0]?.searchParams.get('categoryGroupId')).toBe('g1'); + expect(calls[1]?.searchParams.has('categoryGroupId')).toBe(false); + }); + + it('does not look up a category without categorySlug', async () => { + await get(route); + await get(`${route}?categorySlug=`); + expect(callsTo(categoryPath)).toHaveLength(0); + expect(callsTo(pipePath)[1]?.searchParams.has('categorySlug')).toBe(false); + }); + + it('looks up the category by slug, forwards the slug and returns the category', async () => { + const res = await get(`${route}?categorySlug=api-clients`); + expect(res.statusCode).toBe(200); + expect(callsTo(categoryPath)).toHaveLength(1); + expect(callsTo(categoryPath)[0]?.searchParams.get('slug')).toBe('api-clients'); + expect(callsTo(pipePath)[0]?.searchParams.get('categorySlug')).toBe('api-clients'); + expect(res.json().category).toEqual(category); + }); + + it('leaves category out of the body when categorySlug is absent', async () => { + const res = await get(route); + expect(res.json()).not.toHaveProperty('category'); + }); + + it('answers 404 for an unknown category slug without calling the collections pipe', async () => { + categories = []; + const res = await get(`${route}?categorySlug=nope`); + expect(res.statusCode).toBe(404); + expect(res.json().code).toBe('not_found'); + expect(callsTo(pipePath)).toHaveLength(0); + }); + + it('answers 503 for a malformed category row', async () => { + categories = [{ ...category, categoryGroupSlug: null }]; + const res = await get(`${route}?categorySlug=api-clients`); + expect(res.statusCode).toBe(503); + expect(res.body).toContain('upstream_unavailable'); + }); +}); + +describe('paging', () => { + it('returns a nextCursor when a full page is read', async () => { + rows = numbered(2); + const res = await get(`${route}?pageSize=2`); + expect(res.json().data).toHaveLength(2); + expect(res.json().pageSize).toBe(2); + expect(res.json().nextCursor).toBe(cursorOf(2)); + expect(callsTo(pipePath)[0]?.searchParams.get('pageSize')).toBe('2'); + }); + + it('returns a null nextCursor on a short last page', async () => { + rows = numbered(1); + const res = await get(`${route}?pageSize=2`); + expect(res.json().nextCursor).toBeNull(); + }); + + it('continues from a cursor by reading the matching pipe page', async () => { + rows = numbered(2); + const res = await get(`${route}?pageSize=2&cursor=${cursorOf(4)}`); + expect(res.statusCode).toBe(200); + const pages = callsTo(pipePath).map((call) => call.searchParams.get('page')); + expect(pages).toEqual(['2']); + expect(res.json().nextCursor).toBe(cursorOf(6)); + }); + + describe('with a page-aware pipe', () => { + const ids = (res: { json: () => { data: { id: string }[] } }) => + res.json().data.map((item) => item.id); + + beforeEach(() => { + const all = numbered(5); + mockFetch.mockImplementation( + tinybirdStub((url) => { + if (url.pathname === categoryPath) { + return categories; + } + const size = Number(url.searchParams.get('pageSize')); + const page = Number(url.searchParams.get('page')); + return all.slice(page * size, page * size + size); + }), + ); + }); + + it('returns the rows after a mid-page cursor from two pipe pages', async () => { + const res = await get(`${route}?pageSize=2&cursor=${cursorOf(1)}`); + const pages = callsTo(pipePath).map((call) => call.searchParams.get('page')); + expect(pages.sort()).toEqual(['0', '1']); + expect(ids(res)).toEqual(['c1', 'c2']); + expect(res.json().nextCursor).toBe(cursorOf(3)); + }); + + it('returns the last rows and a null nextCursor near the end', async () => { + const res = await get(`${route}?pageSize=2&cursor=${cursorOf(3)}`); + expect(ids(res)).toEqual(['c3', 'c4']); + expect(res.json().nextCursor).toBeNull(); + }); + + it('walks every row once by following nextCursor', async () => { + const seen: string[] = []; + let cursor: string | null = null; + for (let i = 0; i < 10; i += 1) { + const res = await get(`${route}?pageSize=2${cursor ? `&cursor=${cursor}` : ''}`); + seen.push(...ids(res)); + cursor = res.json().nextCursor; + if (!cursor) { + break; + } + } + expect(seen).toEqual(['c0', 'c1', 'c2', 'c3', 'c4']); + expect(cursor).toBeNull(); + }); + }); + + it('answers 400 for an invalid cursor before calling a pipe', async () => { + const res = await get(`${route}?cursor=not-a-cursor!`); + expect(res.statusCode).toBe(400); + expect(res.json().code).toBe('invalid_request'); + expect(pipeCalls()).toHaveLength(0); + }); + + it('answers 400 for a page size out of range', async () => { + expect((await get(`${route}?pageSize=0`)).statusCode).toBe(400); + expect((await get(`${route}?pageSize=201`)).statusCode).toBe(400); + }); +}); + +describe('malformed rows', () => { + it.each([ + ['a missing collection id', row({ id: undefined })], + ['a null slug', row({ slug: null })], + ['a missing project count', row({ projectCount: undefined })], + ['a non-array topProjects', row({ topProjects: 'x' })], + [ + 'the 9 slot project tuple', + row({ topProjects: [['p1', 1, 'n', 'l', 1, 1, 1, 'd', 'active']] }), + ], + [ + 'a null project slug', + row({ topProjects: [['p1', 1, 'n', 'l', 1, 1, 1, 'd', null, 'active']] }), + ], + ['a negative totalContributors', row({ totalContributors: -1 })], + ['a fractional projectCount', row({ projectCount: 2.5 })], + [ + 'a fractional project count', + row({ topProjects: [['p1', 1.5, 'n', 'l', 1, 1, 1, 'd', 's', 'active']] }), + ], + ])('answers 503 upstream_unavailable for %s', async (_label, bad) => { + rows = [bad]; + const res = await get(route); + expect(res.statusCode).toBe(503); + expect(res.body).toContain('upstream_unavailable'); + }); +}); + +describe('OpenAPI', () => { + it('lists the route under the OSS Index tag', async () => { + const spec = (await get('/v1-alpha/openapi.json')).json(); + const op = spec.paths[route]?.get; + expect(op?.tags).toEqual(['OSS Index']); + const names = op?.parameters?.map((param) => param.name) ?? []; + expect(names).toEqual( + expect.arrayContaining(['categorySlug', 'categoryGroupId', 'sort', 'cursor', 'pageSize']), + ); + }); +}); diff --git a/api/tests/oss-index.test.ts b/api/tests/oss-index.test.ts index 3df2e8793..632bc57a8 100644 --- a/api/tests/oss-index.test.ts +++ b/api/tests/oss-index.test.ts @@ -3,10 +3,12 @@ import { describe, expect, it } from 'vitest'; import { + isCollectionProjectTuple, isTopCollectionTuple, isCategoryTopProjectTuple, isTopProjectTuple, toCategoryTopProject, + toCollectionProject, toTopCollection, toTopProject, } from '../src/lib/oss-index.js'; @@ -116,3 +118,49 @@ describe('tuple guards', () => { expect(isTopProjectTuple(value)).toBe(false); }); }); + +const collectionProjectTuple = [ + 'p1', + 30, + 'Node.js', + 'https://logo.test/n.png', + 2400, + 0.8, + 86, + 'Runtime', + 'nodejs-node', + 'active', +]; + +describe('collection project tuple', () => { + it('is accepted when well formed', () => { + expect(isCollectionProjectTuple(collectionProjectTuple)).toBe(true); + }); + + it.each([ + ['not an array', 'x'], + ['the 9 slot tuple', projectTuple], + ['too long', [...collectionProjectTuple, 1]], + ['a null slug', [...collectionProjectTuple.slice(0, 8), null, 'active']], + ['a numeric status', [...collectionProjectTuple.slice(0, 9), 1]], + ['a null logo', ['p1', 30, 'n', null, 1, 1, 1, 'd', 's', 'active']], + ['a string health score', ['p1', 30, 'n', 'l', 1, 1, '86', 'd', 's', 'active']], + ])('is rejected when %s', (_label, value) => { + expect(isCollectionProjectTuple(value)).toBe(false); + }); + + it('is mapped with the slug and logo renamed to logoUrl', () => { + expect(toCollectionProject(collectionProjectTuple as never)).toEqual({ + id: 'p1', + count: 30, + name: 'Node.js', + slug: 'nodejs-node', + logoUrl: 'https://logo.test/n.png', + description: 'Runtime', + softwareValue: 2400, + avgScore: 0.8, + healthScore: 86, + status: 'active', + }); + }); +});