diff --git a/scverse-stats/collectors/linkedinCollector.ts b/scverse-stats/collectors/linkedinCollector.ts new file mode 100644 index 0000000..d81f29a --- /dev/null +++ b/scverse-stats/collectors/linkedinCollector.ts @@ -0,0 +1,51 @@ +import "dotenv/config"; +import { LinkedInDataSchema } from "../types"; +import { saveJson } from "../utils"; + +const COMPANY = "scverse"; +const PROFILE_URL = `https://www.linkedin.com/company/${COMPANY}/`; + +// LinkedIn serves the follower count to logged-out visitors, but only to clients it takes for a browser. +const BROWSER_USER_AGENT = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"; + +// Deliberately does not match abbreviated counts such as "10K followers": a missing number is better +// than a wrong one, and the field is optional downstream. +const FOLLOWERS = /([\d,.]+)\s*followers/i; + +export async function collectLinkedInStats(): Promise { + console.log("Collecting LinkedIn stats..."); + + // There is no public API for this, and LinkedIn blocks some datacentre ranges outright, so a + // failure here must not take the whole run down. Write nothing and the combiner omits the field. + try { + const response = await fetch(PROFILE_URL, { + headers: { + "user-agent": BROWSER_USER_AGENT, + "accept-language": "en-US,en;q=0.9", + }, + }); + + if (!response.ok) { + console.warn(`LinkedIn returned ${response.status}, skipping follower count`); + return; + } + + const match = FOLLOWERS.exec(await response.text()); + if (!match) { + console.warn("Could not find a follower count on the LinkedIn page, skipping"); + return; + } + + const validated = LinkedInDataSchema.parse({ + followers_count: Number(match[1].replace(/[,.]/g, "")), + company: COMPANY, + timestamp: new Date().toISOString(), + }); + + await saveJson("linkedin.json", validated); + console.log(`Followers: ${validated.followers_count}`); + } catch (error) { + console.warn(`Could not collect LinkedIn stats: ${error}`); + } +} diff --git a/scverse-stats/combiner.ts b/scverse-stats/combiner.ts index 1461517..b9f8e9c 100644 --- a/scverse-stats/combiner.ts +++ b/scverse-stats/combiner.ts @@ -6,6 +6,7 @@ export async function combineStats(): Promise { const githubData = await loadJson("github.json"); const zulipData = await loadJson("zulip.json"); const blueskyData = await loadJson("bluesky.json"); + const linkedinData = await loadJson("linkedin.json"); const ecosystemData = await loadJson("ecosystem.json"); const citationsData = await loadJson("citations.json"); const pepyData = await loadJson("pepy.json"); @@ -18,6 +19,10 @@ export async function combineStats(): Promise { combinedStats.bluesky_followers = blueskyData.followers_count; } + if (linkedinData) { + combinedStats.linkedin_followers = linkedinData.followers_count; + } + if (zulipData) { combinedStats.zulip_users = zulipData.active_users; combinedStats.core_team_size = zulipData.core_team_size; diff --git a/scverse-stats/index.ts b/scverse-stats/index.ts index ef2d758..1c75975 100644 --- a/scverse-stats/index.ts +++ b/scverse-stats/index.ts @@ -2,6 +2,7 @@ import "dotenv/config"; import { collectGitHubStats } from "./collectors/gitHubCollector"; import { collectZulipStats } from "./collectors/zulipCollector"; import { collectBlueskyStats } from "./collectors/blueskyCollector"; +import { collectLinkedInStats } from "./collectors/linkedinCollector"; import { collectEcosystemStats } from "./collectors/ecosystemCollector"; import { collectCitationStats } from "./collectors/citationsCollector"; import { collectPepyStats } from "./collectors/pepyCollector"; @@ -14,6 +15,7 @@ async function main() { collectGitHubStats(), collectZulipStats(), collectBlueskyStats(), + collectLinkedInStats(), collectEcosystemStats(), collectCitationStats(), collectPepyStats(), diff --git a/scverse-stats/types.ts b/scverse-stats/types.ts index f934b28..0933b22 100644 --- a/scverse-stats/types.ts +++ b/scverse-stats/types.ts @@ -38,6 +38,15 @@ export const BlueskyDataSchema = z.object({ export type BlueskyData = z.infer; +// LinkedIn schemas +export const LinkedInDataSchema = z.object({ + followers_count: z.number(), + company: z.string(), + timestamp: z.string(), +}); + +export type LinkedInData = z.infer; + // Ecosystem packages schemas export const EcosystemPackageSchema = z.object({ name: z.string(), @@ -120,6 +129,7 @@ export type CitationsData = z.infer; export const CombinedStatsSchema = z.object({ timestamp: z.string(), bluesky_followers: z.number().optional(), + linkedin_followers: z.number().optional(), zulip_users: z.number(), core_team_size: z.number(), github: z