From 51ba47b297c649f622de21c845e28fb25c75c3fc Mon Sep 17 00:00:00 2001 From: junaiddshaukat Date: Sun, 1 Nov 2026 14:18:06 +0500 Subject: [PATCH] feat(feed): find more LinkedIn posts and import links in bulk /feed had stalled at about 119 posts: 5 searches a day, 10 results each, from a fixed list of 20 queries that mostly found posts we already had. - Ask Olostep for 25 results per search (the API maximum), limited to linkedin.com - Grow the search list to about 99: 40 program, topic and year searches, plus one search per known member, so every member found leads to their other Dev Weekends posts - Remember where the collector stopped (FeedCollectorState), so the daily cron and admin runs work through the whole list; the cursor only moves when some searches succeed - Add a "Deep search" admin button (40 searches, about 200 credits, with a confirmation) for a one-time backfill, capped at 60 per request - Add bulk import: admins paste LinkedIn post links and they go live Co-Authored-By: Claude Opus 5.5 --- app/admin/dashboard/feed/page.tsx | 100 ++++++++++++++++++--- app/api/admin/feed/route.ts | 31 +++++-- lib/linkedin-feed.ts | 141 +++++++++++++++++++++++------- models/FeedCollectorState.ts | 19 ++++ models/LinkedInPost.ts | 4 +- 5 files changed, 246 insertions(+), 49 deletions(-) create mode 100644 models/FeedCollectorState.ts diff --git a/app/admin/dashboard/feed/page.tsx b/app/admin/dashboard/feed/page.tsx index afe2f32..07f090d 100644 --- a/app/admin/dashboard/feed/page.tsx +++ b/app/admin/dashboard/feed/page.tsx @@ -15,7 +15,7 @@ interface FeedPost { excerpt: string; topic: string; postedAt?: string; - source: 'seed' | 'olostep-search-index' | 'community-submission'; + source: 'seed' | 'olostep-search-index' | 'community-submission' | 'admin-import'; status: 'approved' | 'pending' | 'hidden'; createdAt: string; } @@ -30,8 +30,13 @@ const SOURCE_LABELS: Record = { seed: 'Starter list', 'olostep-search-index': 'Found by Olostep', 'community-submission': 'Submitted', + 'admin-import': 'Imported', }; +/** Olostep charges about 5 credits per search */ +const CREDITS_PER_SEARCH = 5; +const DEEP_SEARCHES = 40; + export default function AdminFeedPage() { const [posts, setPosts] = useState([]); const [loading, setLoading] = useState(true); @@ -39,6 +44,9 @@ export default function AdminFeedPage() { const [processingId, setProcessingId] = useState(null); const [collecting, setCollecting] = useState(false); const [collectMessage, setCollectMessage] = useState(''); + const [importText, setImportText] = useState(''); + const [importing, setImporting] = useState(false); + const [importMessage, setImportMessage] = useState(''); const fetchPosts = async () => { try { @@ -84,18 +92,28 @@ export default function AdminFeedPage() { } }; - const collect = async () => { + const collect = async (searches?: number) => { + if ( + searches && + !confirm(`Run ${searches} Olostep searches? This uses about ${searches * CREDITS_PER_SEARCH} credits and can take a few minutes.`) + ) { + return; + } setCollecting(true); setCollectMessage(''); try { - const res = await fetch('/api/admin/feed', { method: 'POST' }); + const res = await fetch('/api/admin/feed', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ action: 'collect', searches }), + }); const data = await res.json(); if (!res.ok) { setCollectMessage(data.error ?? 'Collection failed.'); return; } setCollectMessage( - `Ran ${data.queries} searches${data.failedQueries ? ` (${data.failedQueries} failed)` : ''}: ${data.added} new posts, ${data.enriched} updated.` + `Ran ${data.queries} of ${data.searchListSize} searches${data.failedQueries ? ` (${data.failedQueries} failed)` : ''}: ${data.added} new posts, ${data.enriched} updated.` ); await fetchPosts(); } catch { @@ -105,6 +123,32 @@ export default function AdminFeedPage() { } }; + const importLinks = async () => { + setImporting(true); + setImportMessage(''); + try { + const res = await fetch('/api/admin/feed', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ action: 'import', urls: importText }), + }); + const data = await res.json(); + if (!res.ok) { + setImportMessage(data.error ?? 'Import failed.'); + return; + } + setImportMessage( + `${data.added} added, ${data.alreadyThere} already on the feed${data.invalid ? `, ${data.invalid} not LinkedIn post links` : ''}.` + ); + if (data.added > 0) setImportText(''); + await fetchPosts(); + } catch { + setImportMessage('Import failed.'); + } finally { + setImporting(false); + } + }; + const counts = Object.fromEntries(TABS.map(({ value }) => [value, posts.filter((post) => post.status === value).length])); const shown = posts.filter((post) => post.status === tab); @@ -125,16 +169,52 @@ export default function AdminFeedPage() { LinkedIn posts shown on /feed. Submissions wait here for review.

-
- -

Runs Olostep searches; uses credits.

+
+
+ + +
+

+ Each search uses about {CREDITS_PER_SEARCH} Olostep credits. Runs continue where the last one stopped. +

{collectMessage &&

{collectMessage}

}
+ + + Add posts by link +

+ Paste LinkedIn post links, one per line or separated by spaces, e.g. collected from WhatsApp groups. They go live straight away. +

+ + + +