feat(feed): find more LinkedIn posts and import links in bulk

app/admin/dashboard/feed/page.tsx

@@ -15,7 +15,7 @@ interface FeedPost {
   excerpt: string;
   topic: string;
   postedAt?: string;
-  source: 'seed' | 'olostep-search-index' | 'community-submission';
+  source: 'seed' | 'olostep-search-index' | 'community-submission' | 'admin-import';
   status: 'approved' | 'pending' | 'hidden';
   createdAt: string;
 }
@@ -30,15 +30,23 @@ const SOURCE_LABELS: Record<FeedPost['source'], string> = {
   seed: 'Starter list',
   'olostep-search-index': 'Found by Olostep',
   'community-submission': 'Submitted',
+  'admin-import': 'Imported',
 };
 
+/** Olostep charges about 5 credits per search */
+const CREDITS_PER_SEARCH = 5;
+const DEEP_SEARCHES = 40;
+
 export default function AdminFeedPage() {
   const [posts, setPosts] = useState<FeedPost[]>([]);
   const [loading, setLoading] = useState(true);
   const [tab, setTab] = useState<FeedPost['status']>('pending');
   const [processingId, setProcessingId] = useState<string | null>(null);
   const [collecting, setCollecting] = useState(false);
   const [collectMessage, setCollectMessage] = useState('');
+  const [importText, setImportText] = useState('');
+  const [importing, setImporting] = useState(false);
+  const [importMessage, setImportMessage] = useState('');
 
   const fetchPosts = async () => {
     try {
@@ -84,18 +92,28 @@ export default function AdminFeedPage() {
     }
   };
 
-  const collect = async () => {
+  const collect = async (searches?: number) => {
+    if (
+      searches &&
+      !confirm(`Run ${searches} Olostep searches? This uses about ${searches * CREDITS_PER_SEARCH} credits and can take a few minutes.`)
+    ) {
+      return;
+    }
     setCollecting(true);
     setCollectMessage('');
     try {
-      const res = await fetch('/api/admin/feed', { method: 'POST' });
+      const res = await fetch('/api/admin/feed', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ action: 'collect', searches }),
+      });
       const data = await res.json();
       if (!res.ok) {
         setCollectMessage(data.error ?? 'Collection failed.');
         return;
       }
       setCollectMessage(
-        `Ran ${data.queries} searches${data.failedQueries ? ` (${data.failedQueries} failed)` : ''}: ${data.added} new posts, ${data.enriched} updated.`
+        `Ran ${data.queries} of ${data.searchListSize} searches${data.failedQueries ? ` (${data.failedQueries} failed)` : ''}: ${data.added} new posts, ${data.enriched} updated.`
       );
       await fetchPosts();
     } catch {
@@ -105,6 +123,32 @@ export default function AdminFeedPage() {
     }
   };
 
+  const importLinks = async () => {
+    setImporting(true);
+    setImportMessage('');
+    try {
+      const res = await fetch('/api/admin/feed', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ action: 'import', urls: importText }),
+      });
+      const data = await res.json();
+      if (!res.ok) {
+        setImportMessage(data.error ?? 'Import failed.');
+        return;
+      }
+      setImportMessage(
+        `${data.added} added, ${data.alreadyThere} already on the feed${data.invalid ? `, ${data.invalid} not LinkedIn post links` : ''}.`
+      );
+      if (data.added > 0) setImportText('');
+      await fetchPosts();
+    } catch {
+      setImportMessage('Import failed.');
+    } finally {
+      setImporting(false);
+    }
+  };
+
   const counts = Object.fromEntries(TABS.map(({ value }) => [value, posts.filter((post) => post.status === value).length]));
   const shown = posts.filter((post) => post.status === tab);
 
@@ -125,16 +169,52 @@ export default function AdminFeedPage() {
             LinkedIn posts shown on <a href="/feed" target="_blank" className="underline">/feed</a>. Submissions wait here for review.
           </p>
         </div>
-        <div className="text-right">
-          <Button onClick={collect} disabled={collecting}>
-            {collecting ? <Loader2 className="h-4 w-4 animate-spin" /> : <Search className="h-4 w-4" />}
-            Find new posts
-          </Button>
-          <p className="mt-1 text-xs text-muted-foreground">Runs Olostep searches; uses credits.</p>
+        <div className="max-w-sm text-right">
+          <div className="flex flex-wrap justify-end gap-2">
+            <Button onClick={() => collect()} disabled={collecting}>
+              {collecting ? <Loader2 className="h-4 w-4 animate-spin" /> : <Search className="h-4 w-4" />}
+              Find new posts
+            </Button>
+            <Button variant="outline" onClick={() => collect(DEEP_SEARCHES)} disabled={collecting}>
+              Deep search ({DEEP_SEARCHES})
+            </Button>
+          </div>
+          <p className="mt-1 text-xs text-muted-foreground">
+            Each search uses about {CREDITS_PER_SEARCH} Olostep credits. Runs continue where the last one stopped.
+          </p>
           {collectMessage && <p className="mt-2 text-sm">{collectMessage}</p>}
         </div>
       </div>
 
+      <Card>
+        <CardHeader className="pb-2">
+          <CardTitle className="text-lg">Add posts by link</CardTitle>
+          <p className="text-sm text-muted-foreground">
+            Paste LinkedIn post links, one per line or separated by spaces, e.g. collected from WhatsApp groups. They go live straight away.
+          </p>
+        </CardHeader>
+        <CardContent className="space-y-3">
+          <label htmlFor="feed-import" className="sr-only">
+            LinkedIn post links
+          </label>
+          <textarea
+            id="feed-import"
+            value={importText}
+            onChange={(event) => setImportText(event.target.value)}
+            rows={4}
+            placeholder="https://www.linkedin.com/posts/..."
+            className="w-full rounded-md border border-input bg-background px-3 py-2 text-sm"
+          />
+          <div className="flex flex-wrap items-center gap-3">
+            <Button onClick={importLinks} disabled={importing || !importText.trim()}>
+              {importing && <Loader2 className="h-4 w-4 animate-spin" />}
+              Add posts
+            </Button>
+            {importMessage && <p className="text-sm">{importMessage}</p>}
+          </div>
+        </CardContent>
+      </Card>
+
       <div className="flex flex-wrap gap-2">
         {TABS.map(({ value, label }) => (
           <Button key={value} variant={tab === value ? 'default' : 'outline'} onClick={() => setTab(value)}>

app/api/admin/feed/route.ts

@@ -3,7 +3,7 @@ import { cookies } from 'next/headers';
 import { revalidatePath } from 'next/cache';
 import { verify } from 'jsonwebtoken';
 import connectDB from '@/lib/db';
-import { collectFromOlostep } from '@/lib/linkedin-feed';
+import { collectFromOlostep, importPostUrls } from '@/lib/linkedin-feed';
 import { LinkedInPost } from '@/models/LinkedInPost';
 
 export const maxDuration = 300;
@@ -42,23 +42,42 @@ export async function GET() {
   }
 }
 
-/** Runs the Olostep collector on demand ("Find new posts" in the admin dashboard) */
-export async function POST() {
+/** Most searches one admin request may run: 60 searches is about 300 Olostep credits */
+const MAX_SEARCHES_PER_REQUEST = 60;
+
+/**
+ * Admin actions on the feed:
+ * - `{ action: "collect", searches?: number }` runs Olostep searches ("Find new posts", "Deep search")
+ * - `{ action: "import", urls: string }` adds pasted LinkedIn post links, live straight away
+ */
+export async function POST(request: Request) {
   try {
     if (!(await getAdminUser())) {
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
     }
 
+    const body = await request.json().catch(() => ({}));
+
+    if (body.action === 'import') {
+      if (typeof body.urls !== 'string' || !body.urls.trim()) {
+        return NextResponse.json({ error: 'Paste at least one LinkedIn post link.' }, { status: 400 });
+      }
+      const result = await importPostUrls(body.urls.slice(0, 100_000));
+      if (result.added > 0) revalidatePath('/feed');
+      return NextResponse.json(result);
+    }
+
     const apiKey = process.env.OLOSTEP_API_KEY;
     if (!apiKey) {
       return NextResponse.json({ error: 'OLOSTEP_API_KEY is not set on the server.' }, { status: 500 });
     }
 
-    const result = await collectFromOlostep(apiKey, Number(process.env.FEED_COLLECT_QUERIES) || 5);
+    const requested = Number(body.searches) || Number(process.env.FEED_COLLECT_QUERIES) || 5;
+    const result = await collectFromOlostep(apiKey, Math.min(requested, MAX_SEARCHES_PER_REQUEST));
     if (result.added > 0 || result.enriched > 0) revalidatePath('/feed');
     return NextResponse.json(result);
   } catch (error) {
-    console.error('LinkedIn feed collection failed:', error);
-    return NextResponse.json({ error: 'Collection failed' }, { status: 500 });
+    console.error('LinkedIn feed admin action failed:', error);
+    return NextResponse.json({ error: 'Something went wrong' }, { status: 500 });
   }
 }

lib/linkedin-feed.ts

@@ -7,9 +7,10 @@
  * scraping. Excerpts are search snippets, not full post content.
  */
 import connectDB from '@/lib/db'
+import { FeedCollectorState } from '@/models/FeedCollectorState'
 import { LinkedInPost, type LinkedInPostSource, type LinkedInPostStatus } from '@/models/LinkedInPost'
 import starterPosts from '@/data/linkedin-posts.json'
-import type { FeedPost } from './linkedin-feed-shared'
+import { OFFICIAL_HANDLES, type FeedPost } from './linkedin-feed-shared'
 
 const ACTIVITY_RE = /(?:^|[-_/])activity-(\d{16,21})(?:-|$)/
 
@@ -33,29 +34,64 @@ const KNOWN_AUTHORS: Record<string, string> = {
   arslanchoudhary: 'Arslan Choudhary',
 }
 
-/** Olostep search queries, rotated across daily runs */
-const SEARCH_QUERIES = [
+/** Results per Olostep search; 25 is the API maximum */
+const RESULTS_PER_SEARCH = 25
+const SEARCHES_IN_PARALLEL = 8
+
+/** Searches about programs, topics and years. Member searches are added on top (see searchQueries). */
+const TOPIC_SEARCHES = [
   '"#DevWeekends"',
-  '"#DevWeekends" "fellowship"',
-  '"#DevWeekends" "certificate"',
-  '"#DevWeekends" "DSA"',
-  '"#DevWeekends" "MERN"',
-  '"#DevWeekends" "open source"',
-  '"#DevWeekends" "mentor"',
-  '"#DevWeekends" "AI"',
-  '"#DevWeekends" "selected"',
-  '"#DevWeekends" "completed"',
-  '"#DevWeekends" "2026"',
-  '"#DevWeekends" "2025"',
-  '"#DevWeekends" "project"',
-  '"#DevWeekends" "journey"',
-  '"#DevWeekends" "React"',
-  '"#DevWeekends" "Python"',
-  '"#DevWeekends" "backend"',
-  '"#DevWeekends" "workshop"',
-  '"DevWeekendsFellowship"',
-  '"#DevWeekends" "DSOC"',
-].map((terms) => `site:linkedin.com/posts/ ${terms}`)
+  '"Dev Weekends"',
+  '"#DevWeekendsFellowship"',
+  '"Dev Weekends Fellowship" 2026',
+  '"Dev Weekends Fellowship" 2025',
+  '"Dev Weekends Fellowship" 2024',
+  '"Dev Weekends Fellowship" 2023',
+  '"Dev Weekends" certificate',
+  '"Dev Weekends" selected',
+  '"Dev Weekends" completed',
+  '"Dev Weekends" mentor',
+  '"Dev Weekends" mentorship',
+  '"Dev Weekends" DSA',
+  '"Dev Weekends" MERN',
+  '"Dev Weekends" "full stack"',
+  '"Dev Weekends" "AI engineering"',
+  '"Dev Weekends" DSOC',
+  '"Dev Weekends" "summer of code"',
+  '"Dev Weekends" GSoC',
+  '"Dev Weekends" LFX',
+  '"Dev Weekends" "open source"',
+  '"Dev Weekends" MindMaster',
+  '"Dev Weekends" "Tech Talk"',
+  '"Dev Weekends" FYP',
+  '"Dev Weekends" roadshow',
+  '"Dev Weekends" "Weekend Tech Grind"',
+  '"Dev Weekends" ICPC',
+  '"Dev Weekends" bootcamp',
+  '"Dev Weekends" DevOps',
+  '"Dev Weekends" Kubernetes',
+  '"Dev Weekends" "system design"',
+  '"Dev Weekends" internship',
+  '"Dev Weekends" "remote job"',
+  '"Dev Weekends" session',
+  '"Dev Weekends" workshop',
+  '"Dev Weekends" ambassador',
+  '"Dev Weekends" project',
+  '"Dev Weekends" journey',
+  '"Dev Weekends" LeetCode',
+  '"Dev Weekends" Python',
+]
+
+/**
+ * The full search list: topic searches, then one search per known member, so
+ * every member we find leads to their other Dev Weekends posts.
+ */
+function searchQueries(memberNames: string[]) {
+  return [
+    ...TOPIC_SEARCHES,
+    ...memberNames.map((name) => `"${name.replace(/"/g, '')}" "Dev Weekends"`),
+  ].map((terms) => `site:linkedin.com/posts ${terms}`)
+}
 
 /** Returns the canonical post URL and activity ID, or null for anything that isn't a public LinkedIn post URL */
 export function canonicalize(raw: unknown): { url: string; activityId: string } | null {
@@ -168,7 +204,7 @@ async function searchOlostep(apiKey: string, query: string): Promise<OlostepLink
   const response = await fetch('https://api.olostep.com/v1/searches', {
     method: 'POST',
     headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
-    body: JSON.stringify({ query, limit: 10 }),
+    body: JSON.stringify({ query, limit: RESULTS_PER_SEARCH, include_domains: ['linkedin.com'] }),
     signal: AbortSignal.timeout(75_000),
     cache: 'no-store',
   })
@@ -177,15 +213,41 @@ async function searchOlostep(apiKey: string, query: string): Promise<OlostepLink
   return data?.result?.links ?? []
 }
 
-/** Runs a few Olostep searches and saves new posts. Rotates queries by day so each run covers different ground. */
+/**
+ * Runs the next `maxQueries` searches from the search list and saves new posts.
+ * Remembers where it stopped, so consecutive runs (daily cron, admin buttons)
+ * work through the whole list instead of repeating the same searches.
+ */
 export async function collectFromOlostep(apiKey: string, maxQueries = 5) {
-  const count = Math.min(Math.max(1, maxQueries), SEARCH_QUERIES.length)
-  const day = Math.floor(Date.now() / 86_400_000)
-  const start = (day * count) % SEARCH_QUERIES.length
-  const queries = Array.from({ length: count }, (_, i) => SEARCH_QUERIES[(start + i) % SEARCH_QUERIES.length])
+  await connectDB()
+  const memberNames: string[] = await LinkedInPost.distinct('author', {
+    status: 'approved',
+    handle: { $nin: [...OFFICIAL_HANDLES] },
+  })
+  // A fallback name can't find anyone's posts, so it would only waste credits
+  const allQueries = searchQueries(memberNames.filter((name) => name !== 'Community member').sort())
+  const count = Math.min(Math.max(1, maxQueries), allQueries.length)
+
+  const state = await FeedCollectorState.findById('linkedin').lean<{ cursor: number }>()
+  const start = (state?.cursor ?? 0) % allQueries.length
+  const queries = Array.from({ length: count }, (_, i) => allQueries[(start + i) % allQueries.length])
 
-  const results = await Promise.allSettled(queries.map((query) => searchOlostep(apiKey, query)))
+  const results: PromiseSettledResult<OlostepLink[]>[] = []
+  for (let i = 0; i < queries.length; i += SEARCHES_IN_PARALLEL) {
+    const batch = queries.slice(i, i + SEARCHES_IN_PARALLEL)
+    results.push(...(await Promise.allSettled(batch.map((query) => searchOlostep(apiKey, query)))))
+  }
   const failed = results.filter((r) => r.status === 'rejected').length
+
+  // Only move on when some searches worked, so a bad key or an outage doesn't skip part of the list
+  if (failed < queries.length) {
+    await FeedCollectorState.updateOne(
+      { _id: 'linkedin' },
+      { cursor: (start + count) % allQueries.length },
+      { upsert: true }
+    )
+  }
+
   const posts = results
     .flatMap((r) => (r.status === 'fulfilled' ? r.value : []))
     .filter(mentionsDevWeekends)
@@ -196,7 +258,24 @@ export async function collectFromOlostep(apiKey: string, maxQueries = 5) {
 
   // Search results are trusted enough to publish directly; visitor submissions are not
   const saved = await savePosts(posts, 'approved')
-  return { queries: queries.length, failedQueries: failed, candidates: posts.length, ...saved }
+  return {
+    queries: queries.length,
+    failedQueries: failed,
+    candidates: posts.length,
+    ...saved,
+    searchListSize: allQueries.length,
+  }
+}
+
+/** Adds LinkedIn post links pasted by an admin. Admins are trusted, so the posts go live straight away. */
+export async function importPostUrls(text: string) {
+  const urls = text.split(/[\s,]+/).filter(Boolean)
+  const posts = urls
+    .map((url) => makePost(url, '', '', 'admin-import'))
+    .filter((post): post is PostInput => post !== null)
+  const unique = [...new Map(posts.map((post) => [post.activityId, post])).values()]
+  const saved = await savePosts(unique, 'approved')
+  return { links: urls.length, invalid: urls.length - posts.length, ...saved, alreadyThere: unique.length - saved.added }
 }
 
 /** Inserts new posts and fills in missing excerpts. Never overwrites a post's status or curated fields. */