githubBacklog.ts

// src/libs/githubBacklog.ts
/**
 * GitHub Projects backlog fetcher.
 *
 * Fetches issues in the "Planning", "On Deck", and "In progress" status
 * columns from the datum-cloud GitHub Projects board (project #22, org
 * `datum-cloud`) via the GraphQL Projects v2 API.
 *
 * Cache strategy (stale-while-revalidate):
 *  - Redis (when REDIS_URL is set): key `${redisKeyPrefix}github:backlog`
 *  - File (`.cache/github-backlog.json`): fallback when Redis is absent/down,
 *    and — critically — pre-populated by `npm run build:cache` at image build
 *    time (see `scripts/warmup-cache.ts` and the Dockerfile's `.cache` COPY),
 *    so the very first request after a deploy already has warm data instead
 *    of paying for a live GitHub fetch.
 *  - Staleness is tracked via a stored `updatedAt` timestamp, not either
 *    driver's own expiry (a long safety-net TTL is set on the Redis entry so
 *    an abandoned key doesn't linger forever, but it never drives the
 *    request-facing staleness check).
 *  - A request never blocks on GitHub, period. A stale cache returns the
 *    last-known-good items immediately and fires a de-duplicated background
 *    refresh. An empty cache (no entry in either driver — true cold start, or
 *    a corrupted/empty cache file) returns an empty list immediately and
 *    fires a de-duplicated background populate instead of blocking on it; the
 *    client's live-update poller (`BacklogLiveUpdate.astro`) swaps in real
 *    data once that finishes.
 *  - Before paying for a full paginated refetch, a background refresh first
 *    checks GitHub's own `ProjectV2.updatedAt` (one cheap field, one round
 *    trip). If the board hasn't changed since our last full fetch, it skips
 *    pagination entirely and just extends our cache's freshness.
 *
 * Auth: uses existing APP_ID / APP_PRIVATE_KEY / APP_INSTALLATION_ID env vars
 * via the `graph()` helper in `src/libs/github.ts`.
 */

import { FileCacheDriver } from '@datum-cloud/strapi-revalidate';
import { graph } from './github';
import { isRedisReady, redisClient, redisKeyPrefix } from './strapi/_runtime';

/**
 * Project "Status" options shown on the roadmap — the same names as the board's
 * columns. The project's other statuses (e.g. Backlog, Done) are not shown.
 */
export type GitHubBacklogStatus = 'Planning' | 'On Deck' | 'In progress';

/** Roadmap ordering (most advanced first) — used for the flat markdown export. */
export const INCLUDED_BACKLOG_STATUSES: GitHubBacklogStatus[] = [
  'In progress',
  'On Deck',
  'Planning',
];

/** Case/whitespace-insensitive lookup so "On deck" or " Planning " still match. */
const STATUS_BY_NORMALIZED_NAME = new Map<string, GitHubBacklogStatus>(
  INCLUDED_BACKLOG_STATUSES.map((status) => [normalizeStatusName(status), status])
);

function normalizeStatusName(name: string): string {
  return name.trim().replace(/\s+/g, ' ').toLowerCase();
}

/** Map a raw GitHub Project status name to a roadmap status, or null if it isn't shown. */
export function toBacklogStatus(name: string | null | undefined): GitHubBacklogStatus | null {
  return name ? (STATUS_BY_NORMALIZED_NAME.get(normalizeStatusName(name)) ?? null) : null;
}

export interface GitHubBacklogItem {
  number: number;
  title: string;
  url: string;
  labels: string[];
  status: GitHubBacklogStatus;
  /** Raw markdown body of the issue (rendered on demand by the detail endpoint). */
  body: string;
  /** Plain-text, single-paragraph summary of `body` for list rows. */
  excerpt: string;
}

/** Roadmap page columns, in display order. */
export type RoadmapColumnKey = 'planning' | 'onDeck' | 'inProgress';

export interface RoadmapColumn {
  key: RoadmapColumnKey;
  title: string;
  items: GitHubBacklogItem[];
}

/** GitHub Project status → roadmap column. */
const STATUS_COLUMN: Record<GitHubBacklogStatus, RoadmapColumnKey> = {
  Planning: 'planning',
  'On Deck': 'onDeck',
  'In progress': 'inProgress',
};

export const GITHUB_BACKLOG_PROJECT_URL = 'https://github.com/orgs/datum-cloud/projects/22';

/**
 * Bump when the cached item shape changes. Older entries are rejected so the
 * "board unchanged" shortcut in `triggerBackgroundRefresh` can't keep serving
 * items that lack the new fields.
 */
const CACHE_SCHEMA_VERSION = 3;

interface CachedBacklog {
  schemaVersion: number;
  items: GitHubBacklogItem[];
  updatedAt: number;
  /** GitHub's own `ProjectV2.updatedAt` as of this fetch — lets a background
   *  refresh skip full pagination when the board hasn't changed. */
  projectUpdatedAt: string;
}

const GITHUB_ORG = 'datum-cloud';
const GITHUB_PROJECT_NUMBER = 22;
const CACHE_TTL_SECONDS = 30 * 60; // 30 minutes — staleness threshold, not a hard expiry
const REDIS_SAFETY_TTL_SECONDS = 60 * 60 * 24 * 7; // 7 days — backstop so an abandoned key clears eventually

const REDIS_CACHE_KEY = `${redisKeyPrefix}github:backlog`;

/** Logical cache name for admin force-regen (`POST /api/cache/strapi`) and the file cache key. */
export const GITHUB_BACKLOG_CACHE_KEY = 'github-backlog';

/** Persistent fallback — same `.cache` dir Strapi's file driver uses, baked into the deploy image. */
const fileCache = new FileCacheDriver({ dir: '.cache' });

/** De-dupes concurrent background refresh triggers within a single process. */
let refreshInFlight: Promise<void> | null = null;

const PROJECT_UPDATED_AT_QUERY = `
  query GetProjectUpdatedAt($org: String!, $projectNumber: Int!) {
    organization(login: $org) {
      projectV2(number: $projectNumber) {
        updatedAt
      }
    }
  }
`;

const BACKLOG_QUERY = `
  query GetProjectBacklog($org: String!, $projectNumber: Int!, $cursor: String) {
    organization(login: $org) {
      projectV2(number: $projectNumber) {
        updatedAt
        items(first: 50, after: $cursor) {
          nodes {
            content {
              ... on Issue {
                number
                title
                url
                body
                labels(first: 5) {
                  nodes {
                    name
                  }
                }
              }
            }
            fieldValueByName(name: "Status") {
              ... on ProjectV2ItemFieldSingleSelectValue {
                name
              }
            }
          }
          pageInfo {
            hasNextPage
            endCursor
          }
        }
      }
    }
  }
`;

interface GitHubLabel {
  name: string;
}

interface GitHubIssueContent {
  number?: number;
  title?: string;
  url?: string;
  body?: string;
  labels?: { nodes: GitHubLabel[] };
}

interface GitHubProjectItem {
  content: GitHubIssueContent | null;
  fieldValueByName: { name: string } | null;
}

interface GitHubProjectItemsPage {
  nodes: GitHubProjectItem[];
  pageInfo: { hasNextPage: boolean; endCursor: string | null };
}

interface GitHubProjectResponse {
  organization: {
    projectV2: {
      updatedAt: string;
      items: GitHubProjectItemsPage;
    };
  };
}

interface GitHubProjectUpdatedAtResponse {
  organization: {
    projectV2: {
      updatedAt: string;
    };
  };
}

function isGitHubBacklogStatus(value: string | undefined): value is GitHubBacklogStatus {
  return INCLUDED_BACKLOG_STATUSES.includes(value as GitHubBacklogStatus);
}

function isValidCachedBacklogItem(item: unknown): item is GitHubBacklogItem {
  if (!item || typeof item !== 'object') return false;
  const record = item as Record<string, unknown>;
  return (
    typeof record.number === 'number' &&
    typeof record.title === 'string' &&
    typeof record.url === 'string' &&
    Array.isArray(record.labels) &&
    typeof record.body === 'string' &&
    typeof record.excerpt === 'string' &&
    isGitHubBacklogStatus(record.status as string | undefined)
  );
}

function isValidCachedBacklog(items: unknown): items is GitHubBacklogItem[] {
  return Array.isArray(items) && items.every(isValidCachedBacklogItem);
}

function isValidCachedEntry(value: unknown): value is CachedBacklog {
  if (!value || typeof value !== 'object') return false;
  const record = value as Record<string, unknown>;
  return (
    record.schemaVersion === CACHE_SCHEMA_VERSION &&
    typeof record.updatedAt === 'number' &&
    typeof record.projectUpdatedAt === 'string' &&
    isValidCachedBacklog(record.items)
  );
}

/**
 * Reduce an issue's markdown body to a short plain-text summary: the first
 * paragraph of prose, with headings, code, images, HTML and inline markdown
 * syntax removed.
 */
export function makeExcerpt(body: string, maxLength = 160): string {
  const cleaned = body
    .replace(/\r\n?/g, '\n')
    .replace(/```[\s\S]*?```/g, '')
    .replace(/<!--[\s\S]*?-->/g, '')
    .replace(/<[^>]+>/g, '')
    .replace(/!\[[^\]]*\]\([^)]*\)/g, '')
    .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1');

  for (const block of cleaned.split(/\n{2,}/)) {
    const text = block
      .split('\n')
      .filter((line) => !/^\s{0,3}#{1,6}\s/.test(line))
      .join(' ')
      .replace(/^[\s>*+-]+/, '')
      .replace(/^\d+\.\s+/, '')
      .replace(/[*_`~]/g, '')
      .replace(/\s+/g, ' ')
      .trim();
    if (!text) continue;
    return text.length > maxLength ? `${text.slice(0, maxLength - 1).trimEnd()}…` : text;
  }
  return '';
}

/** Split backlog items into the three roadmap columns (display order). */
export function groupBacklogByColumn(items: GitHubBacklogItem[]): RoadmapColumn[] {
  const buckets: Record<RoadmapColumnKey, GitHubBacklogItem[]> = {
    planning: [],
    onDeck: [],
    inProgress: [],
  };
  for (const item of items) buckets[STATUS_COLUMN[item.status]].push(item);

  const column = (key: RoadmapColumnKey, title: string): RoadmapColumn => ({
    key,
    title,
    items: buckets[key],
  });

  return [
    column('planning', 'Planning'),
    column('onDeck', 'On Deck'),
    column('inProgress', 'In progress'),
  ];
}

function isStale(updatedAt: number): boolean {
  return Date.now() - updatedAt >= CACHE_TTL_SECONDS * 1000;
}

function sortBacklogItems(items: GitHubBacklogItem[]): GitHubBacklogItem[] {
  return [...items].sort((a, b) => {
    const statusDiff =
      INCLUDED_BACKLOG_STATUSES.indexOf(a.status) - INCLUDED_BACKLOG_STATUSES.indexOf(b.status);
    if (statusDiff !== 0) return statusDiff;
    return a.number - b.number;
  });
}

/** Read the cached backlog entry: Redis first, then the persistent file fallback. */
async function readCache(): Promise<CachedBacklog | null> {
  if (redisClient && isRedisReady()) {
    try {
      const raw = await redisClient.get(REDIS_CACHE_KEY);
      if (raw) {
        const parsed = JSON.parse(raw) as unknown;
        if (isValidCachedEntry(parsed)) return parsed;
        console.warn('[githubBacklog] Invalid Redis backlog data detected, falling back');
      }
    } catch (err) {
      console.warn('[githubBacklog] Redis read error:', err);
    }
  }

  try {
    const fileEntry = await fileCache.get<unknown>(GITHUB_BACKLOG_CACHE_KEY);
    if (fileEntry && isValidCachedEntry(fileEntry)) return fileEntry;
  } catch (err) {
    console.warn('[githubBacklog] File cache read error:', err);
  }

  return null;
}

/** Write the backlog entry to the file cache (always) and Redis (when available). */
async function writeCache(entry: CachedBacklog): Promise<void> {
  try {
    await fileCache.set(GITHUB_BACKLOG_CACHE_KEY, entry);
  } catch (err) {
    console.warn('[githubBacklog] File cache write error:', err);
  }

  if (!redisClient || !isRedisReady()) return;
  try {
    await redisClient.set(REDIS_CACHE_KEY, JSON.stringify(entry), 'EX', REDIS_SAFETY_TTL_SECONDS);
  } catch (err) {
    console.warn('[githubBacklog] Redis write error:', err);
  }
}

/** Cheap single-field check: has the board changed since our last full fetch? */
async function fetchProjectUpdatedAt(): Promise<string | null> {
  try {
    const data = await graph(PROJECT_UPDATED_AT_QUERY, {
      org: GITHUB_ORG,
      projectNumber: GITHUB_PROJECT_NUMBER,
    });
    const response = data as GitHubProjectUpdatedAtResponse | null;
    return response?.organization?.projectV2?.updatedAt ?? null;
  } catch (err) {
    console.warn('[githubBacklog] Failed to check project updatedAt:', err);
    return null;
  }
}

/**
 * Fires a de-duplicated background refresh; never throws, never blocks the caller.
 * Checks GitHub's own `ProjectV2.updatedAt` first — if the board is unchanged since
 * `cachedProjectUpdatedAt`, skips the full paginated refetch and just extends
 * freshness on the existing cached items.
 */
function triggerBackgroundRefresh(cachedProjectUpdatedAt: string): void {
  if (refreshInFlight) return;

  refreshInFlight = (async () => {
    try {
      const latestProjectUpdatedAt = await fetchProjectUpdatedAt();

      if (latestProjectUpdatedAt && latestProjectUpdatedAt === cachedProjectUpdatedAt) {
        const cached = await readCache();
        if (cached) {
          await writeCache({ ...cached, updatedAt: Date.now() });
        }
        return;
      }

      const { items, projectUpdatedAt } = await fetchFromGitHub();
      await writeCache({
        schemaVersion: CACHE_SCHEMA_VERSION,
        items,
        updatedAt: Date.now(),
        projectUpdatedAt: projectUpdatedAt ?? cachedProjectUpdatedAt,
      });
    } catch (err) {
      console.error('[githubBacklog] Background refresh failed, keeping stale cache:', err);
    } finally {
      refreshInFlight = null;
    }
  })();
}

/**
 * Fires a de-duplicated background fetch to populate an empty cache; never
 * throws, never blocks the caller. Used when there's no prior data at all to
 * compare against (true cold start, or a corrupted/empty cache file) — on
 * failure it simply leaves the cache empty so the next request retries.
 */
function triggerBackgroundPopulate(): void {
  if (refreshInFlight) return;

  refreshInFlight = (async () => {
    try {
      const { items, projectUpdatedAt } = await fetchFromGitHub();
      await writeCache({
        schemaVersion: CACHE_SCHEMA_VERSION,
        items,
        updatedAt: Date.now(),
        projectUpdatedAt: projectUpdatedAt ?? '',
      });
    } catch (err) {
      console.error('[githubBacklog] Background populate failed:', err);
    } finally {
      refreshInFlight = null;
    }
  })();
}

/**
 * Fetch all backlog pages from GitHub Projects v2 API.
 * @throws When a GraphQL call errors, or the API returns no project data (e.g.
 *   missing/invalid GitHub App credentials) — callers must not mistake that for
 *   a legitimately empty backlog and must not let it overwrite a good cache.
 */
async function fetchFromGitHub(): Promise<{
  items: GitHubBacklogItem[];
  projectUpdatedAt: string | null;
}> {
  const items: GitHubBacklogItem[] = [];
  let cursor: string | null = null;
  let projectUpdatedAt: string | null = null;

  while (true) {
    const variables: Record<string, unknown> = {
      org: GITHUB_ORG,
      projectNumber: GITHUB_PROJECT_NUMBER,
    };
    if (cursor) variables.cursor = cursor;

    const data: unknown = await graph(BACKLOG_QUERY, variables);

    const response = data as GitHubProjectResponse | null;
    const project = response?.organization?.projectV2;
    const page = project?.items;
    if (!project || !page) {
      throw new Error(
        '[githubBacklog] GitHub Projects API returned no data — check APP_ID/APP_PRIVATE_KEY/APP_INSTALLATION_ID'
      );
    }

    if (project.updatedAt) projectUpdatedAt = project.updatedAt;

    for (const node of page.nodes) {
      const status = toBacklogStatus(node.fieldValueByName?.name);
      const content = node.content;

      // Draft items and PRs have no issue number, so they are skipped.
      if (!status || !content || typeof content.number !== 'number') {
        continue;
      }

      items.push({
        number: content.number,
        title: content.title ?? '',
        url: content.url ?? `https://github.com/${GITHUB_ORG}`,
        labels: (content.labels?.nodes ?? []).map((l) => l.name),
        status,
        body: content.body ?? '',
        excerpt: makeExcerpt(content.body ?? ''),
      });
    }

    if (!page.pageInfo.hasNextPage || !page.pageInfo.endCursor) break;
    cursor = page.pageInfo.endCursor;
  }

  return { items: sortBacklogItems(items), projectUpdatedAt };
}

/** Returns true when a backlog cache entry exists at all (fresh or stale). */
export async function isGitHubBacklogCached(): Promise<boolean> {
  return (await readCache()) !== null;
}

/** Clear Redis and file backlog cache without refetching. */
export async function clearGitHubBacklogCache(): Promise<void> {
  if (redisClient && isRedisReady()) {
    try {
      await redisClient.del(REDIS_CACHE_KEY);
    } catch (err) {
      console.warn('[githubBacklog] Redis delete error:', err);
    }
  }
  try {
    await fileCache.delete(GITHUB_BACKLOG_CACHE_KEY);
  } catch (err) {
    console.warn('[githubBacklog] File cache delete error:', err);
  }
}

/**
 * Fetch fresh items from GitHub Projects and replace the cache.
 * Fetches before touching the cache — on failure, the existing (working) cache
 * is left completely untouched instead of being cleared out from under it.
 * @throws When the GitHub fetch fails — the cache is guaranteed unchanged
 */
export async function forceRegenerateGitHubBacklog(): Promise<GitHubBacklogItem[]> {
  const { items, projectUpdatedAt } = await fetchFromGitHub();
  await writeCache({
    schemaVersion: CACHE_SCHEMA_VERSION,
    items,
    updatedAt: Date.now(),
    projectUpdatedAt: projectUpdatedAt ?? '',
  });
  return items;
}

/**
 * Fetch Planning, On Deck, and In progress items from the datum-cloud GitHub Projects board.
 *
 * Stale-while-revalidate: a request never blocks on GitHub, ever. A stale cache
 * returns the last-known-good items immediately and fires a de-duplicated
 * background refresh. An empty cache (true cold start, or a corrupted/empty
 * cache file) returns an empty list immediately and fires a de-duplicated
 * background populate — the client's live-update poller swaps in real data
 * once that completes, without the request ever waiting on it.
 *
 * Never throws.
 */
export async function fetchGitHubBacklog(): Promise<GitHubBacklogItem[]> {
  const cached = await readCache();

  if (cached) {
    if (isStale(cached.updatedAt)) {
      triggerBackgroundRefresh(cached.projectUpdatedAt);
    }
    return cached.items;
  }

  triggerBackgroundPopulate();
  return [];
}

/** Look up a single backlog item by issue number from the cached backlog. Never calls GitHub. */
export async function getGitHubBacklogItem(number: number): Promise<GitHubBacklogItem | null> {
  const cached = await readCache();
  return cached?.items.find((item) => item.number === number) ?? null;
}

/**
 * Returns the timestamp (ms) of the last successful backlog refresh, or null when
 * no cache entry exists yet. Cheap — reads the cache only, never calls GitHub.
 */
export async function getGitHubBacklogUpdatedAt(): Promise<number | null> {
  const meta = await getGitHubBacklogMeta();
  return meta.updatedAt;
}

/** Lightweight backlog cache status for live-update polling. */
export async function getGitHubBacklogMeta(): Promise<{
  updatedAt: number | null;
  isRefreshing: boolean;
}> {
  const cached = await readCache();
  return {
    updatedAt: cached?.updatedAt ?? null,
    isRefreshing: refreshInFlight !== null,
  };
}