From 966ed99866eff3782823678cd4ddafa77ddd101d Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Thu, 24 Sep 2026 05:46:02 +0000 Subject: [PATCH] feat: operator-only user export (moshcode export users [--clean]) - app: GET /api/admin/users/export?format=csv|json, gated by a new ADMIN_EMAILS allowlist (API key or session). Returns email, display_name, created_at (ISO), id and signup_method for accounts with an email, plus counts of the rest. password_hash and coinpay_sub are never selected. - cli: `moshcode export users [--clean] [--format csv|json] [-o file]` and `/export users` in the pit. Files are 0600 and backed up before overwrite; the pit always writes a file and prints only its path and counts. - --clean pipes the CSV through cli-tools' email-cleaner (fails if it is not on PATH), keeps the valid rows, and writes .rejected.csv with reasons. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 32 +++ apps/pwa/.env.example | 2 + apps/pwa/README.md | 4 + apps/pwa/src/config.mjs | 16 ++ apps/pwa/src/lib/user-export.mjs | 94 +++++++ apps/pwa/src/routes/admin.mjs | 52 ++++ apps/pwa/src/server.mjs | 2 + apps/pwa/test/admin-users-export.test.mjs | 122 +++++++++ bin/moshcode.mjs | 7 + src/cli-schema.mjs | 34 +++ src/export-users.mjs | 308 ++++++++++++++++++++++ src/tui.mjs | 6 + test/export-users.test.mjs | 269 +++++++++++++++++++ 13 files changed, 948 insertions(+) create mode 100644 apps/pwa/src/lib/user-export.mjs create mode 100644 apps/pwa/src/routes/admin.mjs create mode 100644 apps/pwa/test/admin-users-export.test.mjs create mode 100644 src/export-users.mjs create mode 100644 test/export-users.test.mjs diff --git a/README.md b/README.md index 362757b3..e5acca62 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ or miss one that does. A test fails the build when it drifts. | `moshcode logout` | account | clear the logged-in account | | `moshcode save` | account | save this machine's pit settings to your account | | `moshcode load` | account | bring your saved pit settings onto this machine | +| `moshcode export` | account | operator only: export the app's users as CSV, optionally cleaned | | `moshcode console` | account | serve or connect to the browser terminal | | `moshcode dns` | hosting | resolve Moshpit names on this machine | | `moshcode name` | hosting | prove you hold a Moshpit name, so an app can use it as your identity | @@ -1592,6 +1593,37 @@ MOSHCODE_NO_AUTOSYNC=1 moshcode # turn it off for this pit MOSHCODE_AUTOSYNC_MS=900000 moshcode # every fifteen minutes instead ``` +## Exporting users (`moshcode export users`, operators only) + +The app's operators can pull the list of accounts that signed up with an email, +without touching the database: + +```sh +moshcode export users -o users.csv # email, display_name, created_at, id, signup_method +moshcode export users --format json -o users.json # the same rows, plus counts +moshcode export users --clean -o users.csv # only the addresses worth mailing +``` + +Who counts as an operator is decided by the app, not the CLI: the account you +are logged in as (`moshcode login`) must be listed in `ADMIN_EMAILS` on +app.moshcode.sh. Everyone else gets a 403. Accounts with no email (passkey or +CoinPay sign-ups that never added one) are counted, never listed. + +`--clean` pipes the CSV through `email-cleaner` from +[cli-tools](https://github.com/profullstack/cli-tools) +(`moshcode install cli-tools`). If it is not on your PATH the export fails +instead of quietly skipping the cleaning. The valid rows come out in the same +columns; with `-o users.csv` the rejected ones go to `users.rejected.csv` with a +`reasons` column. Any of email-cleaner's own flags given after `--clean` +(`--allow-role`, `--allow-disposable`, `--allow-duplicates`, `--allow-unlikely`, +`--allow-no-website`, `--no-dns`, `--fix-typos`) are passed straight through. + +Files are written owner-readable only (0600), and a file already at that path +is copied to `.bak-NNN.csv` first. Counts and paths go to stderr, so +stdout stays pure CSV for a pipe. In the pit, `/export users` always writes a +file (`~/.moshcode/exports/` unless you give `-o`) and prints only where it +went and the counts, never the addresses. + ## Browser terminal (`moshcode console`) A real terminal in the browser — arrow keys, history, full-screen TUIs — because diff --git a/apps/pwa/.env.example b/apps/pwa/.env.example index 062236d9..9e32662f 100644 --- a/apps/pwa/.env.example +++ b/apps/pwa/.env.example @@ -11,6 +11,8 @@ DATABASE_AUTH_TOKEN= # auth SESSION_SECRET=change-me-32-bytes-hex +# operators: account emails allowed to call /api/admin/* (comma-separated; unset = nobody) +ADMIN_EMAILS= # CLI ↔ app signed approval ingest (must match the moshcode CLI's MOSHCODE_WEBHOOK_SECRET) MOSHCODE_WEBHOOK_SECRET=change-me diff --git a/apps/pwa/README.md b/apps/pwa/README.md index 78e68e0c..f2800a8d 100644 --- a/apps/pwa/README.md +++ b/apps/pwa/README.md @@ -32,6 +32,9 @@ CLI: `doppler run -- npm start`. + token), `SESSION_SECRET`, `MOSHCODE_WEBHOOK_SECRET`, `RESEND_API_KEY`, `PUBLIC_ORIGIN=https://app.moshcode.sh`, `MCP_PUBLIC_ORIGIN=https://moshcode.sh`, and the `COINPAY_*` values. +- `ADMIN_EMAILS`: comma-separated account emails allowed to call `/api/admin/*` + (the operator user export behind `moshcode export users`). Unset means + nobody: every admin route answers 403. - Point the domain **app.moshcode.sh** at the service. Root `moshcode.sh` stays a marketing site, but must proxy `/.well-known/oauth-*`, `/oauth/*`, `/device`, and `/api/v1/mcp/*` to this service so canonical MCP URLs work at the apex. @@ -51,6 +54,7 @@ CLI: `doppler run -- npm start`. | `GET/POST /oauth/authorize` | user | authorize one MCP client, share, and scope set | | `POST /oauth/device_authorization` | MCP client | begin RFC 8628 device authorization | | `POST /webhooks/coinpay` | CoinPay | confirm a top-up → credit balance | +| `GET /api/admin/users/export?format=csv\|json` | operator (Bearer key or session, `ADMIN_EMAILS`) | every account with an email: email, display_name, created_at, id, signup_method; counts of the rest | | `GET /healthz` | Railway | health check | ## Wiring the CLI diff --git a/apps/pwa/src/config.mjs b/apps/pwa/src/config.mjs index c3d22c5d..e2ae91c6 100644 --- a/apps/pwa/src/config.mjs +++ b/apps/pwa/src/config.mjs @@ -149,5 +149,21 @@ export const config = { get coinpayLoginEnabled() { return Boolean(this.coinpay.oauth.authorizeUrl && this.coinpay.oauth.clientId); }, + /** + * Operators: the accounts allowed to reach /api/admin/*. + * + * `ADMIN_EMAILS`, a comma- or space-separated list of account emails. Read + * on every call rather than once at boot, so a test (or a restart-free env + * change on a dev box) sees the current value. Unset means nobody is an + * operator, which is the safe default: the admin routes answer 403 to all. + */ + get adminEmails() { + return parseAdminEmails(process.env.ADMIN_EMAILS); + }, secure: (process.env.NODE_ENV || "development") === "production", }; + +/** `"a@x.com, B@y.com"` to a Set of lowercased addresses. Blank entries dropped. */ +export function parseAdminEmails(value = "") { + return new Set(String(value || "").split(/[\s,;]+/).map((s) => s.trim().toLowerCase()).filter((s) => s.includes("@"))); +} diff --git a/apps/pwa/src/lib/user-export.mjs b/apps/pwa/src/lib/user-export.mjs new file mode 100644 index 00000000..a37b05df --- /dev/null +++ b/apps/pwa/src/lib/user-export.mjs @@ -0,0 +1,94 @@ +// The operator's user export: who signed up, with what, and when. +// +// Deliberately narrow. The query below names the columns it reads, and the two +// secrets on the users row (password_hash, coinpay_sub) are only ever asked +// about as "is it set", never selected. A `SELECT *` here would put a password +// hash one refactor away from a CSV on somebody's laptop. +import { all } from "../db.mjs"; +import { config } from "../config.mjs"; + +/** Column order of the CSV, and of each JSON row. The CLI relies on it. */ +export const EXPORT_COLUMNS = ["email", "display_name", "created_at", "id", "signup_method"]; + +/** Is this account an operator? Only an allowlisted email is. */ +export function isAdmin(user, allow = config.adminEmails) { + const email = String(user?.email || "").trim().toLowerCase(); + return Boolean(email) && allow.has(email); +} + +/** + * How the account was created, from what the row carries. + * + * CoinPay first: a CoinPay account never has a password, and its email (when + * it has one) was added later. Then password, which only the email form sets. + * An account with neither was created by a passkey ceremony; "unknown" is for + * a row that has none of the three, which should not exist but would otherwise + * be reported as something it is not. + */ +export function signupMethod({ has_coinpay, has_password, has_passkey }) { + if (Number(has_coinpay)) return "coinpay"; + if (Number(has_password)) return "password"; + if (Number(has_passkey)) return "passkey"; + return "unknown"; +} + +/** + * One CSV field. Quoted when it has to be, and a display name that starts like + * a spreadsheet formula gets a leading apostrophe: names are typed by strangers, + * and this file is exactly the kind that gets opened in a spreadsheet. + */ +export function csvField(value, { neutralize = false } = {}) { + let s = value === null || value === undefined ? "" : String(value); + if (neutralize && /^[=+\-@\t\r]/.test(s)) s = `'${s}`; + return /[",\r\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; +} + +/** Rows (objects keyed by EXPORT_COLUMNS, plus any extra columns) to CSV text. */ +export function toCsv(rows, columns = EXPORT_COLUMNS) { + const lines = [columns.join(",")]; + for (const row of rows) { + lines.push(columns.map((c) => csvField(row[c], { neutralize: c === "display_name" })).join(",")); + } + return `${lines.join("\n")}\n`; +} + +/** + * Every account with an email, oldest first, plus counts of the ones without. + * + * `{ users: [{ email, display_name, created_at, id, signup_method }], counts }` + * where created_at is ISO 8601. Accounts without an email are counted, by + * signup method, and never listed: there is nothing on them to export. + */ +export async function exportUsers() { + const rows = await all( + `SELECT u.id, u.email, u.display_name, u.created_at, + CASE WHEN u.password_hash IS NOT NULL AND u.password_hash <> '' THEN 1 ELSE 0 END AS has_password, + CASE WHEN u.coinpay_sub IS NOT NULL AND u.coinpay_sub <> '' THEN 1 ELSE 0 END AS has_coinpay, + CASE WHEN EXISTS (SELECT 1 FROM webauthn_credentials w WHERE w.user_id = u.id) THEN 1 ELSE 0 END AS has_passkey + FROM users u + ORDER BY u.created_at ASC, u.id ASC`, + ); + const users = []; + const withoutEmail = { total: 0, password: 0, passkey: 0, coinpay: 0, unknown: 0 }; + for (const r of rows) { + const method = signupMethod(r); + const email = String(r.email ?? "").trim(); + if (!email) { + withoutEmail.total += 1; + withoutEmail[method] += 1; + continue; + } + const created = Number(r.created_at); + users.push({ + email, + display_name: r.display_name ?? "", + created_at: Number.isFinite(created) ? new Date(created).toISOString() : "", + id: r.id, + signup_method: method, + }); + } + return { + users, + counts: { total: rows.length, with_email: users.length, without_email: withoutEmail.total, without_email_by_method: withoutEmail }, + }; +} diff --git a/apps/pwa/src/routes/admin.mjs b/apps/pwa/src/routes/admin.mjs new file mode 100644 index 00000000..9a0a4663 --- /dev/null +++ b/apps/pwa/src/routes/admin.mjs @@ -0,0 +1,52 @@ +// Operator-only routes. +// +// GET /api/admin/users/export?format=csv|json every account with an email +// +// "Operator" is an allowlist, ADMIN_EMAILS, checked against the account behind +// the request: a CLI API key (what `moshcode export users` sends) or the +// browser session. Nobody else gets anything but a 403, and an unset allowlist +// means that is everybody. +import { Router } from "express"; +import { bearer, userForApiKey } from "../lib/apikey.mjs"; +import { EXPORT_COLUMNS, exportUsers, isAdmin, toCsv } from "../lib/user-export.mjs"; + +export const adminRouter = Router(); + +/** + * Resolve the caller and require an operator. + * + * A Bearer header is authoritative when present: a request that sends a bad + * key is refused even if it also carries a session cookie, so a script never + * succeeds by accident on the strength of a browser login. + */ +export async function requireAdmin(req, res, next) { + try { + const sent = bearer(req); + const user = sent ? await userForApiKey(sent) : req.user; + if (!user) return res.status(401).json({ error: "sign in, or send an API key: Authorization: Bearer mck_..." }); + if (!isAdmin(user)) return res.status(403).json({ error: "operator access is required" }); + req.adminUser = user; + next(); + } catch (error) { + next(error); + } +} + +adminRouter.get("/api/admin/users/export", requireAdmin, async (req, res, next) => { + try { + const format = String(req.query.format || "csv").toLowerCase(); + if (!["csv", "json"].includes(format)) return res.status(400).json({ error: "format must be csv or json" }); + const { users, counts } = await exportUsers(); + // An address list is the last thing a shared cache should keep. + res.set("Cache-Control", "no-store"); + console.log(`[admin] ${req.adminUser.email} exported ${users.length} users (${format})`); + if (format === "json") return res.json({ columns: EXPORT_COLUMNS, users, counts }); + res.set("X-Users-Total", String(counts.total)); + res.set("X-Users-With-Email", String(counts.with_email)); + res.set("X-Users-Without-Email", String(counts.without_email)); + res.set("Content-Disposition", `attachment; filename="moshcode-users-${new Date().toISOString().slice(0, 10)}.csv"`); + res.type("text/csv; charset=utf-8").send(toCsv(users)); + } catch (error) { + next(error); + } +}); diff --git a/apps/pwa/src/server.mjs b/apps/pwa/src/server.mjs index f928b8a4..d7ef306f 100644 --- a/apps/pwa/src/server.mjs +++ b/apps/pwa/src/server.mjs @@ -19,6 +19,7 @@ import { pagesRouter } from "./routes/pages.mjs"; import { settingsSyncRouter } from "./routes/settings-sync.mjs"; import { moshpitRouter } from "./routes/moshpit.mjs"; import { socialsRouter } from "./routes/socials.mjs"; +import { adminRouter } from "./routes/admin.mjs"; import { MAX_BATCH, MAX_PUBLISH_BYTES } from "./lib/moshpit-content.mjs"; import { endExpiredLeases, expireOffers } from "./moshpit.mjs"; @@ -81,6 +82,7 @@ app.use(coinpayRouter); app.use(approvalsRouter); app.use(creditsRouter); app.use(cliRouter); // /cli/authorize, /cli/token, /api/me +app.use(adminRouter); // /api/admin/*, ADMIN_EMAILS operators only app.use(sessionsRouter); // /sessions (live CLI mirror) + /api/sessions app.use(organizationsRouter); // organizations, teams, membership, and session sharing app.use(mcpOAuthBrowserRouter); // /oauth/authorize — logged-in consent + CSRF diff --git a/apps/pwa/test/admin-users-export.test.mjs b/apps/pwa/test/admin-users-export.test.mjs new file mode 100644 index 00000000..e0b5d07d --- /dev/null +++ b/apps/pwa/test/admin-users-export.test.mjs @@ -0,0 +1,122 @@ +// GET /api/admin/users/export: operators only, and never a secret column. +// +// Boots the real router against a throwaway libsql file, the same way +// settings-sync.test.mjs does, and skips cleanly when the PWA dependencies are +// not installed (the root `node --test` in CI never installs them). +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { createRequire } from "node:module"; +import test from "node:test"; + +const require = createRequire(import.meta.url); +let deps = null; +try { + deps = { express: require("express"), cookieParser: require("cookie-parser") }; +} catch { + deps = null; +} +const skip = deps ? false : "PWA dependencies are not installed"; + +const workdir = mkdtempSync(path.join(tmpdir(), "moshcode-admin-export-test-")); +process.env.DATABASE_URL = `file:${path.join(workdir, "test.db")}`; +process.env.SESSION_SECRET = "test-secret"; + +let booted = null; +async function boot() { + if (booted) return booted; + const { migrate } = await import("../src/migrate.mjs"); + await migrate(); + const { run } = await import("../src/db.mjs"); + const { sessionMiddleware, csrfGuard } = await import("../src/lib/session.mjs"); + const { adminRouter } = await import("../src/routes/admin.mjs"); + const { createApiKey } = await import("../src/lib/apikey.mjs"); + + const app = deps.express(); + app.use(deps.express.json()); + app.use(deps.cookieParser()); + app.use(sessionMiddleware); + app.use(csrfGuard); + app.use(adminRouter); + const server = await new Promise((resolve) => { const s = app.listen(0, "127.0.0.1", () => resolve(s)); }); + const base = `http://127.0.0.1:${server.address().port}`; + + // The operator, signed up with a password. + await run(`INSERT INTO users (id, email, password_hash, display_name, created_at) VALUES ('op','Boss@Example.com','scrypt$SECRETHASH','boss',1700000000000)`); + // A regular user. + await run(`INSERT INTO users (id, email, password_hash, display_name, created_at) VALUES ('u1','fan@example.org','scrypt$OTHERHASH','=cmd|calc,"x"',1700000001000)`); + // A passkey account that later added an email. + await run(`INSERT INTO users (id, email, display_name, created_at) VALUES ('pk','key@example.net','keys',1700000002000)`); + await run(`INSERT INTO webauthn_credentials (id, user_id, public_key, created_at) VALUES ('cred1','pk','pub',1)`); + // CoinPay accounts: one with an email, one without. + await run(`INSERT INTO users (id, email, coinpay_sub, display_name, created_at) VALUES ('cp','coin@example.com','coinpay-SUB-SECRET','coin',1700000003000)`); + await run(`INSERT INTO users (id, coinpay_sub, display_name, created_at) VALUES ('cp2','coinpay-SUB-2','anon',1700000004000)`); + // A passkey account with no email at all. + await run(`INSERT INTO users (id, display_name, created_at) VALUES ('pk2','ghost',1700000005000)`); + await run(`INSERT INTO webauthn_credentials (id, user_id, public_key, created_at) VALUES ('cred2','pk2','pub',1)`); + + const opKey = (await createApiKey("op", "cli")).plaintext; + const userKey = (await createApiKey("u1", "cli")).plaintext; + const call = (key, route) => fetch(`${base}${route}`, { headers: key ? { authorization: `Bearer ${key}` } : {} }); + booted = { server, call, opKey, userKey }; + return booted; +} + +test.after(() => booted?.server.close()); + +test("no key is a 401, a non-operator is a 403, and an unset allowlist refuses everyone", { skip }, async () => { + const { call, opKey, userKey } = await boot(); + process.env.ADMIN_EMAILS = "boss@example.com"; + assert.equal((await call(null, "/api/admin/users/export")).status, 401); + assert.equal((await call("mck_not_a_key", "/api/admin/users/export")).status, 401); + const denied = await call(userKey, "/api/admin/users/export?format=json"); + assert.equal(denied.status, 403); + const body = await denied.text(); + assert.ok(!body.includes("@"), "a refusal must not leak an address"); + + delete process.env.ADMIN_EMAILS; + assert.equal((await call(opKey, "/api/admin/users/export")).status, 403, "unset ADMIN_EMAILS means nobody is an operator"); + process.env.ADMIN_EMAILS = "someone-else@example.com"; + assert.equal((await call(opKey, "/api/admin/users/export")).status, 403); +}); + +test("the CSV has the documented columns, only emailed rows, and no secrets", { skip }, async () => { + const { call, opKey } = await boot(); + process.env.ADMIN_EMAILS = " other@x.io , BOSS@example.com "; + const res = await call(opKey, "/api/admin/users/export?format=csv"); + assert.equal(res.status, 200); + assert.match(res.headers.get("content-type"), /text\/csv/); + assert.equal(res.headers.get("cache-control"), "no-store"); + assert.equal(res.headers.get("x-users-total"), "6"); + assert.equal(res.headers.get("x-users-without-email"), "2"); + const text = await res.text(); + const lines = text.trimEnd().split("\n"); + assert.equal(lines[0], "email,display_name,created_at,id,signup_method"); + assert.equal(lines.length, 5, "four accounts have an email"); + assert.equal(lines[1], "Boss@Example.com,boss,2023-11-14T22:13:20.000Z,op,password"); + // A formula-looking name is neutralised and quoted, not executed. + assert.equal(lines[2], `fan@example.org,"'=cmd|calc,""x""",2023-11-14T22:13:21.000Z,u1,password`); + assert.equal(lines[3], "key@example.net,keys,2023-11-14T22:13:22.000Z,pk,passkey"); + assert.equal(lines[4], "coin@example.com,coin,2023-11-14T22:13:23.000Z,cp,coinpay"); + for (const secret of ["SECRETHASH", "OTHERHASH", "coinpay-SUB", "password_hash", "coinpay_sub"]) { + assert.ok(!text.includes(secret), `the export leaked ${secret}`); + } +}); + +test("JSON carries the same rows plus counts of users without an email", { skip }, async () => { + const { call, opKey } = await boot(); + process.env.ADMIN_EMAILS = "boss@example.com"; + const res = await call(opKey, "/api/admin/users/export?format=json"); + assert.equal(res.status, 200); + const body = await res.json(); + assert.deepEqual(body.columns, ["email", "display_name", "created_at", "id", "signup_method"]); + assert.equal(body.users.length, 4); + for (const u of body.users) assert.deepEqual(Object.keys(u), body.columns); + assert.deepEqual(body.counts, { + total: 6, with_email: 4, without_email: 2, + without_email_by_method: { total: 2, password: 0, passkey: 1, coinpay: 1, unknown: 0 }, + }); + assert.ok(!JSON.stringify(body).includes("SECRET")); + assert.equal((await call(opKey, "/api/admin/users/export?format=xml")).status, 400); +}); diff --git a/bin/moshcode.mjs b/bin/moshcode.mjs index 9c50c603..d9506484 100755 --- a/bin/moshcode.mjs +++ b/bin/moshcode.mjs @@ -747,6 +747,13 @@ async function main() { return; } if (cmd === "logout") { logout(); return; } + // Operator-only, and the app is what enforces that: a non-operator key gets + // a 403 back. Lazy like fleet: nothing else needs it. + if (cmd === "export") { + const { exportCommand } = await import("../src/export-users.mjs"); + process.exitCode = await exportCommand(rest); + return; + } if (cmd === "save") { process.exitCode = await saveCommand(rest); return; } if (cmd === "load") { process.exitCode = await loadCommand(rest); return; } if (cmd === "run") { diff --git a/src/cli-schema.mjs b/src/cli-schema.mjs index c3bd4438..b0e3b7bd 100644 --- a/src/cli-schema.mjs +++ b/src/cli-schema.mjs @@ -470,6 +470,38 @@ export const CORE_CLI_COMMANDS = [ note: "refuses rather than overwriting a local file you edited since the last sync — " + "`moshcode save` to keep it, or --force to replace it.", }, + { + name: "export", + group: "account", + description: "operator only: export the app's users as CSV, optionally cleaned", + synopsis: [ + ["moshcode export users [--format csv|json] [-o file]", ""], + ["moshcode export users --clean [cleaner flags] [-o file]", ""], + ], + flags: [ + ["--clean", "run the list through cli-tools' email-cleaner and keep the valid rows", ""], + ["--format ", "output format", "csv"], + ["-o, --output ", "write here (mode 0600) instead of stdout; an existing file is backed up first", ""], + ["--allow-role", "email-cleaner: keep role addresses (after --clean)", ""], + ["--allow-disposable", "email-cleaner: keep disposable domains (after --clean)", ""], + ["--allow-duplicates", "email-cleaner: keep duplicates (after --clean)", ""], + ["--allow-unlikely", "email-cleaner: keep unlikely-looking addresses (after --clean)", ""], + ["--allow-no-website", "email-cleaner: keep domains with no website (after --clean)", ""], + ["--no-dns", "email-cleaner: skip the MX lookups (after --clean)", ""], + ["--fix-typos", "email-cleaner: correct gmial.com and friends (after --clean)", ""], + ], + examples: [ + ["moshcode export users -o users.csv", "the whole list"], + ["moshcode export users --clean -o users.csv", "plus users.rejected.csv"], + ["moshcode export users --clean --no-dns", "offline, CSV on stdout"], + ], + seeAlso: ["login", "whoami"], + note: "needs `moshcode login` as an account listed in ADMIN_EMAILS on app.moshcode.sh; everyone else gets a 403. " + + "columns: email, display_name, created_at (ISO), id, signup_method (password, passkey or coinpay). " + + "accounts without an email are counted, never listed. --clean fails rather than skipping when email-cleaner " + + "is not on PATH (`moshcode install cli-tools`). counts go to stderr, so stdout stays pure CSV. " + + "in the pit, /export users always writes a file (~/.moshcode/exports/ unless -o) and prints only its path and the counts.", + }, { name: "console", group: "account", @@ -1793,6 +1825,8 @@ export const PIT_COMMANDS = [ { name: "whoami", cli: "whoami", description: "who this machine is logged in as" }, // Dispatched since forever and missing from /help until now. { name: "logout", cli: "logout", description: "clear the logged-in account" }, + { name: "export", args: "users [--clean] [--format csv|json] [-o file]", cli: "export", + description: "operator only: export users to a file, optionally cleaned" }, { name: "save", args: "[--dry-run] [--force]", cli: "save", description: "save this pit's settings to your moshcode.sh account" }, { name: "load", args: "[--dry-run] [--force]", cli: "load", diff --git a/src/export-users.mjs b/src/export-users.mjs new file mode 100644 index 00000000..c2036307 --- /dev/null +++ b/src/export-users.mjs @@ -0,0 +1,308 @@ +// `moshcode export users [--clean]` and `/export users`: the operator's list of +// signed-up accounts, from app.moshcode.sh's /api/admin/users/export. +// +// The app decides who may have it (ADMIN_EMAILS there); this side only asks +// with the logged-in account's key, so a non-operator gets the app's 403 and +// nothing else. `--clean` hands the CSV to cli-tools' `email-cleaner` and keeps +// the rows it calls valid. There is no silent fallback: an export that says it +// was cleaned and was not is worse than one that fails. +// +// Addresses never reach the screen from the pit. The CLI writes CSV to stdout +// only when asked to (no -o, not in the pit), so it can be piped; every human +// line (counts, where the file went) goes to stderr. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { loadCreds } from "./auth.mjs"; + +/** Column order of the export, matching the app's EXPORT_COLUMNS. */ +export const EXPORT_COLUMNS = ["email", "display_name", "created_at", "id", "signup_method"]; + +/** email-cleaner's own flags that `--clean` passes through untouched. */ +export const CLEANER_FLAGS = [ + "--allow-role", "--allow-disposable", "--allow-duplicates", "--allow-unlikely", + "--allow-no-website", "--no-dns", "--fix-typos", +]; + +const USAGE = "usage: export users [--clean [cleaner flags]] [--format csv|json] [-o file]"; +const DEFAULT_API = "https://app.moshcode.sh"; + +/* ------------------------------------------------------------------ parsing */ + +export function parseExportArgs(argv = []) { + const opts = { subject: null, clean: false, format: "csv", output: null, cleanerFlags: [], error: null }; + const fail = (msg) => ({ ...opts, error: msg }); + for (let i = 0; i < argv.length; i++) { + const a = String(argv[i]); + if (a === "--clean") opts.clean = true; + else if (a === "--format" || a.startsWith("--format=")) { + const v = a.includes("=") ? a.slice(a.indexOf("=") + 1) : argv[++i]; + if (!["csv", "json"].includes(String(v || "").toLowerCase())) return fail(`--format takes csv or json. ${USAGE}`); + opts.format = String(v).toLowerCase(); + } else if (a === "-o" || a === "--output" || a.startsWith("--output=")) { + const v = a.includes("=") ? a.slice(a.indexOf("=") + 1) : argv[++i]; + if (!v || String(v).startsWith("-")) return fail(`${a} needs a file name. ${USAGE}`); + opts.output = String(v); + } else if (CLEANER_FLAGS.includes(a)) { + if (!opts.clean) return fail(`${a} is an email-cleaner flag, so it goes after --clean. ${USAGE}`); + if (!opts.cleanerFlags.includes(a)) opts.cleanerFlags.push(a); + } else if (a.startsWith("-")) return fail(`unknown option ${a}. ${USAGE}`); + else if (!opts.subject) opts.subject = a.toLowerCase(); + else return fail(`unexpected argument ${a}. ${USAGE}`); + } + if (!opts.subject) return fail(USAGE); + if (opts.subject !== "users") return fail(`nothing called "${opts.subject}" to export; only users. ${USAGE}`); + return opts; +} + +/* --------------------------------------------------------------------- CSV */ + +export function csvField(value, { neutralize = false } = {}) { + let s = value === null || value === undefined ? "" : String(value); + if (neutralize && /^[=+\-@\t\r]/.test(s)) s = `'${s}`; + return /[",\r\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; +} + +export function toCsv(rows, columns = EXPORT_COLUMNS) { + const lines = [columns.join(",")]; + for (const row of rows) { + lines.push(columns.map((c) => csvField(row[c], { neutralize: c === "display_name" })).join(",")); + } + return `${lines.join("\n")}\n`; +} + +/* ----------------------------------------------------------- email-cleaner */ + +/** The first executable called `name` on `PATH`, or null. */ +export function findOnPath(name, envPath = process.env.PATH || "") { + for (const dir of String(envPath).split(path.delimiter).filter(Boolean)) { + const candidate = path.join(dir, name); + try { + if (fs.statSync(candidate).isFile()) { + fs.accessSync(candidate, fs.constants.X_OK); + return candidate; + } + } catch { /* keep looking */ } + } + return null; +} + +const lower = (s) => String(s ?? "").trim().toLowerCase(); + +/** + * Pair email-cleaner's verdicts with the rows that went in. + * + * The cleaner reports `input` (what it read), `email` (what it made of it, + * which `--fix-typos` may have corrected) and sometimes `row`. Matching is by + * the address first, because that is unambiguous whatever `row` counts from, + * and then by `row` read as a data row, a line number, or a zero-based index, + * accepted only when that row's address agrees. A verdict that matches no row + * still comes out, carrying just its address, rather than vanishing. + */ +export function matchCleaned(rows, result) { + const byEmail = new Map(rows.map((r, i) => [lower(r.email), i])); + const used = new Set(); + const locate = (entry) => { + for (const key of [lower(entry.input), lower(entry.email)]) { + const i = byEmail.get(key); + if (i !== undefined && !used.has(i)) return i; + } + const inputText = lower(entry.input); + const n = Number(entry.row); + if (Number.isInteger(n)) { + for (const i of [n - 1, n - 2, n]) { + const r = rows[i]; + if (!r || used.has(i)) continue; + const e = lower(r.email); + if (e && (e === lower(entry.email) || e === inputText || inputText.split(/[,;\s"]+/).includes(e))) return i; + } + } + for (const [i, r] of rows.entries()) { + if (!used.has(i) && inputText && inputText.split(/[,;\s"]+/).includes(lower(r.email))) return i; + } + return -1; + }; + const take = (entry) => { + const i = locate(entry); + if (i >= 0) used.add(i); + return i >= 0 ? rows[i] : null; + }; + + const kept = []; + for (const entry of result.valid || []) { + const row = take(entry); + const email = String(entry.email || row?.email || "").trim(); + kept.push(row ? { ...row, email } : { email, display_name: entry.name ?? "", created_at: "", id: "", signup_method: "" }); + } + const rejected = []; + for (const entry of result.invalid || []) { + const row = take(entry); + const reasons = Array.isArray(entry.reasons) ? entry.reasons.map(String) : []; + rejected.push({ + ...(row || { email: String(entry.email || entry.input || ""), display_name: "", created_at: "", id: "", signup_method: "" }), + reasons: reasons.join(";"), + suggestion: entry.suggestion ?? "", + _reasons: reasons, + }); + } + const unaccounted = rows.length - used.size; + return { kept, rejected, unaccounted: unaccounted > 0 ? unaccounted : 0 }; +} + +/** Run email-cleaner over `csv`. `{ ok, result }` or `{ ok: false, error }`. */ +export function runCleaner(bin, csv, flags = [], { spawnImpl = spawnSync, env = process.env } = {}) { + const res = spawnImpl(bin, ["-", "--format", "json", "--report", ...flags], { + input: csv, encoding: "utf8", env, maxBuffer: 512 * 1024 * 1024, + }); + if (res.error) return { ok: false, error: `could not run email-cleaner: ${res.error.message}` }; + let result = null; + try { result = JSON.parse(res.stdout || ""); } catch { result = null; } + if (!result || !Array.isArray(result.valid) || !Array.isArray(result.invalid)) { + const why = String(res.stderr || "").trim().split("\n")[0] || `exit ${res.status ?? res.signal}`; + return { ok: false, error: `email-cleaner did not return its JSON report (${why})` }; + } + return { ok: true, result }; +} + +/* -------------------------------------------------------------------- files */ + +/** Copy `file` to `.bak-NNN` beside it, if it exists. Returns the copy's path. */ +export function backupBeside(file) { + if (!fs.existsSync(file)) return null; + const ext = path.extname(file); + const stem = file.slice(0, file.length - ext.length); + let n = 1; + let backup; + do { backup = `${stem}.bak-${String(n).padStart(3, "0")}${ext}`; n += 1; } while (fs.existsSync(backup)); + fs.copyFileSync(file, backup); + return backup; +} + +/** Write `text` to `file` readable by its owner only, backing up what was there. */ +export function writePrivate(file, text) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const backup = backupBeside(file); + fs.writeFileSync(file, text, { mode: 0o600 }); + fs.chmodSync(file, 0o600); + return backup; +} + +/** `users.csv` to `users.rejected.csv`. */ +export function rejectedPath(file) { + const ext = path.extname(file); + return `${ext ? file.slice(0, -ext.length) : file}.rejected.csv`; +} + +function defaultFile(home, format, now) { + const stamp = now.toISOString().replace(/\.\d+Z$/, "").replace(/[:T]/g, (c) => (c === "T" ? "-" : "")); + return path.join(home, ".moshcode", "exports", `users-${stamp}.${format}`); +} + +/* ------------------------------------------------------------------ command */ + +function tally(rejected) { + const byReason = {}; + for (const r of rejected) { + const reasons = r._reasons.length ? r._reasons : ["unspecified"]; + for (const reason of reasons) byReason[reason] = (byReason[reason] || 0) + 1; + } + return byReason; +} + +const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`; + +/** + * Returns an exit code. `pit` means the session UI: output always goes to a + * file (default ~/.moshcode/exports/users-