proxy: fetch through our paid proxies (@profullstack/proxy), 0.56.0

README.md

@@ -36,6 +36,7 @@ TypeScript, installed as executables on `PATH`.
 | [`openinstall`](#openinstall) | Give a repo an idempotent bin/install.sh: systemd, nginx + TLS, postgres, redis |
 | [`openmcp`](#openmcp) | The OpenMCP catalog of MCP relays: list, find a tool, call it, register your own |
 | [`shorten`](#shorten) | Mint a short link on the pit, and follow it from `/f/<code>` |
+| [`proxy`](#proxy) | Fetch through our paid residential proxies (Proxiware, Webshare); also a local forward proxy, an MCP server and a TUI |
 | [`sysupdate`](#sysupdate) | Update this box: apt lists, apt packages, snaps |
 | [`scorecard`](#scorecard) | One weekly number for the whole fleet: traffic, channels, posts and ads, week over week |
 | [`users-dump`](#users-dump) | Every user account across the fleet, as one CSV |
@@ -452,6 +453,10 @@ carries the same masked previews, not the values.
 | `porkbun_secret` | `PORKBUN_SECRET_API_KEY` | `porkbun` |
 | `moshcode` | `MOSHCODE_API_KEY` | `shorten` |
 | `resend` | `RESEND_API_KEY` | `mail`, as the fallback sender |
+| `proxiware` | `PROXIWARE_API_KEY` | `proxy status` |
+| `proxiware_proxy_user`, `proxiware_proxy_password` | `PROXIWARE_PROXY_USER`, `PROXIWARE_PROXY_PASSWORD` | `proxy` |
+| `webshare` | `WEBSHARE_API_KEY` | `proxy` (status, and the proxy login when no user/password is set) |
+| `webshare_proxy_user`, `webshare_proxy_password` | `WEBSHARE_PROXY_USER`, `WEBSHARE_PROXY_PASSWORD` | `proxy` |
 
 A key earns a row here by being read by a command in this repository, not by
 being a key the team owns. The vault holds more than twice as many; the rest
@@ -2079,6 +2084,36 @@ the only thing on stdout, so it pipes.
 
 The same thing lives inside moshcode as `/shorten`; this is the copy that pipes.
 
+### `proxy`
+
+Call the web through the residential proxies we pay for. Everything past the
+keys is [`@profullstack/proxy`](https://github.com/profullstack/proxy), run
+in-process, so `proxy --help` is its help.
+
+```sh
+proxy https://example.com                    # body to stdout, US residential exit (Proxiware)
+proxy -i -c gb https://example.com           # status + headers, UK exit
+proxy -s job42 --ttl 10 https://example.com  # sticky: same exit IP for 10 minutes
+proxy -p webshare https://example.com        # a specific provider
+proxy ip                                     # 50.25.147.254  US North Carolina …  AS19108 Optimum  (proxiware)
+proxy status                                 # credit, plan, renewal, expiry per provider
+export HTTPS_PROXY="$(proxy url --reveal)"   # hand the URL to another tool
+proxy serve --port 8888                      # local forward proxy holding the credentials + JSON API
+proxy mcp                                    # MCP server on stdio
+proxy tui                                    # accounts and a live exit test
+```
+
+**Keys need no setup on a box that can reach the team vault.** The first run
+with none stored pulls the proxy keys (only those) from the same vault
+`cli-tools config pull` reads, and caches them in the credential store; later
+runs are local. Shell variables still win, one by one. To set one by hand:
+`cli-tools config set proxiware_proxy_password`.
+
+The default User-Agent is `curl/8.5.0 (+@profullstack/proxy)` because ESPN, for
+one, allowlists curl-like agents and refuses browser and runtime defaults
+whatever the exit IP. Exit codes follow curl where it has one: 22 for a non-2xx
+answer, plus 3 for no credentials.
+
 ### `users-dump`
 
 Every user account across the fleet, as one CSV: `name,email,site,last_login,created_at,source`.

bin/cli-tools.ts

@@ -100,6 +100,8 @@ Keys (config set <key>):
   anthropic   ANTHROPIC_API_KEY   generate-names
   perplexity  PERPLEXITY_API_KEY  ask-web
   elevenlabs  ELEVENLABS_API_KEY  tts
+  proxiware   PROXIWARE_API_KEY   proxy (status); proxiware_proxy_user/_password: traffic
+  webshare    WEBSHARE_API_KEY    proxy (status, and the login when no user/password)
 
 Options:
   --force   link: take over a symlink owned by another checkout

bin/proxy.ts

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+/**
+ * proxy — call the web through our paid proxies (Proxiware, Webshare).
+ *
+ *   proxy https://example.com          body to stdout, US residential exit
+ *   proxy -i -c gb https://example.com
+ *   proxy ip | url | status | providers
+ *   proxy serve [--port 8888]          local forward proxy + JSON API
+ *   proxy mcp                          MCP server on stdio
+ *   proxy tui                          accounts and a live exit test
+ *
+ * Everything past the keys is @profullstack/proxy, run in-process: `proxy
+ * --help` is its help. The keys come from the shell, then the credential store,
+ * then (once, cached) the team vault — the same keys `cli-tools config pull`
+ * imports. See src/proxy.ts.
+ */
+
+import { run } from '@profullstack/proxy/cli';
+import { isMain } from '../src/is-main.ts';
+import { proxyEnv } from '../src/proxy.ts';
+
+for (const stream of [process.stdout, process.stderr]) {
+  stream.on('error', (error: NodeJS.ErrnoException) => {
+    if (error.code === 'EPIPE') process.exit(0);
+    throw error;
+  });
+}
+
+if (isMain(import.meta.url)) {
+  const argv = process.argv.slice(2);
+  const quiet = argv.length === 0 || argv.includes('-h') || argv.includes('--help') || argv.includes('-v') || argv.includes('--version');
+  const { env, source } = quiet
+    ? { env: process.env as Record<string, string | undefined>, source: 'env' as const }
+    : proxyEnv({ log: (line) => process.stderr.write(`${line}\n`) });
+
+  if (source === 'none') {
+    process.stderr.write(
+      'proxy: no proxy credentials. Pull them with `cli-tools config pull`, or set one:\n' +
+        '  cli-tools config set proxiware_proxy_user\n' +
+        '  cli-tools config set proxiware_proxy_password\n' +
+        '  cli-tools config set webshare        # Webshare API key (login is read from it)\n',
+    );
+  }
+
+  run(argv, { env })
+    .then((code) => {
+      process.exitCode = code;
+    })
+    .catch((error: Error) => {
+      process.stderr.write(`proxy: ${error.message}\n`);
+      process.exitCode = 1;
+    });
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/cli-tools",
-  "version": "0.55.0",
+  "version": "0.56.0",
   "private": true,
   "description": "Local command-line tools, in TypeScript, exposed on PATH.",
   "type": "module",
@@ -33,6 +33,7 @@
   },
   "dependencies": {
     "@profullstack/hqtui": "^0.6.0",
+    "@profullstack/proxy": "^0.1.0",
     "@resvg/resvg-js": "^2.6.2",
     "axe-core": "^4.13.0",
     "imapflow": "^1.7.8",

src/credentials.ts

@@ -41,6 +41,16 @@
   // ~/.moshcode/credentials.json and that is what gets picked up. This is for a
   // box that has the key but not moshcode.
   moshcode: 'MOSHCODE_API_KEY',
+  // Read by `proxy` (@profullstack/proxy). The proxy login is what traffic
+  // needs; the API keys only add `proxy status`. Webshare can do without its
+  // user/password: given the API key, the package reads the login from the
+  // account, which survives a password change in the dashboard.
+  proxiware: 'PROXIWARE_API_KEY',
+  proxiware_proxy_user: 'PROXIWARE_PROXY_USER',
+  proxiware_proxy_password: 'PROXIWARE_PROXY_PASSWORD',
+  webshare: 'WEBSHARE_API_KEY',
+  webshare_proxy_user: 'WEBSHARE_PROXY_USER',
+  webshare_proxy_password: 'WEBSHARE_PROXY_PASSWORD',
 };
 
 export type Source = 'env' | 'file' | 'unset';

src/proxy.ts

@@ -0,0 +1,92 @@
+/**
+ * Keys for `proxy`, the @profullstack/proxy command.
+ *
+ * The package reads credentials from an environment record and nothing else,
+ * so this side's whole job is to build that record: the shell first, then the
+ * credential store, and on a box that has neither, one pull of the team vault
+ * (the same one `cli-tools config pull` reads), cached so the next run is
+ * local. Only the proxy variables are taken from the pull; the rest of the
+ * vault is not this command's business.
+ */
+
+import { KNOWN_KEYS, loadStored, resolveCredentials, saveStored } from './credentials.ts';
+import { pullVault, vaultTarget, type VaultTarget } from './vault.ts';
+
+export const PROXY_VARIABLES = [
+  'PROXIWARE_API_KEY',
+  'PROXIWARE_PROXY_USER',
+  'PROXIWARE_PROXY_PASSWORD',
+  'WEBSHARE_API_KEY',
+  'WEBSHARE_PROXY_USER',
+  'WEBSHARE_PROXY_PASSWORD',
+] as const;
+
+// The store only keeps what KNOWN_KEYS names; a variable missing from there
+// would be pulled every run and never cached.
+for (const variable of PROXY_VARIABLES) {
+  if (!Object.values(KNOWN_KEYS).includes(variable)) {
+    throw new Error(`proxy: ${variable} is not in KNOWN_KEYS`);
+  }
+}
+
+type Env = Record<string, string | undefined>;
+
+/** Enough to send traffic through at least one provider. */
+export function hasProxyLogin(env: Env): boolean {
+  return Boolean(
+    (env.PROXIWARE_PROXY_USER && env.PROXIWARE_PROXY_PASSWORD) ||
+      (env.WEBSHARE_PROXY_USER && env.WEBSHARE_PROXY_PASSWORD) ||
+      env.WEBSHARE_API_KEY,
+  );
+}
+
+export interface ProxyEnvDeps {
+  env?: NodeJS.ProcessEnv;
+  pull?: (target: VaultTarget) => Record<string, string>;
+  log?: (line: string) => void;
+}
+
+/**
+ * The environment to hand the package: process env, with stored keys for any
+ * proxy variable the shell does not set, pulling the vault once if there is no
+ * usable login at all. Returns the record and where the keys came from.
+ */
+export function proxyEnv(deps: ProxyEnvDeps = {}): { env: Env; source: 'env' | 'store' | 'vault' | 'none' } {
+  const env = deps.env ?? process.env;
+  const log = deps.log ?? (() => {});
+  const merged: Env = { ...env, ...pickDefined(resolveCredentials(env), PROXY_VARIABLES) };
+  for (const variable of PROXY_VARIABLES) if (env[variable]) merged[variable] = env[variable];
+
+  if (hasProxyLogin(merged)) {
+    const fromShell = PROXY_VARIABLES.some((variable) => env[variable]);
+    return { env: merged, source: fromShell ? 'env' : 'store' };
+  }
+
+  const target = vaultTarget(env);
+  log(`proxy: no proxy keys on this machine; pulling them from ${target.team}/${target.project}--${target.env}…`);
+  let vault: Record<string, string>;
+  try {
+    vault = (deps.pull ?? pullVault)(target);
+  } catch (error) {
+    log(`proxy: ${(error as Error).message}`);
+    return { env: merged, source: 'none' };
+  }
+
+  const found = pickDefined(vault, PROXY_VARIABLES);
+  if (Object.keys(found).length === 0) {
+    log(`proxy: ${target.project}--${target.env} has none of ${PROXY_VARIABLES.join(', ')}`);
+    return { env: merged, source: 'none' };
+  }
+  const path = saveStored({ ...loadStored(env), ...found }, env);
+  log(`proxy: cached ${Object.keys(found).join(', ')} in ${path}`);
+  return { env: { ...merged, ...found, ...pickDefined(env, PROXY_VARIABLES) }, source: 'vault' };
+}
+
+function pickDefined(record: Env, keys: readonly string[]): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const key of keys) {
+    const value = record[key];
+    if (value) out[key] = value;
+  }
+  return out;
+}

src/registry.ts

@@ -62,6 +62,7 @@ const SUMMARIES: Record<string, string> = {
   'gh-pulse': 'What moved on GitHub, last hour to all time, ranked, with traffic: by email, TUI or JSON',
   hqtui: 'Every vital of this box, in the terminal: sockets, HTTP, sessions, services',
   porkbun: 'Read and change DNS at Porkbun, and un-park a domain',
+  proxy: 'Fetch through our paid proxies (Proxiware, Webshare); serve, MCP and TUI too',
   scorecard: 'One weekly number for the whole fleet: traffic, channels, posts and ads, week over week',
   shorten: 'Mint a short link on the pit, and follow it from /f/<code>',
   sysupdate: 'Update this box: apt lists, apt packages, snaps',