diff --git a/README.md b/README.md index a223c41..9005bd4 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ TypeScript, installed as executables on `PATH`. | [`openinstall`](#openinstall) | Give a repo an idempotent bin/install.sh: systemd, nginx + TLS, postgres, redis | | [`openmcp`](#openmcp) | The OpenMCP catalog of MCP relays: list, find a tool, call it, register your own | | [`shorten`](#shorten) | Mint a short link on the pit, and follow it from `/f/` | +| [`proxy`](#proxy) | Fetch through our paid residential proxies (Proxiware, Webshare); also a local forward proxy, an MCP server and a TUI | | [`sysupdate`](#sysupdate) | Update this box: apt lists, apt packages, snaps | | [`scorecard`](#scorecard) | One weekly number for the whole fleet: traffic, channels, posts and ads, week over week | | [`users-dump`](#users-dump) | Every user account across the fleet, as one CSV | @@ -452,6 +453,10 @@ carries the same masked previews, not the values. | `porkbun_secret` | `PORKBUN_SECRET_API_KEY` | `porkbun` | | `moshcode` | `MOSHCODE_API_KEY` | `shorten` | | `resend` | `RESEND_API_KEY` | `mail`, as the fallback sender | +| `proxiware` | `PROXIWARE_API_KEY` | `proxy status` | +| `proxiware_proxy_user`, `proxiware_proxy_password` | `PROXIWARE_PROXY_USER`, `PROXIWARE_PROXY_PASSWORD` | `proxy` | +| `webshare` | `WEBSHARE_API_KEY` | `proxy` (status, and the proxy login when no user/password is set) | +| `webshare_proxy_user`, `webshare_proxy_password` | `WEBSHARE_PROXY_USER`, `WEBSHARE_PROXY_PASSWORD` | `proxy` | A key earns a row here by being read by a command in this repository, not by being a key the team owns. The vault holds more than twice as many; the rest @@ -2079,6 +2084,36 @@ the only thing on stdout, so it pipes. The same thing lives inside moshcode as `/shorten`; this is the copy that pipes. +### `proxy` + +Call the web through the residential proxies we pay for. Everything past the +keys is [`@profullstack/proxy`](https://github.com/profullstack/proxy), run +in-process, so `proxy --help` is its help. + +```sh +proxy https://example.com # body to stdout, US residential exit (Proxiware) +proxy -i -c gb https://example.com # status + headers, UK exit +proxy -s job42 --ttl 10 https://example.com # sticky: same exit IP for 10 minutes +proxy -p webshare https://example.com # a specific provider +proxy ip # 50.25.147.254 US North Carolina … AS19108 Optimum (proxiware) +proxy status # credit, plan, renewal, expiry per provider +export HTTPS_PROXY="$(proxy url --reveal)" # hand the URL to another tool +proxy serve --port 8888 # local forward proxy holding the credentials + JSON API +proxy mcp # MCP server on stdio +proxy tui # accounts and a live exit test +``` + +**Keys need no setup on a box that can reach the team vault.** The first run +with none stored pulls the proxy keys (only those) from the same vault +`cli-tools config pull` reads, and caches them in the credential store; later +runs are local. Shell variables still win, one by one. To set one by hand: +`cli-tools config set proxiware_proxy_password`. + +The default User-Agent is `curl/8.5.0 (+@profullstack/proxy)` because ESPN, for +one, allowlists curl-like agents and refuses browser and runtime defaults +whatever the exit IP. Exit codes follow curl where it has one: 22 for a non-2xx +answer, plus 3 for no credentials. + ### `users-dump` Every user account across the fleet, as one CSV: `name,email,site,last_login,created_at,source`. diff --git a/bin/cli-tools.ts b/bin/cli-tools.ts index b107edc..d20fa6d 100755 --- a/bin/cli-tools.ts +++ b/bin/cli-tools.ts @@ -100,6 +100,8 @@ Keys (config set ): anthropic ANTHROPIC_API_KEY generate-names perplexity PERPLEXITY_API_KEY ask-web elevenlabs ELEVENLABS_API_KEY tts + proxiware PROXIWARE_API_KEY proxy (status); proxiware_proxy_user/_password: traffic + webshare WEBSHARE_API_KEY proxy (status, and the login when no user/password) Options: --force link: take over a symlink owned by another checkout diff --git a/bin/proxy.ts b/bin/proxy.ts new file mode 100755 index 0000000..35c5927 --- /dev/null +++ b/bin/proxy.ts @@ -0,0 +1,53 @@ +#!/usr/bin/env node +/** + * proxy — call the web through our paid proxies (Proxiware, Webshare). + * + * proxy https://example.com body to stdout, US residential exit + * proxy -i -c gb https://example.com + * proxy ip | url | status | providers + * proxy serve [--port 8888] local forward proxy + JSON API + * proxy mcp MCP server on stdio + * proxy tui accounts and a live exit test + * + * Everything past the keys is @profullstack/proxy, run in-process: `proxy + * --help` is its help. The keys come from the shell, then the credential store, + * then (once, cached) the team vault — the same keys `cli-tools config pull` + * imports. See src/proxy.ts. + */ + +import { run } from '@profullstack/proxy/cli'; +import { isMain } from '../src/is-main.ts'; +import { proxyEnv } from '../src/proxy.ts'; + +for (const stream of [process.stdout, process.stderr]) { + stream.on('error', (error: NodeJS.ErrnoException) => { + if (error.code === 'EPIPE') process.exit(0); + throw error; + }); +} + +if (isMain(import.meta.url)) { + const argv = process.argv.slice(2); + const quiet = argv.length === 0 || argv.includes('-h') || argv.includes('--help') || argv.includes('-v') || argv.includes('--version'); + const { env, source } = quiet + ? { env: process.env as Record, source: 'env' as const } + : proxyEnv({ log: (line) => process.stderr.write(`${line}\n`) }); + + if (source === 'none') { + process.stderr.write( + 'proxy: no proxy credentials. Pull them with `cli-tools config pull`, or set one:\n' + + ' cli-tools config set proxiware_proxy_user\n' + + ' cli-tools config set proxiware_proxy_password\n' + + ' cli-tools config set webshare # Webshare API key (login is read from it)\n', + ); + } + + run(argv, { env }) + .then((code) => { + process.exitCode = code; + }) + .catch((error: Error) => { + process.stderr.write(`proxy: ${error.message}\n`); + process.exitCode = 1; + }); +} diff --git a/package.json b/package.json index 17955d5..3ea6d12 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/cli-tools", - "version": "0.55.0", + "version": "0.56.0", "private": true, "description": "Local command-line tools, in TypeScript, exposed on PATH.", "type": "module", @@ -33,6 +33,7 @@ }, "dependencies": { "@profullstack/hqtui": "^0.6.0", + "@profullstack/proxy": "^0.1.0", "@resvg/resvg-js": "^2.6.2", "axe-core": "^4.13.0", "imapflow": "^1.7.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5f59bf1..9169f9a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@profullstack/hqtui': specifier: ^0.6.0 version: 0.6.0 + '@profullstack/proxy': + specifier: ^0.1.0 + version: 0.1.0 '@resvg/resvg-js': specifier: ^2.6.2 version: 2.6.2 @@ -390,6 +393,11 @@ packages: engines: {bun: '>=1.1', node: '>=22.6'} hasBin: true + '@profullstack/proxy@0.1.0': + resolution: {integrity: sha512-XobCE8NPNAzLcSEC8e8yXZI53/LCGLRTP3BLJKTIsQOHeyNxnH1oyKUSDB7SE1TwmWnCQnvjCkFLFMeKxshoaw==} + engines: {node: '>=22.19'} + hasBin: true + '@resvg/resvg-js-android-arm-eabi@2.6.2': resolution: {integrity: sha512-FrJibrAk6v29eabIPgcTUMPXiEz8ssrAk7TXxsiZzww9UTQ1Z5KAbFJs+Z0Ez+VZTYgnE5IQJqBcoSiMebtPHA==} engines: {node: '>= 10'} @@ -994,6 +1002,10 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici@8.11.2: + resolution: {integrity: sha512-u4UB2/IrKdU6lFxumHmmo1a3fCQO5tzQllRorfoRS63txhrB7xTpSn1PftwC4qEHkOaqP95fCWW4lJzwErwzhQ==} + engines: {node: '>=22.19.0'} + vite@8.2.1: resolution: {integrity: sha512-EU/eS7BH3XROHh2YnBefjM6DBKA6ZeMZEYQbj7NLWg5wHYlhB8B/Mayd5XsgWq+NFYccDOTemRpdETWR6Ka/lw==} engines: {node: ^20.19.0 || >=22.12.0} @@ -1283,6 +1295,10 @@ snapshots: '@profullstack/hqtui@0.6.0': {} + '@profullstack/proxy@0.1.0': + dependencies: + undici: 8.11.2 + '@resvg/resvg-js-android-arm-eabi@2.6.2': optional: true @@ -1837,6 +1853,8 @@ snapshots: undici-types@6.21.0: {} + undici@8.11.2: {} + vite@8.2.1(@types/node@22.20.1)(esbuild@0.28.2)(tsx@4.23.12): dependencies: lightningcss: 1.33.0 diff --git a/src/credentials.ts b/src/credentials.ts index d89c79b..faa5a7a 100644 --- a/src/credentials.ts +++ b/src/credentials.ts @@ -41,6 +41,16 @@ export const KNOWN_KEYS: Record = { // ~/.moshcode/credentials.json and that is what gets picked up. This is for a // box that has the key but not moshcode. moshcode: 'MOSHCODE_API_KEY', + // Read by `proxy` (@profullstack/proxy). The proxy login is what traffic + // needs; the API keys only add `proxy status`. Webshare can do without its + // user/password: given the API key, the package reads the login from the + // account, which survives a password change in the dashboard. + proxiware: 'PROXIWARE_API_KEY', + proxiware_proxy_user: 'PROXIWARE_PROXY_USER', + proxiware_proxy_password: 'PROXIWARE_PROXY_PASSWORD', + webshare: 'WEBSHARE_API_KEY', + webshare_proxy_user: 'WEBSHARE_PROXY_USER', + webshare_proxy_password: 'WEBSHARE_PROXY_PASSWORD', }; export type Source = 'env' | 'file' | 'unset'; diff --git a/src/proxy.ts b/src/proxy.ts new file mode 100644 index 0000000..e09e4b8 --- /dev/null +++ b/src/proxy.ts @@ -0,0 +1,92 @@ +/** + * Keys for `proxy`, the @profullstack/proxy command. + * + * The package reads credentials from an environment record and nothing else, + * so this side's whole job is to build that record: the shell first, then the + * credential store, and on a box that has neither, one pull of the team vault + * (the same one `cli-tools config pull` reads), cached so the next run is + * local. Only the proxy variables are taken from the pull; the rest of the + * vault is not this command's business. + */ + +import { KNOWN_KEYS, loadStored, resolveCredentials, saveStored } from './credentials.ts'; +import { pullVault, vaultTarget, type VaultTarget } from './vault.ts'; + +export const PROXY_VARIABLES = [ + 'PROXIWARE_API_KEY', + 'PROXIWARE_PROXY_USER', + 'PROXIWARE_PROXY_PASSWORD', + 'WEBSHARE_API_KEY', + 'WEBSHARE_PROXY_USER', + 'WEBSHARE_PROXY_PASSWORD', +] as const; + +// The store only keeps what KNOWN_KEYS names; a variable missing from there +// would be pulled every run and never cached. +for (const variable of PROXY_VARIABLES) { + if (!Object.values(KNOWN_KEYS).includes(variable)) { + throw new Error(`proxy: ${variable} is not in KNOWN_KEYS`); + } +} + +type Env = Record; + +/** Enough to send traffic through at least one provider. */ +export function hasProxyLogin(env: Env): boolean { + return Boolean( + (env.PROXIWARE_PROXY_USER && env.PROXIWARE_PROXY_PASSWORD) || + (env.WEBSHARE_PROXY_USER && env.WEBSHARE_PROXY_PASSWORD) || + env.WEBSHARE_API_KEY, + ); +} + +export interface ProxyEnvDeps { + env?: NodeJS.ProcessEnv; + pull?: (target: VaultTarget) => Record; + log?: (line: string) => void; +} + +/** + * The environment to hand the package: process env, with stored keys for any + * proxy variable the shell does not set, pulling the vault once if there is no + * usable login at all. Returns the record and where the keys came from. + */ +export function proxyEnv(deps: ProxyEnvDeps = {}): { env: Env; source: 'env' | 'store' | 'vault' | 'none' } { + const env = deps.env ?? process.env; + const log = deps.log ?? (() => {}); + const merged: Env = { ...env, ...pickDefined(resolveCredentials(env), PROXY_VARIABLES) }; + for (const variable of PROXY_VARIABLES) if (env[variable]) merged[variable] = env[variable]; + + if (hasProxyLogin(merged)) { + const fromShell = PROXY_VARIABLES.some((variable) => env[variable]); + return { env: merged, source: fromShell ? 'env' : 'store' }; + } + + const target = vaultTarget(env); + log(`proxy: no proxy keys on this machine; pulling them from ${target.team}/${target.project}--${target.env}…`); + let vault: Record; + try { + vault = (deps.pull ?? pullVault)(target); + } catch (error) { + log(`proxy: ${(error as Error).message}`); + return { env: merged, source: 'none' }; + } + + const found = pickDefined(vault, PROXY_VARIABLES); + if (Object.keys(found).length === 0) { + log(`proxy: ${target.project}--${target.env} has none of ${PROXY_VARIABLES.join(', ')}`); + return { env: merged, source: 'none' }; + } + const path = saveStored({ ...loadStored(env), ...found }, env); + log(`proxy: cached ${Object.keys(found).join(', ')} in ${path}`); + return { env: { ...merged, ...found, ...pickDefined(env, PROXY_VARIABLES) }, source: 'vault' }; +} + +function pickDefined(record: Env, keys: readonly string[]): Record { + const out: Record = {}; + for (const key of keys) { + const value = record[key]; + if (value) out[key] = value; + } + return out; +} diff --git a/src/registry.ts b/src/registry.ts index 99a33bc..85cf766 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -62,6 +62,7 @@ const SUMMARIES: Record = { 'gh-pulse': 'What moved on GitHub, last hour to all time, ranked, with traffic: by email, TUI or JSON', hqtui: 'Every vital of this box, in the terminal: sockets, HTTP, sessions, services', porkbun: 'Read and change DNS at Porkbun, and un-park a domain', + proxy: 'Fetch through our paid proxies (Proxiware, Webshare); serve, MCP and TUI too', scorecard: 'One weekly number for the whole fleet: traffic, channels, posts and ads, week over week', shorten: 'Mint a short link on the pit, and follow it from /f/', sysupdate: 'Update this box: apt lists, apt packages, snaps', diff --git a/test/proxy.test.ts b/test/proxy.test.ts new file mode 100644 index 0000000..0018967 --- /dev/null +++ b/test/proxy.test.ts @@ -0,0 +1,104 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { keyVariable, loadStored, saveStored } from '../src/credentials.ts'; +import { hasProxyLogin, proxyEnv, PROXY_VARIABLES } from '../src/proxy.ts'; + +const dirs: string[] = []; + +afterEach(async () => { + await Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +async function sandbox(extra: Record = {}): Promise { + const dir = await mkdtemp(join(tmpdir(), 'proxy-')); + dirs.push(dir); + return { XDG_CONFIG_HOME: dir, ...extra } as NodeJS.ProcessEnv; +} + +const neverPull = () => { + throw new Error('should not pull'); +}; + +describe('proxy keys', () => { + it('are config-settable by friendly name', () => { + expect(keyVariable('proxiware')).toBe('PROXIWARE_API_KEY'); + expect(keyVariable('proxiware_proxy_user')).toBe('PROXIWARE_PROXY_USER'); + expect(keyVariable('proxiware-proxy-password')).toBe('PROXIWARE_PROXY_PASSWORD'); + expect(keyVariable('webshare')).toBe('WEBSHARE_API_KEY'); + expect(keyVariable('WEBSHARE_PROXY_PASSWORD')).toBe('WEBSHARE_PROXY_PASSWORD'); + }); + + it('hasProxyLogin wants a full login, or a Webshare API key', () => { + expect(hasProxyLogin({})).toBe(false); + expect(hasProxyLogin({ PROXIWARE_PROXY_USER: 'u' })).toBe(false); + expect(hasProxyLogin({ PROXIWARE_PROXY_USER: 'u', PROXIWARE_PROXY_PASSWORD: 'p' })).toBe(true); + expect(hasProxyLogin({ WEBSHARE_API_KEY: 'k' })).toBe(true); + expect(hasProxyLogin({ PROXIWARE_API_KEY: 'k' })).toBe(false); + }); +}); + +describe('proxyEnv', () => { + it('uses the shell when it has a login, without touching the vault', async () => { + const env = await sandbox({ PROXIWARE_PROXY_USER: 'u', PROXIWARE_PROXY_PASSWORD: 'p' }); + const result = proxyEnv({ env, pull: neverPull }); + expect(result.source).toBe('env'); + expect(result.env.PROXIWARE_PROXY_USER).toBe('u'); + }); + + it('uses the store, and the shell still wins per variable', async () => { + const env = await sandbox({ PROXIWARE_PROXY_PASSWORD: 'from-shell' }); + saveStored({ PROXIWARE_PROXY_USER: 'stored-user', PROXIWARE_PROXY_PASSWORD: 'stored-pass', PROXIWARE_API_KEY: 'k' }, env); + const result = proxyEnv({ env, pull: neverPull }); + expect(result.env.PROXIWARE_PROXY_USER).toBe('stored-user'); + expect(result.env.PROXIWARE_PROXY_PASSWORD).toBe('from-shell'); + expect(result.env.PROXIWARE_API_KEY).toBe('k'); + }); + + it('pulls the vault once when nothing is set, caches only the proxy keys', async () => { + const env = await sandbox(); + const lines: string[] = []; + let pulls = 0; + const pull = () => { + pulls++; + return { + PROXIWARE_PROXY_USER: 'vault-user', + PROXIWARE_PROXY_PASSWORD: 'VAULT-PASS-SECRET', + PROXIWARE_API_KEY: 'VAULT-KEY-SECRET', + OPENAI_API_KEY: 'not-ours', + }; + }; + const first = proxyEnv({ env, pull, log: (line) => lines.push(line) }); + expect(first.source).toBe('vault'); + expect(first.env.PROXIWARE_PROXY_PASSWORD).toBe('VAULT-PASS-SECRET'); + const stored = loadStored(env); + expect(Object.keys(stored).sort()).toEqual(['PROXIWARE_API_KEY', 'PROXIWARE_PROXY_PASSWORD', 'PROXIWARE_PROXY_USER']); + expect(lines.join('\n')).toMatch(/cached .*PROXIWARE_PROXY_USER/); + expect(lines.join('\n')).not.toMatch(/SECRET/); + + const second = proxyEnv({ env, pull }); + expect(second.source).toBe('store'); + expect(pulls).toBe(1); + }); + + it('reports none when the vault fails or has no proxy keys', async () => { + const env = await sandbox(); + expect(proxyEnv({ env, pull: () => ({ OPENAI_API_KEY: 'x' }) }).source).toBe('none'); + expect( + proxyEnv({ + env, + pull: () => { + throw new Error('logicsrc is not installed'); + }, + }).source, + ).toBe('none'); + }); + + it('covers every variable the package reads', () => { + expect([...PROXY_VARIABLES].sort()).toEqual( + ['PROXIWARE_API_KEY', 'PROXIWARE_PROXY_PASSWORD', 'PROXIWARE_PROXY_USER', 'WEBSHARE_API_KEY', 'WEBSHARE_PROXY_PASSWORD', 'WEBSHARE_PROXY_USER'], + ); + }); +});