Add a live, read-only CLI smoke test

README.md

@@ -1195,6 +1195,32 @@ The published and deployed checks are release gates. They are expected to fail b
 published or the documentation deployment reaches production. The scenario catalog is stored in
 `evals/claude-code-name-lookup.json` so the same prompt variants remain visible and reviewable.
 
+### Testing
+
+```bash
+npm test           # Unit and behavioral tests with a mocked API (no credentials)
+npm run check      # Typecheck, build, tests and the public artifact guard
+DM_API_URL=https://api-staging.v2.dealmachine.com/v1 DM_API_KEY=<key> npm run smoke:live
+```
+
+`npm test` never calls an API. `smoke:live` builds the CLI and runs the real `dm` binary against
+the API named by `DM_API_URL`, authenticated with `DM_API_KEY`. Both are required; the key is never
+printed. Each Command runs with a temporary home directory, so the smoke never reads or changes
+your own `dm login` credentials. It runs only read-only, credit-free Commands:
+
+| Command                                                     | Checks                                                     |
+| ----------------------------------------------------------- | ---------------------------------------------------------- |
+| `dm --version`                                              | Prints the package version without a network request       |
+| `dm account --json`                                         | Organization, user auth type and plan are present          |
+| `dm filters --source-type properties --per-page 5 --json`   | Filters have IDs, names, operators and pagination          |
+| `dm fields --source-type properties --per-page 5 --json`    | Fields have IDs, names and pagination                      |
+| `dm properties search --body <ZIP 78704> --estimate-cost --json` | Returns an estimate with no records and no charged credits |
+
+Requests pass through a local relay that refuses anything outside those read-only Endpoints
+(property search must carry `estimate_cost: true`) and confirms each request sends
+`X-DealMachine-Source: cli` and the CLI `User-Agent`. The smoke prints one `PASS` or `FAIL` line per
+Command and exits 0 only when every Command passes.
+
 ### Standalone Binary
 
 The compiled `dist/index.js` includes a `#!/usr/bin/env node` shebang and is declared in `package.json` under `bin.dm`. When installed globally via npm, it becomes available as `dm` on the PATH.

docs/development.md

@@ -13,7 +13,7 @@ npm run dev
 
 The [Command reference](commands.md) describes CLI usage and JSON output. Tests under `tests/` exercise request behavior and output. `test:package` installs both npm archives into an empty temporary project and checks the executable, module import and project-local Claude Code Playbook installation. It does not write to your personal agent setup or call the API.
 
-For local API work, start the API in its owning checkout, then use `DM_API_URL=http://localhost:3001/v1 npm start -- account`. Supply your development API key through the approved local environment or `dm login`; never commit it. Normal local build and tests need no credentials. Production Commands can read or mutate live data and consume credits, so choose an environment deliberately.
+For local API work, start the API in its owning checkout, then use `DM_API_URL=http://localhost:3001/v1 npm start -- account`. Supply your development API key through the approved local environment or `dm login`; never commit it. Normal local build and tests need no credentials. `npm run smoke:live` runs the built binary against a real API with `DM_API_URL` and `DM_API_KEY`, using only read-only, credit-free Commands and a temporary home directory; see the README's Testing section. Production Commands can read or mutate live data and consume credits, so choose an environment deliberately.
 
 The public agent plugin manifests and `skills/dealmachine` are retained in this repository. The hosted MCP server is a separate service. This extraction does not make MCP implementation or docs-site deployment part of CLI publication.
 

package.json

@@ -29,6 +29,7 @@
     "eval:cold-start:published": "node scripts/eval-cold-start.mjs published",
     "eval:cold-start:deployed": "node scripts/eval-cold-start.mjs deployed",
     "eval:cold-start:all": "npm run build && node scripts/eval-cold-start.mjs all",
+    "smoke:live": "npm run build && node scripts/smoke-live.mjs",
     "prepublishOnly": "npm run check",
     "typecheck": "tsc --noEmit --composite false --incremental false",
     "check": "npm run typecheck && npm run build && npm test && npm run check:artifact",

scripts/smoke-live-lib.mjs

@@ -0,0 +1,172 @@
+/**
+ * Helpers for the live CLI smoke (scripts/smoke-live.mjs).
+ *
+ * Kept separate so the credential seeding, request allowlist, redaction and
+ * response shape checks can be unit tested without contacting an API.
+ */
+
+import { createCipheriv, createHash, randomBytes } from 'node:crypto';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+
+// These mirror the encrypted-file credential store in src/lib/config.ts. The
+// unit test reads a seeded home through the real readConfig, so a format
+// change there fails the test instead of silently breaking the smoke.
+const FALLBACK_ENCRYPTION_CONTEXT = 'dealmachine-cli-api-key-v1';
+const AES_ALGORITHM = 'aes-256-gcm';
+const AES_IV_BYTES = 12;
+
+/**
+ * Write a CLI config under `homeDir` that uses the encrypted-file credential
+ * store. This never calls the macOS Keychain or Windows DPAPI, so the smoke
+ * cannot read or overwrite a developer's stored key.
+ */
+export function seedIsolatedConfig(homeDir, apiKey, identity = {}) {
+  const configDir = path.join(homeDir, '.dealmachine');
+  fs.mkdirSync(configDir, { recursive: true, mode: 0o700 });
+
+  const username = identity.username ?? safeUsername();
+  const hostname = identity.hostname ?? os.hostname();
+  const key = createHash('sha256')
+    .update([FALLBACK_ENCRYPTION_CONTEXT, username, hostname, configDir].join('\0'))
+    .digest();
+  const iv = randomBytes(AES_IV_BYTES);
+  const cipher = createCipheriv(AES_ALGORITHM, key, iv);
+  const encrypted = Buffer.concat([cipher.update(apiKey, 'utf-8'), cipher.final()]);
+  const payload = Buffer.concat([iv, encrypted, cipher.getAuthTag()]).toString('base64');
+
+  const credentialFile = path.join(configDir, 'api-key.enc');
+  fs.writeFileSync(credentialFile, payload, { mode: 0o600, encoding: 'utf-8' });
+  fs.writeFileSync(
+    path.join(configDir, 'config.json'),
+    JSON.stringify(
+      {
+        configVersion: 2,
+        keyId: 'live-smoke',
+        organizationId: 0,
+        organizationName: 'Live smoke',
+        organizationSlug: '',
+        credentialStore: 'encrypted-file',
+        credentialFile,
+      },
+      null,
+      2
+    ),
+    { mode: 0o600, encoding: 'utf-8' }
+  );
+  return configDir;
+}
+
+function safeUsername() {
+  try {
+    return os.userInfo().username;
+  } catch {
+    return 'unknown-user';
+  }
+}
+
+/** Replace every occurrence of each secret with a fixed marker. */
+export function redact(text, secrets) {
+  let output = String(text ?? '');
+  for (const secret of secrets) {
+    if (secret && secret.length >= 4) output = output.split(secret).join('[redacted]');
+  }
+  return output;
+}
+
+/**
+ * Requests the smoke may forward to the live API. Anything else is refused by
+ * the local relay before it leaves the machine.
+ */
+export function checkAllowedRequest(method, pathWithQuery, body) {
+  const pathname = new URL(pathWithQuery, 'http://relay.local').pathname;
+  if (method === 'GET' && ['/account', '/fields', '/filters'].includes(pathname)) {
+    return { allowed: true };
+  }
+  if (method === 'POST' && pathname === '/properties/search') {
+    if (body && typeof body === 'object' && body.estimate_cost === true) {
+      return { allowed: true };
+    }
+    return { allowed: false, reason: 'property search without estimate_cost=true' };
+  }
+  return { allowed: false, reason: `${method} ${pathname} is not on the read-only allowlist` };
+}
+
+/** Parse JSON from CLI stdout. Throws with a short reason. */
+export function parseJsonOutput(stdout) {
+  try {
+    return JSON.parse(stdout);
+  } catch {
+    throw new Error('stdout was not valid JSON');
+  }
+}
+
+function fail(reason) {
+  throw new Error(reason);
+}
+
+function isObject(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function checkPagination(pagination) {
+  if (!isObject(pagination)) fail('missing pagination object');
+  for (const key of ['page', 'per_page', 'total_results', 'total_pages']) {
+    if (typeof pagination[key] !== 'number') fail(`pagination.${key} is not a number`);
+  }
+}
+
+/** Shape checks return a short success reason or throw with the failure. */
+export function assertAccountShape(json) {
+  const data = isObject(json) ? json.data : undefined;
+  if (!isObject(data)) fail('missing data object');
+  if (!isObject(data.organization)) fail('missing data.organization');
+  if (typeof data.organization.id !== 'number') fail('data.organization.id is not a number');
+  if (typeof data.organization.name !== 'string') fail('data.organization.name is not a string');
+  if (!isObject(data.user) || typeof data.user.authType !== 'string') {
+    fail('missing data.user.authType');
+  }
+  if (!isObject(data.plan)) fail('missing data.plan');
+  return `organization ${data.organization.id}, auth ${data.user.authType}`;
+}
+
+function assertCatalogShape(json, idKey, extraCheck) {
+  if (!isObject(json) || !Array.isArray(json.data)) fail('missing data array');
+  if (json.data.length === 0) fail('data array is empty');
+  for (const item of json.data) {
+    if (!isObject(item) || typeof item[idKey] !== 'string') fail(`item without string ${idKey}`);
+    if (typeof item.name !== 'string') fail(`${item[idKey]} has no name`);
+    extraCheck?.(item);
+  }
+  checkPagination(json.pagination);
+  return `${json.data.length} of ${json.pagination.total_results} returned`;
+}
+
+export function assertFiltersShape(json) {
+  return assertCatalogShape(json, 'filter_id', (item) => {
+    if (!Array.isArray(item.allowed_operators)) {
+      fail(`${item.filter_id} has no allowed_operators array`);
+    }
+  });
+}
+
+export function assertFieldsShape(json) {
+  return assertCatalogShape(json, 'field_id');
+}
+
+export function assertEstimateShape(json) {
+  if (!isObject(json)) fail('response is not an object');
+  if ('data' in json || 'credits' in json) fail('response contains records or charged credits');
+  const estimate = json.estimated_credits;
+  if (!isObject(estimate)) fail('missing estimated_credits');
+  if (typeof estimate.this_page !== 'number') fail('estimated_credits.this_page is not a number');
+  if (typeof estimate.total_all_pages !== 'number') {
+    fail('estimated_credits.total_all_pages is not a number');
+  }
+  if (!isObject(json.totals) || typeof json.totals.properties !== 'number') {
+    fail('missing totals.properties');
+  }
+  checkPagination(json.pagination);
+  return `${json.totals.properties} properties, ${estimate.this_page} credits estimated for page 1`;
+}