diff --git a/apps/web/src/lib/assets.js b/apps/web/src/lib/assets.js
new file mode 100644
index 0000000..362b16f
--- /dev/null
+++ b/apps/web/src/lib/assets.js
@@ -0,0 +1,31 @@
+import { readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+/** Where the static files live. */
+export const PUBLIC = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'public');
+
+/**
+ * Content-hash version for cache busting: computed once per process.
+ *
+ * Synchronous on purpose. The Layout is rendered by code that has to answer
+ * with a string, not a promise (the x402 gateway's sales page hook), so the
+ * one thing the shell needs from disk is read the plain way. It happens once
+ * per asset per process; every later call is a map lookup.
+ */
+const versions = new Map();
+export function assetVersion(name) {
+ if (versions.has(name)) return versions.get(name);
+ try {
+ const hash = new Bun.CryptoHasher('sha1')
+ .update(readFileSync(join(PUBLIC, name)))
+ .digest('hex')
+ .slice(0, 10);
+ versions.set(name, hash);
+ return hash;
+ } catch {
+ return 'dev';
+ }
+}
+
+export const assetUrl = (name) => `/${name}?v=${assetVersion(name)}`;
diff --git a/apps/web/src/lib/gate.js b/apps/web/src/lib/gate.js
index 6a50a31..2ccf90e 100644
--- a/apps/web/src/lib/gate.js
+++ b/apps/web/src/lib/gate.js
@@ -2,6 +2,7 @@ import { createThrottle, memoryStore } from '@profullstack/throttle';
import { createGateway } from '@profullstack/x402-gateway';
import { config } from '@r4ck/config';
import * as accounts from '@r4ck/db/accounts';
+import { renderCrawl } from '../views/crawl.jsx';
import { Denied } from './http.js';
/**
@@ -39,6 +40,8 @@ export const gateway = createGateway({
contact: config.x402.contact,
openPaths: OPEN_PATHS,
chargeSpoofedBrowsers: false,
+ // /crawl in the site's own shell instead of the gateway's bare page.
+ page: renderCrawl,
benefits: [
'No hourly allowance on the API, the CLI or MCP',
'Bulk CSV and JSON export of any query',
diff --git a/apps/web/src/routes/static.js b/apps/web/src/routes/static.js
index c2aa480..ce60a9e 100644
--- a/apps/web/src/routes/static.js
+++ b/apps/web/src/routes/static.js
@@ -1,11 +1,10 @@
-import { dirname, join } from 'node:path';
-import { fileURLToPath } from 'node:url';
+import { join } from 'node:path';
import { config } from '@r4ck/config';
import { sql } from '@r4ck/db';
+import { assetVersion, PUBLIC } from '../lib/assets.js';
import { gateway } from '../lib/gate.js';
import { llmsTxt, skillMd } from '../lib/llms.js';
-const PUBLIC = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'public');
const TYPES = {
css: 'text/css; charset=utf-8',
js: 'text/javascript; charset=utf-8',
@@ -29,26 +28,11 @@ const FILES = [
'fonts/GeistMono.woff2',
];
-/** Content-hash version for cache busting: computed once per process. */
-const versions = new Map();
-export async function assetVersion(name) {
- if (versions.has(name)) return versions.get(name);
- try {
- const buf = await Bun.file(join(PUBLIC, name)).arrayBuffer();
- const hash = new Bun.CryptoHasher('sha1').update(buf).digest('hex').slice(0, 10);
- versions.set(name, hash);
- return hash;
- } catch {
- return 'dev';
- }
-}
-export const assetUrl = async (name) => `/${name}?v=${await assetVersion(name)}`;
-
async function serve(c, name, { cache = 'public, max-age=3600' } = {}) {
const file = Bun.file(join(PUBLIC, name));
if (!(await file.exists())) return c.notFound();
const ext = name.split('.').pop();
- const immutable = c.req.query('v') && c.req.query('v') === (await assetVersion(name));
+ const immutable = c.req.query('v') && c.req.query('v') === assetVersion(name);
return new Response(file, {
headers: {
'content-type': TYPES[ext] ?? 'application/octet-stream',
diff --git a/apps/web/src/views/Layout.jsx b/apps/web/src/views/Layout.jsx
index 01297ae..5b53c89 100644
--- a/apps/web/src/views/Layout.jsx
+++ b/apps/web/src/views/Layout.jsx
@@ -1,13 +1,16 @@
import { config } from '@r4ck/config';
import { raw } from 'hono/html';
-import { assetUrl } from '../routes/static.js';
+import { assetUrl } from '../lib/assets.js';
/**
* The one HTML shell. Dark ground by default, a light theme on request,
* a sticky top bar on wide screens and a bottom bar on phones, and the
* command palette that turns any page into the search box.
+ *
+ * Synchronous: a page whose children are plain markup renders to a string
+ * with no await, which is what the x402 gateway's sales page hook needs.
*/
-export async function Layout({
+export function Layout({
title,
description,
user,
@@ -25,9 +28,9 @@ export async function Layout({
const desc =
description ??
'Every VPS, cloud, bare metal, GPU and PaaS offer for sale, searchable by spec, price and place. Same query on the page, the API, the CLI and MCP.';
- const css = await assetUrl('styles.css');
- const js = await assetUrl('app.js');
- const vendor = await assetUrl('vendor-webauthn.js');
+ const css = assetUrl('styles.css');
+ const js = assetUrl('app.js');
+ const vendor = assetUrl('vendor-webauthn.js');
const url = `${config.siteUrl}${canonical ?? path}`;
const nav = [
['/servers', 'Servers', 'search'],
diff --git a/apps/web/src/views/crawl.jsx b/apps/web/src/views/crawl.jsx
new file mode 100644
index 0000000..92dc78f
--- /dev/null
+++ b/apps/web/src/views/crawl.jsx
@@ -0,0 +1,258 @@
+import { Layout } from './Layout.jsx';
+
+/**
+ * The x402 sales page, /crawl, in the site's own shell.
+ *
+ * The gateway ships a bare page of its own, unstyled and shell-less, and that
+ * is what /crawl looked like until now. This is the same page with the same
+ * words, rendered through the Layout like every other route, so the one URL a
+ * refused crawler may read looks like the site it was refused from. Every
+ * number on it comes from the gateway's context; nothing is typed twice.
+ *
+ * Two audiences reach it and they need different first sentences. A training
+ * crawler is here because it is on a list. A heavy reader is here because it
+ * ran out of the free allowance, and telling that one it is a training crawler
+ * is both wrong and insulting. The price is the same; the argument is not.
+ */
+export function Crawl({ ctx }) {
+ const {
+ siteName,
+ siteUrl,
+ buyUrl,
+ price,
+ minutes,
+ header,
+ enabled,
+ offer,
+ training = [],
+ retrieval = [],
+ contact,
+ days = 1,
+ total = price,
+ maxDays = 30,
+ quota = null,
+ benefits = null,
+ } = ctx;
+
+ const throttled = Boolean(quota?.exceeded);
+ const window =
+ minutes === 1440
+ ? 'one day'
+ : minutes % 1440 === 0
+ ? `${minutes / 1440} days`
+ : minutes === 60
+ ? 'one hour'
+ : minutes % 60 === 0
+ ? `${minutes / 60} hours`
+ : `${minutes} minutes`;
+ const networks = (offer?.accepts ?? []).map((a) => a.network).join(', ');
+
+ return (
+ }
+ >
+
+
+ {throttled
+ ? 'You have used up the free allowance.'
+ : 'Training crawlers pay for access here.'}
+
+ {throttled ? (
+
+ {quota.requests} requests every {quota.windowSeconds} seconds are free, no key and no
+ account, and that is not changing. You have gone past it
+ {quota.resetSeconds ? `, and it resets in ${quota.resetSeconds} seconds` : ''}. A pass
+ lifts the limit rather than waiting it out.
+
+ ) : (
+
+ People read {siteName} free. So do search engines and the
+ retrieval crawlers behind AI answers, because they send readers back. A crawler that
+ copies pages into a training corpus sends nobody back, so it pays for the time it
+ spends.
+
+ )}
+
+
+
+ x402 pass
+
+ {days > 1 ? total : price}{' '}
+
+ for {days > 1 ? `${days} × ${window}` : window} of requests
+
+
+ {days > 1 ? (
+
+ This offer is for {days} days at {price} a day. The plain page at {buyUrl}{' '}
+ quotes one.
+
+ ) : (
+
+ Want longer? Add {'?days='} to this URL for an offer of up to {maxDays}{' '}
+ days at {price} a day, or simply pay a whole multiple of the price: the pass lasts as
+ many days as you paid for.
+
+ )}
+ {!enabled ? (
+
+ Payments are not switched on here yet. The offer below is empty until
+ the operator configures a payout address, so for now this crawler is simply refused.
+
+ ) : null}
+ {benefits?.length ? (
+
+ {benefits.map((b) => (
+ - {b}
+ ))}
+
+ ) : null}
+
+
+ {throttled ? (
+
+ Before you reach for a proxy pool
+
+ Spreading the same crawl over rotating addresses works, and it is the expensive way to
+ do this. Residential bandwidth is sold by the gigabyte, you still fetch every page one
+ at a time, and the bill starts on the first day. A pass is {price} a day, flat, with no
+ rotation to maintain and nothing to keep working. We would rather sell you access than
+ play that game, which is why the limit answers with a price instead of a refusal.
+
+
+ ) : null}
+
+
+ How it works
+
+ -
+ Any page you fetch answers
402 Payment Required. This page, fetched with{' '}
+ Accept: application/json, returns the x402 offer: USDC, exact{' '}
+ scheme, on {networks || 'Base, Polygon or Ethereum'}.
+
+ -
+ Sign the payment and retry with the proof in an
X-PAYMENT header. The
+ response is a JSON receipt carrying a pass.
+
+ -
+ Send the pass in
{header} on every request until it expires: {window} per
+ day paid, so a proof for three times the price buys three. When it expires, buy another.
+ The sale is the pass, not the page: fetch the page again with the pass.
+
+
+
+
+
+ Pay with the CoinPay CLI
+
+ Settlement is by CoinPay: the buyer's USDC goes straight to the site's wallet and
+ CoinPay's relayer pays the gas, so you need USDC and nothing else.
+
+
+ {`npm install -g @profullstack/coinpay
+coinpay x402 pay ${buyUrl} --output pass.json
+# or a week at once:
+coinpay x402 pay "${buyUrl}?days=7" --output pass.json`}
+
+
+ The command fetches this page, reads the offer, opens a browser tab to approve the payment
+ with the CoinPay Wallet extension or any EIP-6963 wallet (MetaMask, Rabby, Coinbase
+ Wallet), and writes the receipt to pass.json. Then:
+
+
+ {`PASS=$(node -p "require('./pass.json').pass")
+curl -H "${header}: $PASS" ${siteUrl}/`}
+
+
+
+
+ Pay from your own x402 client
+
+ {`curl -sS -H "Accept: application/json" ${buyUrl}
+# 402 with { "x402Version": 2, "accepts": [ ... ] }
+# sign an EIP-3009 transferWithAuthorization for one entry, then:
+curl -sS -H "X-PAYMENT: " ${buyUrl}
+# 200 with { "ok": true, "pass": "cp_...", "expires_at": "...", "days": 1, "header": "${header}" }`}
+
+
+ The days a proof buys are read off the value it authorizes: a whole multiple of the
+ one-day amount, up to {maxDays}. {'?days='} only changes what the offer
+ quotes, so a standard client that pays exactly what is asked gets n days.
+
+
+ The proof is x402 v2 in CoinPay's dialect:{' '}
+
+ {
+ '{ x402Version: 2, scheme: "exact", network: "", payload: { signature, authorization } }'
+ }
+
+ , base64-encoded. A proof is single-use; retrying with the same one returns the same pass,
+ not a second charge.
+
+
+
+
+ Who pays and who does not
+
+
+
+ |
+ Charged
+ |
+ {training.join(', ')} |
+
+
+ |
+ Free, named in robots.txt
+ |
+ {retrieval.join(', ')} |
+
+
+ |
+ Free
+ |
+
+ Everyone else: people, Googlebot, Applebot, Bingbot and any crawler not on the first
+ line.
+ |
+
+
+
+
+ If your crawler is on the first line and you believe it should not be, or you want more
+ than an hour at a time,{' '}
+ {contact ? get in touch : 'contact the site'}.
+
+
+
+
+ The offer, verbatim
+
+ {JSON.stringify(offer, null, 2)}
+
+
+
+
+ Served by @profullstack/x402-gateway. This page is noindex and is the one URL a
+ refused crawler may read.
+
+
+ );
+}
+
+/**
+ * The page as a string, for the gateway's `page` hook.
+ *
+ * The hook is synchronous: the gateway hands what it returns straight to a
+ * Response. The Layout and this view are plain markup, so the render never
+ * yields a promise; the check below turns a future async slip into a loud
+ * failure in the test rather than "[object Promise]" on a live page.
+ */
+export function renderCrawl(ctx) {
+ const html = ().toString();
+ if (typeof html !== 'string') throw new Error('the crawl page rendered asynchronously');
+ return `${html}`;
+}
diff --git a/test/crawl.test.js b/test/crawl.test.js
new file mode 100644
index 0000000..438ea43
--- /dev/null
+++ b/test/crawl.test.js
@@ -0,0 +1,78 @@
+import { describe, expect, test } from 'bun:test';
+import { renderCrawl } from '../apps/web/src/views/crawl.jsx';
+
+/**
+ * /crawl is the x402 gateway's sales page. It used to be the gateway's own
+ * bare HTML; now it renders through the site Layout, and the gateway's hook
+ * needs a string back, not a promise.
+ */
+const ctx = {
+ siteName: 'r4ck',
+ siteUrl: 'https://r4ck.dev',
+ buyUrl: 'https://r4ck.dev/crawl',
+ price: '1.00 USD',
+ minutes: 1440,
+ days: 1,
+ total: '1.00 USD',
+ maxDays: 30,
+ header: 'x-crawl-pass',
+ enabled: true,
+ offer: { x402Version: 2, accepts: [{ network: 'eip155:8453' }, { network: 'eip155:1' }] },
+ training: ['GPTBot', 'ClaudeBot'],
+ retrieval: ['OAI-SearchBot', 'Claude-User'],
+ contact: 'mailto:hello@profullstack.com',
+ benefits: ['No hourly allowance on the API', 'Every field on every row'],
+ quota: null,
+};
+
+describe('/crawl page', () => {
+ test('renders synchronously inside the site shell', () => {
+ const html = renderCrawl(ctx);
+ expect(typeof html).toBe('string');
+ expect(html.startsWith('')).toBe(true);
+ expect(html).toContain('Crawl pass · r4ck');
+ expect(html).toContain('class="topbar"');
+ expect(html).toContain('class="footer"');
+ expect(html).toContain('/styles.css?v=');
+ expect(html).toContain(' {
+ const html = renderCrawl({
+ ...ctx,
+ quota: { requests: 100, windowSeconds: 60, used: 101, resetSeconds: 42, exceeded: true },
+ });
+ expect(html).toContain('You have used up the free allowance.');
+ expect(html).toContain('it resets in 42 seconds');
+ expect(html).toContain('Before you reach for a proxy pool');
+ expect(html).not.toContain('Training crawlers pay for access here.');
+ });
+
+ test('quotes a multi-day offer', () => {
+ const html = renderCrawl({ ...ctx, days: 7, total: '7.00 USD' });
+ expect(html).toContain('7.00 USD');
+ expect(html).toContain('7 × one day');
+ expect(html).toContain('This offer is for 7 days at 1.00 USD a day.');
+ });
+
+ test('escapes what it is given', () => {
+ const html = renderCrawl({ ...ctx, siteName: '' });
+ expect(html).not.toContain('');
+ expect(html).toContain('<script>x</script>');
+ });
+
+ test('the gateway serves it', async () => {
+ const { gateway } = await import('../apps/web/src/lib/gate.js');
+ const html = gateway.page();
+ expect(html).toContain('class="topbar"');
+ expect(html).toContain('Crawl pass · r4ck');
+ });
+});