diff --git a/.github/workflows/translate.yml b/.github/workflows/translate.yml index d631c13..21983fd 100644 --- a/.github/workflows/translate.yml +++ b/.github/workflows/translate.yml @@ -39,9 +39,11 @@ jobs: run: | BASE="${{ github.event.pull_request.base.sha }}" # Only source .md files — exclude already-translated ones (e.g. index.es.md) + # and MCP runbooks (git/raw only, not on the public site). CHANGED=$(git diff --name-only "$BASE" HEAD -- docs/ \ | grep '\.md$' \ | grep -vE '\.[a-z]{2,3}\.md$' \ + | grep -vE '(^|/)runbooks\.md$|(^|/).*-runbook\.md$' \ || true) echo "Source files changed:" echo "$CHANGED" diff --git a/docs/runbooks.md b/docs/runbooks.md new file mode 100644 index 0000000..1eb4b09 --- /dev/null +++ b/docs/runbooks.md @@ -0,0 +1,24 @@ +--- +description: Machine-readable index of onboarding runbooks for the DoubleZero MCP. Not linked in the docs nav. +search: + exclude: true +--- + +# Runbooks + +The DoubleZero MCP loads this file from GitHub raw +(`https://raw.githubusercontent.com/malbeclabs/docs/main/docs/runbooks.md`) +to discover walkthroughs. Humans use [Connect your AI](mcp.md); do not add this +page to the site nav. + +To register a runbook, add a list item under **Index** in this shape: + +```markdown +- `service-id` — [Human title](page-slug.md) +``` + +## Index + +- `solana-ibrl` — [Connect validator (IBRL Mainnet)](solana-ibrl-runbook.md) +- `solana-shreds-publisher` — [Publish shreds (Edge)](solana-shreds-publisher-runbook.md) +- `solana-shreds` — [Subscribe to shreds (Edge)](solana-shreds-runbook.md) diff --git a/docs/solana-ibrl-runbook.md b/docs/solana-ibrl-runbook.md new file mode 100644 index 0000000..947afff --- /dev/null +++ b/docs/solana-ibrl-runbook.md @@ -0,0 +1,160 @@ +--- +description: LLM-oriented runbook — connect a Solana Mainnet-Beta validator to DoubleZero in IBRL mode. Served to the MCP via GitHub raw; not published on the docs site. +--- + +# Connect validator (IBRL Mainnet) — runbook + +This page is for the DoubleZero MCP (`get_onboarding_runbook`) via GitHub raw. It is +not published on the docs site. + +1. Connect the [DoubleZero MCP](mcp.md) (`https://data.doublezero.xyz/api/mcp`). +2. Tell it this is a **Solana Mainnet-Beta validator**, the Linux host (or SSH), and where the validator identity keypair lives. +3. Walk the steps below in order. Prefer to do it by hand? Use the [human guide](DZ Mainnet-beta Connection.md). + +**What success looks like:** `doublezero status` shows tunnel **up**, User Type **IBRL**, Network **mainnet-beta**. `Tunnel src` and `Doublezero IP` match the host public IPv4. + +IBRL does not require restarting validator clients; it uses the existing public IP. + +--- + +## Prerequisites + +| Need | Notes | +|------|--------| +| Linux/amd64 host | Install DoubleZero **on the validator host**, not in a container. | +| Public IPv4, no NAT | Gossip IP must match this host. | +| Solana CLI on `$PATH` | For `solana sign-offchain-message`. | +| Validator identity keypair | Readable by the user running the commands (often under the `sol` user). | +| ≥1 SOL on the identity | Passport / onchain request. | +| GRE (IP proto 47) + BGP | BGP on `169.254.0.0/16` tcp/179. | +| `doublezero-solana` | `sudo apt update && sudo apt install doublezero-solana` (or distro equivalent). | + +The Validator ID is checked against Solana gossip to determine the target IP. A junk ID on the same IP is ignored; only the primary ID in gossip is used. + +--- + +## Steps + +### 1. Install the client + +Follow [setup](setup.md) if `doublezero` is not installed. Mainnet packages: + +```bash +curl -1sLf https://dl.cloudsmith.io/public/malbeclabs/doublezero/setup.deb.sh | sudo -E bash +sudo apt-get install doublezero +``` + +Rocky / RHEL: use `setup.rpm.sh` and `sudo yum install doublezero`. + +**Verify:** `sudo systemctl status doublezerod` is active. Back up `~/.config/doublezero/id.json`. + +### 2. Point the daemon at mainnet-beta + +```bash +DESIRED_DOUBLEZERO_ENV=mainnet-beta \ + && sudo mkdir -p /etc/systemd/system/doublezerod.service.d \ + && echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/doublezerod -sock-file /run/doublezerod/doublezerod.sock -env $DESIRED_DOUBLEZERO_ENV" | sudo tee /etc/systemd/system/doublezerod.service.d/override.conf > /dev/null \ + && sudo systemctl daemon-reload \ + && sudo systemctl restart doublezerod \ + && doublezero config set --env $DESIRED_DOUBLEZERO_ENV > /dev/null \ + && echo "✅ doublezerod configured for environment $DESIRED_DOUBLEZERO_ENV" +``` + +Wait ~30s, then `doublezero latency` should list mainnet devices. + +### 3. Open UDP 44880 on `doublezero0` + +```bash +sudo iptables -A INPUT -i doublezero0 -p udp --dport 44880 -j ACCEPT +sudo iptables -A OUTPUT -o doublezero0 -p udp --dport 44880 -j ACCEPT +``` + +UFW: `sudo ufw allow in on doublezero0 to any port 44880 proto udp` and the matching `out` rule. Also allow GRE and BGP as in [setup](setup.md). + +### 4. Confirm DoubleZero ID and primary validator + +The DoubleZero ID from setup on the **primary** must be on every backup (`~/.config/doublezero/id.json`). + +```bash +doublezero address +doublezero-solana passport find-validator -u mainnet-beta +``` + +Expect the primary: in gossip, in the leader schedule, “can connect as a primary”. On backups, run the same `find-validator`; they should **not** be leader-scheduled. + +One machine only: omit `--backup-validator-ids` / `backup_ids=` from later commands. + +### 5. Prepare the access message (primary) + +On the primary (active stake, identity in gossip): + +```bash +doublezero-solana passport prepare-validator-access -u mainnet-beta \ + --doublezero-address \ + --primary-validator-id \ + --backup-validator-ids ,, +``` + +Drop `--backup-validator-ids` if there are no backups (max 3). Copy the `solana sign-offchain-message …` line from the output. + +### 6. Sign with the validator identity key + +On the primary, run the printed command (identity keypair, **not** only the DoubleZero key): + +```bash +solana sign-offchain-message \ + service_key=,backup_ids=,, \ + -k +``` + +**Produces:** a signature string. Carry it into the next step. + +### 7. Request validator access + +```bash +doublezero-solana passport request-validator-access -k -u mainnet-beta \ + --primary-validator-id \ + --backup-validator-ids ,, \ + --signature \ + --doublezero-address +``` + +Wait for Sentinel to validate and create the access pass. Optional: the agent can call **`check_edge_access`** with `pubkey` (`doublezero address`) and the host public IP until the pass is present. + +### 8. Connect IBRL + +```bash +doublezero connect ibrl +``` + +Wait ~1 minute for GRE. Until then, status may be `down` / `Unknown`. + +```bash +doublezero status +``` + +**Pass:** `up`, User Type `IBRL`, Network `mainnet-beta`, tunnel typically `doublezero0`. + +```bash +ip route +``` + +Expect BGP-learned routes via `doublezero0`. + +--- + +## Gotchas + +1. **Wrong env.** Testnet packages / `DESIRED_DOUBLEZERO_ENV=testnet` will not land on mainnet-beta. +2. **Identity not in gossip.** Junk IDs on the same IP cannot register the machine. +3. **Backups must share the primary DoubleZero ID.** Copy `id.json`; do not keygen a second identity. +4. **Sign with the validator identity**, not the DoubleZero key. +5. **Status down for ~1 minute** after `connect ibrl` is normal while GRE comes up. + +--- + +## See also + +- [Validator Mainnet-Beta Connection](DZ Mainnet-beta Connection.md) +- [Setup](setup.md) +- Next: [Publish shreds (Edge)](solana-shreds-publisher-runbook.md) diff --git a/docs/solana-shreds-publisher-runbook.md b/docs/solana-shreds-publisher-runbook.md new file mode 100644 index 0000000..2bda63c --- /dev/null +++ b/docs/solana-shreds-publisher-runbook.md @@ -0,0 +1,88 @@ +--- +description: LLM-oriented runbook — configure a connected Solana validator to publish leader shreds to DoubleZero Edge. Served to the MCP via GitHub raw; not published on the docs site. +--- + +# Publish shreds (Edge) — runbook + +This page is for the DoubleZero MCP (`get_onboarding_runbook`) via GitHub raw. It is +not published on the docs site. + +1. Connect the [DoubleZero MCP](mcp.md) (`https://data.doublezero.xyz/api/mcp`). +2. Confirm the validator is **already on DoubleZero IBRL** (mainnet-beta). If not, finish [Connect validator (IBRL Mainnet)](solana-ibrl-runbook.md) first. +3. Walk the steps below. Human guide: [Validator Multicast Connection](Validator Multicast Connection.md). + +**What success looks like:** validator sends leader shreds to `233.84.178.1:7733`, multicast publish on `edge-solana-shreds` is up, and [publisher-check](https://data.doublezero.xyz/dz/publisher-check) shows publishing after at least one leader slot. + +This path is for **validators**. Trading firms that want to *subscribe* use [Subscribe to shreds](solana-shreds-runbook.md). + +--- + +## Prerequisites + +| Need | Notes | +|------|--------| +| IBRL tunnel already up | [IBRL Mainnet runbook](solana-ibrl-runbook.md) / [human guide](DZ Mainnet-beta Connection.md). | +| Supported client | Jito-Agave **3.1.9+**, JitoBam 3.1.9+, Frankendancer, or Harmonic **3.1.11+**. Other versions will not publish. | +| Restart window | Adding the shred destination requires a validator restart. | + +--- + +## Steps + +### 1. Point the client at the Edge shred group + +**Jito-Agave (v3.1.9+) and Harmonic (3.1.11+)** — in the validator start script add: + +```text +--shred-receiver-address 233.84.178.1:7733 +``` + +You can send to Jito and `edge-solana-shreds` at the same time. Restart the validator. + +**Frankendancer** — in `config.toml`: + +```toml +[tiles.shred] +additional_shred_destinations_leader = [ "233.84.178.1:7733", ] +``` + +Restart the validator. + +### 2. Publish on the multicast group + +```bash +doublezero connect ibrl && doublezero connect multicast --publish edge-solana-shreds +``` + +**Verify:** `doublezero status` still IBRL/up, and the user is a publisher on `edge-solana-shreds`. + +Live group IPs: `doublezero multicast group list`. All shred feeds use UDP **`7733`**; the IP selects the feed. + +| Feed | Address | +|------|---------| +| `edge-solana-shreds` (leader) | `233.84.178.1:7733` | +| `edge-solana-retrans-eu` | `233.84.178.12:7733` | +| `edge-solana-retrans-apac` | `233.84.178.13:7733` | +| `edge-solana-retrans-amer` | `233.84.178.14:7733` | + +### 3. Confirm publishing + +Open [publisher-check](https://data.doublezero.xyz/dz/publisher-check). You will not see confirmation until the validator has published leader shreds for **at least one slot**. + +Healthy: outbound spikes aligned with leader slots (sawtooth). Steady outbound with no slot pattern is **retransmit** (bad). + +--- + +## Gotchas + +1. **Wrong client version.** Not 3.1.9+ / 3.1.11+ → nothing useful on the wire. +2. **Retransmit flag left on.** Remove `--shred-retransmit-receiver-address` from Jito-Agave. Check the **No Retransmit Shreds** column on publisher-check (2-epoch vs recent-slot views). +3. **Not yet a leader.** Dashboard stays empty until a leader slot. +4. **IBRL not up.** Do not start here; finish IBRL first. + +--- + +## See also + +- [Validator Multicast Connection](Validator Multicast Connection.md) +- [Validator Rewards](Validator Rewards.md) diff --git a/docs/solana-shreds-runbook.md b/docs/solana-shreds-runbook.md new file mode 100644 index 0000000..a54f5ed --- /dev/null +++ b/docs/solana-shreds-runbook.md @@ -0,0 +1,145 @@ +--- +description: LLM-oriented runbook — buy an Edge shreds seat and receive Solana shred multicast on doublezero1. Served to the MCP via GitHub raw; not published on the docs site. +--- + +# Subscribe to shreds (Edge) — runbook + +This page is for the DoubleZero MCP (`get_onboarding_runbook`) via GitHub raw. It is +not published on the docs site. + +1. Connect the [DoubleZero MCP](mcp.md) (`https://data.doublezero.xyz/api/mcp`). +2. Tell it the Linux host that will **receive** shreds (or SSH), the wallet/keypair for `doublezero-solana`, and which feed (leader vs retransmit). +3. Walk the steps below in order. Human guide: [Edge Subscriber Connection](Edge Subscriber Connection.md). + +**What success looks like:** seat allocated for the current epoch, `doublezero status` shows the tunnel up, UDP shreds on `doublezero1` port `7733` (leader group `233.84.178.1`). + +By connecting, the user agrees to the [DoubleZero Terms of Use](https://doublezero.xyz/terms-protocol). Data is for internal use and may not be retransmitted. + +--- + +## Prerequisites + +| Need | Notes | +|------|--------| +| Linux/amd64 host | Public IPv4, no NAT. On AWS: disable ENI source/dest check. | +| Solana CLI + `doublezero-solana` | `sudo apt update && sudo apt install doublezero-solana` | +| Wallet | `~/.config/solana/id.json` (or `--keypair`). Needs **SOL** (fees) + **USDC** (seat escrow). | +| USDC mint | `EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v` | +| Firewall | GRE, BGP (`169.254.0.0/16` tcp/179), PIM, UDP `7733` on `doublezero1`, UDP `44880` on `doublezero0`. | + +--- + +## Steps + +### 1. Install client + packages + +Follow [setup](setup.md), then: + +```bash +sudo apt update && sudo apt install doublezero-solana +``` + +Back up `~/.config/doublezero/id.json`. + +### 2. Firewall + +```bash +sudo iptables -A OUTPUT -p gre -j ACCEPT +sudo iptables -A INPUT -i doublezero1 -s 169.254.0.0/16 -d 169.254.0.0/16 -p tcp --dport 179 -j ACCEPT +sudo iptables -A OUTPUT -o doublezero1 -s 169.254.0.0/16 -d 169.254.0.0/16 -p tcp --dport 179 -j ACCEPT +sudo iptables -A OUTPUT -o doublezero1 -p pim -j ACCEPT +sudo iptables -A INPUT -i doublezero1 -p udp --dport 7733 -j ACCEPT +sudo iptables -A INPUT -i doublezero0 -p udp --dport 44880 -j ACCEPT +``` + +UFW variants: human guide. + +### 3. Enable the reconciler + +Required so seats auto-provision the tunnel. + +```bash +doublezero enable +``` + +### 4. Wallet + +```bash +solana-keygen new # if needed — writes ~/.config/solana/id.json; back it up +solana address +``` + +Fund SOL and USDC. + +### 5. Pick device + price + +```bash +doublezero latency +doublezero-solana shreds price +doublezero-solana shreds price --device-code +``` + +Note lowest-latency **device code** and epoch price (base + premium). Prefer funding **>1 epoch**. Pricing UI: [devices](https://data.doublezero.xyz/dz/shreds/devices). + +### 6. Buy a seat (blocking) + +On the receiving host: + +```bash +curl -4 -s ifconfig.me; echo +``` + +```bash +doublezero-solana shreds pay \ + --device-code \ + --client-ip \ + --amount +``` + +`--client-ip` must be the public IPv4 of the machine that will receive shreds. `--amount` is USDC decimal (e.g. `100`) and must meet the epoch price. + +If less than 10% of the epoch remains, the CLI warns. `--accept-partial-epoch` takes the remainder now; otherwise wait. Underfunded escrow at settlement → seat gone, tunnel torn down, **tenure lost**. + +Once allocated, the daemon brings the GRE tunnel up. + +```bash +doublezero status +doublezero-solana shreds list +``` + +### 7. Confirm shreds + +Leader shreds: `233.84.178.1:7733` on `doublezero1`. Discover groups with `doublezero multicast group list`. + +| Feed | Group | Address | +|------|-------|---------| +| Leader | `edge-solana-shreds` | `233.84.178.1:7733` | +| Root | `edge-solana-root` | `233.84.178.16:7733` | +| Retransmit EU | `edge-solana-retrans-eu` | `233.84.178.12:7733` | +| Retransmit APAC | `edge-solana-retrans-apac` | `233.84.178.13:7733` | +| Retransmit AMER | `edge-solana-retrans-amer` | `233.84.178.14:7733` | + +Port `5765` is a publisher heartbeat — not shreds. Traffic is GRE-encapsulated; some pipelines (XDP deshredders) must strip GRE. + +```bash +sudo tcpdump -ni doublezero1 host 233.84.178.1 and udp port 7733 +``` + +--- + +## Gotchas + +1. **Reconciler off.** Without `doublezero enable`, paying does not bring the tunnel up. +2. **`--client-ip` ≠ daemon IP.** Auto-discovery must match the seat. +3. **`Multicast user already exists`.** Disconnect first: `doublezero disconnect`, then retry `shreds pay`. +4. **Amount below current price.** Re-check `shreds price` and increase `--amount`. +5. **Seat not allocated after pay.** Late epoch (next epoch), device full (higher tenure), or withdraw before settlement. +6. **Keep escrow funded.** Top up with another `shreds pay`; do not let balance drop below epoch price. + +--- + +## See also + +- [Edge Subscriber Connection](Edge Subscriber Connection.md) +- [Support](support.md) +- Scoreboard / seats: [data.doublezero.xyz](https://data.doublezero.xyz/dz/shreds/scoreboard) diff --git a/hooks/exclude_runbooks.py b/hooks/exclude_runbooks.py new file mode 100644 index 0000000..8cbeb76 --- /dev/null +++ b/hooks/exclude_runbooks.py @@ -0,0 +1,25 @@ +"""Keep MCP runbook markdown in git (GitHub raw) but out of the MkDocs site. + +mkdocs-static-i18n copies unsuffixed English pages into each locale as +``runbooks.zh.md`` (etc.). Those copies do not match ``exclude_docs: runbooks.md``, +so they would still be built and fail link-check. Drop every locale variant here. +""" + +from __future__ import annotations + + +def _is_mcp_runbook(src_uri: str) -> bool: + name = src_uri.replace("\\", "/").rsplit("/", 1)[-1] + if not name.endswith(".md"): + return False + stem = name[:-3] + if stem == "runbooks" or stem.startswith("runbooks."): + return True + return "-runbook" in stem + + +def on_files(files, config): + for file in list(files): + if _is_mcp_runbook(file.src_uri): + files.remove(file) + return files diff --git a/mkdocs.yml b/mkdocs.yml index 9fc9210..22b4437 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,6 +4,17 @@ site_description: >- Documentation for connecting to and operating on the Malbec Labs / DoubleZero network — validator and tenant setup, multicast, rewards, geolocation, and contributor operations. +# Machine-oriented runbooks stay in git for MCP / GitHub raw, but are not built +# into the public docs site. +exclude_docs: | + runbooks.md + runbooks.*.md + solana-ibrl-runbook.md + solana-ibrl-runbook.*.md + solana-shreds-runbook.md + solana-shreds-runbook.*.md + solana-shreds-publisher-runbook.md + solana-shreds-publisher-runbook.*.md theme: name: material custom_dir: overrides @@ -136,5 +147,6 @@ extra_javascript: - javascripts/mathjax.js - path: https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js hooks: + - hooks/exclude_runbooks.py - hooks/emit_markdown.py - hooks/emit_well_known.py diff --git a/scripts/translate.py b/scripts/translate.py index 406797b..502d661 100644 --- a/scripts/translate.py +++ b/scripts/translate.py @@ -44,6 +44,11 @@ def translate(content: str, lang_name: str) -> str: return message.content[0].text +def _is_runbook(filepath: str) -> bool: + name = filepath.replace("\\", "/").rsplit("/", 1)[-1] + return name == "runbooks.md" or name.endswith("-runbook.md") + + def main(): changed_files = os.environ.get("CHANGED_FILES", "").strip().splitlines() target_langs = [ @@ -53,6 +58,7 @@ def main(): ] source_files = [f.strip() for f in changed_files if f.strip() and f.endswith(".md")] + source_files = [f for f in source_files if not _is_runbook(f)] if not source_files: print("No changed source files.")