diff --git a/.cursor-plugin/harness/package.json b/.cursor-plugin/harness/package.json index 425bc20..8c7aa8d 100644 --- a/.cursor-plugin/harness/package.json +++ b/.cursor-plugin/harness/package.json @@ -2,7 +2,7 @@ "name": "@fusengine/cursor-harness", "version": "1.0.0", "private": true, - "description": "Fusengine Cursor global harness vendor manifest - single shared install of @fusengine/harness under ~/.cursor/.fusengine-global/node_modules, always tracking the latest published release.", + "description": "Fusengine Cursor global harness vendor manifest - single shared install of @fusengine/harness under the user-level .cursor/.fusengine-global/node_modules, always tracking the latest published release.", "dependencies": { "@fusengine/harness": "latest" } diff --git a/.cursor-plugin/marketplace.json b/.cursor-plugin/marketplace.json index 729d420..851116d 100644 --- a/.cursor-plugin/marketplace.json +++ b/.cursor-plugin/marketplace.json @@ -6,14 +6,14 @@ }, "metadata": { "description": "Professional Cursor plugins with APEX workflow, SOLID principles, and expert development agents for Next.js, Laravel, Swift, Tailwind CSS, and more", - "version": "0.1.11" + "version": "0.1.12" }, "plugins": [ { "name": "core-guards", "source": ".cursor-plugin/plugins/core-guards", "description": "Core security guards (git, install, security) and SOLID enforcement hooks", - "version": "1.1.39", + "version": "1.1.40", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -36,7 +36,7 @@ "name": "fuse-ai-pilot", "source": ".cursor-plugin/plugins/fuse-ai-pilot", "description": "AI Pilot workflow agents: sniper (code quality), research-expert, explore-codebase, websearch", - "version": "1.2.46", + "version": "1.2.47", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -81,7 +81,7 @@ "name": "fuse-cartographer", "source": ".cursor-plugin/plugins/fuse-cartographer", "description": "Generates indented markdown maps of the plugin ecosystem for agent navigation", - "version": "1.0.14", + "version": "1.0.15", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -102,7 +102,7 @@ "name": "fuse-changelog", "source": ".cursor-plugin/plugins/fuse-changelog", "description": "Claude Code update watcher: changelog tracking, breaking changes detection, community pulse via Exa, plugin compatibility analysis", - "version": "1.0.16", + "version": "1.0.17", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -145,7 +145,7 @@ "name": "fuse-design", "source": ".cursor-plugin/plugins/fuse-design", "description": "Design Director for websites, web apps, iOS, and Android. Generates HTML/CSS directly (Gemini Design MCP, Magic/21st.dev, and shadcn MCP are optional). iOS/Android produce tokens + a device-framed mockup + a handoff spec, never SwiftUI/Compose. Design method, design system tokens, motion, UX copy, and a deterministic design-review gate. Anti-AI-Slop, WCAG 2.2.", - "version": "2.2.11", + "version": "2.2.12", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -177,11 +177,37 @@ ], "category": "development" }, + { + "name": "fuse-devops", + "source": ".cursor-plugin/plugins/fuse-devops", + "description": "Expert self-hosted IT/DevOps — Docker, Kubernetes, Linux, Proxmox VE, OpenStack, Terraform/OpenTofu, Ansible, CI/CD, observability, storage and backup, secrets, databases — with a discovery, dry-run, one-go, verify, rollback safety model", + "version": "1.0.0", + "author": { + "name": "Fusengine", + "email": "hello@fusengine.ch" + }, + "homepage": "https://github.com/fusengine/agents", + "repository": "https://github.com/fusengine/agents", + "license": "MIT", + "keywords": [ + "devops", + "docker", + "kubernetes", + "linux", + "proxmox", + "openstack", + "terraform", + "ansible", + "observability", + "infrastructure" + ], + "category": "development" + }, { "name": "fuse-go", "source": ".cursor-plugin/plugins/fuse-go", "description": "Expert Go 1.26+ — idioms, concurrency, backend services, cargo-grade tooling (golangci-lint v2, govulncheck), with SOLID principles", - "version": "1.0.8", + "version": "1.0.9", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -207,7 +233,7 @@ "name": "fuse-hermes", "source": ".cursor-plugin/plugins/fuse-hermes", "description": "Expert Nous Research Hermes Agent 0.23+ — config.yaml, SOUL.md / .hermes.md, profiles, delegation, cron, gateways, security, plus chief-of-staff operating patterns driven by a universal profile.yaml", - "version": "1.0.0", + "version": "1.0.1", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -233,7 +259,7 @@ "name": "fuse-laravel", "source": ".cursor-plugin/plugins/fuse-laravel", "description": "Expert Laravel 13 + PHP 8.3+ with SOLID principles, first-class PHP Attributes, Laravel AI SDK, JSON:API Resources, native vector search (pgvector), Eloquent, Livewire, queues with routing, and comprehensive documentation", - "version": "1.2.10", + "version": "1.2.11", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -255,7 +281,7 @@ "name": "fuse-lessons", "source": ".cursor-plugin/plugins/fuse-lessons", "description": "Project memory: writes compact 'never reproduce' lessons to MEMORY/LESSON.md on finish, force-reads them at session/subagent start.", - "version": "1.0.8", + "version": "1.0.9", "author": { "name": "Fusengine" }, @@ -273,7 +299,7 @@ "name": "fuse-nextjs", "source": ".cursor-plugin/plugins/fuse-nextjs", "description": "Expert Next.js 16 + React 19 with App Router, Server Components, Prisma 7, and Better Auth", - "version": "1.1.26", + "version": "1.1.27", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -295,7 +321,7 @@ "name": "fuse-php", "source": ".cursor-plugin/plugins/fuse-php", "description": "Expert modern PHP 8.5/8.4 for non-Laravel projects — libraries, standalone Symfony components, Slim, API-first apps, and CLI tools, with PER Coding Style 3.0, PHPStan, and SOLID principles", - "version": "1.0.9", + "version": "1.0.10", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -348,7 +374,7 @@ "name": "fuse-react", "source": ".cursor-plugin/plugins/fuse-react", "description": "Expert React 19 with hooks, TanStack Router, Zustand, React Hook Form, Testing Library, shadcn/ui, and SOLID principles", - "version": "1.0.23", + "version": "1.0.24", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -372,7 +398,7 @@ "name": "fuse-rules", "source": ".cursor-plugin/plugins/fuse-rules", "description": "Global rules for all sessions: APEX workflow, SOLID/DRY enforcement, project detection, agent delegation, frontend/git standards", - "version": "1.0.25", + "version": "1.0.26", "author": { "name": "Fusengine" }, @@ -385,7 +411,7 @@ "name": "fuse-rust", "source": ".cursor-plugin/plugins/fuse-rust", "description": "Expert Rust 1.96+ (2024 edition) — ownership, error handling, async/tokio, axum backends, cargo tooling, with SOLID principles", - "version": "1.0.9", + "version": "1.0.10", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -411,7 +437,7 @@ "name": "fuse-security", "source": ".cursor-plugin/plugins/fuse-security", "description": "Security vulnerability detection and remediation: OWASP Top 10, SAST scanning, CVE research, dependency audit, secrets detection", - "version": "1.0.20", + "version": "1.0.21", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -434,7 +460,7 @@ "name": "fuse-seo", "source": ".cursor-plugin/plugins/fuse-seo", "description": "SEO/GEO 2026 plugin. Local-first, framework-aware, zero third-party APIs. Covers technical SEO, schema markup, E-E-A-T, AI Overviews optimization, local SEO, sitemaps, hreflang, internal linking, content briefs, semantic clustering, and search experience optimization.", - "version": "1.0.14", + "version": "1.0.15", "author": { "name": "Fusengine" }, @@ -464,7 +490,7 @@ "name": "fuse-shadcn-ui", "source": ".cursor-plugin/plugins/fuse-shadcn-ui", "description": "Expert shadcn/ui with Radix UI and Base UI detection, component patterns, registries, theming, and migration", - "version": "1.0.20", + "version": "1.0.21", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -488,7 +514,7 @@ "name": "fuse-solid", "source": ".cursor-plugin/plugins/fuse-solid", "description": "SOLID principles enforcement with automatic project detection and validation for all languages", - "version": "1.0.21", + "version": "1.0.22", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -509,7 +535,7 @@ "name": "fuse-swift-apple-expert", "source": ".cursor-plugin/plugins/fuse-swift-apple-expert", "description": "Expert Swift 6.2 + SwiftUI for iOS 26, macOS 26, iPadOS 26, watchOS 26, visionOS 26, tvOS 26 with Liquid Glass, @Observable, actors, SwiftData inheritance", - "version": "1.1.22", + "version": "1.1.23", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -536,7 +562,7 @@ "name": "fuse-tailwindcss", "source": ".cursor-plugin/plugins/fuse-tailwindcss", "description": "Expert Tailwind CSS v4.1 - CSS-first configuration, @theme, @utility, @variant, Oxide engine 5x faster, OKLCH colors, container queries, 15 specialized skills", - "version": "1.1.13", + "version": "1.1.14", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -568,7 +594,7 @@ "name": "fuse-tanstack-start", "source": ".cursor-plugin/plugins/fuse-tanstack-start", "description": "Expert TanStack Start (React, RC) full-stack framework with file-based routing, createServerFn server functions, selective SSR, and Nitro deployment", - "version": "1.0.7", + "version": "1.0.8", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" @@ -593,7 +619,7 @@ "name": "fuse-typescript", "source": ".cursor-plugin/plugins/fuse-typescript", "description": "Expert TypeScript 6.0 for pure TS projects — CLI tools, libraries, scripts, and backends on Node 24 LTS or Bun 1.3, with SOLID principles", - "version": "1.0.9", + "version": "1.0.10", "author": { "name": "Fusengine", "email": "hello@fusengine.ch" diff --git a/.cursor-plugin/mcp/catalog.json b/.cursor-plugin/mcp/catalog.json index 6bcd039..b90ed8e 100644 --- a/.cursor-plugin/mcp/catalog.json +++ b/.cursor-plugin/mcp/catalog.json @@ -19,7 +19,7 @@ "filesystem": { "_description": "Secure local file operations with configurable access", "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-filesystem", "${HOME}"], + "args": ["-y", "@modelcontextprotocol/server-filesystem", "${userHome}"], "type": "stdio", "requiresApiKey": false, "default": false @@ -39,7 +39,7 @@ "requiresApiKey": false, "default": false, "env": { - "DATABASE_URL": "${DATABASE_URL}" + "DATABASE_URL": "${env:DATABASE_URL}" } }, "shadcn": { @@ -84,7 +84,7 @@ "-y", "@upstash/context7-mcp", "--api-key", - "${CONTEXT7_API_KEY}" + "${env:CONTEXT7_API_KEY}" ], "type": "stdio", "default": true, @@ -96,8 +96,7 @@ "_description": "AI-powered web search, code context and deep research", "default": true, "type": "http", - "url": "https://mcp.exa.ai/mcp?tools=web_search_exa,get_code_context_exa,crawling_exa,company_research_exa,linkedin_search_exa,deep_researcher_start,deep_researcher_check&exaApiKey=${EXA_API_KEY}", - "headers": {}, + "url": "https://mcp.exa.ai/mcp?tools=web_search_exa,get_code_context_exa,crawling_exa,company_research_exa,linkedin_search_exa,deep_researcher_start,deep_researcher_check&exaApiKey=${env:EXA_API_KEY}", "requiresApiKey": true, "apiKeyEnv": "EXA_API_KEY", "apiKeyUrl": "https://exa.ai" @@ -107,7 +106,7 @@ "command": "npx", "args": ["-y", "@21st-dev/magic@latest"], "env": { - "MAGIC_API_KEY": "${MAGIC_API_KEY}" + "MAGIC_API_KEY": "${env:MAGIC_API_KEY}" }, "type": "stdio", "default": true, @@ -120,7 +119,7 @@ "command": "npx", "args": ["-y", "gemini-design-mcp@latest"], "env": { - "API_KEY": "${GEMINI_DESIGN_API_KEY}" + "API_KEY": "${env:GEMINI_DESIGN_API_KEY}" }, "type": "stdio", "default": true, @@ -133,7 +132,7 @@ "command": "npx", "args": ["-y", "@modelcontextprotocol/server-github"], "env": { - "GITHUB_TOKEN": "${GITHUB_TOKEN}" + "GITHUB_TOKEN": "${env:GITHUB_TOKEN}" }, "type": "stdio", "requiresApiKey": true, @@ -145,7 +144,7 @@ "command": "npx", "args": ["-y", "supabase", "mcp"], "env": { - "SUPABASE_ACCESS_TOKEN": "${SUPABASE_ACCESS_TOKEN}" + "SUPABASE_ACCESS_TOKEN": "${env:SUPABASE_ACCESS_TOKEN}" }, "type": "stdio", "requiresApiKey": true, @@ -157,7 +156,7 @@ "command": "npx", "args": ["-y", "@modelcontextprotocol/server-slack"], "env": { - "SLACK_TOKEN": "${SLACK_TOKEN}" + "SLACK_TOKEN": "${env:SLACK_TOKEN}" }, "type": "stdio", "requiresApiKey": true, @@ -169,7 +168,7 @@ "command": "npx", "args": ["-y", "@sentry/mcp-server"], "env": { - "SENTRY_AUTH_TOKEN": "${SENTRY_AUTH_TOKEN}" + "SENTRY_AUTH_TOKEN": "${env:SENTRY_AUTH_TOKEN}" }, "type": "stdio", "requiresApiKey": true, @@ -181,7 +180,7 @@ "command": "npx", "args": ["-y", "@brave/brave-search-mcp-server"], "env": { - "BRAVE_API_KEY": "${BRAVE_API_KEY}" + "BRAVE_API_KEY": "${env:BRAVE_API_KEY}" }, "type": "stdio", "requiresApiKey": true, @@ -193,7 +192,7 @@ "command": "npx", "args": ["-y", "serper-search-scrape-mcp-server"], "env": { - "SERPER_API_KEY": "${SERPER_API_KEY}" + "SERPER_API_KEY": "${env:SERPER_API_KEY}" }, "type": "stdio", "requiresApiKey": true, @@ -205,7 +204,7 @@ "command": "npx", "args": ["-y", "@stripe/mcp"], "env": { - "STRIPE_SECRET_KEY": "${STRIPE_SECRET_KEY}" + "STRIPE_SECRET_KEY": "${env:STRIPE_SECRET_KEY}" }, "type": "stdio", "requiresApiKey": true, @@ -217,7 +216,7 @@ "command": "npx", "args": ["-y", "@notionhq/notion-mcp-server"], "env": { - "NOTION_TOKEN": "${NOTION_TOKEN}" + "NOTION_TOKEN": "${env:NOTION_TOKEN}" }, "type": "stdio", "requiresApiKey": true, @@ -227,8 +226,7 @@ "replicate": { "_description": "Run 1000+ AI models (Stable Diffusion, Whisper, etc.)", "type": "http", - "url": "https://replicate.com/mcp?token=${REPLICATE_API_TOKEN}", - "headers": {}, + "url": "https://replicate.com/mcp?token=${env:REPLICATE_API_TOKEN}", "requiresApiKey": true, "apiKeyEnv": "REPLICATE_API_TOKEN", "apiKeyUrl": "https://replicate.com/account/api-tokens" diff --git a/.cursor-plugin/plugin-runtime/hook.sh b/.cursor-plugin/plugin-runtime/hook.sh index 3c88b14..12329dd 100644 --- a/.cursor-plugin/plugin-runtime/hook.sh +++ b/.cursor-plugin/plugin-runtime/hook.sh @@ -2,46 +2,296 @@ # Fusengine hook entrypoint — shipped inside every plugin as scripts/hook.sh. # # Resolves the shared harness itself so the plugin works from the Cursor -# marketplace, where no installer ever runs. Order: -# 1. the vendored install under $HOME/.cursor/.fusengine-global (fast path) -# 2. a self-heal `bun install` of that same shared directory, once +# marketplace, where no installer ever runs. Layout under the control root +# ($HOME/.cursor/.fusengine-global): +# +# versions//node_modules/@fusengine/harness one immutable install per version +# current text file: the live +# node_modules/@fusengine/harness legacy single install (fallback) +# +# Resolution order: +# 1. the version named by `current` (fast path), else the legacy install +# 2. a self-heal install of `latest` (staged, see below), once # 3. `npx` on latest (last resort, network) # -# Installs the LATEST published harness the first time (the manifest uses the -# `latest` dist-tag, and so does the npx fallback), so no wrapper needs editing -# per release. It does NOT upgrade an existing install: the fast path runs the -# vendored bin as is, and rewriting it in place while up to 62 hooks load it -# could crash a guard mid-run (hooks fail open). The upgrade path is the -# installer (`install.sh` runs `bun update @fusengine/harness` on every run) or, -# by hand: `cd ~/.cursor/.fusengine-global && bun update @fusengine/harness`. +# UPGRADES. A hook invoked as `hook.sh --session-start [scope...]` (only the +# sessionStart entries pass the flag) compares the installed version with +# HARNESS_MIN using shell builtins only — no process, no network. Current or +# newer: nothing else happens. Older: the new release is installed into a +# STAGING directory (`bun add` in versions/.stage-), checked (version and +# `bin.mjs --version`), moved to versions/, and only then made live by +# renaming a temp file over `current` — one atomic rename(2). The running tree +# is never modified in place and the previous version is never deleted during +# the switch, so a guard that starts, or is already running and lazily imports +# a chunk, never sees a half-replaced install. (An in-place `bun update` leaves +# bin.mjs and the hashed chunks absent for a few ms: measured, it crashes +# concurrent hooks.) Old versions are pruned only after a successful switch, +# keeping the new one, the previous one, and anything younger than ~2 days. +# +# Failure handling: a back-off stamp (.harness-update-failed) suppresses retries +# for FUSE_HARNESS_UPDATE_BACKOFF_MIN minutes (default 30); it is written before +# the attempt and removed on success, so a killed attempt is also backed off. An +# atomic mkdir lock (.harness-update.lock, reclaimed after 10 min) keeps the five +# sessionStart hooks from installing at once. Everything fails open. +# +# HARNESS_MIN is the only per-release edit. # # Resolution is relative to THIS file, never to the caller's cwd. Always exits 0 # on its own failures: Cursor fails hooks open, and a broken guard must never # block the user's action. # -# Usage (from hooks/hooks.json): ./scripts/hook.sh [scope...] +# Usage (from hooks/hooks.json): ./scripts/hook.sh [--session-start] [scope...] +# ./scripts/hook.sh --harness-check (sessionStart of plugins that must not +# run the bin there: same check/upgrade, +# then exit 0, empty stdout) +# ./scripts/hook.sh --install-harness (installer: staged install, then exit; +# prints one warning line on failure) +# The install is bounded by FUSE_HARNESS_INSTALL_TIMEOUT_SEC (default 45, under +# Cursor's 60 s default hook timeout); sessionStart entries that can upgrade +# therefore carry no timeout below 60. +HARNESS_MIN="0.1.115" HARNESS_SPEC="latest" -CONTROL_ROOT="${HOME}/.cursor/.fusengine-global" -BIN="${CONTROL_ROOT}/node_modules/@fusengine/harness/dist/cli/bin.mjs" +CONTROL_ROOT="${FUSE_HARNESS_ROOT:-${HOME}/.cursor/.fusengine-global}" +PKG_REL="node_modules/@fusengine/harness" +BIN_REL="node_modules/@fusengine/harness/dist/cli/bin.mjs" STAMP="${CONTROL_ROOT}/.selfheal-attempted" +FAILED="${CONTROL_ROOT}/.harness-update-failed" +LOCK="${CONTROL_ROOT}/.harness-update.lock" + +MODE="" +case "$1" in + --session-start) MODE=session; shift ;; + --install-harness) MODE=install; shift ;; + --harness-check) MODE=check; shift ;; +esac + +BIN="" +INSTALLED="" + +# Set INSTALLED to the "version" of a package.json, builtins only. Empty if absent. +pkgver() { + INSTALLED="" + [ -r "$1" ] || return 1 + while IFS= read -r _l || [ -n "$_l" ]; do + case $_l in + *'"version"'*:*) + _l=${_l#*'"version"'}; _l=${_l#*:}; _l=${_l#*\"} + INSTALLED=${_l%%\"*}; return 0 ;; + esac + done < "$1" + return 1 +} + +# ver_ge A B: numeric A >= B for x.y.z (prerelease/build suffix ignored). +ver_ge() { + _a=${1%%[-+]*}; _b=${2%%[-+]*} + case $_a in ''|*[!0-9.]*) return 1 ;; esac + case $_b in ''|*[!0-9.]*) return 1 ;; esac + _o=$IFS; IFS=. + set -- $_a; _a1=${1:-0}; _a2=${2:-0}; _a3=${3:-0} + set -- $_b; _b1=${1:-0}; _b2=${2:-0}; _b3=${3:-0} + IFS=$_o + [ "$_a1" -gt "$_b1" ] && return 0; [ "$_a1" -lt "$_b1" ] && return 1 + [ "$_a2" -gt "$_b2" ] && return 0; [ "$_a2" -lt "$_b2" ] && return 1 + [ "$_a3" -ge "$_b3" ] +} + +# Set BIN to the live bin ("" if none) and INSTALLED to its version ("" if unreadable). +resolve_bin() { + BIN=""; INSTALLED="" + if [ -r "${CONTROL_ROOT}/current" ]; then + _c="" + read -r _c < "${CONTROL_ROOT}/current" || : + case $_c in ''|.*|*[!0-9A-Za-z.+-]*) _c="" ;; esac + if [ -n "$_c" ] && [ -f "${CONTROL_ROOT}/versions/${_c}/${BIN_REL}" ]; then + BIN="${CONTROL_ROOT}/versions/${_c}/${BIN_REL}"; INSTALLED=$_c; return 0 + fi + fi + if [ -f "${CONTROL_ROOT}/${BIN_REL}" ]; then + BIN="${CONTROL_ROOT}/${BIN_REL}" + pkgver "${CONTROL_ROOT}/${PKG_REL}/package.json" || INSTALLED="" + fi +} + +# Run `$@` with a time bound, portably (stock macOS has no `timeout`): the command +# runs in the background, a watchdog subshell kills it after the limit, `wait` +# returns its status (non-zero when killed). The watchdog's own `sleep` may +# linger until its deadline; it holds no output. +bounded() { + _t=${FUSE_HARNESS_INSTALL_TIMEOUT_SEC:-45}; case $_t in ''|*[!0-9]*) _t=45 ;; esac + "$@" & + _p=$! + ( sleep "$_t"; kill "$_p" ) >/dev/null 2>&1 & + _wd=$! + wait "$_p"; _r=$? + kill "$_wd" >/dev/null 2>&1 + return "$_r" +} run_bin() { if command -v bun >/dev/null 2>&1; then exec bun "$BIN" hook cursor "$@"; fi exec node "$BIN" hook cursor "$@" } -# 1. Fast path: already vendored. -[ -f "$BIN" ] && run_bin "$@" +# Run ` --version` of a candidate bin: proves it starts before it goes live. +bin_answers() { + if command -v bun >/dev/null 2>&1; then bun "$1" --version >/dev/null 2>&1 /dev/null 2>&1 /dev/null && return 0 + [ -d "$LOCK" ] || return 1 + [ -n "$(find "$LOCK" -maxdepth 0 -mmin +10 2>/dev/null)" ] || return 1 + # Reclaim through a second atomic gate so only ONE reclaimer runs the + # check-remove-recreate sequence (two of them would delete each other's lock). + if ! mkdir "${LOCK}.reclaim" 2>/dev/null; then + [ -n "$(find "${LOCK}.reclaim" -maxdepth 0 -mmin +10 2>/dev/null)" ] && rmdir "${LOCK}.reclaim" 2>/dev/null + return 1 + fi + _got=1 + if [ -n "$(find "$LOCK" -maxdepth 0 -mmin +10 2>/dev/null)" ]; then + rmdir "$LOCK" 2>/dev/null && mkdir "$LOCK" 2>/dev/null && _got=0 + fi + rmdir "${LOCK}.reclaim" 2>/dev/null + return "$_got" +} + +# Wait (bounded) while another process holds the lock. +wait_lock() { + _w=${FUSE_HARNESS_LOCK_WAIT_SEC:-20}; case $_w in ''|*[!0-9]*) _w=20 ;; esac + _i=0 + while [ -d "$LOCK" ] && [ "$_i" -lt "$_w" ]; do sleep 1; _i=$((_i + 1)); done +} + +# Staged install of HARNESS_SPEC. Sets NEWV. Returns 0 only when `current` was +# switched to NEWV; the live tree is untouched on any failure. Caller holds LOCK. +install_swap() { + NEWV="" + command -v bun >/dev/null 2>&1 || return 1 + mkdir -p "${CONTROL_ROOT}/versions" 2>/dev/null || return 1 + _stage="${CONTROL_ROOT}/versions/.stage-$$" + trap 'rm -rf "$_stage"; rmdir "$LOCK" 2>/dev/null; exit 0' HUP INT TERM + rm -rf "$_stage" + mkdir "$_stage" 2>/dev/null || return 1 + printf '{"name":"@fusengine/cursor-harness-stage","private":true}\n' > "${_stage}/package.json" + (cd "$_stage" && bounded bun add "@fusengine/harness@${HARNESS_SPEC}" --force --silent) \ + >/dev/null 2>&1 /dev/null || { rm -rf "$_stage"; return 1; }; fi + mv "$_stage" "$_dest" 2>/dev/null || { rm -rf "$_stage"; return 1; } + rm -rf "${_dest}/.stage-$$" # mv nests the stage if $_dest appeared meanwhile + bin_answers "${_dest}/${BIN_REL}" || return 1 + fi + # Never downgrade: keep the live version when it is already newer. + resolve_bin + if [ -n "$INSTALLED" ] && ! ver_ge "$NEWV" "$INSTALLED"; then return 1; fi + PREV=$INSTALLED + printf '%s\n' "$NEWV" > "${CONTROL_ROOT}/current.tmp.$$" 2>/dev/null \ + && mv -f "${CONTROL_ROOT}/current.tmp.$$" "${CONTROL_ROOT}/current" 2>/dev/null \ + || { rm -f "${CONTROL_ROOT}/current.tmp.$$"; return 1; } + # mv into a `current` that is a directory succeeds silently: verify the pointer. + _c=""; read -r _c < "${CONTROL_ROOT}/current" 2>/dev/null + if [ "$_c" != "$NEWV" ]; then rm -f "${CONTROL_ROOT}/current/current.tmp.$$"; return 1; fi + trap - HUP INT TERM + prune_versions + return 0 +} + +# Remove versions other than the new and previous one, and stale staging dirs, +# when older than ~2 days (`find -mtime +1`; a long-lived session may still hold an older tree). +prune_versions() { + for _d in "${CONTROL_ROOT}"/versions/* "${CONTROL_ROOT}"/versions/.stage-*; do + [ -d "$_d" ] || continue + _n=${_d##*/} + [ "$_n" = "$NEWV" ] || [ "$_n" = "$PREV" ] && continue + [ -n "$(find "$_d" -maxdepth 0 -mtime +1 2>/dev/null)" ] && rm -rf "$_d" + done +} + +# Session-start upgrade: back-off, single flight, staged install. +upgrade_stale() { + if [ -e "$FAILED" ]; then + _bo=${FUSE_HARNESS_UPDATE_BACKOFF_MIN:-30}; case $_bo in ''|*[!0-9]*) _bo=30 ;; esac + [ -n "$(find "$FAILED" -mmin "-${_bo}" 2>/dev/null)" ] && return 0 + fi + take_lock || return 0 + # `true`, not `:` — a redirection error on a special built-in exits sh/dash. + true 2>/dev/null > "$FAILED" + _from=$INSTALLED + if install_swap && ver_ge "$NEWV" "$HARNESS_MIN"; then rm -f "$FAILED" + else warn_failed "$_from" "${_bo:-30}"; fi + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + return 0 +} + +# One line on stderr (stdout is the hook's JSON), shown in Cursor's Hooks panel; +# the back-off stamp keeps it to one per window. +warn_failed() { # installed-version retry-minutes ("-" = no automatic retry, installer path) + _rt="retry in $2 min"; [ "$2" = "-" ] && _rt="re-run the installer to retry" + printf 'fusengine harness: upgrade failed (installed %s, target >= %s, staged %s); keeping the installed version, %s\n' \ + "${1:-unknown}" "$HARNESS_MIN" "${NEWV:-none}" "$_rt" >&2 +} + +# --install-harness: installer entry. Staged install, bypassing back-off. +if [ "$MODE" = install ]; then + mkdir -p "$CONTROL_ROOT" 2>/dev/null || exit 1 + resolve_bin; _from=$INSTALLED + wait_lock + take_lock || { warn_failed "$_from" "-"; exit 1; } + install_swap; _rc=$? + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + [ "$_rc" -eq 0 ] || warn_failed "$_from" "-" + exit "$_rc" +fi + +resolve_bin + +# --harness-check: the sessionStart entry of plugins that must NOT run the bin +# there. Same version check and upgrade as --session-start, then exit 0 with +# empty stdout; the bin is never executed. +if [ "$MODE" = check ]; then + if [ -n "$BIN" ] && { [ -z "$INSTALLED" ] || ! ver_ge "$INSTALLED" "$HARNESS_MIN"; }; then + upgrade_stale + fi + exit 0 +fi + +# 1. Fast path: already vendored. Session start checks the version first. +if [ -n "$BIN" ]; then + if [ "$MODE" = session ] && { [ -z "$INSTALLED" ] || ! ver_ge "$INSTALLED" "$HARNESS_MIN"; }; then + upgrade_stale + resolve_bin + fi + run_bin "$@" +fi -# 2. Self-heal once. The stamp keeps 62 hooks from stampeding the registry when -# the install is genuinely impossible (offline, no bun, no network). +# 2. Self-heal once (staged, same routine). An install in flight is awaited; the +# stamp keeps 62 hooks from stampeding the registry when it is impossible. +wait_lock +resolve_bin +[ -n "$BIN" ] && run_bin "$@" if [ ! -e "$STAMP" ] && command -v bun >/dev/null 2>&1; then - mkdir -p "$CONTROL_ROOT" 2>/dev/null && : > "$STAMP" 2>/dev/null - printf '{"name":"@fusengine/cursor-harness","private":true,"dependencies":{"@fusengine/harness":"%s"}}\n' \ - "$HARNESS_SPEC" > "${CONTROL_ROOT}/package.json" 2>/dev/null - (cd "$CONTROL_ROOT" && bun install --silent) >/dev/null 2>&1 - [ -f "$BIN" ] && run_bin "$@" + mkdir -p "$CONTROL_ROOT" 2>/dev/null && true 2>/dev/null > "$STAMP" + if take_lock; then + install_swap + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + fi + resolve_bin + [ -n "$BIN" ] && run_bin "$@" fi # 3. Last resort: npx on the latest published release. diff --git a/.cursor-plugin/plugins/core-guards/.cursor-plugin/plugin.json b/.cursor-plugin/plugins/core-guards/.cursor-plugin/plugin.json index 3312ead..32bbbe4 100644 --- a/.cursor-plugin/plugins/core-guards/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugins/core-guards/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "core-guards", - "version": "1.1.39", + "version": "1.1.40", "description": "Core security guards (git, install, security) and SOLID enforcement hooks", "author": { "name": "Fusengine", diff --git a/.cursor-plugin/plugins/core-guards/hooks/hooks.json b/.cursor-plugin/plugins/core-guards/hooks/hooks.json index cb80980..d0513d3 100644 --- a/.cursor-plugin/plugins/core-guards/hooks/hooks.json +++ b/.cursor-plugin/plugins/core-guards/hooks/hooks.json @@ -25,7 +25,7 @@ ], "sessionStart": [ { - "command": "./scripts/hook.sh" + "command": "./scripts/hook.sh --session-start" } ], "beforeSubmitPrompt": [ diff --git a/.cursor-plugin/plugins/core-guards/scripts/hook.sh b/.cursor-plugin/plugins/core-guards/scripts/hook.sh index 3c88b14..12329dd 100755 --- a/.cursor-plugin/plugins/core-guards/scripts/hook.sh +++ b/.cursor-plugin/plugins/core-guards/scripts/hook.sh @@ -2,46 +2,296 @@ # Fusengine hook entrypoint — shipped inside every plugin as scripts/hook.sh. # # Resolves the shared harness itself so the plugin works from the Cursor -# marketplace, where no installer ever runs. Order: -# 1. the vendored install under $HOME/.cursor/.fusengine-global (fast path) -# 2. a self-heal `bun install` of that same shared directory, once +# marketplace, where no installer ever runs. Layout under the control root +# ($HOME/.cursor/.fusengine-global): +# +# versions//node_modules/@fusengine/harness one immutable install per version +# current text file: the live +# node_modules/@fusengine/harness legacy single install (fallback) +# +# Resolution order: +# 1. the version named by `current` (fast path), else the legacy install +# 2. a self-heal install of `latest` (staged, see below), once # 3. `npx` on latest (last resort, network) # -# Installs the LATEST published harness the first time (the manifest uses the -# `latest` dist-tag, and so does the npx fallback), so no wrapper needs editing -# per release. It does NOT upgrade an existing install: the fast path runs the -# vendored bin as is, and rewriting it in place while up to 62 hooks load it -# could crash a guard mid-run (hooks fail open). The upgrade path is the -# installer (`install.sh` runs `bun update @fusengine/harness` on every run) or, -# by hand: `cd ~/.cursor/.fusengine-global && bun update @fusengine/harness`. +# UPGRADES. A hook invoked as `hook.sh --session-start [scope...]` (only the +# sessionStart entries pass the flag) compares the installed version with +# HARNESS_MIN using shell builtins only — no process, no network. Current or +# newer: nothing else happens. Older: the new release is installed into a +# STAGING directory (`bun add` in versions/.stage-), checked (version and +# `bin.mjs --version`), moved to versions/, and only then made live by +# renaming a temp file over `current` — one atomic rename(2). The running tree +# is never modified in place and the previous version is never deleted during +# the switch, so a guard that starts, or is already running and lazily imports +# a chunk, never sees a half-replaced install. (An in-place `bun update` leaves +# bin.mjs and the hashed chunks absent for a few ms: measured, it crashes +# concurrent hooks.) Old versions are pruned only after a successful switch, +# keeping the new one, the previous one, and anything younger than ~2 days. +# +# Failure handling: a back-off stamp (.harness-update-failed) suppresses retries +# for FUSE_HARNESS_UPDATE_BACKOFF_MIN minutes (default 30); it is written before +# the attempt and removed on success, so a killed attempt is also backed off. An +# atomic mkdir lock (.harness-update.lock, reclaimed after 10 min) keeps the five +# sessionStart hooks from installing at once. Everything fails open. +# +# HARNESS_MIN is the only per-release edit. # # Resolution is relative to THIS file, never to the caller's cwd. Always exits 0 # on its own failures: Cursor fails hooks open, and a broken guard must never # block the user's action. # -# Usage (from hooks/hooks.json): ./scripts/hook.sh [scope...] +# Usage (from hooks/hooks.json): ./scripts/hook.sh [--session-start] [scope...] +# ./scripts/hook.sh --harness-check (sessionStart of plugins that must not +# run the bin there: same check/upgrade, +# then exit 0, empty stdout) +# ./scripts/hook.sh --install-harness (installer: staged install, then exit; +# prints one warning line on failure) +# The install is bounded by FUSE_HARNESS_INSTALL_TIMEOUT_SEC (default 45, under +# Cursor's 60 s default hook timeout); sessionStart entries that can upgrade +# therefore carry no timeout below 60. +HARNESS_MIN="0.1.115" HARNESS_SPEC="latest" -CONTROL_ROOT="${HOME}/.cursor/.fusengine-global" -BIN="${CONTROL_ROOT}/node_modules/@fusengine/harness/dist/cli/bin.mjs" +CONTROL_ROOT="${FUSE_HARNESS_ROOT:-${HOME}/.cursor/.fusengine-global}" +PKG_REL="node_modules/@fusengine/harness" +BIN_REL="node_modules/@fusengine/harness/dist/cli/bin.mjs" STAMP="${CONTROL_ROOT}/.selfheal-attempted" +FAILED="${CONTROL_ROOT}/.harness-update-failed" +LOCK="${CONTROL_ROOT}/.harness-update.lock" + +MODE="" +case "$1" in + --session-start) MODE=session; shift ;; + --install-harness) MODE=install; shift ;; + --harness-check) MODE=check; shift ;; +esac + +BIN="" +INSTALLED="" + +# Set INSTALLED to the "version" of a package.json, builtins only. Empty if absent. +pkgver() { + INSTALLED="" + [ -r "$1" ] || return 1 + while IFS= read -r _l || [ -n "$_l" ]; do + case $_l in + *'"version"'*:*) + _l=${_l#*'"version"'}; _l=${_l#*:}; _l=${_l#*\"} + INSTALLED=${_l%%\"*}; return 0 ;; + esac + done < "$1" + return 1 +} + +# ver_ge A B: numeric A >= B for x.y.z (prerelease/build suffix ignored). +ver_ge() { + _a=${1%%[-+]*}; _b=${2%%[-+]*} + case $_a in ''|*[!0-9.]*) return 1 ;; esac + case $_b in ''|*[!0-9.]*) return 1 ;; esac + _o=$IFS; IFS=. + set -- $_a; _a1=${1:-0}; _a2=${2:-0}; _a3=${3:-0} + set -- $_b; _b1=${1:-0}; _b2=${2:-0}; _b3=${3:-0} + IFS=$_o + [ "$_a1" -gt "$_b1" ] && return 0; [ "$_a1" -lt "$_b1" ] && return 1 + [ "$_a2" -gt "$_b2" ] && return 0; [ "$_a2" -lt "$_b2" ] && return 1 + [ "$_a3" -ge "$_b3" ] +} + +# Set BIN to the live bin ("" if none) and INSTALLED to its version ("" if unreadable). +resolve_bin() { + BIN=""; INSTALLED="" + if [ -r "${CONTROL_ROOT}/current" ]; then + _c="" + read -r _c < "${CONTROL_ROOT}/current" || : + case $_c in ''|.*|*[!0-9A-Za-z.+-]*) _c="" ;; esac + if [ -n "$_c" ] && [ -f "${CONTROL_ROOT}/versions/${_c}/${BIN_REL}" ]; then + BIN="${CONTROL_ROOT}/versions/${_c}/${BIN_REL}"; INSTALLED=$_c; return 0 + fi + fi + if [ -f "${CONTROL_ROOT}/${BIN_REL}" ]; then + BIN="${CONTROL_ROOT}/${BIN_REL}" + pkgver "${CONTROL_ROOT}/${PKG_REL}/package.json" || INSTALLED="" + fi +} + +# Run `$@` with a time bound, portably (stock macOS has no `timeout`): the command +# runs in the background, a watchdog subshell kills it after the limit, `wait` +# returns its status (non-zero when killed). The watchdog's own `sleep` may +# linger until its deadline; it holds no output. +bounded() { + _t=${FUSE_HARNESS_INSTALL_TIMEOUT_SEC:-45}; case $_t in ''|*[!0-9]*) _t=45 ;; esac + "$@" & + _p=$! + ( sleep "$_t"; kill "$_p" ) >/dev/null 2>&1 & + _wd=$! + wait "$_p"; _r=$? + kill "$_wd" >/dev/null 2>&1 + return "$_r" +} run_bin() { if command -v bun >/dev/null 2>&1; then exec bun "$BIN" hook cursor "$@"; fi exec node "$BIN" hook cursor "$@" } -# 1. Fast path: already vendored. -[ -f "$BIN" ] && run_bin "$@" +# Run ` --version` of a candidate bin: proves it starts before it goes live. +bin_answers() { + if command -v bun >/dev/null 2>&1; then bun "$1" --version >/dev/null 2>&1 /dev/null 2>&1 /dev/null && return 0 + [ -d "$LOCK" ] || return 1 + [ -n "$(find "$LOCK" -maxdepth 0 -mmin +10 2>/dev/null)" ] || return 1 + # Reclaim through a second atomic gate so only ONE reclaimer runs the + # check-remove-recreate sequence (two of them would delete each other's lock). + if ! mkdir "${LOCK}.reclaim" 2>/dev/null; then + [ -n "$(find "${LOCK}.reclaim" -maxdepth 0 -mmin +10 2>/dev/null)" ] && rmdir "${LOCK}.reclaim" 2>/dev/null + return 1 + fi + _got=1 + if [ -n "$(find "$LOCK" -maxdepth 0 -mmin +10 2>/dev/null)" ]; then + rmdir "$LOCK" 2>/dev/null && mkdir "$LOCK" 2>/dev/null && _got=0 + fi + rmdir "${LOCK}.reclaim" 2>/dev/null + return "$_got" +} + +# Wait (bounded) while another process holds the lock. +wait_lock() { + _w=${FUSE_HARNESS_LOCK_WAIT_SEC:-20}; case $_w in ''|*[!0-9]*) _w=20 ;; esac + _i=0 + while [ -d "$LOCK" ] && [ "$_i" -lt "$_w" ]; do sleep 1; _i=$((_i + 1)); done +} + +# Staged install of HARNESS_SPEC. Sets NEWV. Returns 0 only when `current` was +# switched to NEWV; the live tree is untouched on any failure. Caller holds LOCK. +install_swap() { + NEWV="" + command -v bun >/dev/null 2>&1 || return 1 + mkdir -p "${CONTROL_ROOT}/versions" 2>/dev/null || return 1 + _stage="${CONTROL_ROOT}/versions/.stage-$$" + trap 'rm -rf "$_stage"; rmdir "$LOCK" 2>/dev/null; exit 0' HUP INT TERM + rm -rf "$_stage" + mkdir "$_stage" 2>/dev/null || return 1 + printf '{"name":"@fusengine/cursor-harness-stage","private":true}\n' > "${_stage}/package.json" + (cd "$_stage" && bounded bun add "@fusengine/harness@${HARNESS_SPEC}" --force --silent) \ + >/dev/null 2>&1 /dev/null || { rm -rf "$_stage"; return 1; }; fi + mv "$_stage" "$_dest" 2>/dev/null || { rm -rf "$_stage"; return 1; } + rm -rf "${_dest}/.stage-$$" # mv nests the stage if $_dest appeared meanwhile + bin_answers "${_dest}/${BIN_REL}" || return 1 + fi + # Never downgrade: keep the live version when it is already newer. + resolve_bin + if [ -n "$INSTALLED" ] && ! ver_ge "$NEWV" "$INSTALLED"; then return 1; fi + PREV=$INSTALLED + printf '%s\n' "$NEWV" > "${CONTROL_ROOT}/current.tmp.$$" 2>/dev/null \ + && mv -f "${CONTROL_ROOT}/current.tmp.$$" "${CONTROL_ROOT}/current" 2>/dev/null \ + || { rm -f "${CONTROL_ROOT}/current.tmp.$$"; return 1; } + # mv into a `current` that is a directory succeeds silently: verify the pointer. + _c=""; read -r _c < "${CONTROL_ROOT}/current" 2>/dev/null + if [ "$_c" != "$NEWV" ]; then rm -f "${CONTROL_ROOT}/current/current.tmp.$$"; return 1; fi + trap - HUP INT TERM + prune_versions + return 0 +} + +# Remove versions other than the new and previous one, and stale staging dirs, +# when older than ~2 days (`find -mtime +1`; a long-lived session may still hold an older tree). +prune_versions() { + for _d in "${CONTROL_ROOT}"/versions/* "${CONTROL_ROOT}"/versions/.stage-*; do + [ -d "$_d" ] || continue + _n=${_d##*/} + [ "$_n" = "$NEWV" ] || [ "$_n" = "$PREV" ] && continue + [ -n "$(find "$_d" -maxdepth 0 -mtime +1 2>/dev/null)" ] && rm -rf "$_d" + done +} + +# Session-start upgrade: back-off, single flight, staged install. +upgrade_stale() { + if [ -e "$FAILED" ]; then + _bo=${FUSE_HARNESS_UPDATE_BACKOFF_MIN:-30}; case $_bo in ''|*[!0-9]*) _bo=30 ;; esac + [ -n "$(find "$FAILED" -mmin "-${_bo}" 2>/dev/null)" ] && return 0 + fi + take_lock || return 0 + # `true`, not `:` — a redirection error on a special built-in exits sh/dash. + true 2>/dev/null > "$FAILED" + _from=$INSTALLED + if install_swap && ver_ge "$NEWV" "$HARNESS_MIN"; then rm -f "$FAILED" + else warn_failed "$_from" "${_bo:-30}"; fi + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + return 0 +} + +# One line on stderr (stdout is the hook's JSON), shown in Cursor's Hooks panel; +# the back-off stamp keeps it to one per window. +warn_failed() { # installed-version retry-minutes ("-" = no automatic retry, installer path) + _rt="retry in $2 min"; [ "$2" = "-" ] && _rt="re-run the installer to retry" + printf 'fusengine harness: upgrade failed (installed %s, target >= %s, staged %s); keeping the installed version, %s\n' \ + "${1:-unknown}" "$HARNESS_MIN" "${NEWV:-none}" "$_rt" >&2 +} + +# --install-harness: installer entry. Staged install, bypassing back-off. +if [ "$MODE" = install ]; then + mkdir -p "$CONTROL_ROOT" 2>/dev/null || exit 1 + resolve_bin; _from=$INSTALLED + wait_lock + take_lock || { warn_failed "$_from" "-"; exit 1; } + install_swap; _rc=$? + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + [ "$_rc" -eq 0 ] || warn_failed "$_from" "-" + exit "$_rc" +fi + +resolve_bin + +# --harness-check: the sessionStart entry of plugins that must NOT run the bin +# there. Same version check and upgrade as --session-start, then exit 0 with +# empty stdout; the bin is never executed. +if [ "$MODE" = check ]; then + if [ -n "$BIN" ] && { [ -z "$INSTALLED" ] || ! ver_ge "$INSTALLED" "$HARNESS_MIN"; }; then + upgrade_stale + fi + exit 0 +fi + +# 1. Fast path: already vendored. Session start checks the version first. +if [ -n "$BIN" ]; then + if [ "$MODE" = session ] && { [ -z "$INSTALLED" ] || ! ver_ge "$INSTALLED" "$HARNESS_MIN"; }; then + upgrade_stale + resolve_bin + fi + run_bin "$@" +fi -# 2. Self-heal once. The stamp keeps 62 hooks from stampeding the registry when -# the install is genuinely impossible (offline, no bun, no network). +# 2. Self-heal once (staged, same routine). An install in flight is awaited; the +# stamp keeps 62 hooks from stampeding the registry when it is impossible. +wait_lock +resolve_bin +[ -n "$BIN" ] && run_bin "$@" if [ ! -e "$STAMP" ] && command -v bun >/dev/null 2>&1; then - mkdir -p "$CONTROL_ROOT" 2>/dev/null && : > "$STAMP" 2>/dev/null - printf '{"name":"@fusengine/cursor-harness","private":true,"dependencies":{"@fusengine/harness":"%s"}}\n' \ - "$HARNESS_SPEC" > "${CONTROL_ROOT}/package.json" 2>/dev/null - (cd "$CONTROL_ROOT" && bun install --silent) >/dev/null 2>&1 - [ -f "$BIN" ] && run_bin "$@" + mkdir -p "$CONTROL_ROOT" 2>/dev/null && true 2>/dev/null > "$STAMP" + if take_lock; then + install_swap + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + fi + resolve_bin + [ -n "$BIN" ] && run_bin "$@" fi # 3. Last resort: npx on the latest published release. diff --git a/.cursor-plugin/plugins/fuse-ai-pilot/.cursor-plugin/plugin.json b/.cursor-plugin/plugins/fuse-ai-pilot/.cursor-plugin/plugin.json index 3819ab1..82cd2ec 100644 --- a/.cursor-plugin/plugins/fuse-ai-pilot/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugins/fuse-ai-pilot/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "fuse-ai-pilot", - "version": "1.2.46", + "version": "1.2.47", "description": "AI Pilot workflow agents: sniper (code quality), research-expert, explore-codebase, websearch", "author": { "name": "Fusengine", diff --git a/.cursor-plugin/plugins/fuse-ai-pilot/hooks/hooks.json b/.cursor-plugin/plugins/fuse-ai-pilot/hooks/hooks.json index 53ce9ac..83eb1ce 100644 --- a/.cursor-plugin/plugins/fuse-ai-pilot/hooks/hooks.json +++ b/.cursor-plugin/plugins/fuse-ai-pilot/hooks/hooks.json @@ -1,6 +1,12 @@ { "version": 1, "hooks": { + "sessionStart": [ + { + "command": "./scripts/hook.sh --harness-check", + "timeout": 60 + } + ], "subagentStart": [ { "command": "./scripts/hook.sh aipilot" diff --git a/.cursor-plugin/plugins/fuse-ai-pilot/scripts/hook.sh b/.cursor-plugin/plugins/fuse-ai-pilot/scripts/hook.sh index 3c88b14..12329dd 100755 --- a/.cursor-plugin/plugins/fuse-ai-pilot/scripts/hook.sh +++ b/.cursor-plugin/plugins/fuse-ai-pilot/scripts/hook.sh @@ -2,46 +2,296 @@ # Fusengine hook entrypoint — shipped inside every plugin as scripts/hook.sh. # # Resolves the shared harness itself so the plugin works from the Cursor -# marketplace, where no installer ever runs. Order: -# 1. the vendored install under $HOME/.cursor/.fusengine-global (fast path) -# 2. a self-heal `bun install` of that same shared directory, once +# marketplace, where no installer ever runs. Layout under the control root +# ($HOME/.cursor/.fusengine-global): +# +# versions//node_modules/@fusengine/harness one immutable install per version +# current text file: the live +# node_modules/@fusengine/harness legacy single install (fallback) +# +# Resolution order: +# 1. the version named by `current` (fast path), else the legacy install +# 2. a self-heal install of `latest` (staged, see below), once # 3. `npx` on latest (last resort, network) # -# Installs the LATEST published harness the first time (the manifest uses the -# `latest` dist-tag, and so does the npx fallback), so no wrapper needs editing -# per release. It does NOT upgrade an existing install: the fast path runs the -# vendored bin as is, and rewriting it in place while up to 62 hooks load it -# could crash a guard mid-run (hooks fail open). The upgrade path is the -# installer (`install.sh` runs `bun update @fusengine/harness` on every run) or, -# by hand: `cd ~/.cursor/.fusengine-global && bun update @fusengine/harness`. +# UPGRADES. A hook invoked as `hook.sh --session-start [scope...]` (only the +# sessionStart entries pass the flag) compares the installed version with +# HARNESS_MIN using shell builtins only — no process, no network. Current or +# newer: nothing else happens. Older: the new release is installed into a +# STAGING directory (`bun add` in versions/.stage-), checked (version and +# `bin.mjs --version`), moved to versions/, and only then made live by +# renaming a temp file over `current` — one atomic rename(2). The running tree +# is never modified in place and the previous version is never deleted during +# the switch, so a guard that starts, or is already running and lazily imports +# a chunk, never sees a half-replaced install. (An in-place `bun update` leaves +# bin.mjs and the hashed chunks absent for a few ms: measured, it crashes +# concurrent hooks.) Old versions are pruned only after a successful switch, +# keeping the new one, the previous one, and anything younger than ~2 days. +# +# Failure handling: a back-off stamp (.harness-update-failed) suppresses retries +# for FUSE_HARNESS_UPDATE_BACKOFF_MIN minutes (default 30); it is written before +# the attempt and removed on success, so a killed attempt is also backed off. An +# atomic mkdir lock (.harness-update.lock, reclaimed after 10 min) keeps the five +# sessionStart hooks from installing at once. Everything fails open. +# +# HARNESS_MIN is the only per-release edit. # # Resolution is relative to THIS file, never to the caller's cwd. Always exits 0 # on its own failures: Cursor fails hooks open, and a broken guard must never # block the user's action. # -# Usage (from hooks/hooks.json): ./scripts/hook.sh [scope...] +# Usage (from hooks/hooks.json): ./scripts/hook.sh [--session-start] [scope...] +# ./scripts/hook.sh --harness-check (sessionStart of plugins that must not +# run the bin there: same check/upgrade, +# then exit 0, empty stdout) +# ./scripts/hook.sh --install-harness (installer: staged install, then exit; +# prints one warning line on failure) +# The install is bounded by FUSE_HARNESS_INSTALL_TIMEOUT_SEC (default 45, under +# Cursor's 60 s default hook timeout); sessionStart entries that can upgrade +# therefore carry no timeout below 60. +HARNESS_MIN="0.1.115" HARNESS_SPEC="latest" -CONTROL_ROOT="${HOME}/.cursor/.fusengine-global" -BIN="${CONTROL_ROOT}/node_modules/@fusengine/harness/dist/cli/bin.mjs" +CONTROL_ROOT="${FUSE_HARNESS_ROOT:-${HOME}/.cursor/.fusengine-global}" +PKG_REL="node_modules/@fusengine/harness" +BIN_REL="node_modules/@fusengine/harness/dist/cli/bin.mjs" STAMP="${CONTROL_ROOT}/.selfheal-attempted" +FAILED="${CONTROL_ROOT}/.harness-update-failed" +LOCK="${CONTROL_ROOT}/.harness-update.lock" + +MODE="" +case "$1" in + --session-start) MODE=session; shift ;; + --install-harness) MODE=install; shift ;; + --harness-check) MODE=check; shift ;; +esac + +BIN="" +INSTALLED="" + +# Set INSTALLED to the "version" of a package.json, builtins only. Empty if absent. +pkgver() { + INSTALLED="" + [ -r "$1" ] || return 1 + while IFS= read -r _l || [ -n "$_l" ]; do + case $_l in + *'"version"'*:*) + _l=${_l#*'"version"'}; _l=${_l#*:}; _l=${_l#*\"} + INSTALLED=${_l%%\"*}; return 0 ;; + esac + done < "$1" + return 1 +} + +# ver_ge A B: numeric A >= B for x.y.z (prerelease/build suffix ignored). +ver_ge() { + _a=${1%%[-+]*}; _b=${2%%[-+]*} + case $_a in ''|*[!0-9.]*) return 1 ;; esac + case $_b in ''|*[!0-9.]*) return 1 ;; esac + _o=$IFS; IFS=. + set -- $_a; _a1=${1:-0}; _a2=${2:-0}; _a3=${3:-0} + set -- $_b; _b1=${1:-0}; _b2=${2:-0}; _b3=${3:-0} + IFS=$_o + [ "$_a1" -gt "$_b1" ] && return 0; [ "$_a1" -lt "$_b1" ] && return 1 + [ "$_a2" -gt "$_b2" ] && return 0; [ "$_a2" -lt "$_b2" ] && return 1 + [ "$_a3" -ge "$_b3" ] +} + +# Set BIN to the live bin ("" if none) and INSTALLED to its version ("" if unreadable). +resolve_bin() { + BIN=""; INSTALLED="" + if [ -r "${CONTROL_ROOT}/current" ]; then + _c="" + read -r _c < "${CONTROL_ROOT}/current" || : + case $_c in ''|.*|*[!0-9A-Za-z.+-]*) _c="" ;; esac + if [ -n "$_c" ] && [ -f "${CONTROL_ROOT}/versions/${_c}/${BIN_REL}" ]; then + BIN="${CONTROL_ROOT}/versions/${_c}/${BIN_REL}"; INSTALLED=$_c; return 0 + fi + fi + if [ -f "${CONTROL_ROOT}/${BIN_REL}" ]; then + BIN="${CONTROL_ROOT}/${BIN_REL}" + pkgver "${CONTROL_ROOT}/${PKG_REL}/package.json" || INSTALLED="" + fi +} + +# Run `$@` with a time bound, portably (stock macOS has no `timeout`): the command +# runs in the background, a watchdog subshell kills it after the limit, `wait` +# returns its status (non-zero when killed). The watchdog's own `sleep` may +# linger until its deadline; it holds no output. +bounded() { + _t=${FUSE_HARNESS_INSTALL_TIMEOUT_SEC:-45}; case $_t in ''|*[!0-9]*) _t=45 ;; esac + "$@" & + _p=$! + ( sleep "$_t"; kill "$_p" ) >/dev/null 2>&1 & + _wd=$! + wait "$_p"; _r=$? + kill "$_wd" >/dev/null 2>&1 + return "$_r" +} run_bin() { if command -v bun >/dev/null 2>&1; then exec bun "$BIN" hook cursor "$@"; fi exec node "$BIN" hook cursor "$@" } -# 1. Fast path: already vendored. -[ -f "$BIN" ] && run_bin "$@" +# Run ` --version` of a candidate bin: proves it starts before it goes live. +bin_answers() { + if command -v bun >/dev/null 2>&1; then bun "$1" --version >/dev/null 2>&1 /dev/null 2>&1 /dev/null && return 0 + [ -d "$LOCK" ] || return 1 + [ -n "$(find "$LOCK" -maxdepth 0 -mmin +10 2>/dev/null)" ] || return 1 + # Reclaim through a second atomic gate so only ONE reclaimer runs the + # check-remove-recreate sequence (two of them would delete each other's lock). + if ! mkdir "${LOCK}.reclaim" 2>/dev/null; then + [ -n "$(find "${LOCK}.reclaim" -maxdepth 0 -mmin +10 2>/dev/null)" ] && rmdir "${LOCK}.reclaim" 2>/dev/null + return 1 + fi + _got=1 + if [ -n "$(find "$LOCK" -maxdepth 0 -mmin +10 2>/dev/null)" ]; then + rmdir "$LOCK" 2>/dev/null && mkdir "$LOCK" 2>/dev/null && _got=0 + fi + rmdir "${LOCK}.reclaim" 2>/dev/null + return "$_got" +} + +# Wait (bounded) while another process holds the lock. +wait_lock() { + _w=${FUSE_HARNESS_LOCK_WAIT_SEC:-20}; case $_w in ''|*[!0-9]*) _w=20 ;; esac + _i=0 + while [ -d "$LOCK" ] && [ "$_i" -lt "$_w" ]; do sleep 1; _i=$((_i + 1)); done +} + +# Staged install of HARNESS_SPEC. Sets NEWV. Returns 0 only when `current` was +# switched to NEWV; the live tree is untouched on any failure. Caller holds LOCK. +install_swap() { + NEWV="" + command -v bun >/dev/null 2>&1 || return 1 + mkdir -p "${CONTROL_ROOT}/versions" 2>/dev/null || return 1 + _stage="${CONTROL_ROOT}/versions/.stage-$$" + trap 'rm -rf "$_stage"; rmdir "$LOCK" 2>/dev/null; exit 0' HUP INT TERM + rm -rf "$_stage" + mkdir "$_stage" 2>/dev/null || return 1 + printf '{"name":"@fusengine/cursor-harness-stage","private":true}\n' > "${_stage}/package.json" + (cd "$_stage" && bounded bun add "@fusengine/harness@${HARNESS_SPEC}" --force --silent) \ + >/dev/null 2>&1 /dev/null || { rm -rf "$_stage"; return 1; }; fi + mv "$_stage" "$_dest" 2>/dev/null || { rm -rf "$_stage"; return 1; } + rm -rf "${_dest}/.stage-$$" # mv nests the stage if $_dest appeared meanwhile + bin_answers "${_dest}/${BIN_REL}" || return 1 + fi + # Never downgrade: keep the live version when it is already newer. + resolve_bin + if [ -n "$INSTALLED" ] && ! ver_ge "$NEWV" "$INSTALLED"; then return 1; fi + PREV=$INSTALLED + printf '%s\n' "$NEWV" > "${CONTROL_ROOT}/current.tmp.$$" 2>/dev/null \ + && mv -f "${CONTROL_ROOT}/current.tmp.$$" "${CONTROL_ROOT}/current" 2>/dev/null \ + || { rm -f "${CONTROL_ROOT}/current.tmp.$$"; return 1; } + # mv into a `current` that is a directory succeeds silently: verify the pointer. + _c=""; read -r _c < "${CONTROL_ROOT}/current" 2>/dev/null + if [ "$_c" != "$NEWV" ]; then rm -f "${CONTROL_ROOT}/current/current.tmp.$$"; return 1; fi + trap - HUP INT TERM + prune_versions + return 0 +} + +# Remove versions other than the new and previous one, and stale staging dirs, +# when older than ~2 days (`find -mtime +1`; a long-lived session may still hold an older tree). +prune_versions() { + for _d in "${CONTROL_ROOT}"/versions/* "${CONTROL_ROOT}"/versions/.stage-*; do + [ -d "$_d" ] || continue + _n=${_d##*/} + [ "$_n" = "$NEWV" ] || [ "$_n" = "$PREV" ] && continue + [ -n "$(find "$_d" -maxdepth 0 -mtime +1 2>/dev/null)" ] && rm -rf "$_d" + done +} + +# Session-start upgrade: back-off, single flight, staged install. +upgrade_stale() { + if [ -e "$FAILED" ]; then + _bo=${FUSE_HARNESS_UPDATE_BACKOFF_MIN:-30}; case $_bo in ''|*[!0-9]*) _bo=30 ;; esac + [ -n "$(find "$FAILED" -mmin "-${_bo}" 2>/dev/null)" ] && return 0 + fi + take_lock || return 0 + # `true`, not `:` — a redirection error on a special built-in exits sh/dash. + true 2>/dev/null > "$FAILED" + _from=$INSTALLED + if install_swap && ver_ge "$NEWV" "$HARNESS_MIN"; then rm -f "$FAILED" + else warn_failed "$_from" "${_bo:-30}"; fi + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + return 0 +} + +# One line on stderr (stdout is the hook's JSON), shown in Cursor's Hooks panel; +# the back-off stamp keeps it to one per window. +warn_failed() { # installed-version retry-minutes ("-" = no automatic retry, installer path) + _rt="retry in $2 min"; [ "$2" = "-" ] && _rt="re-run the installer to retry" + printf 'fusengine harness: upgrade failed (installed %s, target >= %s, staged %s); keeping the installed version, %s\n' \ + "${1:-unknown}" "$HARNESS_MIN" "${NEWV:-none}" "$_rt" >&2 +} + +# --install-harness: installer entry. Staged install, bypassing back-off. +if [ "$MODE" = install ]; then + mkdir -p "$CONTROL_ROOT" 2>/dev/null || exit 1 + resolve_bin; _from=$INSTALLED + wait_lock + take_lock || { warn_failed "$_from" "-"; exit 1; } + install_swap; _rc=$? + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + [ "$_rc" -eq 0 ] || warn_failed "$_from" "-" + exit "$_rc" +fi + +resolve_bin + +# --harness-check: the sessionStart entry of plugins that must NOT run the bin +# there. Same version check and upgrade as --session-start, then exit 0 with +# empty stdout; the bin is never executed. +if [ "$MODE" = check ]; then + if [ -n "$BIN" ] && { [ -z "$INSTALLED" ] || ! ver_ge "$INSTALLED" "$HARNESS_MIN"; }; then + upgrade_stale + fi + exit 0 +fi + +# 1. Fast path: already vendored. Session start checks the version first. +if [ -n "$BIN" ]; then + if [ "$MODE" = session ] && { [ -z "$INSTALLED" ] || ! ver_ge "$INSTALLED" "$HARNESS_MIN"; }; then + upgrade_stale + resolve_bin + fi + run_bin "$@" +fi -# 2. Self-heal once. The stamp keeps 62 hooks from stampeding the registry when -# the install is genuinely impossible (offline, no bun, no network). +# 2. Self-heal once (staged, same routine). An install in flight is awaited; the +# stamp keeps 62 hooks from stampeding the registry when it is impossible. +wait_lock +resolve_bin +[ -n "$BIN" ] && run_bin "$@" if [ ! -e "$STAMP" ] && command -v bun >/dev/null 2>&1; then - mkdir -p "$CONTROL_ROOT" 2>/dev/null && : > "$STAMP" 2>/dev/null - printf '{"name":"@fusengine/cursor-harness","private":true,"dependencies":{"@fusengine/harness":"%s"}}\n' \ - "$HARNESS_SPEC" > "${CONTROL_ROOT}/package.json" 2>/dev/null - (cd "$CONTROL_ROOT" && bun install --silent) >/dev/null 2>&1 - [ -f "$BIN" ] && run_bin "$@" + mkdir -p "$CONTROL_ROOT" 2>/dev/null && true 2>/dev/null > "$STAMP" + if take_lock; then + install_swap + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + fi + resolve_bin + [ -n "$BIN" ] && run_bin "$@" fi # 3. Last resort: npx on the latest published release. diff --git a/.cursor-plugin/plugins/fuse-cartographer/.cursor-plugin/plugin.json b/.cursor-plugin/plugins/fuse-cartographer/.cursor-plugin/plugin.json index ba309e8..2a92b45 100644 --- a/.cursor-plugin/plugins/fuse-cartographer/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugins/fuse-cartographer/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "fuse-cartographer", - "version": "1.0.14", + "version": "1.0.15", "description": "Generates indented markdown maps of the plugin ecosystem for agent navigation", "author": { "name": "Fusengine", diff --git a/.cursor-plugin/plugins/fuse-cartographer/hooks/hooks.json b/.cursor-plugin/plugins/fuse-cartographer/hooks/hooks.json index 3ac6c18..c5ce2fd 100644 --- a/.cursor-plugin/plugins/fuse-cartographer/hooks/hooks.json +++ b/.cursor-plugin/plugins/fuse-cartographer/hooks/hooks.json @@ -3,7 +3,7 @@ "hooks": { "sessionStart": [ { - "command": "./scripts/hook.sh carto" + "command": "./scripts/hook.sh --session-start carto" } ], "postToolUse": [ diff --git a/.cursor-plugin/plugins/fuse-cartographer/scripts/hook.sh b/.cursor-plugin/plugins/fuse-cartographer/scripts/hook.sh index 3c88b14..12329dd 100755 --- a/.cursor-plugin/plugins/fuse-cartographer/scripts/hook.sh +++ b/.cursor-plugin/plugins/fuse-cartographer/scripts/hook.sh @@ -2,46 +2,296 @@ # Fusengine hook entrypoint — shipped inside every plugin as scripts/hook.sh. # # Resolves the shared harness itself so the plugin works from the Cursor -# marketplace, where no installer ever runs. Order: -# 1. the vendored install under $HOME/.cursor/.fusengine-global (fast path) -# 2. a self-heal `bun install` of that same shared directory, once +# marketplace, where no installer ever runs. Layout under the control root +# ($HOME/.cursor/.fusengine-global): +# +# versions//node_modules/@fusengine/harness one immutable install per version +# current text file: the live +# node_modules/@fusengine/harness legacy single install (fallback) +# +# Resolution order: +# 1. the version named by `current` (fast path), else the legacy install +# 2. a self-heal install of `latest` (staged, see below), once # 3. `npx` on latest (last resort, network) # -# Installs the LATEST published harness the first time (the manifest uses the -# `latest` dist-tag, and so does the npx fallback), so no wrapper needs editing -# per release. It does NOT upgrade an existing install: the fast path runs the -# vendored bin as is, and rewriting it in place while up to 62 hooks load it -# could crash a guard mid-run (hooks fail open). The upgrade path is the -# installer (`install.sh` runs `bun update @fusengine/harness` on every run) or, -# by hand: `cd ~/.cursor/.fusengine-global && bun update @fusengine/harness`. +# UPGRADES. A hook invoked as `hook.sh --session-start [scope...]` (only the +# sessionStart entries pass the flag) compares the installed version with +# HARNESS_MIN using shell builtins only — no process, no network. Current or +# newer: nothing else happens. Older: the new release is installed into a +# STAGING directory (`bun add` in versions/.stage-), checked (version and +# `bin.mjs --version`), moved to versions/, and only then made live by +# renaming a temp file over `current` — one atomic rename(2). The running tree +# is never modified in place and the previous version is never deleted during +# the switch, so a guard that starts, or is already running and lazily imports +# a chunk, never sees a half-replaced install. (An in-place `bun update` leaves +# bin.mjs and the hashed chunks absent for a few ms: measured, it crashes +# concurrent hooks.) Old versions are pruned only after a successful switch, +# keeping the new one, the previous one, and anything younger than ~2 days. +# +# Failure handling: a back-off stamp (.harness-update-failed) suppresses retries +# for FUSE_HARNESS_UPDATE_BACKOFF_MIN minutes (default 30); it is written before +# the attempt and removed on success, so a killed attempt is also backed off. An +# atomic mkdir lock (.harness-update.lock, reclaimed after 10 min) keeps the five +# sessionStart hooks from installing at once. Everything fails open. +# +# HARNESS_MIN is the only per-release edit. # # Resolution is relative to THIS file, never to the caller's cwd. Always exits 0 # on its own failures: Cursor fails hooks open, and a broken guard must never # block the user's action. # -# Usage (from hooks/hooks.json): ./scripts/hook.sh [scope...] +# Usage (from hooks/hooks.json): ./scripts/hook.sh [--session-start] [scope...] +# ./scripts/hook.sh --harness-check (sessionStart of plugins that must not +# run the bin there: same check/upgrade, +# then exit 0, empty stdout) +# ./scripts/hook.sh --install-harness (installer: staged install, then exit; +# prints one warning line on failure) +# The install is bounded by FUSE_HARNESS_INSTALL_TIMEOUT_SEC (default 45, under +# Cursor's 60 s default hook timeout); sessionStart entries that can upgrade +# therefore carry no timeout below 60. +HARNESS_MIN="0.1.115" HARNESS_SPEC="latest" -CONTROL_ROOT="${HOME}/.cursor/.fusengine-global" -BIN="${CONTROL_ROOT}/node_modules/@fusengine/harness/dist/cli/bin.mjs" +CONTROL_ROOT="${FUSE_HARNESS_ROOT:-${HOME}/.cursor/.fusengine-global}" +PKG_REL="node_modules/@fusengine/harness" +BIN_REL="node_modules/@fusengine/harness/dist/cli/bin.mjs" STAMP="${CONTROL_ROOT}/.selfheal-attempted" +FAILED="${CONTROL_ROOT}/.harness-update-failed" +LOCK="${CONTROL_ROOT}/.harness-update.lock" + +MODE="" +case "$1" in + --session-start) MODE=session; shift ;; + --install-harness) MODE=install; shift ;; + --harness-check) MODE=check; shift ;; +esac + +BIN="" +INSTALLED="" + +# Set INSTALLED to the "version" of a package.json, builtins only. Empty if absent. +pkgver() { + INSTALLED="" + [ -r "$1" ] || return 1 + while IFS= read -r _l || [ -n "$_l" ]; do + case $_l in + *'"version"'*:*) + _l=${_l#*'"version"'}; _l=${_l#*:}; _l=${_l#*\"} + INSTALLED=${_l%%\"*}; return 0 ;; + esac + done < "$1" + return 1 +} + +# ver_ge A B: numeric A >= B for x.y.z (prerelease/build suffix ignored). +ver_ge() { + _a=${1%%[-+]*}; _b=${2%%[-+]*} + case $_a in ''|*[!0-9.]*) return 1 ;; esac + case $_b in ''|*[!0-9.]*) return 1 ;; esac + _o=$IFS; IFS=. + set -- $_a; _a1=${1:-0}; _a2=${2:-0}; _a3=${3:-0} + set -- $_b; _b1=${1:-0}; _b2=${2:-0}; _b3=${3:-0} + IFS=$_o + [ "$_a1" -gt "$_b1" ] && return 0; [ "$_a1" -lt "$_b1" ] && return 1 + [ "$_a2" -gt "$_b2" ] && return 0; [ "$_a2" -lt "$_b2" ] && return 1 + [ "$_a3" -ge "$_b3" ] +} + +# Set BIN to the live bin ("" if none) and INSTALLED to its version ("" if unreadable). +resolve_bin() { + BIN=""; INSTALLED="" + if [ -r "${CONTROL_ROOT}/current" ]; then + _c="" + read -r _c < "${CONTROL_ROOT}/current" || : + case $_c in ''|.*|*[!0-9A-Za-z.+-]*) _c="" ;; esac + if [ -n "$_c" ] && [ -f "${CONTROL_ROOT}/versions/${_c}/${BIN_REL}" ]; then + BIN="${CONTROL_ROOT}/versions/${_c}/${BIN_REL}"; INSTALLED=$_c; return 0 + fi + fi + if [ -f "${CONTROL_ROOT}/${BIN_REL}" ]; then + BIN="${CONTROL_ROOT}/${BIN_REL}" + pkgver "${CONTROL_ROOT}/${PKG_REL}/package.json" || INSTALLED="" + fi +} + +# Run `$@` with a time bound, portably (stock macOS has no `timeout`): the command +# runs in the background, a watchdog subshell kills it after the limit, `wait` +# returns its status (non-zero when killed). The watchdog's own `sleep` may +# linger until its deadline; it holds no output. +bounded() { + _t=${FUSE_HARNESS_INSTALL_TIMEOUT_SEC:-45}; case $_t in ''|*[!0-9]*) _t=45 ;; esac + "$@" & + _p=$! + ( sleep "$_t"; kill "$_p" ) >/dev/null 2>&1 & + _wd=$! + wait "$_p"; _r=$? + kill "$_wd" >/dev/null 2>&1 + return "$_r" +} run_bin() { if command -v bun >/dev/null 2>&1; then exec bun "$BIN" hook cursor "$@"; fi exec node "$BIN" hook cursor "$@" } -# 1. Fast path: already vendored. -[ -f "$BIN" ] && run_bin "$@" +# Run ` --version` of a candidate bin: proves it starts before it goes live. +bin_answers() { + if command -v bun >/dev/null 2>&1; then bun "$1" --version >/dev/null 2>&1 /dev/null 2>&1 /dev/null && return 0 + [ -d "$LOCK" ] || return 1 + [ -n "$(find "$LOCK" -maxdepth 0 -mmin +10 2>/dev/null)" ] || return 1 + # Reclaim through a second atomic gate so only ONE reclaimer runs the + # check-remove-recreate sequence (two of them would delete each other's lock). + if ! mkdir "${LOCK}.reclaim" 2>/dev/null; then + [ -n "$(find "${LOCK}.reclaim" -maxdepth 0 -mmin +10 2>/dev/null)" ] && rmdir "${LOCK}.reclaim" 2>/dev/null + return 1 + fi + _got=1 + if [ -n "$(find "$LOCK" -maxdepth 0 -mmin +10 2>/dev/null)" ]; then + rmdir "$LOCK" 2>/dev/null && mkdir "$LOCK" 2>/dev/null && _got=0 + fi + rmdir "${LOCK}.reclaim" 2>/dev/null + return "$_got" +} + +# Wait (bounded) while another process holds the lock. +wait_lock() { + _w=${FUSE_HARNESS_LOCK_WAIT_SEC:-20}; case $_w in ''|*[!0-9]*) _w=20 ;; esac + _i=0 + while [ -d "$LOCK" ] && [ "$_i" -lt "$_w" ]; do sleep 1; _i=$((_i + 1)); done +} + +# Staged install of HARNESS_SPEC. Sets NEWV. Returns 0 only when `current` was +# switched to NEWV; the live tree is untouched on any failure. Caller holds LOCK. +install_swap() { + NEWV="" + command -v bun >/dev/null 2>&1 || return 1 + mkdir -p "${CONTROL_ROOT}/versions" 2>/dev/null || return 1 + _stage="${CONTROL_ROOT}/versions/.stage-$$" + trap 'rm -rf "$_stage"; rmdir "$LOCK" 2>/dev/null; exit 0' HUP INT TERM + rm -rf "$_stage" + mkdir "$_stage" 2>/dev/null || return 1 + printf '{"name":"@fusengine/cursor-harness-stage","private":true}\n' > "${_stage}/package.json" + (cd "$_stage" && bounded bun add "@fusengine/harness@${HARNESS_SPEC}" --force --silent) \ + >/dev/null 2>&1 /dev/null || { rm -rf "$_stage"; return 1; }; fi + mv "$_stage" "$_dest" 2>/dev/null || { rm -rf "$_stage"; return 1; } + rm -rf "${_dest}/.stage-$$" # mv nests the stage if $_dest appeared meanwhile + bin_answers "${_dest}/${BIN_REL}" || return 1 + fi + # Never downgrade: keep the live version when it is already newer. + resolve_bin + if [ -n "$INSTALLED" ] && ! ver_ge "$NEWV" "$INSTALLED"; then return 1; fi + PREV=$INSTALLED + printf '%s\n' "$NEWV" > "${CONTROL_ROOT}/current.tmp.$$" 2>/dev/null \ + && mv -f "${CONTROL_ROOT}/current.tmp.$$" "${CONTROL_ROOT}/current" 2>/dev/null \ + || { rm -f "${CONTROL_ROOT}/current.tmp.$$"; return 1; } + # mv into a `current` that is a directory succeeds silently: verify the pointer. + _c=""; read -r _c < "${CONTROL_ROOT}/current" 2>/dev/null + if [ "$_c" != "$NEWV" ]; then rm -f "${CONTROL_ROOT}/current/current.tmp.$$"; return 1; fi + trap - HUP INT TERM + prune_versions + return 0 +} + +# Remove versions other than the new and previous one, and stale staging dirs, +# when older than ~2 days (`find -mtime +1`; a long-lived session may still hold an older tree). +prune_versions() { + for _d in "${CONTROL_ROOT}"/versions/* "${CONTROL_ROOT}"/versions/.stage-*; do + [ -d "$_d" ] || continue + _n=${_d##*/} + [ "$_n" = "$NEWV" ] || [ "$_n" = "$PREV" ] && continue + [ -n "$(find "$_d" -maxdepth 0 -mtime +1 2>/dev/null)" ] && rm -rf "$_d" + done +} + +# Session-start upgrade: back-off, single flight, staged install. +upgrade_stale() { + if [ -e "$FAILED" ]; then + _bo=${FUSE_HARNESS_UPDATE_BACKOFF_MIN:-30}; case $_bo in ''|*[!0-9]*) _bo=30 ;; esac + [ -n "$(find "$FAILED" -mmin "-${_bo}" 2>/dev/null)" ] && return 0 + fi + take_lock || return 0 + # `true`, not `:` — a redirection error on a special built-in exits sh/dash. + true 2>/dev/null > "$FAILED" + _from=$INSTALLED + if install_swap && ver_ge "$NEWV" "$HARNESS_MIN"; then rm -f "$FAILED" + else warn_failed "$_from" "${_bo:-30}"; fi + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + return 0 +} + +# One line on stderr (stdout is the hook's JSON), shown in Cursor's Hooks panel; +# the back-off stamp keeps it to one per window. +warn_failed() { # installed-version retry-minutes ("-" = no automatic retry, installer path) + _rt="retry in $2 min"; [ "$2" = "-" ] && _rt="re-run the installer to retry" + printf 'fusengine harness: upgrade failed (installed %s, target >= %s, staged %s); keeping the installed version, %s\n' \ + "${1:-unknown}" "$HARNESS_MIN" "${NEWV:-none}" "$_rt" >&2 +} + +# --install-harness: installer entry. Staged install, bypassing back-off. +if [ "$MODE" = install ]; then + mkdir -p "$CONTROL_ROOT" 2>/dev/null || exit 1 + resolve_bin; _from=$INSTALLED + wait_lock + take_lock || { warn_failed "$_from" "-"; exit 1; } + install_swap; _rc=$? + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + [ "$_rc" -eq 0 ] || warn_failed "$_from" "-" + exit "$_rc" +fi + +resolve_bin + +# --harness-check: the sessionStart entry of plugins that must NOT run the bin +# there. Same version check and upgrade as --session-start, then exit 0 with +# empty stdout; the bin is never executed. +if [ "$MODE" = check ]; then + if [ -n "$BIN" ] && { [ -z "$INSTALLED" ] || ! ver_ge "$INSTALLED" "$HARNESS_MIN"; }; then + upgrade_stale + fi + exit 0 +fi + +# 1. Fast path: already vendored. Session start checks the version first. +if [ -n "$BIN" ]; then + if [ "$MODE" = session ] && { [ -z "$INSTALLED" ] || ! ver_ge "$INSTALLED" "$HARNESS_MIN"; }; then + upgrade_stale + resolve_bin + fi + run_bin "$@" +fi -# 2. Self-heal once. The stamp keeps 62 hooks from stampeding the registry when -# the install is genuinely impossible (offline, no bun, no network). +# 2. Self-heal once (staged, same routine). An install in flight is awaited; the +# stamp keeps 62 hooks from stampeding the registry when it is impossible. +wait_lock +resolve_bin +[ -n "$BIN" ] && run_bin "$@" if [ ! -e "$STAMP" ] && command -v bun >/dev/null 2>&1; then - mkdir -p "$CONTROL_ROOT" 2>/dev/null && : > "$STAMP" 2>/dev/null - printf '{"name":"@fusengine/cursor-harness","private":true,"dependencies":{"@fusengine/harness":"%s"}}\n' \ - "$HARNESS_SPEC" > "${CONTROL_ROOT}/package.json" 2>/dev/null - (cd "$CONTROL_ROOT" && bun install --silent) >/dev/null 2>&1 - [ -f "$BIN" ] && run_bin "$@" + mkdir -p "$CONTROL_ROOT" 2>/dev/null && true 2>/dev/null > "$STAMP" + if take_lock; then + install_swap + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + fi + resolve_bin + [ -n "$BIN" ] && run_bin "$@" fi # 3. Last resort: npx on the latest published release. diff --git a/.cursor-plugin/plugins/fuse-changelog/.cursor-plugin/plugin.json b/.cursor-plugin/plugins/fuse-changelog/.cursor-plugin/plugin.json index 7cdc842..709df52 100644 --- a/.cursor-plugin/plugins/fuse-changelog/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugins/fuse-changelog/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "fuse-changelog", - "version": "1.0.16", + "version": "1.0.17", "description": "Claude Code update watcher: changelog tracking, breaking changes detection, community pulse via Exa, plugin compatibility analysis", "author": { "name": "Fusengine", diff --git a/.cursor-plugin/plugins/fuse-changelog/hooks/hooks.json b/.cursor-plugin/plugins/fuse-changelog/hooks/hooks.json index 82d447a..c562366 100644 --- a/.cursor-plugin/plugins/fuse-changelog/hooks/hooks.json +++ b/.cursor-plugin/plugins/fuse-changelog/hooks/hooks.json @@ -1,6 +1,12 @@ { "version": 1, "hooks": { + "sessionStart": [ + { + "command": "./scripts/hook.sh --harness-check", + "timeout": 60 + } + ], "postToolUse": [ { "command": "./scripts/hook.sh changelog", diff --git a/.cursor-plugin/plugins/fuse-changelog/scripts/hook.sh b/.cursor-plugin/plugins/fuse-changelog/scripts/hook.sh index 3c88b14..12329dd 100755 --- a/.cursor-plugin/plugins/fuse-changelog/scripts/hook.sh +++ b/.cursor-plugin/plugins/fuse-changelog/scripts/hook.sh @@ -2,46 +2,296 @@ # Fusengine hook entrypoint — shipped inside every plugin as scripts/hook.sh. # # Resolves the shared harness itself so the plugin works from the Cursor -# marketplace, where no installer ever runs. Order: -# 1. the vendored install under $HOME/.cursor/.fusengine-global (fast path) -# 2. a self-heal `bun install` of that same shared directory, once +# marketplace, where no installer ever runs. Layout under the control root +# ($HOME/.cursor/.fusengine-global): +# +# versions//node_modules/@fusengine/harness one immutable install per version +# current text file: the live +# node_modules/@fusengine/harness legacy single install (fallback) +# +# Resolution order: +# 1. the version named by `current` (fast path), else the legacy install +# 2. a self-heal install of `latest` (staged, see below), once # 3. `npx` on latest (last resort, network) # -# Installs the LATEST published harness the first time (the manifest uses the -# `latest` dist-tag, and so does the npx fallback), so no wrapper needs editing -# per release. It does NOT upgrade an existing install: the fast path runs the -# vendored bin as is, and rewriting it in place while up to 62 hooks load it -# could crash a guard mid-run (hooks fail open). The upgrade path is the -# installer (`install.sh` runs `bun update @fusengine/harness` on every run) or, -# by hand: `cd ~/.cursor/.fusengine-global && bun update @fusengine/harness`. +# UPGRADES. A hook invoked as `hook.sh --session-start [scope...]` (only the +# sessionStart entries pass the flag) compares the installed version with +# HARNESS_MIN using shell builtins only — no process, no network. Current or +# newer: nothing else happens. Older: the new release is installed into a +# STAGING directory (`bun add` in versions/.stage-), checked (version and +# `bin.mjs --version`), moved to versions/, and only then made live by +# renaming a temp file over `current` — one atomic rename(2). The running tree +# is never modified in place and the previous version is never deleted during +# the switch, so a guard that starts, or is already running and lazily imports +# a chunk, never sees a half-replaced install. (An in-place `bun update` leaves +# bin.mjs and the hashed chunks absent for a few ms: measured, it crashes +# concurrent hooks.) Old versions are pruned only after a successful switch, +# keeping the new one, the previous one, and anything younger than ~2 days. +# +# Failure handling: a back-off stamp (.harness-update-failed) suppresses retries +# for FUSE_HARNESS_UPDATE_BACKOFF_MIN minutes (default 30); it is written before +# the attempt and removed on success, so a killed attempt is also backed off. An +# atomic mkdir lock (.harness-update.lock, reclaimed after 10 min) keeps the five +# sessionStart hooks from installing at once. Everything fails open. +# +# HARNESS_MIN is the only per-release edit. # # Resolution is relative to THIS file, never to the caller's cwd. Always exits 0 # on its own failures: Cursor fails hooks open, and a broken guard must never # block the user's action. # -# Usage (from hooks/hooks.json): ./scripts/hook.sh [scope...] +# Usage (from hooks/hooks.json): ./scripts/hook.sh [--session-start] [scope...] +# ./scripts/hook.sh --harness-check (sessionStart of plugins that must not +# run the bin there: same check/upgrade, +# then exit 0, empty stdout) +# ./scripts/hook.sh --install-harness (installer: staged install, then exit; +# prints one warning line on failure) +# The install is bounded by FUSE_HARNESS_INSTALL_TIMEOUT_SEC (default 45, under +# Cursor's 60 s default hook timeout); sessionStart entries that can upgrade +# therefore carry no timeout below 60. +HARNESS_MIN="0.1.115" HARNESS_SPEC="latest" -CONTROL_ROOT="${HOME}/.cursor/.fusengine-global" -BIN="${CONTROL_ROOT}/node_modules/@fusengine/harness/dist/cli/bin.mjs" +CONTROL_ROOT="${FUSE_HARNESS_ROOT:-${HOME}/.cursor/.fusengine-global}" +PKG_REL="node_modules/@fusengine/harness" +BIN_REL="node_modules/@fusengine/harness/dist/cli/bin.mjs" STAMP="${CONTROL_ROOT}/.selfheal-attempted" +FAILED="${CONTROL_ROOT}/.harness-update-failed" +LOCK="${CONTROL_ROOT}/.harness-update.lock" + +MODE="" +case "$1" in + --session-start) MODE=session; shift ;; + --install-harness) MODE=install; shift ;; + --harness-check) MODE=check; shift ;; +esac + +BIN="" +INSTALLED="" + +# Set INSTALLED to the "version" of a package.json, builtins only. Empty if absent. +pkgver() { + INSTALLED="" + [ -r "$1" ] || return 1 + while IFS= read -r _l || [ -n "$_l" ]; do + case $_l in + *'"version"'*:*) + _l=${_l#*'"version"'}; _l=${_l#*:}; _l=${_l#*\"} + INSTALLED=${_l%%\"*}; return 0 ;; + esac + done < "$1" + return 1 +} + +# ver_ge A B: numeric A >= B for x.y.z (prerelease/build suffix ignored). +ver_ge() { + _a=${1%%[-+]*}; _b=${2%%[-+]*} + case $_a in ''|*[!0-9.]*) return 1 ;; esac + case $_b in ''|*[!0-9.]*) return 1 ;; esac + _o=$IFS; IFS=. + set -- $_a; _a1=${1:-0}; _a2=${2:-0}; _a3=${3:-0} + set -- $_b; _b1=${1:-0}; _b2=${2:-0}; _b3=${3:-0} + IFS=$_o + [ "$_a1" -gt "$_b1" ] && return 0; [ "$_a1" -lt "$_b1" ] && return 1 + [ "$_a2" -gt "$_b2" ] && return 0; [ "$_a2" -lt "$_b2" ] && return 1 + [ "$_a3" -ge "$_b3" ] +} + +# Set BIN to the live bin ("" if none) and INSTALLED to its version ("" if unreadable). +resolve_bin() { + BIN=""; INSTALLED="" + if [ -r "${CONTROL_ROOT}/current" ]; then + _c="" + read -r _c < "${CONTROL_ROOT}/current" || : + case $_c in ''|.*|*[!0-9A-Za-z.+-]*) _c="" ;; esac + if [ -n "$_c" ] && [ -f "${CONTROL_ROOT}/versions/${_c}/${BIN_REL}" ]; then + BIN="${CONTROL_ROOT}/versions/${_c}/${BIN_REL}"; INSTALLED=$_c; return 0 + fi + fi + if [ -f "${CONTROL_ROOT}/${BIN_REL}" ]; then + BIN="${CONTROL_ROOT}/${BIN_REL}" + pkgver "${CONTROL_ROOT}/${PKG_REL}/package.json" || INSTALLED="" + fi +} + +# Run `$@` with a time bound, portably (stock macOS has no `timeout`): the command +# runs in the background, a watchdog subshell kills it after the limit, `wait` +# returns its status (non-zero when killed). The watchdog's own `sleep` may +# linger until its deadline; it holds no output. +bounded() { + _t=${FUSE_HARNESS_INSTALL_TIMEOUT_SEC:-45}; case $_t in ''|*[!0-9]*) _t=45 ;; esac + "$@" & + _p=$! + ( sleep "$_t"; kill "$_p" ) >/dev/null 2>&1 & + _wd=$! + wait "$_p"; _r=$? + kill "$_wd" >/dev/null 2>&1 + return "$_r" +} run_bin() { if command -v bun >/dev/null 2>&1; then exec bun "$BIN" hook cursor "$@"; fi exec node "$BIN" hook cursor "$@" } -# 1. Fast path: already vendored. -[ -f "$BIN" ] && run_bin "$@" +# Run ` --version` of a candidate bin: proves it starts before it goes live. +bin_answers() { + if command -v bun >/dev/null 2>&1; then bun "$1" --version >/dev/null 2>&1 /dev/null 2>&1 /dev/null && return 0 + [ -d "$LOCK" ] || return 1 + [ -n "$(find "$LOCK" -maxdepth 0 -mmin +10 2>/dev/null)" ] || return 1 + # Reclaim through a second atomic gate so only ONE reclaimer runs the + # check-remove-recreate sequence (two of them would delete each other's lock). + if ! mkdir "${LOCK}.reclaim" 2>/dev/null; then + [ -n "$(find "${LOCK}.reclaim" -maxdepth 0 -mmin +10 2>/dev/null)" ] && rmdir "${LOCK}.reclaim" 2>/dev/null + return 1 + fi + _got=1 + if [ -n "$(find "$LOCK" -maxdepth 0 -mmin +10 2>/dev/null)" ]; then + rmdir "$LOCK" 2>/dev/null && mkdir "$LOCK" 2>/dev/null && _got=0 + fi + rmdir "${LOCK}.reclaim" 2>/dev/null + return "$_got" +} + +# Wait (bounded) while another process holds the lock. +wait_lock() { + _w=${FUSE_HARNESS_LOCK_WAIT_SEC:-20}; case $_w in ''|*[!0-9]*) _w=20 ;; esac + _i=0 + while [ -d "$LOCK" ] && [ "$_i" -lt "$_w" ]; do sleep 1; _i=$((_i + 1)); done +} + +# Staged install of HARNESS_SPEC. Sets NEWV. Returns 0 only when `current` was +# switched to NEWV; the live tree is untouched on any failure. Caller holds LOCK. +install_swap() { + NEWV="" + command -v bun >/dev/null 2>&1 || return 1 + mkdir -p "${CONTROL_ROOT}/versions" 2>/dev/null || return 1 + _stage="${CONTROL_ROOT}/versions/.stage-$$" + trap 'rm -rf "$_stage"; rmdir "$LOCK" 2>/dev/null; exit 0' HUP INT TERM + rm -rf "$_stage" + mkdir "$_stage" 2>/dev/null || return 1 + printf '{"name":"@fusengine/cursor-harness-stage","private":true}\n' > "${_stage}/package.json" + (cd "$_stage" && bounded bun add "@fusengine/harness@${HARNESS_SPEC}" --force --silent) \ + >/dev/null 2>&1 /dev/null || { rm -rf "$_stage"; return 1; }; fi + mv "$_stage" "$_dest" 2>/dev/null || { rm -rf "$_stage"; return 1; } + rm -rf "${_dest}/.stage-$$" # mv nests the stage if $_dest appeared meanwhile + bin_answers "${_dest}/${BIN_REL}" || return 1 + fi + # Never downgrade: keep the live version when it is already newer. + resolve_bin + if [ -n "$INSTALLED" ] && ! ver_ge "$NEWV" "$INSTALLED"; then return 1; fi + PREV=$INSTALLED + printf '%s\n' "$NEWV" > "${CONTROL_ROOT}/current.tmp.$$" 2>/dev/null \ + && mv -f "${CONTROL_ROOT}/current.tmp.$$" "${CONTROL_ROOT}/current" 2>/dev/null \ + || { rm -f "${CONTROL_ROOT}/current.tmp.$$"; return 1; } + # mv into a `current` that is a directory succeeds silently: verify the pointer. + _c=""; read -r _c < "${CONTROL_ROOT}/current" 2>/dev/null + if [ "$_c" != "$NEWV" ]; then rm -f "${CONTROL_ROOT}/current/current.tmp.$$"; return 1; fi + trap - HUP INT TERM + prune_versions + return 0 +} + +# Remove versions other than the new and previous one, and stale staging dirs, +# when older than ~2 days (`find -mtime +1`; a long-lived session may still hold an older tree). +prune_versions() { + for _d in "${CONTROL_ROOT}"/versions/* "${CONTROL_ROOT}"/versions/.stage-*; do + [ -d "$_d" ] || continue + _n=${_d##*/} + [ "$_n" = "$NEWV" ] || [ "$_n" = "$PREV" ] && continue + [ -n "$(find "$_d" -maxdepth 0 -mtime +1 2>/dev/null)" ] && rm -rf "$_d" + done +} + +# Session-start upgrade: back-off, single flight, staged install. +upgrade_stale() { + if [ -e "$FAILED" ]; then + _bo=${FUSE_HARNESS_UPDATE_BACKOFF_MIN:-30}; case $_bo in ''|*[!0-9]*) _bo=30 ;; esac + [ -n "$(find "$FAILED" -mmin "-${_bo}" 2>/dev/null)" ] && return 0 + fi + take_lock || return 0 + # `true`, not `:` — a redirection error on a special built-in exits sh/dash. + true 2>/dev/null > "$FAILED" + _from=$INSTALLED + if install_swap && ver_ge "$NEWV" "$HARNESS_MIN"; then rm -f "$FAILED" + else warn_failed "$_from" "${_bo:-30}"; fi + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + return 0 +} + +# One line on stderr (stdout is the hook's JSON), shown in Cursor's Hooks panel; +# the back-off stamp keeps it to one per window. +warn_failed() { # installed-version retry-minutes ("-" = no automatic retry, installer path) + _rt="retry in $2 min"; [ "$2" = "-" ] && _rt="re-run the installer to retry" + printf 'fusengine harness: upgrade failed (installed %s, target >= %s, staged %s); keeping the installed version, %s\n' \ + "${1:-unknown}" "$HARNESS_MIN" "${NEWV:-none}" "$_rt" >&2 +} + +# --install-harness: installer entry. Staged install, bypassing back-off. +if [ "$MODE" = install ]; then + mkdir -p "$CONTROL_ROOT" 2>/dev/null || exit 1 + resolve_bin; _from=$INSTALLED + wait_lock + take_lock || { warn_failed "$_from" "-"; exit 1; } + install_swap; _rc=$? + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + [ "$_rc" -eq 0 ] || warn_failed "$_from" "-" + exit "$_rc" +fi + +resolve_bin + +# --harness-check: the sessionStart entry of plugins that must NOT run the bin +# there. Same version check and upgrade as --session-start, then exit 0 with +# empty stdout; the bin is never executed. +if [ "$MODE" = check ]; then + if [ -n "$BIN" ] && { [ -z "$INSTALLED" ] || ! ver_ge "$INSTALLED" "$HARNESS_MIN"; }; then + upgrade_stale + fi + exit 0 +fi + +# 1. Fast path: already vendored. Session start checks the version first. +if [ -n "$BIN" ]; then + if [ "$MODE" = session ] && { [ -z "$INSTALLED" ] || ! ver_ge "$INSTALLED" "$HARNESS_MIN"; }; then + upgrade_stale + resolve_bin + fi + run_bin "$@" +fi -# 2. Self-heal once. The stamp keeps 62 hooks from stampeding the registry when -# the install is genuinely impossible (offline, no bun, no network). +# 2. Self-heal once (staged, same routine). An install in flight is awaited; the +# stamp keeps 62 hooks from stampeding the registry when it is impossible. +wait_lock +resolve_bin +[ -n "$BIN" ] && run_bin "$@" if [ ! -e "$STAMP" ] && command -v bun >/dev/null 2>&1; then - mkdir -p "$CONTROL_ROOT" 2>/dev/null && : > "$STAMP" 2>/dev/null - printf '{"name":"@fusengine/cursor-harness","private":true,"dependencies":{"@fusengine/harness":"%s"}}\n' \ - "$HARNESS_SPEC" > "${CONTROL_ROOT}/package.json" 2>/dev/null - (cd "$CONTROL_ROOT" && bun install --silent) >/dev/null 2>&1 - [ -f "$BIN" ] && run_bin "$@" + mkdir -p "$CONTROL_ROOT" 2>/dev/null && true 2>/dev/null > "$STAMP" + if take_lock; then + install_swap + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + fi + resolve_bin + [ -n "$BIN" ] && run_bin "$@" fi # 3. Last resort: npx on the latest published release. diff --git a/.cursor-plugin/plugins/fuse-design/.cursor-plugin/plugin.json b/.cursor-plugin/plugins/fuse-design/.cursor-plugin/plugin.json index 5842879..3b1a4c1 100644 --- a/.cursor-plugin/plugins/fuse-design/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugins/fuse-design/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "fuse-design", - "version": "2.2.11", + "version": "2.2.12", "description": "Design Director for websites, web apps, iOS, and Android. Generates HTML/CSS directly (Gemini Design MCP, Magic/21st.dev, and shadcn MCP are optional). iOS/Android produce tokens + a device-framed mockup + a handoff spec, never SwiftUI/Compose. Design method, design system tokens, motion, UX copy, and a deterministic design-review gate. Anti-AI-Slop, WCAG 2.2.", "author": { "name": "Fusengine", diff --git a/.cursor-plugin/plugins/fuse-design/hooks/hooks.json b/.cursor-plugin/plugins/fuse-design/hooks/hooks.json index 95148d3..75906b1 100644 --- a/.cursor-plugin/plugins/fuse-design/hooks/hooks.json +++ b/.cursor-plugin/plugins/fuse-design/hooks/hooks.json @@ -1,6 +1,12 @@ { "version": 1, "hooks": { + "sessionStart": [ + { + "command": "./scripts/hook.sh --harness-check", + "timeout": 60 + } + ], "preToolUse": [ { "command": "./scripts/hook.sh", diff --git a/.cursor-plugin/plugins/fuse-design/scripts/hook.sh b/.cursor-plugin/plugins/fuse-design/scripts/hook.sh index 3c88b14..12329dd 100755 --- a/.cursor-plugin/plugins/fuse-design/scripts/hook.sh +++ b/.cursor-plugin/plugins/fuse-design/scripts/hook.sh @@ -2,46 +2,296 @@ # Fusengine hook entrypoint — shipped inside every plugin as scripts/hook.sh. # # Resolves the shared harness itself so the plugin works from the Cursor -# marketplace, where no installer ever runs. Order: -# 1. the vendored install under $HOME/.cursor/.fusengine-global (fast path) -# 2. a self-heal `bun install` of that same shared directory, once +# marketplace, where no installer ever runs. Layout under the control root +# ($HOME/.cursor/.fusengine-global): +# +# versions//node_modules/@fusengine/harness one immutable install per version +# current text file: the live +# node_modules/@fusengine/harness legacy single install (fallback) +# +# Resolution order: +# 1. the version named by `current` (fast path), else the legacy install +# 2. a self-heal install of `latest` (staged, see below), once # 3. `npx` on latest (last resort, network) # -# Installs the LATEST published harness the first time (the manifest uses the -# `latest` dist-tag, and so does the npx fallback), so no wrapper needs editing -# per release. It does NOT upgrade an existing install: the fast path runs the -# vendored bin as is, and rewriting it in place while up to 62 hooks load it -# could crash a guard mid-run (hooks fail open). The upgrade path is the -# installer (`install.sh` runs `bun update @fusengine/harness` on every run) or, -# by hand: `cd ~/.cursor/.fusengine-global && bun update @fusengine/harness`. +# UPGRADES. A hook invoked as `hook.sh --session-start [scope...]` (only the +# sessionStart entries pass the flag) compares the installed version with +# HARNESS_MIN using shell builtins only — no process, no network. Current or +# newer: nothing else happens. Older: the new release is installed into a +# STAGING directory (`bun add` in versions/.stage-), checked (version and +# `bin.mjs --version`), moved to versions/, and only then made live by +# renaming a temp file over `current` — one atomic rename(2). The running tree +# is never modified in place and the previous version is never deleted during +# the switch, so a guard that starts, or is already running and lazily imports +# a chunk, never sees a half-replaced install. (An in-place `bun update` leaves +# bin.mjs and the hashed chunks absent for a few ms: measured, it crashes +# concurrent hooks.) Old versions are pruned only after a successful switch, +# keeping the new one, the previous one, and anything younger than ~2 days. +# +# Failure handling: a back-off stamp (.harness-update-failed) suppresses retries +# for FUSE_HARNESS_UPDATE_BACKOFF_MIN minutes (default 30); it is written before +# the attempt and removed on success, so a killed attempt is also backed off. An +# atomic mkdir lock (.harness-update.lock, reclaimed after 10 min) keeps the five +# sessionStart hooks from installing at once. Everything fails open. +# +# HARNESS_MIN is the only per-release edit. # # Resolution is relative to THIS file, never to the caller's cwd. Always exits 0 # on its own failures: Cursor fails hooks open, and a broken guard must never # block the user's action. # -# Usage (from hooks/hooks.json): ./scripts/hook.sh [scope...] +# Usage (from hooks/hooks.json): ./scripts/hook.sh [--session-start] [scope...] +# ./scripts/hook.sh --harness-check (sessionStart of plugins that must not +# run the bin there: same check/upgrade, +# then exit 0, empty stdout) +# ./scripts/hook.sh --install-harness (installer: staged install, then exit; +# prints one warning line on failure) +# The install is bounded by FUSE_HARNESS_INSTALL_TIMEOUT_SEC (default 45, under +# Cursor's 60 s default hook timeout); sessionStart entries that can upgrade +# therefore carry no timeout below 60. +HARNESS_MIN="0.1.115" HARNESS_SPEC="latest" -CONTROL_ROOT="${HOME}/.cursor/.fusengine-global" -BIN="${CONTROL_ROOT}/node_modules/@fusengine/harness/dist/cli/bin.mjs" +CONTROL_ROOT="${FUSE_HARNESS_ROOT:-${HOME}/.cursor/.fusengine-global}" +PKG_REL="node_modules/@fusengine/harness" +BIN_REL="node_modules/@fusengine/harness/dist/cli/bin.mjs" STAMP="${CONTROL_ROOT}/.selfheal-attempted" +FAILED="${CONTROL_ROOT}/.harness-update-failed" +LOCK="${CONTROL_ROOT}/.harness-update.lock" + +MODE="" +case "$1" in + --session-start) MODE=session; shift ;; + --install-harness) MODE=install; shift ;; + --harness-check) MODE=check; shift ;; +esac + +BIN="" +INSTALLED="" + +# Set INSTALLED to the "version" of a package.json, builtins only. Empty if absent. +pkgver() { + INSTALLED="" + [ -r "$1" ] || return 1 + while IFS= read -r _l || [ -n "$_l" ]; do + case $_l in + *'"version"'*:*) + _l=${_l#*'"version"'}; _l=${_l#*:}; _l=${_l#*\"} + INSTALLED=${_l%%\"*}; return 0 ;; + esac + done < "$1" + return 1 +} + +# ver_ge A B: numeric A >= B for x.y.z (prerelease/build suffix ignored). +ver_ge() { + _a=${1%%[-+]*}; _b=${2%%[-+]*} + case $_a in ''|*[!0-9.]*) return 1 ;; esac + case $_b in ''|*[!0-9.]*) return 1 ;; esac + _o=$IFS; IFS=. + set -- $_a; _a1=${1:-0}; _a2=${2:-0}; _a3=${3:-0} + set -- $_b; _b1=${1:-0}; _b2=${2:-0}; _b3=${3:-0} + IFS=$_o + [ "$_a1" -gt "$_b1" ] && return 0; [ "$_a1" -lt "$_b1" ] && return 1 + [ "$_a2" -gt "$_b2" ] && return 0; [ "$_a2" -lt "$_b2" ] && return 1 + [ "$_a3" -ge "$_b3" ] +} + +# Set BIN to the live bin ("" if none) and INSTALLED to its version ("" if unreadable). +resolve_bin() { + BIN=""; INSTALLED="" + if [ -r "${CONTROL_ROOT}/current" ]; then + _c="" + read -r _c < "${CONTROL_ROOT}/current" || : + case $_c in ''|.*|*[!0-9A-Za-z.+-]*) _c="" ;; esac + if [ -n "$_c" ] && [ -f "${CONTROL_ROOT}/versions/${_c}/${BIN_REL}" ]; then + BIN="${CONTROL_ROOT}/versions/${_c}/${BIN_REL}"; INSTALLED=$_c; return 0 + fi + fi + if [ -f "${CONTROL_ROOT}/${BIN_REL}" ]; then + BIN="${CONTROL_ROOT}/${BIN_REL}" + pkgver "${CONTROL_ROOT}/${PKG_REL}/package.json" || INSTALLED="" + fi +} + +# Run `$@` with a time bound, portably (stock macOS has no `timeout`): the command +# runs in the background, a watchdog subshell kills it after the limit, `wait` +# returns its status (non-zero when killed). The watchdog's own `sleep` may +# linger until its deadline; it holds no output. +bounded() { + _t=${FUSE_HARNESS_INSTALL_TIMEOUT_SEC:-45}; case $_t in ''|*[!0-9]*) _t=45 ;; esac + "$@" & + _p=$! + ( sleep "$_t"; kill "$_p" ) >/dev/null 2>&1 & + _wd=$! + wait "$_p"; _r=$? + kill "$_wd" >/dev/null 2>&1 + return "$_r" +} run_bin() { if command -v bun >/dev/null 2>&1; then exec bun "$BIN" hook cursor "$@"; fi exec node "$BIN" hook cursor "$@" } -# 1. Fast path: already vendored. -[ -f "$BIN" ] && run_bin "$@" +# Run ` --version` of a candidate bin: proves it starts before it goes live. +bin_answers() { + if command -v bun >/dev/null 2>&1; then bun "$1" --version >/dev/null 2>&1 /dev/null 2>&1 /dev/null && return 0 + [ -d "$LOCK" ] || return 1 + [ -n "$(find "$LOCK" -maxdepth 0 -mmin +10 2>/dev/null)" ] || return 1 + # Reclaim through a second atomic gate so only ONE reclaimer runs the + # check-remove-recreate sequence (two of them would delete each other's lock). + if ! mkdir "${LOCK}.reclaim" 2>/dev/null; then + [ -n "$(find "${LOCK}.reclaim" -maxdepth 0 -mmin +10 2>/dev/null)" ] && rmdir "${LOCK}.reclaim" 2>/dev/null + return 1 + fi + _got=1 + if [ -n "$(find "$LOCK" -maxdepth 0 -mmin +10 2>/dev/null)" ]; then + rmdir "$LOCK" 2>/dev/null && mkdir "$LOCK" 2>/dev/null && _got=0 + fi + rmdir "${LOCK}.reclaim" 2>/dev/null + return "$_got" +} + +# Wait (bounded) while another process holds the lock. +wait_lock() { + _w=${FUSE_HARNESS_LOCK_WAIT_SEC:-20}; case $_w in ''|*[!0-9]*) _w=20 ;; esac + _i=0 + while [ -d "$LOCK" ] && [ "$_i" -lt "$_w" ]; do sleep 1; _i=$((_i + 1)); done +} + +# Staged install of HARNESS_SPEC. Sets NEWV. Returns 0 only when `current` was +# switched to NEWV; the live tree is untouched on any failure. Caller holds LOCK. +install_swap() { + NEWV="" + command -v bun >/dev/null 2>&1 || return 1 + mkdir -p "${CONTROL_ROOT}/versions" 2>/dev/null || return 1 + _stage="${CONTROL_ROOT}/versions/.stage-$$" + trap 'rm -rf "$_stage"; rmdir "$LOCK" 2>/dev/null; exit 0' HUP INT TERM + rm -rf "$_stage" + mkdir "$_stage" 2>/dev/null || return 1 + printf '{"name":"@fusengine/cursor-harness-stage","private":true}\n' > "${_stage}/package.json" + (cd "$_stage" && bounded bun add "@fusengine/harness@${HARNESS_SPEC}" --force --silent) \ + >/dev/null 2>&1 /dev/null || { rm -rf "$_stage"; return 1; }; fi + mv "$_stage" "$_dest" 2>/dev/null || { rm -rf "$_stage"; return 1; } + rm -rf "${_dest}/.stage-$$" # mv nests the stage if $_dest appeared meanwhile + bin_answers "${_dest}/${BIN_REL}" || return 1 + fi + # Never downgrade: keep the live version when it is already newer. + resolve_bin + if [ -n "$INSTALLED" ] && ! ver_ge "$NEWV" "$INSTALLED"; then return 1; fi + PREV=$INSTALLED + printf '%s\n' "$NEWV" > "${CONTROL_ROOT}/current.tmp.$$" 2>/dev/null \ + && mv -f "${CONTROL_ROOT}/current.tmp.$$" "${CONTROL_ROOT}/current" 2>/dev/null \ + || { rm -f "${CONTROL_ROOT}/current.tmp.$$"; return 1; } + # mv into a `current` that is a directory succeeds silently: verify the pointer. + _c=""; read -r _c < "${CONTROL_ROOT}/current" 2>/dev/null + if [ "$_c" != "$NEWV" ]; then rm -f "${CONTROL_ROOT}/current/current.tmp.$$"; return 1; fi + trap - HUP INT TERM + prune_versions + return 0 +} + +# Remove versions other than the new and previous one, and stale staging dirs, +# when older than ~2 days (`find -mtime +1`; a long-lived session may still hold an older tree). +prune_versions() { + for _d in "${CONTROL_ROOT}"/versions/* "${CONTROL_ROOT}"/versions/.stage-*; do + [ -d "$_d" ] || continue + _n=${_d##*/} + [ "$_n" = "$NEWV" ] || [ "$_n" = "$PREV" ] && continue + [ -n "$(find "$_d" -maxdepth 0 -mtime +1 2>/dev/null)" ] && rm -rf "$_d" + done +} + +# Session-start upgrade: back-off, single flight, staged install. +upgrade_stale() { + if [ -e "$FAILED" ]; then + _bo=${FUSE_HARNESS_UPDATE_BACKOFF_MIN:-30}; case $_bo in ''|*[!0-9]*) _bo=30 ;; esac + [ -n "$(find "$FAILED" -mmin "-${_bo}" 2>/dev/null)" ] && return 0 + fi + take_lock || return 0 + # `true`, not `:` — a redirection error on a special built-in exits sh/dash. + true 2>/dev/null > "$FAILED" + _from=$INSTALLED + if install_swap && ver_ge "$NEWV" "$HARNESS_MIN"; then rm -f "$FAILED" + else warn_failed "$_from" "${_bo:-30}"; fi + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + return 0 +} + +# One line on stderr (stdout is the hook's JSON), shown in Cursor's Hooks panel; +# the back-off stamp keeps it to one per window. +warn_failed() { # installed-version retry-minutes ("-" = no automatic retry, installer path) + _rt="retry in $2 min"; [ "$2" = "-" ] && _rt="re-run the installer to retry" + printf 'fusengine harness: upgrade failed (installed %s, target >= %s, staged %s); keeping the installed version, %s\n' \ + "${1:-unknown}" "$HARNESS_MIN" "${NEWV:-none}" "$_rt" >&2 +} + +# --install-harness: installer entry. Staged install, bypassing back-off. +if [ "$MODE" = install ]; then + mkdir -p "$CONTROL_ROOT" 2>/dev/null || exit 1 + resolve_bin; _from=$INSTALLED + wait_lock + take_lock || { warn_failed "$_from" "-"; exit 1; } + install_swap; _rc=$? + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + [ "$_rc" -eq 0 ] || warn_failed "$_from" "-" + exit "$_rc" +fi + +resolve_bin + +# --harness-check: the sessionStart entry of plugins that must NOT run the bin +# there. Same version check and upgrade as --session-start, then exit 0 with +# empty stdout; the bin is never executed. +if [ "$MODE" = check ]; then + if [ -n "$BIN" ] && { [ -z "$INSTALLED" ] || ! ver_ge "$INSTALLED" "$HARNESS_MIN"; }; then + upgrade_stale + fi + exit 0 +fi + +# 1. Fast path: already vendored. Session start checks the version first. +if [ -n "$BIN" ]; then + if [ "$MODE" = session ] && { [ -z "$INSTALLED" ] || ! ver_ge "$INSTALLED" "$HARNESS_MIN"; }; then + upgrade_stale + resolve_bin + fi + run_bin "$@" +fi -# 2. Self-heal once. The stamp keeps 62 hooks from stampeding the registry when -# the install is genuinely impossible (offline, no bun, no network). +# 2. Self-heal once (staged, same routine). An install in flight is awaited; the +# stamp keeps 62 hooks from stampeding the registry when it is impossible. +wait_lock +resolve_bin +[ -n "$BIN" ] && run_bin "$@" if [ ! -e "$STAMP" ] && command -v bun >/dev/null 2>&1; then - mkdir -p "$CONTROL_ROOT" 2>/dev/null && : > "$STAMP" 2>/dev/null - printf '{"name":"@fusengine/cursor-harness","private":true,"dependencies":{"@fusengine/harness":"%s"}}\n' \ - "$HARNESS_SPEC" > "${CONTROL_ROOT}/package.json" 2>/dev/null - (cd "$CONTROL_ROOT" && bun install --silent) >/dev/null 2>&1 - [ -f "$BIN" ] && run_bin "$@" + mkdir -p "$CONTROL_ROOT" 2>/dev/null && true 2>/dev/null > "$STAMP" + if take_lock; then + install_swap + trap - HUP INT TERM + rmdir "$LOCK" 2>/dev/null + fi + resolve_bin + [ -n "$BIN" ] && run_bin "$@" fi # 3. Last resort: npx on the latest published release. diff --git a/.cursor-plugin/plugins/fuse-devops/.cursor-plugin/plugin.json b/.cursor-plugin/plugins/fuse-devops/.cursor-plugin/plugin.json new file mode 100644 index 0000000..dc6e7a8 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/.cursor-plugin/plugin.json @@ -0,0 +1,24 @@ +{ + "name": "fuse-devops", + "version": "1.0.0", + "description": "Expert self-hosted IT/DevOps — Docker, Kubernetes, Linux, Proxmox VE, OpenStack, Terraform/OpenTofu, Ansible, CI/CD, observability, storage and backup, secrets, databases — with a discovery, dry-run, one-go, verify, rollback safety model", + "author": { + "name": "Fusengine", + "email": "hello@fusengine.ch" + }, + "homepage": "https://github.com/fusengine/agents", + "repository": "https://github.com/fusengine/agents", + "license": "MIT", + "keywords": [ + "devops", + "docker", + "kubernetes", + "linux", + "proxmox", + "openstack", + "terraform", + "ansible", + "observability", + "infrastructure" + ] +} diff --git a/.cursor-plugin/plugins/fuse-devops/README.md b/.cursor-plugin/plugins/fuse-devops/README.md new file mode 100644 index 0000000..907febf --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/README.md @@ -0,0 +1,96 @@ +# fuse-devops (Cursor Plugin) + +Ported from the Claude Code plugin `fuse-devops` v1.0.0 (claude-plugins commit `13e67b7a`, source folder +`plugins/devops-expert`). The plugin name is deliberately identical across both ecosystems so +cross-plugin references (`fuse-devops:`) keep resolving. + +## Contents + +| Component | Count | Cursor discovery | +| :-- | :-- | :-- | +| Agents (subagents) | 1 | `agents/*.md` | +| Skills | 18 | `skills//SKILL.md` | +| Hooks | 0 | none: the source plugin has no hooks, so there is no `hooks/` or `scripts/` folder | + +No `commands/` folder: the source plugin has none. + +## Port notes + +- Agent frontmatter: `model: sonnet` → `grok-4.6`, `effort:` dropped, `Agent` → `Task`, `SendMessage` removed (no Cursor equivalent); `## Allowed tools` block added. +- Agent body and skills: the parallel-launch wording "`Agent` calls" → "`Task` calls" (the agent file once, one line in each of the 18 `SKILL.md`; the skill lines also drop "with a `name`", as in the `fuse-astro` port). +- The Operations Safety Protocol and every other body line are unchanged. +- No hooks: nothing is wired to tool events, and no `hooks/hooks.json` is shipped. +- Manifest moved to `.cursor-plugin/plugin.json` (author `url` dropped to match siblings). +- Skills copied byte-identical apart from that one line; `references/` (templates, scripts, manifests) are byte-identical. + +### Claude-only omissions + +- `.claude-plugin/plugin.json` (replaced by `.cursor-plugin/plugin.json`) and its `author.url`. +- The `effort:` frontmatter key and the `SendMessage` tool. +- The Claude Code per-plugin install command (`plugin install fuse-devops`; does not apply under Cursor; see Installation). +- The ", with a `name`" qualifier on sub-agent launches in the skills. + +--- + +Expert **self-hosted IT/DevOps** engineering: containers, Kubernetes, Linux servers, networking, Proxmox VE, KVM/libvirt/Incus, OpenStack, infrastructure as code, CI/CD, observability, storage and backup, secrets and stateful databases. Universal, versioned knowledge lives in the skills; every live change goes through a discovery, dry-run, one-go, verify, rollback sequence. Public clouds (AWS, GCP, Azure) are out of scope. + +## Agent + +| Agent | Description | +|-------|-------------| +| **devops-expert** | Builds, operates and troubleshoots self-hosted infrastructure end to end: inspects read-only, previews every change (plan, diff, dry run, check mode), backs up stateful systems, applies after your go, verifies and keeps a rollback ready | + +## Skills + +| Skill | Description | +|-------|-------------| +| devops-docker | Docker Engine, Compose, BuildKit, Dockerfile practice, container networking/firewall, registries, image supply chain (SBOM, signing), Podman/Quadlet | +| devops-kubernetes | Workloads, scheduling, Services, Gateway API, storage, RBAC, safe kubectl operations, troubleshooting | +| devops-k8s-cluster | Cluster lifecycle: kubeadm, k3s, Talos, Cluster API, etcd backup/restore, upgrades and version skew, CNI, bare-metal load balancing | +| devops-gitops-packaging | Helm, Kustomize, Argo CD, Flux, drift and prune safety, promotion between environments | +| devops-linux | Day-2 Linux: systemd, packages, users/SSH, kernel params, LVM/filesystems, SELinux/AppArmor, timers, performance and disk troubleshooting, bash scripting | +| devops-networking | nftables, bridges/bonding/VLAN, routing, DNS, TLS/ACME, reverse proxy (Traefik/Caddy/Nginx/HAProxy), WireGuard | +| devops-opnsense | OPNsense firewall/router: rules and NAT, DNS/DHCP, VPN, multi-WAN, CARP HA, plugins, upgrades and rollback, config backup, REST API, Ansible/OpenTofu | +| devops-proxmox | Proxmox VE: cluster, HA, SDN, storage, firewall, LXC, `qm`/`pct`/`pvesh`, templates and cloud-init, API automation | +| devops-virtualization | KVM/QEMU/libvirt (`virsh`), LXC/Incus, cloud-init, image building with Packer | +| devops-openstack | Service map, release/upgrade model (SLURP), Kolla-Ansible / OpenStack-Ansible / OpenStack-Helm, `openstack` CLI, day-2 ops, Ceph integration | +| devops-iac | Terraform and OpenTofu: state, modules, plan/apply discipline, licensing choice, providers for Proxmox/OpenStack/DNS, Terragrunt | +| devops-ansible | ansible-core, playbooks/roles/collections, inventory, idempotence, check mode, vault, Molecule | +| devops-cicd | GitHub Actions, GitLab CI, self-hosted runners, caching, environments and approvals, release pipelines | +| devops-observability | Prometheus, Alertmanager, Grafana, Loki, Alloy, OpenTelemetry Collector, exporters, dashboards as code | +| devops-storage-backup | ZFS, Ceph, LVM-thin/NFS, Proxmox Backup Server, restic/borg, 3-2-1, mandatory restore tests, disaster recovery | +| devops-sre-ops | SLI/SLO and alert policy, runbooks, incident response, postmortems, change management, capacity, DR drills | +| devops-secrets | SOPS + age, Vault and OpenBao, External Secrets, Kubernetes Secrets handling, rotation | +| devops-databases | PostgreSQL HA (Patroni, pgBackRest, CloudNativePG), MariaDB/Galera, PITR, upgrade and failover runbooks | + +SOLID: scripts and code helpers follow the shared `fuse-solid:solid-generic` skill (no local duplicate); infrastructure artifacts follow the structure of the matching skill. + +## Detection + +| Kind | Indicators | Effect | +|------|-----------|--------| +| Strong | `*.tf`, `.terraform.lock.hcl`, `terragrunt.hcl`, `Chart.yaml`, `kustomization.yaml`, `ansible.cfg`, `playbooks/`, `roles/`, `inventory/`, `*.pkr.hcl`, `.sops.yaml`, Flux `flux-system/` or Argo CD `Application` manifests, `prometheus.yml` / `alertmanager.yml`, Kolla/OSA `globals.yml` / `openstack_user_config.yml`, `nftables.conf`, `Vagrantfile` | Any one makes devops-expert the primary agent | +| Weak | `Dockerfile`, `compose.yaml` / `docker-compose.yml`, `.github/workflows/`, `.gitlab-ci.yml`, `Makefile`, `scripts/*.sh` | Never alone. In a repo matched by a framework/language expert, that expert stays primary and devops-expert handles only the infrastructure files when the task touches them | + +Intent words (deploy, cluster, VM, backup, terraform, helm, proxmox, openstack) also route to devops-expert. + +## Safety model + +1. **Read-only discovery** — inspect the host, cluster or hypervisor before anything else; the SSH host, kube context and namespace are never guessed. +2. **Plan / dry run** — `terraform plan` / `tofu plan`, `kubectl diff` and `--dry-run=server`, `ansible-playbook --check --diff`, `helm diff` or `helm upgrade --dry-run=server`; one canary host before a fleet. +3. **Backup or snapshot** before changing any stateful system; restore tests are logged. +4. **One go** — you see the plan (adds / changes / destroys), then say go once. +5. **Apply, verify, roll back** — real post-change checks; a failed check rolls back, never stacks a fix. + +Destructive commands (`terraform destroy`, `kubectl delete`, `qm destroy`, `pct destroy`, `zfs destroy`, `ceph osd purge`, `openstack server delete`, `docker system prune`, `rm -rf`) are proposed with their exact effect and run only after your explicit go. Secrets are never printed, committed or passed on a command line. + +## Installation + +This plugin ships with the `fusengine-plugins` Cursor marketplace and is installed by the repository's installer (`install.sh` / `install.ps1`), not by a per-plugin command. Register the marketplace with `agent plugin marketplace add https://github.com/fusengine/cursor-plugins.git`, then run the installer from the marketplace checkout. Full steps (global and project scope, dry run): [docs/getting-started/installation.md](../../../docs/getting-started/installation.md) and the root [README.md](../../../README.md). + +## Usage + +> "Add a Proxmox VM template with cloud-init and provision three nodes with OpenTofu." +> "Why is this k3s node NotReady?" + +For hardening and security audits use `security-expert`; for application code and framework deploys use the framework/language experts. diff --git a/.cursor-plugin/plugins/fuse-devops/agents/devops-expert.md b/.cursor-plugin/plugins/fuse-devops/agents/devops-expert.md new file mode 100644 index 0000000..74abd25 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/agents/devops-expert.md @@ -0,0 +1,163 @@ +--- +name: devops-expert +description: "Use when: building, operating or troubleshooting self-hosted infrastructure — Docker/Compose/Podman, Kubernetes workloads and clusters (kubeadm, k3s, Talos), Helm/Kustomize/Argo CD/Flux, Linux servers (systemd, LVM, SELinux), networking (nftables, DNS, TLS, reverse proxy, WireGuard), Proxmox VE, KVM/libvirt/Incus, OpenStack, Terraform/OpenTofu, Ansible, CI/CD pipelines and runners, Prometheus/Grafana/Loki/OpenTelemetry, ZFS/Ceph/Proxmox Backup Server/restic, SLOs and incident runbooks, SOPS/Vault/OpenBao secrets, PostgreSQL/MariaDB HA — or when a repo holds *.tf, Chart.yaml, kustomization.yaml, ansible.cfg, *.pkr.hcl, .sops.yaml. Do NOT use for: public clouds AWS/GCP/Azure and their managed services (EKS/GKE/AKS, cloud landing zones, FinOps) — out of scope; application code, framework PaaS deploys, language CI gates (framework/language experts); hardening, security audit, secret scanning, CVE and supply-chain audit (security-expert); repo commit/release flow (commit agent)." +model: grok-4.6 +color: yellow +tools: Read, Edit, Write, Bash, Grep, Glob, Task, Skill, mcp__context7__resolve-library-id, mcp__context7__query-docs, mcp__exa__web_search_exa, mcp__exa__get_code_context_exa, mcp__sequential-thinking__sequentialthinking, mcp__fuse-browser__browser_fetch, mcp__fuse-browser__browser_fetch_batch +skills: devops-docker, devops-kubernetes, devops-k8s-cluster, devops-gitops-packaging, devops-linux, devops-networking, devops-opnsense, devops-proxmox, devops-virtualization, devops-openstack, devops-iac, devops-ansible, devops-cicd, devops-observability, devops-storage-backup, devops-sre-ops, devops-secrets, devops-databases, fuse-ai-pilot:fuse-browser-usage +--- + +## Allowed tools + +> Reminder, not a substitute. The `tools:` key in the frontmatter above is the +> authoritative declaration — it is a first-class repeated field of Cursor's internal +> `agent.v1.CustomSubagent` model. It is restated here because runtime enforcement of +> that key is unverified, so this list keeps the constraint in the model's context. + +This agent must restrict itself strictly to the following tools: + +- `Read` +- `Edit` +- `Write` +- `Bash` +- `Grep` +- `Glob` +- `Task` +- `Skill` +- `mcp__context7__resolve-library-id` +- `mcp__context7__query-docs` +- `mcp__exa__web_search_exa` +- `mcp__exa__get_code_context_exa` +- `mcp__sequential-thinking__sequentialthinking` +- `mcp__fuse-browser__browser_fetch` +- `mcp__fuse-browser__browser_fetch_batch` + + +You are an expert IT/DevOps engineer for self-hosted infrastructure: containers (Docker, Compose, Podman), Kubernetes from workloads to cluster lifecycle, Linux servers, networking, Proxmox VE, KVM/libvirt/Incus, OpenStack, infrastructure as code (Terraform, OpenTofu, Ansible), CI/CD, observability, storage and backup, secrets and stateful databases. You read infrastructure the way an operator on call does: what runs, where the state lives, what breaks if this changes. + +Your posture is discovery-first and reversible-by-default: you inspect read-only before you touch anything, you preview every change (plan, diff, dry run, check mode) before it applies, you take a backup or snapshot before changing a stateful system, and every destructive command waits for the user's explicit go. Infrastructure tools move fast and reach end of life fast, so every version, flag and config key you write is confirmed against the official source first — never recalled. + +You own self-hosted infrastructure and its operation specifically; public clouds (AWS, GCP, Azure) are out of scope, hardening and security audits belong to `fuse-security:security-expert`, and application code belongs to the framework and language experts. + + +# DevOps Expert Agent + +Builds, operates and troubleshoots self-hosted infrastructure end to end — from a Dockerfile or a systemd unit to a Proxmox cluster, a Kubernetes control plane or an OpenStack deployment. Exact versions, flags and per-product procedures live in the skills; this file holds the workflow and the safety model. + +## Agent Workflow (MANDATORY) + +Before ANY change, launch in PARALLEL (single message, multiple `Task` calls): + +1. **fuse-ai-pilot:explore-codebase** - Map the infrastructure repo (IaC layout, `*.tf` and state backend, Helm charts / Kustomize overlays, Ansible inventory and roles, compose files, CI workflows, existing runbooks) +2. **fuse-ai-pilot:research-expert** - Verify the current version, lifecycle (EOL) and flags of every product touched, at the official source + +Then call `mcp__context7__query-docs` directly (MCP tool call, not a sub-agent) to confirm CLI flags and config keys. + +Then follow the **Operations Safety Protocol** below for anything that touches a live system. After writing files in a repo, run **fuse-ai-pilot:sniper** for validation. + +--- + +## MANDATORY SKILLS USAGE (CRITICAL) + +**You MUST use your skills for EVERY task.** + +| Task | Required Skill | +|------|----------------| +| Docker Engine, Compose, BuildKit, Dockerfile practice, container networking/firewall, registries, image supply chain (SBOM, signing), Podman/Quadlet | `devops-docker` | +| Kubernetes workloads, scheduling, Services, Gateway API, storage, RBAC, safe kubectl operations, troubleshooting | `devops-kubernetes` | +| Cluster lifecycle: kubeadm, k3s, Talos, Cluster API, etcd backup/restore, upgrades and version skew, CNI, bare-metal load balancing | `devops-k8s-cluster` | +| Helm, Kustomize, Argo CD, Flux, drift and prune safety, promotion between environments | `devops-gitops-packaging` | +| Day-2 Linux: systemd, packages, users/SSH, kernel params, LVM/filesystems, SELinux/AppArmor, timers, performance and disk troubleshooting, bash scripting | `devops-linux` | +| nftables, bridges/bonding/VLAN, routing, DNS, TLS/ACME, reverse proxy (Traefik/Caddy/Nginx/HAProxy), WireGuard | `devops-networking` | +| OPNsense firewall/router: rules and NAT, DNS/DHCP (Unbound, Dnsmasq, Kea), VPN, multi-WAN, CARP HA, plugins, upgrades and rollback, config backup, REST API, Ansible/OpenTofu | `devops-opnsense` | +| Proxmox VE: cluster, HA, SDN, storage, firewall, LXC, `qm`/`pct`/`pvesh`, templates and cloud-init, API automation | `devops-proxmox` | +| KVM/QEMU/libvirt (`virsh`), LXC/Incus, cloud-init, image building with Packer | `devops-virtualization` | +| OpenStack service map, release/upgrade model (SLURP), Kolla-Ansible / OpenStack-Ansible / OpenStack-Helm, `openstack` CLI, day-2 ops, Ceph integration | `devops-openstack` | +| Terraform and OpenTofu: state, modules, plan/apply discipline, licensing choice, providers for Proxmox/OpenStack/DNS, Terragrunt | `devops-iac` | +| ansible-core, playbooks/roles/collections, inventory, idempotence, check mode, vault, Molecule | `devops-ansible` | +| GitHub Actions, GitLab CI, self-hosted runners, caching, environments and approvals, release pipelines | `devops-cicd` | +| Prometheus, Alertmanager, Grafana, Loki, Alloy, OpenTelemetry Collector, exporters, dashboards as code | `devops-observability` | +| ZFS, Ceph, LVM-thin/NFS, Proxmox Backup Server, restic/borg, 3-2-1, mandatory restore tests, disaster recovery | `devops-storage-backup` | +| SLI/SLO and alert policy, runbooks, incident response, postmortems, change management, capacity, DR drills | `devops-sre-ops` | +| SOPS + age, Vault and OpenBao, External Secrets, Kubernetes Secrets handling, rotation | `devops-secrets` | +| PostgreSQL HA (Patroni, pgBackRest, CloudNativePG), MariaDB/Galera, PITR, upgrade and failover runbooks | `devops-databases` | + +**Workflow:** identify the domain(s), load the skill(s), read the target version at the source, follow the skill strictly. A task spanning several domains (e.g. a Proxmox VM built by Terraform, configured by Ansible) loads every matching skill. + +--- + +## Operations Safety Protocol (ZERO TOLERANCE) + +Every change to a live system — local host, remote server over SSH, cluster, hypervisor or private-cloud API — follows this sequence, the same as `fuse-hermes` skill `hermes-deploy` (discovery → dry run → one go → apply → verify → rollback): + +1. **Read-only discovery first.** Inspect before mutating: `systemctl status`, `journalctl`, `kubectl get/describe`, `kubectl config current-context` and namespace, `qm list` / `pct list`, `openstack ... list/show`, `terraform state list`, `zfs list`, `ceph status`. The SSH host comes from the user's words or `~/.ssh/config`, never guessed. Discovery writes nothing. +2. **Plan / dry run before any change.** `terraform plan -out=FILE` / `tofu plan -out=FILE` (then apply that saved plan, re-checking it if the infrastructure moved meanwhile); `kubectl diff -f` and `kubectl apply --dry-run=server`; `ansible-playbook --check --diff` with `--limit` before a wide run; `helm diff` (plugin) or `helm upgrade --dry-run=server`; `zfs destroy -nv` before any ZFS destroy. Multi-host changes run on one canary host first. +3. **Backup or snapshot before changing a stateful system.** VM/CT snapshot or Proxmox Backup Server backup, ZFS snapshot, etcd snapshot, database dump or PITR checkpoint, `terraform state pull` copy. A backup counts only once its existence is verified; restore tests are logged as evidence. +4. **One go.** Show the user what will change (plan summary: adds / changes / destroys, the diff, the target host, context and namespace), what was decided, what stays pending — then wait for the user's explicit go (relayed by the lead when you run as a sub-agent). The go covers that plan only. +5. **Apply** in the planned order, recording each command and its exit status. +6. **Verify** with real checks: service health, `kubectl rollout status`, `terraform plan` showing no drift, monitoring targets up, the endpoint answering. +7. **Rollback** on any failed verification — return to the last good state (snapshot, previous release, `helm rollback`, revert commit through GitOps) before trying anything else. Never stack a second fix on a broken state. + +**Destructive commands** — `terraform destroy` / `tofu destroy`, `kubectl delete` (namespaces, PV, PVC, CRD above all), `qm destroy`, `pct destroy`, `zfs destroy`, `ceph osd purge`, `openstack server delete`, `docker system prune`, `docker volume rm`, `rm -rf`, storage or pool removal — are **proposed** to the user with their exact effect (what is removed, what data is lost, what depends on it, whether a backup exists) and executed **only after the user's explicit go**. `--force` flags, `kubectl apply --prune` and wildcard selectors are never a default. + +**Remote access safety.** No change to `sshd`, firewall or network config over the same connection without a timed automatic revert or console access. No loop over many hosts without a dry run and a canary. + +**Secrets.** Never printed, never committed, never passed on a command line (they land in shell history and process lists): read them from files, environment injected by a secret manager, SOPS/age, Vault/OpenBao or External Secrets. `.env` and secret stores are read as key NAMES only; logs are redacted before being quoted. + +## Delegation boundaries + +- Hardening, compliance, security audits, secret scanning, CVE and supply-chain audits → `fuse-security:security-expert`. +- Application code, framework deploy adapters (Vercel, Netlify, Cloudflare), language CI gates → the framework/language expert (`fuse-nextjs:nextjs-expert`, `fuse-go:go-expert`, `fuse-rust:rust-expert`, ...). +- Commits, PRs, releases of this repo → `fuse-ai-pilot:commit` (via the lead). +- Public clouds (AWS, GCP, Azure) and their managed services → out of scope: say so, do not improvise. + +## SOLID Rules (MANDATORY) + +**Read the `fuse-solid:solid-generic` skill before writing any script or code file** (Python, TypeScript, Go helpers). It applies only to code files; pure infrastructure artifacts (YAML, HCL, Dockerfile, shell, nftables rulesets) follow the per-product structure of the matching skill. Do NOT duplicate SOLID guidance locally (DRY). + +| Rule | Requirement | +|------|-------------| +| Files | code files within `FUSE_SOLID_MAX_LINES` | +| Structure | IaC split into modules/roles/overlays, one concern each; no copy-pasted environment blocks | +| Documentation | every module/role/chart states inputs, outputs and the rollback path | +| Validation | `fuse-ai-pilot:sniper` after changes | + +## Core Rule + +- **Verify Before Writing**: confirm every version, CLI flag, config key and API against the official docs of the version actually installed (read it on the host first) — never from memory. An EOL line is flagged, never recommended silently. +- **Docs > memory**: verification chain **fuse-browser (raw source, release notes) → Context7 (official docs) → Exa (latest practices)**. A key you cannot confirm is left out, not guessed. + +## fuse-browser (ZERO TOLERANCE) + +- **Fast-path ONLY** — `browser_fetch` (one URL) / `browser_fetch_batch` (N URLs) for docs, release notes, changelogs, lifecycle pages: NO browser launch. You have no live-session tools — never attempt browser_open. +- Full guide: invoke skill `fuse-ai-pilot:fuse-browser-usage` (profile: research-docs). + +## Completion Criteria + +- **Done** = versions read on the target and at the source + plan/diff/dry-run output shown + backup or snapshot verified before any stateful change + user go obtained before any live or destructive change + post-change verification passed (health, rollout status, no drift) + rollback command stated + `fuse-ai-pilot:sniper` ZERO errors on files written in a repo. + +## Forbidden + +- **Mutating before read-only discovery**, or applying without a plan / diff / dry run +- **Running a destructive command without the user's explicit go** on its stated effect +- **Changing a stateful system without a verified backup or snapshot** +- **Guessing the SSH host, kube context or namespace**, or trying targets one after another +- **Printing, committing or passing secrets on a command line** +- **Hand-editing live objects managed by GitOps or IaC** — change the repo, then let the tool reconcile +- **Invented flags, keys, versions or provider arguments** — verify or omit +- **Recommending an end-of-life version without flagging it** +- **Stacking a fix on a failed change** instead of rolling back +- **Public-cloud (AWS/GCP/Azure) work, security audits, application code** — delegate or decline as above + +## Output Format + +Report back to the lead with: +- **status**: `done` | `failed` | `blocked` | `awaiting-go` +- **target**: host / cluster context and namespace / Proxmox node / OpenStack project, and where it came from +- **versions**: installed (read on the target) and current at the source, EOL flags +- **plan**: the plan / diff / dry-run summary shown before the go (adds / changes / destroys) +- **files_changed**: files written in the repo and, after the go, changes applied on the target +- **commands**: exact commands proposed, or run with their exit status after the go +- **verification**: backup or snapshot reference, post-change checks, rollback command +- **remaining_issues**: unverified flags, pending user decisions, follow-ups delegated, or `none` +- **sources_verified**: URLs / docs consulted diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/SKILL.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/SKILL.md new file mode 100644 index 0000000..4a938a4 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/SKILL.md @@ -0,0 +1,172 @@ +--- +name: devops-ansible +description: Use when writing, reviewing, testing or running Ansible - choosing ansible-core and community package versions (control node and managed node Python), static and dynamic inventory with group_vars/host_vars, variable precedence, playbooks, roles and collections (ansible-galaxy, requirements.yml, FQCN), idempotency (changed_when/failed_when), handlers, tags, Jinja2 templates, Ansible Vault and no_log, privilege escalation, check mode and diff mode, --limit, serial/rolling updates and max_fail_percentage, block/rescue/always, performance (forks, pipelining, fact caching, strategy), ansible-lint, Molecule, execution environments (ansible-navigator, ansible-builder), project layout, and Ansible versus Terraform/OpenTofu. Not for Terraform/OpenTofu state and modules (devops-iac), Linux host internals (devops-linux), CI pipeline authoring (devops-cicd), secret stores (devops-secrets), Proxmox or OpenStack platform internals (devops-proxmox, devops-openstack) or a security hardening audit (fuse-security:security-expert). +versions: + ansible-core: "2.22.0 (maintained lines 2.22, 2.21, 2.20; 2.19 EOL Nov 2026)" + ansible-community-package: "14.6.0 (ansible-core ~=2.21.6; 15.0.0 in development on 2.22)" + ansible-lint: "26.11.0" + molecule: "26.11.0" + ansible-navigator: "26.11.0" + ansible-builder: "3.2.0 (EE definition schema version 3)" +user-invocable: true +references: + - references/versions-support.md + - references/inventory-variables.md + - references/playbooks-roles-collections.md + - references/idempotency-handlers-errors.md + - references/safe-run-protocol.md + - references/vault-secrets.md + - references/performance-tuning.md + - references/testing-lint-molecule.md + - references/execution-environments.md + - references/ansible-vs-iac-awx.md + - references/templates/ansible.cfg + - references/templates/inventory.yml + - references/templates/group_vars-all.yml + - references/templates/site-rolling-update.yml + - references/templates/requirements.yml + - references/templates/ansible-lint.yml + - references/templates/role-skeleton/tasks/main.yml + - references/templates/role-skeleton/handlers/main.yml + - references/templates/role-skeleton/defaults/main.yml + - references/templates/role-skeleton/meta/main.yml + - references/templates/role-skeleton/templates/service.conf.j2 + - references/templates/molecule/molecule.yml + - references/templates/molecule/inventory/hosts.yml + - references/templates/molecule/converge.yml + - references/templates/molecule/verify.yml + - references/templates/execution-environment.yml + - references/templates/ansible-navigator.yml +related-skills: + - devops-iac + - devops-linux + - devops-secrets + - devops-cicd + - devops-proxmox + - devops-openstack + - devops-virtualization + - devops-docker +--- + + +Covers configuration management with Ansible: picking versions, structuring a +project, inventory and variables, playbooks, roles and collections, idempotent +tasks, handlers and error handling, Vault, safe execution against real hosts +(check mode, diff, limit, rolling batches), performance, testing (ansible-lint, +Molecule) and execution environments. Does not cover Terraform/OpenTofu +(devops-iac), host-level Linux administration (devops-linux), CI pipelines +(devops-cicd) or external secret stores (devops-secrets); AWX/AAP is a pointer only. + + +# DevOps Ansible + +## Agent Workflow (MANDATORY) + +Before ANY playbook, role or inventory change, spawn in parallel, one `Task` call each: + +1. **fuse-ai-pilot:explore-codebase** - find `ansible.cfg`, `inventories/`, `roles/`, `collections/requirements.yml`, `group_vars/`, vault files +2. **fuse-ai-pilot:research-expert** - verify module options for the installed collection versions at docs.ansible.com +3. **mcp__context7__query-docs** - `/websites/ansible_projects_ansible`, `/ansible/ansible-lint`, `/ansible/molecule` + +Then read the environment read-only (`ansible --version`, `ansible-galaxy collection list`, `ansible-inventory --graph`, `ansible-doc `). +After changes, run **fuse-ai-pilot:sniper**; against real hosts follow [safe-run-protocol.md](references/safe-run-protocol.md). + +## Versions (checked 2026-11-10) + +ansible-core 2.22.0 is the latest (PyPI; needs Python >= 3.13 on the control node). The community package `ansible` 14.6.0 pins `ansible-core ~=2.21.6` and needs Python >= 3.12; package 15.0.0 (on 2.22) is unreleased. ansible-core keeps the latest release plus two older ones. Details, Python matrix and sources: [versions-support.md](references/versions-support.md). + +## Overview + +| Area | Decision in one line | +|------|----------------------| +| **Install** | `ansible` package for batteries included; `ansible-core` + pinned `requirements.yml` for reproducible control nodes; EE container for CI | +| **Inventory** | YAML files per environment, `group_vars/` and `host_vars/` next to them; dynamic inventory through plugins, not scripts | +| **Content** | FQCN everywhere; roles for reusable units; collections pinned by version range in `requirements.yml` | +| **Tasks** | Native modules first; `command`/`shell` only with `changed_when`/`failed_when`; `state: present` over `latest` | +| **Run** | `--check --diff --limit` first, then canary host, then `serial` batches with `max_fail_percentage` | +| **Secrets** | Vault with `--vault-id` labels or an external store; `no_log: true` on tasks handling secrets; `diff: false` on secret files | +| **Quality** | `ansible-lint` profile `production`, Molecule with an idempotence step | + +## Critical Rules + +1. **Version first** - `ansible --version` and `ansible-galaxy collection list` before writing a task; module options differ per collection version; unknown option = `ansible-doc ` +2. **Check, diff, limit before any real run** - `ansible-playbook --check --diff --limit `; check mode is a prediction, not a guarantee (modules without check support are skipped) +3. **Production runs are proposed, not run** - show playbook, inventory, `--limit`, tags, `--become` scope and expected changes, wait for the user's go ([safe-run-protocol.md](references/safe-run-protocol.md)) +4. **Canary then rolling** - one host, then `serial: [1, 5, 10]` with `max_fail_percentage`; never `hosts: all` plus `strategy: free` on a stateful tier +5. **Idempotent or it does not ship** - second run reports `changed=0`; `command`/`shell` without `changed_when` is a defect ([idempotency-handlers-errors.md](references/idempotency-handlers-errors.md)) +6. **Never print or commit a secret** - vault passwords out of the repo, `no_log: true`, names and keys only in reports ([vault-secrets.md](references/vault-secrets.md)) +7. **Never invent a module, option or collection** - cite the docs.ansible.com or collection page; unknown = "not verified" + +## Architecture + +``` +infra-ansible/ +├── ansible.cfg # inventory path, roles_path, collections_path, forks, pipelining +├── collections/requirements.yml # pinned collections (and roles) +├── inventories/ +│ ├── production/{hosts.yml,group_vars/,host_vars/} +│ └── staging/{hosts.yml,group_vars/,host_vars/} +├── playbooks/site.yml # imports per-tier playbooks +├── roles//{tasks,handlers,defaults,vars,templates,files,meta}/ +├── molecule/default/ # per-role or per-collection scenarios +├── .ansible-lint # profile and skip/warn lists +└── execution-environment.yml # only when running in an EE +``` + +## Reference Guide + +### Concepts + +| Topic | Reference | When to Consult | +|-------|-----------|-----------------| +| **Versions and support** | [versions-support.md](references/versions-support.md) | Choosing core/package, Python on control and managed nodes | +| **Inventory and variables** | [inventory-variables.md](references/inventory-variables.md) | Static/dynamic inventory, precedence, groups | +| **Playbooks, roles, collections** | [playbooks-roles-collections.md](references/playbooks-roles-collections.md) | Layout, FQCN, galaxy, requirements.yml, Jinja2 | +| **Idempotency and errors** | [idempotency-handlers-errors.md](references/idempotency-handlers-errors.md) | changed_when, handlers, block/rescue/always, tags | +| **Safe execution** | [safe-run-protocol.md](references/safe-run-protocol.md) | ANY run against real hosts, become, reboot, state: absent | +| **Vault and secrets** | [vault-secrets.md](references/vault-secrets.md) | Encrypting values, vault ids, no_log | +| **Performance** | [performance-tuning.md](references/performance-tuning.md) | forks, pipelining, fact caching, strategies | +| **Testing** | [testing-lint-molecule.md](references/testing-lint-molecule.md) | ansible-lint profiles, Molecule, ansible-test | +| **Execution environments** | [execution-environments.md](references/execution-environments.md) | ansible-navigator, ansible-builder | +| **Ansible vs IaC, AWX** | [ansible-vs-iac-awx.md](references/ansible-vs-iac-awx.md) | Tool choice, handoff to devops-iac | + +### Templates + +| Template | When to Use | +|----------|-------------| +| [ansible.cfg](references/templates/ansible.cfg) | New project defaults: forks, pipelining, fact cache | +| [inventory.yml](references/templates/inventory.yml) | YAML inventory with groups, ranges and group vars | +| [group_vars-all.yml](references/templates/group_vars-all.yml) | Variables with the `vault_` indirection pattern | +| [site-rolling-update.yml](references/templates/site-rolling-update.yml) | Rolling update: serial, max_fail_percentage, block/rescue | +| [requirements.yml](references/templates/requirements.yml) | Pinned collections and roles | +| [ansible-lint.yml](references/templates/ansible-lint.yml) | Lint config (copy to `.ansible-lint`) | +| [role-skeleton/](references/templates/role-skeleton/tasks/main.yml) | New role: tasks, handlers, defaults, meta, template | +| [molecule/](references/templates/molecule/molecule.yml) | Ansible-native Molecule scenario | +| [execution-environment.yml](references/templates/execution-environment.yml) | ansible-builder v3 definition | +| [ansible-navigator.yml](references/templates/ansible-navigator.yml) | Navigator settings with an EE | + +## Quick Reference + +```bash +ansible --version; ansible-galaxy collection list # versions first +ansible-inventory -i inventories/staging --graph # what the inventory resolves to +ansible-doc ansible.builtin.template # option names for the installed version +ansible-lint # from the project root +ansible-playbook -i inventories/staging playbooks/site.yml --syntax-check +ansible-playbook -i inventories/staging playbooks/site.yml --check --diff --limit web01 # read-only prediction +ansible-playbook -i inventories/production playbooks/site.yml --limit web01 # after the user's go +``` + +## Best Practices + +### DO +- Pin collections (`version: ">=x,` in the project (no leading dot) +3. `~/.ansible-navigator.` (leading dot) + +JSON (`.json`) or YAML (`.yml`/`.yaml`); only one file per directory, otherwise it errors. Useful settings (all under the `ansible-navigator:` key): `execution-environment.enabled`, `.image`, `.container-engine` (`auto` tries podman, then docker, then container), `.pull.policy` (`always`, `missing`, `never`, `tag`), `.volume-mounts` (`src`, `dest`, `options`), `.environment-variables` (`pass` / `set`), `mode` (`stdout` or interactive), `playbook-artifact` (`enable`, `save-as`), `logging`. CLI equivalents: `--ee`, `--eei `, `--ce`, `--pp`, `--mode stdout`. + +```bash +ansible-navigator run playbooks/site.yml -i inventories/staging --mode stdout --check --diff --limit web01 +``` + +Everything in [safe-run-protocol.md](safe-run-protocol.md) applies unchanged: the EE changes where Ansible runs, not what it may do. SSH keys and vault passwords reach the container through mounts or environment passthrough: mount read-only, pass only what the play needs. + +Molecule can use `ansible-navigator` as its executor backend ([testing-lint-molecule.md](testing-lint-molecule.md)). + +## Sources (read 2026-11-10) + +- https://docs.ansible.com/projects/builder/en/latest/definition/ +- https://docs.ansible.com/projects/navigator/ +- https://docs.ansible.com/projects/navigator/settings/ +- https://docs.ansible.com/projects/ansible/latest/installation_guide/intro_installation.html (EE as control node) +- https://pypi.org/pypi/ansible-builder/json , https://pypi.org/pypi/ansible-navigator/json + +## Not verified + +- `ansible-builder` command names and flags (`build`, tags, `--container-runtime`): not read. +- Whether `ubi9:latest` is a good pin (use a digest); the base image names are the docs' examples. +- The `python:`/`system:` dependency file names (`requirements.txt`, `bindep.txt`) are conventional: the docs sample shows `bindep.txt` and an inline list. +- The versions of ansible-core available for the chosen base image (the builder picks "most recently compatible" when unpinned). +- Hardware, rootless and SELinux (`:Z`) specifics of running EEs; AAP-supplied EE catalogs. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/idempotency-handlers-errors.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/idempotency-handlers-errors.md new file mode 100644 index 0000000..e5aa50e --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/idempotency-handlers-errors.md @@ -0,0 +1,102 @@ +--- +name: idempotency-handlers-errors +description: Writing idempotent tasks, changed_when and failed_when, handlers (notify, listen, flush_handlers, force_handlers), block/rescue/always, ignore_errors versus ignore_unreachable, any_errors_fatal, tags +when-to-use: Writing tasks that use command/shell, designing restart-on-change, handling failures in a play, using tags +keywords: idempotent, changed_when, failed_when, handler, notify, listen, flush_handlers, force_handlers, block, rescue, always, ignore_errors, any_errors_fatal, tags +priority: high +related: safe-run-protocol.md, testing-lint-molecule.md, playbooks-roles-collections.md +--- + +# Idempotency, Handlers, Error Handling + +**Load when:** a task uses `command`/`shell`/`raw`, a service must restart when config changes, or a failure path must be designed. Template: [site-rolling-update.yml](templates/site-rolling-update.yml). + +## Idempotency contract + +A play is idempotent when a second run reports `changed=0`. Molecule checks exactly this (`idempotence` step, [testing-lint-molecule.md](testing-lint-molecule.md)). + +1. Prefer a native module (`ansible.builtin.package`, `template`, `service`, `lineinfile`, `file`, `user`, ...) over `command`/`shell`; the module compares state and reports `changed` correctly. ansible-lint (`command-instead-of-module`, `command-instead-of-shell`) flags the rest. +2. If `command`/`shell` is unavoidable, make it guarded and honest: + ```yaml + - name: Initialise the application database once + ansible.builtin.command: /opt/app/bin/initdb + args: + creates: /var/lib/app/.initdb-done # skipped when the file exists + register: initdb + changed_when: "'initialised' in initdb.stdout" + ``` + `creates`/`removes` are `command` options; `changed_when` decides what counts as a change and therefore whether handlers fire. Rule `no-changed-when` is in the lint `shared` profile. +3. `state: present` for packages in production. `state: latest` (lint rule `package-latest`, `safety` profile) makes every run a potential upgrade. +4. File modes as strings (`"0640"`), explicit `owner`/`group` (`risky-file-permissions`, `risky-octal`). + +## changed_when and failed_when + +```yaml +- name: Check pending migrations + ansible.builtin.command: /opt/app/bin/migrate --plan + register: plan + changed_when: false # read-only command never "changes" + failed_when: plan.rc not in [0, 3] # rc 3 = nothing to do (example) +``` + +- Both take expressions, not `{{ }}`. A **list** of `failed_when` conditions is joined with `and` (all must be true); use one string with `or` for "any". +- `changed_when: false` on every read-only probe keeps recaps honest. + +## Handlers + +- A handler runs once, at the end of the play section (`pre_tasks`, `roles`/`tasks`, `post_tasks`), even if notified many times, and only when a notifying task reported `changed`. +- `listen:` lets several handlers subscribe to one topic ("restart web services"); `listen` topics cannot contain templates. +- `ansible.builtin.meta: flush_handlers` runs notified handlers now - use it before a verification task that depends on the restart. +- **If a later task fails, handlers notified earlier do not run on that host** by default; the config is changed but the service is not restarted. Fix with `force_handlers: true` (play), `--force-handlers`, or `force_handlers = True` in `ansible.cfg`; or run `flush_handlers` from a `rescue` section. Unreachable hosts still cannot run handlers. +- ansible-lint rule `no-handler` flags tasks gated on `when: x.changed` that should be handlers. + +## Failure handling + +| Tool | Does | Careful | +|------|------|---------| +| `ignore_errors: true` | continue after a *failed* task | does not cover undefined variables, connection failures, syntax errors; hides real faults (lint `ignore-errors` in `shared` profile) | +| `ignore_unreachable: true` | continue after UNREACHABLE | the host stays in the play for later tasks | +| `meta: clear_host_errors` | re-activate hosts marked unreachable | | +| `failed_when` | redefine failure | preferred to `ignore_errors` | +| `any_errors_fatal: true` | one failure stops the play on all hosts, after finishing that task in the current batch | play or block level; recoverable with `rescue` | +| `max_fail_percentage: N` | abort when MORE than N% of a batch fail | per `serial` batch; "exceeded, not equaled" (serial 4, stop at 2 failures -> 49) | + +### block / rescue / always + +```yaml +- name: Upgrade with rollback path + block: + - ansible.builtin.include_role: { name: example_service, tasks_from: upgrade.yml } + rescue: + - name: Restore the previous config and restart + ansible.builtin.include_role: { name: example_service, tasks_from: rollback.yml } + - name: Keep the host marked failed so the batch gate sees it + ansible.builtin.fail: + msg: "upgrade failed on {{ inventory_hostname }}, rolled back" + always: + - name: Record the outcome + ansible.builtin.debug: + msg: "finished {{ inventory_hostname }}" +``` + +- `rescue` runs only after a **failed** task in the block; invalid task definitions and unreachable hosts trigger neither `rescue` nor `always`. +- If `rescue` succeeds, Ansible reverts the failed status: the host counts as fine and does **not** count toward `max_fail_percentage` / `any_errors_fatal`, though the recap still shows a failure. End `rescue` with `ansible.builtin.fail` when the batch gate must still trip. +- Tags: selection by `--tags` supersedes block logic; a tagged task in `block` without the tag on `rescue`/`always` leaves those out. + +## Tags + +`--tags`, `--skip-tags`, `--list-tags`. Reserved: `always` (runs unless explicitly skipped), `never` (runs only when named), `tagged`, `untagged`, `all`. A tag on `include_role`/`include_tasks` applies to the include only, not the inner tasks; tag the inner tasks or use `apply:`. Tags narrow *which tasks* run; they are not an authorization boundary: state them in every ACTION block ([safe-run-protocol.md](safe-run-protocol.md)). + +## Sources (read 2026-11-10) + +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_error_handling.html +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_blocks.html +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_handlers.html +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_tags.html +- https://docs.ansible.com/projects/lint/profiles/ + +## Not verified + +- `command` options `creates`/`removes` and the `fail` module `msg` option (standard, not re-read on their module pages today). +- Exact handler execution points with `import_role`/`include_role` handlers in 2.22. +- Behaviour of `ignore_unreachable` combined with `serial` and `max_fail_percentage`. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/inventory-variables.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/inventory-variables.md new file mode 100644 index 0000000..7a4247d --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/inventory-variables.md @@ -0,0 +1,119 @@ +--- +name: inventory-variables +description: Static and dynamic inventory, groups and ranges, multiple sources, group_vars and host_vars, the 22-level variable precedence list and the rules for where to put a variable +when-to-use: Writing or reviewing an inventory, debugging "why does this variable have that value", splitting environments +keywords: inventory, yaml, ini, group_vars, host_vars, children, ungrouped, range, dynamic inventory, keyed_groups, precedence, extra vars, hash_behaviour +priority: high +related: playbooks-roles-collections.md, vault-secrets.md, safe-run-protocol.md +--- + +# Inventory and Variables + +**Load when:** touching inventory files, `group_vars/`, `host_vars/`, or a variable that "has the wrong value". Template: [inventory.yml](templates/inventory.yml). + +## Inventory formats + +INI and YAML are built in; any inventory plugin can add formats and sources (cloud APIs, Proxmox, OpenStack). Two groups always exist: `all` (every host) and `ungrouped` (hosts with no other group). + +```yaml +# inventories/production/hosts.yml +all: + children: + web: + hosts: + web[01:03].example.net: # numeric range; stride: web[01:09:2] + db: + hosts: + db01.example.net: + ansible_host: 10.0.20.11 + vars: + ansible_user: deploy +``` + +- Child groups are declared with `children:` (YAML) or the `:children` suffix (INI). A host or group defined twice is the same entity. +- Several sources can be passed at once (`-i a.yml -i b.yml`), or put in a directory; mixed static and dynamic files in one directory are merged. Within one directory, files load in order, so later files win on conflicting group variables - name files with numeric prefixes (`01-openstack.yml`, `03-static`). +- Prefer **one inventory directory per environment** (`inventories/production`, `inventories/staging`) so variables of one environment cannot leak into another. + +## Dynamic inventory + +Use **inventory plugins** (maintained path); inventory scripts still work through the `script` plugin. A plugin is configured by a small YAML file: + +```yaml +# inventory/dynamic.yml (example from the Molecule docs; collection amazon.aws must be installed) +plugin: amazon.aws.aws_ec2 +regions: + - us-east-1 +filters: + "tag:Environment": "test" +keyed_groups: + - key: tags.Name + prefix: tag +``` + +For Proxmox use `community.proxmox` and for OpenStack `openstack.cloud` inventory plugins; their option names come from `ansible-doc -t inventory ` for the installed version (not read here). Verify with `ansible-inventory -i --graph` before running any play against the result. Caching of inventory plugins is configured separately ([performance-tuning.md](performance-tuning.md)). + +## group_vars and host_vars + +``` +inventories/production/ +├── hosts.yml +├── group_vars/ +│ ├── all.yml # or all/ directory: main.yml + vault.yml +│ └── web.yml +└── host_vars/ + └── db01.example.net.yml +``` + +- Child groups override parent groups; host variables override group variables. If two groups at the same level set the same variable, the last one loaded wins. +- A variables file can be a directory: Ansible reads every file in it alphabetically. Use `group_vars/all/main.yml` plus `group_vars/all/vault.yml` ([vault-secrets.md](vault-secrets.md)). +- Hash variables are **replaced**, not merged, by default (`hash_behaviour=replace`); switching to `merge` is a global behaviour change - avoid it, use `combine` filters explicitly. + +## Variable precedence (least to greatest, docs list read 2026-11-10) + +1. Command-line values (for example `-u my_user`; not variables) +2. Role defaults (`defaults/main.yml`) +3. Inventory file or script group vars +4. Inventory `group_vars/all` +5. Playbook `group_vars/all` +6. Inventory `group_vars/*` +7. Playbook `group_vars/*` +8. Inventory file or script host vars +9. Inventory `host_vars/*` +10. Playbook `host_vars/*` +11. Host facts and cached `set_fact` +12. Play `vars` +13. Play `vars_prompt` +14. Play `vars_files` +15. Role `vars` (`vars/main.yml`) +16. Block vars +17. Task vars +18. `include_vars` +19. Registered vars and `set_fact` +20. Role (and `include_role`) params +21. `include` params +22. Extra vars (`-e`) - always win + +Consequences: +- Put tunable defaults in **role defaults**; a role that must not be overridden uses `vars/main.yml`. +- Put geography and behaviour (region, environment, endpoints) in inventory group vars; map roles to hosts with groups. +- `-e` beats everything, including `set_fact`: never use it for routine configuration, only for one-off overrides that you will state in the report. +- `ansible_user` set as a host var overrides `ansible_ssh_user` set as a group var. +- When a value surprises you: `ansible -i -m ansible.builtin.debug -a 'var=the_var'` shows the final value for that host; `ansible-inventory -i --host ` shows inventory-level vars only (command names not re-read today). + +## Naming + +Prefix role variables with the role name (`nginx_worker_processes`) so they cannot collide across roles; ansible-lint enforces `var_naming_pattern` (default `^[a-z_][a-z0-9_]*$`) and can check loop variable prefixes ([testing-lint-molecule.md](testing-lint-molecule.md)). + +## Sources (read 2026-11-10) + +- https://docs.ansible.com/projects/ansible/latest/inventory_guide/intro_inventory.html +- https://docs.ansible.com/projects/ansible/latest/inventory_guide/intro_dynamic_inventory.html +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_variables.html +- https://docs.ansible.com/projects/molecule/ansible-native/ (dynamic plugin example) + +## Not verified + +- `ansible-inventory --graph/--host` and `ansible -m debug` flags (standard CLI, not re-read on a docs page today). +- Required file name suffix for plugin config files (some plugins demand names such as `*.aws_ec2.yml`): check the plugin page. +- Option names of the Proxmox and OpenStack inventory plugins. +- INI-format syntax details beyond `:children` and ranges. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/performance-tuning.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/performance-tuning.md new file mode 100644 index 0000000..f8537bb --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/performance-tuning.md @@ -0,0 +1,71 @@ +--- +name: performance-tuning +description: Making Ansible runs faster and safer - forks, SSH pipelining and ControlPersist, fact gathering policy and fact caching, strategies (linear, free), throttle, run_once, interpreter discovery +when-to-use: A run is slow, a fleet is large, or you are choosing ansible.cfg defaults and a strategy +keywords: forks, pipelining, ControlPersist, gathering, smart, explicit, fact_caching, jsonfile, strategy free, throttle, run_once, interpreter_python +priority: medium +related: safe-run-protocol.md, idempotency-handlers-errors.md +--- + +# Performance Tuning + +**Load when:** tuning `ansible.cfg` or diagnosing slow plays. Template: [ansible.cfg](templates/ansible.cfg). Measure first (a callback such as `profile_tasks` is optional; run time of the play before and after); speed settings never replace the safety gates in [safe-run-protocol.md](safe-run-protocol.md). + +## Settings that matter + +| Setting | Default (docs) | Effect and caveat | +|---------|----------------|-------------------| +| `forks` (`[defaults]`, `-f`, `ANSIBLE_FORKS`) | **5** | hosts processed in parallel; raise to what the control node CPU/RAM and the targets/bastion tolerate (the docs example uses 30). More forks widen blast radius: combine with `serial` for changes | +| `pipelining` (`[connection]` or `[defaults]`, `ANSIBLE_PIPELINING`) | **False** | pipes modules to the remote Python's stdin instead of copying a temp file: fewer SSH operations, "very significant" speed-up. Conflicts with `become` when `requiretty` is set in sudoers (disable `requiretty`); does not work for modules that transfer files (`copy` and similar); disabled if `ANSIBLE_KEEP_REMOTE_FILES` is on. Also removes the temp-file permission problem when becoming an unprivileged user | +| SSH multiplexing | per `ssh_args` | the Molecule docs show `ssh_args = -o ControlMaster=auto -o ControlPersist=60s` under `[ssh_connection]` | +| `gathering` | `implicit` | `implicit`: gather per play unless `gather_facts: false`; `explicit`: only when asked; `smart`: gather once per host per run, reusing the cache plugin. `smart` and `explicit` use the fact cache | +| `gather_facts: false` | per play | skip the `setup` step for plays that do not use `ansible_facts` | +| `strategy` | `linear` | `free` lets each host run to the end of the play independently | +| `interpreter_python` | `auto` | set a fixed path via `ansible_python_interpreter` for hosts where discovery is slow or wrong; `auto_silent` removes the warning | + +## Fact caching + +```ini +[defaults] +gathering = smart +fact_caching = jsonfile +fact_caching_connection = .cache/facts +fact_caching_timeout = 86400 +``` + +- Fact caching is always on; exactly one cache plugin is active (`ANSIBLE_CACHE_PLUGIN`). Default is `memory` (this run only). `jsonfile` writes one JSON file per host; `fact_caching_timeout` defaults to 86400 s; `fact_caching_connection` is the directory (required for `jsonfile`). +- A playbook must never **require** a cache hit: a miss or a stale entry must not change the outcome. Facts about state that changes (services, packages) go stale; gather fresh facts before decisions that depend on them (`ansible.builtin.setup`). +- Inventory plugins have their own cache (`ANSIBLE_INVENTORY_CACHE_PLUGIN`, `cache_plugin` in the plugin config). +- Cached facts are host data: keep the directory out of the repo and out of shared volumes. + +## Strategies and execution keywords + +- `linear` (default): each task finishes on all hosts of the batch before the next task starts. `free`: no lockstep; fast, but hosts diverge in time. Use `free` only for stateless, independent tasks; never for rolling restarts where ordering across hosts matters. +- `serial`, `throttle`, `order`, `run_once` are keywords, not strategies: + - `throttle: N` (task or block) caps workers for a CPU-heavy task or rate-limited API; it can lower, never raise, the limit set by `forks`/`serial`. + - `run_once: true` runs the task on the first host of the batch and applies the result to all; with `serial` it runs once **per batch**. For once overall use `when: inventory_hostname == ansible_play_hosts_all[0]`. Add `delegate_to` to pick the host. +- `strategy: debug` is for interactive debugging, never for unattended runs. + +## Other levers + +- Fewer, bigger tasks: use module list arguments (`name: [a, b]`) instead of loops over `package`. +- Prefer `ansible.builtin.template`/`copy` over repeated `lineinfile` edits. +- Async/poll for long tasks (`async`, `poll`) - see the Not verified note. +- For thousands of hosts or many users, move runs to AWX/AAP or an EE-based runner ([ansible-vs-iac-awx.md](ansible-vs-iac-awx.md)). + +## Sources (read 2026-11-10) + +- https://docs.ansible.com/projects/ansible/latest/reference_appendices/config.html (`ANSIBLE_PIPELINING`, `DEFAULT_FORKS`, `DEFAULT_GATHERING`) +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_strategies.html +- https://docs.ansible.com/projects/ansible/latest/plugins/cache.html +- https://docs.ansible.com/projects/ansible/latest/collections/ansible/builtin/jsonfile_cache.html +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_privilege_escalation.html (pipelining and become) +- https://docs.ansible.com/projects/ansible/latest/reference_appendices/interpreter_discovery.html +- https://docs.ansible.com/projects/molecule/ansible-native/ (ControlPersist example) + +## Not verified + +- The `async`/`poll` semantics, `gather_subset`, and the `profile_tasks` callback (not read today). +- `ssh_args` default value (the template does not override it for that reason), `ANSIBLE_SSH_RETRIES`, connection `timeout`. +- Real speed-up figures: none measured; the docs only call pipelining "very significant". +- Whether the `[connection]` section key is honoured by every connection plugin (docs: "each connection plugin can override"). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/playbooks-roles-collections.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/playbooks-roles-collections.md new file mode 100644 index 0000000..12ca931 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/playbooks-roles-collections.md @@ -0,0 +1,101 @@ +--- +name: playbooks-roles-collections +description: Playbook and project layout, role directory structure and reuse (roles, include_role, import_role), collections and FQCN, ansible-galaxy and requirements.yml for roles and collections, signature verification, Jinja2 templating rules +when-to-use: Creating a project, a role or a template; installing or pinning collections; deciding import versus include +keywords: playbook, role, include_role, import_role, collection, FQCN, ansible-galaxy, requirements.yml, template, jinja2, meta/main.yml, defaults, vars +priority: high +related: inventory-variables.md, idempotency-handlers-errors.md, testing-lint-molecule.md +--- + +# Playbooks, Roles, Collections + +**Load when:** creating or restructuring Ansible content. Templates: [role-skeleton](templates/role-skeleton/tasks/main.yml), [requirements.yml](templates/requirements.yml), [ansible.cfg](templates/ansible.cfg). + +## Playbook layout + +`playbooks/site.yml` only imports tier playbooks; each tier playbook targets a group and applies roles. Keep plays short, named, and free of inline business logic (that belongs in roles). + +```yaml +- name: Web tier + hosts: web + become: true + roles: + - role: example_service + vars: { example_service_port: 8080 } +``` + +Order inside a play: `pre_tasks` -> handlers flush -> `roles` -> `tasks` -> `post_tasks`; roles listed in `roles:` run before `tasks:`, and their `meta/main.yml` dependencies run first. + +## Role structure (docs, read 2026-11-10) + +| Path | Purpose | +|------|---------| +| `tasks/main.yml` | the tasks | +| `handlers/main.yml` | handlers imported into the play | +| `defaults/main.yml` | very low precedence variables (the role's API) | +| `vars/main.yml` | high precedence variables (internal constants) | +| `templates/`, `files/` | Jinja2 templates and static files | +| `meta/main.yml` | dependencies and optional Galaxy metadata (required only to upload a standalone role) | + +- Variables from `vars/` and `defaults/` enter play scope unless `public` is disabled on `include_role`/`import_role`. +- `import_role` is static (same behaviour as `roles:`); `include_role` is dynamic (evaluated at run time, in order with other tasks). Tags and `when` on an `include_role` apply to the include itself; use `apply:` or tag the inner tasks to filter inside. +- `tasks_from:` selects another task file, e.g. `include_role: { name: x, tasks_from: apt.yml }`, a clean way to split by OS family. +- Nested directories under `defaults/` and `vars/` are read alphabetically. + +## Collections and FQCN + +Always write the fully qualified collection name: `ansible.builtin.copy`, `community.general.ufw`. FQCN avoids name clashes and is required by ansible-lint's `production` profile (`fqcn` rule). In playbooks you may also declare a `collections:` search list, but FQCN is clearer. + +```bash +ansible-galaxy collection install -r collections/requirements.yml # pinned install +ansible-galaxy collection install community.general --upgrade # move to the latest (deliberate) +ansible-galaxy collection list # what is installed, from which path +``` + +Install path: the first writable entry of the configured collections paths, or `-p ./collections`; the `-p` value must be one of the configured paths or Ansible will not find the collection. Version syntax: `ns.name:==1.0.0`, `'ns.name:>=1.0.0,<2.0.0'`. + +**Signature verification** (only for collections from a distribution server that signs): `--keyring ` enables GnuPG verification; `--signature ` adds signatures; `signatures:` key in `requirements.yml`; by default one valid signature suffices (`--required-valid-signature-count`); `--disable-gpg-verify` turns it off (avoid). Git, directory and tarball sources are not signature-verified. + +### requirements.yml + +```yaml +roles: + - name: example.role # galaxy role: exact version; ranges are not supported for roles + version: "1.9.6" +collections: + - name: community.general + version: ">=14.0.0,<15.0.0" +``` + +The file may `include:` other requirement files in large projects. Install roles with `ansible-galaxy install -r requirements.yml` (roles and collections can share one file under `roles:` / `collections:`). Pin Git sources with a commit hash or tag, never a branch name, in production. + +## Jinja2 templating rules + +- Templates (`.j2`) go through `ansible.builtin.template`; set `owner`, `group`, `mode`, and use `validate:` for configs that have a syntax checker (`validate: nginx -t -c %s` style) so a bad file never replaces a good one; `backup: true` keeps the previous copy. +- A value starting with `{{` must be quoted in YAML: `dest: "{{ app_dir }}/conf"`. +- No `{{ }}` inside `when:`, `changed_when:`, `failed_when:` (these are already expressions; ansible-lint flags `no-jinja-when`). +- Use filters (`default`, `combine`, `join`, `bool`) instead of shell post-processing; give every optional variable a `default()` or a role default. +- Secrets in templates: set `no_log: true` on the task and `diff: false` so check/diff mode does not print them. + +## Project hygiene + +- `ansible.cfg` in the project root sets `inventory`, `roles_path`, collections path, forks, pipelining ([performance-tuning.md](performance-tuning.md)). +- One reusable role per concern; environment differences live in inventory vars, not in `when:` ladders inside roles. +- Document each role's `defaults/main.yml` variables with comments; they are the role's public interface. +- Run `ansible-lint` and `ansible-playbook --syntax-check` in CI ([testing-lint-molecule.md](testing-lint-molecule.md)). + +## Sources (read 2026-11-10) + +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_reuse_roles.html +- https://docs.ansible.com/projects/ansible/latest/collections_guide/collections_installing.html +- https://docs.ansible.com/projects/ansible/latest/collections_guide/collections_using_playbooks.html +- https://docs.ansible.com/projects/ansible/latest/galaxy/user_guide.html +- https://docs.ansible.com/projects/lint/profiles/ (fqcn and no-jinja-when rules) +- https://docs.ansible.com/projects/ansible/latest/reference_appendices/config.html (`collections_path`, `roles_path`, config file search order) + +## Not verified + +- `template` options `validate` and `backup` and the `apply:` keyword of `include_role` (standard documented options, not re-read on their module pages today). +- Config-page defaults were read from the docs build "latest" (2.21 line): `collections_path` (ini key, `[defaults]`) defaults to `~/.ansible/collections:/usr/share/ansible/collections` and `roles_path` to `~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles`; confirm with `ansible-config dump` on 2.22. +- Role dependencies syntax in `meta/main.yml` beyond the sentence that dependencies run first. +- Whether `meta/argument_specs.yml` validation exists/behaves as expected in 2.22 (not read). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/safe-run-protocol.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/safe-run-protocol.md new file mode 100644 index 0000000..2f2fc37 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/safe-run-protocol.md @@ -0,0 +1,114 @@ +--- +name: safe-run-protocol +description: How to run Ansible against real hosts - read-only discovery, syntax check, check and diff mode, --limit, canary, serial batches with max_fail_percentage, privilege escalation, and the catalogue of destructive plays and commands in ACTION / EFFECT / PROOF / WAIT-FOR-GO format +when-to-use: Before ANY ansible-playbook or ad-hoc command that is not purely read-only, and whenever state absent, reboot, a service stop/restart, a package removal or a wide become run is on the table +keywords: check mode, diff, limit, canary, serial, max_fail_percentage, become, reboot, state absent, extra vars, step, start-at-task, wait-for-go +priority: critical +related: idempotency-handlers-errors.md, vault-secrets.md, performance-tuning.md +--- + +# Safe Run Protocol + +**Load when:** the agent is about to execute Ansible against anything real. Same pipeline as the rest of the plugin (discovery, dry run, one go, apply, verify, rollback). Template: [site-rolling-update.yml](templates/site-rolling-update.yml). + +## Phase A - read-only discovery (changes nothing) + +```bash +ansible --version # core version, config file, python +ansible-galaxy collection list # installed collection versions +ansible-config dump --only-changed # effective non-default settings +ansible-inventory -i inventories/production --graph # which hosts/groups the run can touch +ansible-playbook -i inventories/production playbooks/site.yml --syntax-check +ansible-lint # when installed +ansible -i inventories/production web --limit web01 -m ansible.builtin.ping # connectivity only +``` + +Stop and report instead of running when: the inventory resolves to more hosts than the request names, the target environment is ambiguous, the playbook has no `name:`d plays, `ansible-lint` reports `syntax-check`/`load-failure`, or a vault password is needed and not provided by the user. + +## Phase B - prediction: check mode and diff mode + +```bash +ansible-playbook -i inventories/production playbooks/site.yml --check --diff --limit web01 +``` + +- `--check` runs modules in prediction mode; modules without check-mode support do not run and report nothing. **A clean check is not a guarantee** (later tasks may depend on earlier changes that did not happen, e.g. a package install a template needs). +- Per task: `check_mode: false` forces a real execution even under `--check` (use only for read-only probes); `check_mode: true` forces prediction. Guard with `when: not ansible_check_mode` or `ignore_errors: "{{ ansible_check_mode }}"` for tasks that cannot predict. +- `--diff` shows before/after for modules that support it (files, templates); it is large, so use it with `--limit` on one host. It can reveal secrets: set `diff: false` on such tasks. +- Show the user the changed-task list from the check output, not a summary of what you hope happens. + +## Phase C - the user's go + +One go covers one ACTION block: playbook, inventory, `--limit`, tags, extra vars, become scope, batch settings. Anything added later (another group, `--tags`, `-e`) gets a new block. + +## Phase D - run: canary, then batches + +1. **Canary**: `--limit ` of the tier, then verify (Phase E). +2. **Batches** in the play: `serial: [1, 5, 10]` and `max_fail_percentage: 30` (a failure count must *exceed* the percentage of the batch to abort). `serial` limits the scope of failures to the batch. +3. `strategy: free` and large `forks` are for stateless fan-out only ([performance-tuning.md](performance-tuning.md)). +4. Snapshot or backup stateful targets first (VM snapshot via devops-proxmox/devops-openstack, database dump); a backup counts only once its existence is checked. +5. Keep `--limit` explicit even when the playbook names a group: the CLI limit is an intersection, never a widening. + +## Phase E - verify, or roll back + +Re-run the read-only checks plus the service-level probe (port open, endpoint status, replication state). Idempotence check: a second `--check --diff` on the changed hosts shows `changed=0`. Any failed check: stop, roll back to the last good state (snapshot, previous template via `backup: true`, previous role version), never stack a second fix on a broken host. + +## Destructive and wide-impact operations + +| Operation | Exact effect | Precondition before proposing | +|-----------|--------------|-------------------------------| +| `ansible-playbook` against an inventory group in production | Applies every non-skipped task to every host matched by both `hosts:` and `--limit`, up to `forks` hosts at once | Check + diff reviewed, canary done, backup, rollback path | +| Any task with `state: absent` (package, user, file, directory, mount, VM, volume, DNS record) | Removes the object; for `file` with a directory, its whole tree; for `user`, optionally its home (`remove`) | List what matches first with a read-only task; data backed up; dependents known | +| `ansible.builtin.reboot` | Reboots the host and waits for it to return; services stop, sessions drop, a host that does not come back fails the batch | Maintenance window, `serial: 1`, out-of-band console, redundancy for the tier | +| `ansible.builtin.service`/`systemd` `state: stopped` or `restarted` | Interrupts the service; connections drop | Load balancer drain, `serial`, health check after | +| `become` on a whole inventory (`-b` or `become: true` at play level) | Every task runs as root (or `become_user`) | Task list reviewed; least privilege for tasks that do not need root; `become` password never in the repo | +| `ansible.builtin.shell`/`command` with `rm`, `dd`, `mkfs`, `iptables -F`, `systemctl isolate` | Whatever the string does, on every host, with no state check | Prefer a module; show the exact line; run on one host | +| `-e` extra vars | Overrides every other variable source (level 22) | State every `-e` in the ACTION block | +| `--start-at-task`, `--step`, `--tags` | Skips tasks that set up what later tasks assume | Review the skipped part; never on production for the first run | +| `ansible-galaxy install --force` / `collection install --upgrade` | Replaces installed role/collection versions on the control node | Pin first; test in staging | +| `ansible-vault rekey` / `decrypt` | Changes key / writes plaintext secrets to disk | See [vault-secrets.md](vault-secrets.md) | +| `strategy: free`, `forks` raised | Hosts proceed independently and in parallel; ordering across hosts is lost | Stateless tier only | + +### Proposal format (one block per action) + +``` +ACTION ansible-playbook -i inventories/production playbooks/site.yml --limit web01 --tags config + (become: true at play level; extra vars: none; vault id: prod@prompt) +EFFECT templates /etc/example/service.conf on web01 (diff attached), notifies handler "restart example" + -> service restarts once, ~5 s of dropped connections on web01 only +BLAST 1 host of 6 in group web; LB drains web01 in pre_tasks +PROOF --check --diff output attached (1 changed task); ansible-lint clean; backup=true keeps the old file +WAIT-FOR-GO +``` + +``` +ACTION ansible-playbook -i inventories/production playbooks/decommission.yml --limit old-db02 + (task: ansible.builtin.package name=postgresql* state=absent; file /var/lib/postgresql state=absent) +EFFECT uninstalls PostgreSQL and DELETES /var/lib/postgresql on old-db02: data unrecoverable without backup +BLAST 1 host; replica of db01 (replication slot must be dropped first) +PROOF backup id and restore test logged; `pg_stat_replication` shows old-db02 detached; --check output attached +WAIT-FOR-GO +``` + +``` +ACTION ansible-playbook -i inventories/production playbooks/kernel.yml --limit web[01:06] (serial: 1, reboot task) +EFFECT each host in turn reboots; ~90 s unavailable per host; batch aborts if more than 30% fail +BLAST 6 hosts, one at a time; the tier keeps 5 of 6 serving +PROOF console access confirmed; --check run clean; rollback = boot previous kernel entry +WAIT-FOR-GO +``` + +## Sources (read 2026-11-10) + +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_checkmode.html +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_strategies.html +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_error_handling.html +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_privilege_escalation.html +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_variables.html (extra vars precedence) +- Context7 `/websites/ansible_projects_ansible` cross-check of `--check --diff --limit`, `serial` lists and `max_fail_percentage` + +## Not verified + +- `ansible-config dump --only-changed`, `ansible-inventory --graph` and `--list-hosts`/`--list-tasks` flags (standard CLI, not on a page read today). +- Effects of `ansible.builtin.reboot`, `file state: absent` and `user remove` (module pages not re-read today); confirm with `ansible-doc `. +- Which modules of the installed collections support check mode: `ansible-doc ` lists `check_mode` support per module. +- Whether `--limit` can ever widen the target set (documented behaviour is an intersection with the play's `hosts:`; verify with `--list-hosts` before the run). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/ansible-lint.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/ansible-lint.yml new file mode 100644 index 0000000..bf6699d --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/ansible-lint.yml @@ -0,0 +1,26 @@ +--- +# Copy to the project root as `.ansible-lint` (run ansible-lint from that root). +# Profiles, each extending the previous: min, basic, moderate, safety, shared, production. +profile: production + +exclude_paths: + - .cache/ + - collections/ + +# Opt-in rules switched on (they are not loaded by default). +enable_list: + - no-log-password + - no-same-owner + - name[prefix] + - args + +# Findings that stay visible but do not fail the run. Prefer .ansible-lint-ignore entries over skip_list. +warn_list: + - experimental + +# Enforce role-prefixed variable names and loop variables. +var_naming_pattern: "^[a-z_][a-z0-9_]*$" +loop_var_prefix: "^(__|{role}_)" + +# true for air-gapped CI: no requirements.yml install, no schema refresh. +offline: false diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/ansible-navigator.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/ansible-navigator.yml new file mode 100644 index 0000000..9100e81 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/ansible-navigator.yml @@ -0,0 +1,22 @@ +--- +# ./ansible-navigator.yml (project dir, no leading dot) or ~/.ansible-navigator.yml (leading dot). +# Only one settings file per directory; ANSIBLE_NAVIGATOR_CONFIG overrides the lookup. +ansible-navigator: + execution-environment: + enabled: true + image: registry.example.net/ee/infra-ee:2026.11.0 # your built EE image, pinned tag + container-engine: podman # auto tries podman, docker, container + pull: + policy: missing # always | missing | never | tag + volume-mounts: + - src: "~/.ssh/known_hosts" + dest: "/root/.ssh/known_hosts" + options: "ro" + environment-variables: + pass: + - ANSIBLE_VAULT_PASSWORD_FILE + mode: stdout + playbook-artifact: + enable: false + logging: + level: warning diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/ansible.cfg b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/ansible.cfg new file mode 100644 index 0000000..7228b71 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/ansible.cfg @@ -0,0 +1,30 @@ +# Project-level ansible.cfg. Search order: ANSIBLE_CONFIG, ./ansible.cfg, ~/.ansible.cfg, /etc/ansible/ansible.cfg +# (first file found wins). Ansible ignores ./ansible.cfg when the directory is world-writable. +# Relative paths below resolve against this file's directory. +# Full reference: `ansible-config init --disabled > ansible.cfg` generates every option, commented out. + +[defaults] +# Default to STAGING so a bare command can never touch production; production needs an explicit -i. +inventory = inventories/staging +roles_path = roles +collections_path = collections + +# Parallelism: default is 5. Raise only after checking control node CPU/RAM and target tolerance. +forks = 10 + +# smart: gather facts once per host per run and reuse the cache plugin. +gathering = smart +fact_caching = jsonfile +fact_caching_connection = .cache/facts +fact_caching_timeout = 86400 + +# Host key checking stays ON (the default). Do not switch it off for production fleets. +host_key_checking = True + +# Pin discovery noise down; set ansible_python_interpreter per group when a host needs a fixed path. +interpreter_python = auto_silent + +[connection] +# Fewer SSH round trips. Needs "requiretty" disabled in sudoers when using become with sudo; +# does not apply to modules that transfer files. +pipelining = True diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/execution-environment.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/execution-environment.yml new file mode 100644 index 0000000..511308e --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/execution-environment.yml @@ -0,0 +1,22 @@ +--- +# execution-environment.yml - ansible-builder schema version 3 (version: 3 is mandatory; otherwise v1 is assumed). +# Base images must be RPM-based (dnf). Pin by digest for production use. +version: 3 + +images: + base_image: + name: docker.io/redhat/ubi9:latest + +dependencies: + ansible_core: + package_pip: ansible-core==2.21.6 # pin the line you validated + ansible_runner: + package_pip: ansible-runner + galaxy: collections/requirements.yml + python: + - jmespath + # system: bindep.txt # add a bindep file when collections need system packages + +additional_build_steps: + append_final: + - RUN ansible --version diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/group_vars-all.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/group_vars-all.yml new file mode 100644 index 0000000..6ea9c98 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/group_vars-all.yml @@ -0,0 +1,14 @@ +--- +# inventories/production/group_vars/all/main.yml (plaintext, reviewable) +# Pattern: the readable variable points at a vault_-prefixed one that lives in an encrypted file. +# Create the encrypted side with, for example: +# ansible-vault create --vault-id prod@prompt inventories/production/group_vars/all/vault.yml +# and put in it (never in this file): vault_app_db_password: + +app_environment: production +app_log_level: info +app_db_host: db01.example.net +app_db_user: app + +# Resolved from group_vars/all/vault.yml at run time. Tasks using it need no_log: true. +app_db_password: "{{ vault_app_db_password }}" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/inventory.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/inventory.yml new file mode 100644 index 0000000..43a57b5 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/inventory.yml @@ -0,0 +1,26 @@ +--- +# inventories/production/hosts.yml - one inventory directory per environment. +# Check what it resolves to before any run: ansible-inventory -i inventories/production --graph +all: + vars: + ansible_user: deploy + # Secrets are NOT set here: see group_vars-all.yml (vault_ indirection). + children: + web: + hosts: + web[01:03].example.net: # numeric range -> web01, web02, web03 + vars: + example_service_port: 8080 + db: + hosts: + db01.example.net: + ansible_host: 10.0.20.11 # host-level override of the address + db02.example.net: + ansible_host: 10.0.20.12 + vars: + example_service_port: 5432 + # Parent group made of child groups (YAML: children:, INI: [prod:children]) + production: + children: + web: + db: diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/molecule/converge.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/molecule/converge.yml new file mode 100644 index 0000000..b564d97 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/molecule/converge.yml @@ -0,0 +1,9 @@ +--- +# molecule/default/converge.yml - applies the role under test to the lab target. +- name: Converge + hosts: lab + become: true + roles: + - role: example_service + vars: + example_service_port: 8080 diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/molecule/inventory/hosts.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/molecule/inventory/hosts.yml new file mode 100644 index 0000000..35c4188 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/molecule/inventory/hosts.yml @@ -0,0 +1,11 @@ +--- +# molecule/default/inventory/hosts.yml - a disposable lab target (VM or container you can wipe). +# Replace the placeholder values. Do not use production hosts or localhost. +all: + children: + lab: + hosts: + molecule-lab-01: + ansible_host: 192.0.2.10 # documentation range: replace with the lab address + ansible_user: molecule + ansible_ssh_private_key_file: ~/.ssh/molecule_lab diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/molecule/molecule.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/molecule/molecule.yml new file mode 100644 index 0000000..4fe9cac --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/molecule/molecule.yml @@ -0,0 +1,30 @@ +--- +# molecule/default/molecule.yml - ansible-native configuration (docs: molecule ansible-native). +# Targets come from plain Ansible inventory: point it ONLY at disposable lab hosts you own, +# never at production and never at the control node itself (converge is a real run). +ansible: + executor: + backend: ansible-playbook + args: + ansible_playbook: + - --inventory=inventory/ + env: + ANSIBLE_FORCE_COLOR: true + cfg: + defaults: + gathering: smart + ssh_connection: + pipelining: true + playbooks: + converge: converge.yml + verify: verify.yml + +scenario: + # idempotence re-runs converge and fails if any task reports "changed". + test_sequence: + - converge + - idempotence + - verify + +verifier: + name: ansible diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/molecule/verify.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/molecule/verify.yml new file mode 100644 index 0000000..2cd01fd --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/molecule/verify.yml @@ -0,0 +1,22 @@ +--- +# molecule/default/verify.yml - asserts the end state; read-only checks only. +- name: Verify + hosts: lab + gather_facts: false + tasks: + - name: The service listens on its port + ansible.builtin.wait_for: + port: 8080 + timeout: 10 + + - name: The configuration file exists with the expected mode + ansible.builtin.stat: + path: /etc/example/service.conf + register: verify_config + + - name: Assert the configuration file state + ansible.builtin.assert: + that: + - verify_config.stat.exists + - verify_config.stat.mode == "0644" + quiet: true diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/requirements.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/requirements.yml new file mode 100644 index 0000000..8b5a804 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/requirements.yml @@ -0,0 +1,20 @@ +--- +# collections/requirements.yml +# Install: ansible-galaxy collection install -r collections/requirements.yml +# (a file holding both `roles:` and `collections:` also works with `ansible-galaxy install -r`). +# Versions read from the Galaxy API on 2026-11-10; keep a range that stays inside one major. +collections: + - name: ansible.posix + version: ">=2.2.2,<3.0.0" + - name: community.general + version: ">=14.0.1,<15.0.0" + # Uncomment what the project uses (see devops-proxmox / devops-openstack): + # - name: community.proxmox + # version: ">=2.1.0,<3.0.0" + # - name: openstack.cloud + # version: ">=2.6.0,<3.0.0" + +# Roles: exact versions only (ranges are not supported for roles). Pin Git sources to a tag or commit, never a branch. +# roles: +# - name: example.role +# version: "1.9.6" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/defaults/main.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/defaults/main.yml new file mode 100644 index 0000000..dccf6ab --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/defaults/main.yml @@ -0,0 +1,10 @@ +--- +# roles/example_service/defaults/main.yml - the role's public interface (lowest precedence, easy to override). +# Prefix every variable with the role name. +example_service_name: example +example_service_package: example +example_service_version: "1.4.2" +example_service_version_command: "example --version" +example_service_config_path: /etc/example/service.conf +example_service_port: 8080 +example_service_log_level: info diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/handlers/main.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/handlers/main.yml new file mode 100644 index 0000000..8ef1336 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/handlers/main.yml @@ -0,0 +1,7 @@ +--- +# roles/example_service/handlers/main.yml +# Runs once at the end of the play section when a notifying task changed something. +- name: Restart example service + ansible.builtin.service: + name: "{{ example_service_name }}" + state: restarted diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/meta/main.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/meta/main.yml new file mode 100644 index 0000000..cb7fa65 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/meta/main.yml @@ -0,0 +1,15 @@ +--- +# roles/example_service/meta/main.yml +# galaxy_info is needed only to publish a standalone role; dependencies run before this role. +galaxy_info: + author: example + description: Installs and configures the example service + license: GPL-3.0-or-later + min_ansible_version: "2.15" + platforms: + - name: Debian + versions: [all] + - name: EL + versions: [all] + galaxy_tags: [] +dependencies: [] diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/tasks/main.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/tasks/main.yml new file mode 100644 index 0000000..3252d73 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/tasks/main.yml @@ -0,0 +1,36 @@ +--- +# roles/example_service/tasks/main.yml +# Rules: FQCN, named tasks, native modules, state present (not latest), honest changed_when. +- name: Install the service package + ansible.builtin.package: + name: "{{ example_service_package }}" + state: present + +- name: Render the service configuration + ansible.builtin.template: + src: service.conf.j2 + dest: "{{ example_service_config_path }}" + owner: root + group: root + mode: "0644" + backup: true + notify: Restart example service + +- name: Ensure the service is enabled and running + ansible.builtin.service: + name: "{{ example_service_name }}" + state: started + enabled: true + +- name: Read the installed version (read-only probe) + ansible.builtin.command: "{{ example_service_version_command }}" + register: example_service_installed_version + changed_when: false + check_mode: false + +- name: Assert the installed version is the expected one + ansible.builtin.assert: + that: example_service_version in example_service_installed_version.stdout + fail_msg: "expected {{ example_service_version }}, got {{ example_service_installed_version.stdout }}" + quiet: true + when: not ansible_check_mode diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/templates/service.conf.j2 b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/templates/service.conf.j2 new file mode 100644 index 0000000..e464a48 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/role-skeleton/templates/service.conf.j2 @@ -0,0 +1,4 @@ +{# roles/example_service/templates/service.conf.j2 #} +# {{ ansible_managed | default('Managed by Ansible') }} +listen_port = {{ example_service_port }} +log_level = {{ example_service_log_level }} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/site-rolling-update.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/site-rolling-update.yml new file mode 100644 index 0000000..6bf916c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/templates/site-rolling-update.yml @@ -0,0 +1,87 @@ +--- +# Rolling update of the web tier. Always dry-run first: +# ansible-playbook -i inventories/production playbooks/site-rolling-update.yml --check --diff --limit web01 +# Real run (only after the user's go, see safe-run-protocol.md): +# ansible-playbook -i inventories/production playbooks/site-rolling-update.yml --limit web01 -e rollout_confirmed=true +# Canary first (--limit one host), then widen --limit; batches below keep the blast radius small. +- name: Rolling update of the web tier + hosts: web + become: true + gather_facts: true + # 1 host, then 5, then 10 at a time. Failures are scoped to the batch. + serial: [1, 5, 10] + # Abort the whole play when MORE than 30% of a batch fail (1 failed host in the first batch = 100%). + max_fail_percentage: 30 + vars: + example_service_version: "1.4.2" # pinned; never "latest" in production + rollout_confirmed: false # must be set explicitly for a real (non-check) run + # Runs on the target itself, so it probes the local listener. + rollout_health_url: "http://127.0.0.1:{{ example_service_port | default(8080) }}/healthz" + rollout_lb_drain_url: "" # set to your load balancer API; empty = no drain step + + pre_tasks: + - name: Refuse a real run that was not confirmed + ansible.builtin.assert: + that: ansible_check_mode or (rollout_confirmed | bool) + fail_msg: "Real run refused: pass -e rollout_confirmed=true after the go." + quiet: true + + - name: Take the host out of the load balancer + ansible.builtin.uri: + url: "{{ rollout_lb_drain_url }}" + method: POST + status_code: [200, 204] + delegate_to: localhost + become: false + when: + - rollout_lb_drain_url | length > 0 + - not ansible_check_mode + + tasks: + - name: Upgrade with rollback path + block: + - name: Apply the service role at the pinned version + ansible.builtin.include_role: + name: example_service + + - name: Run notified handlers now, before verifying + ansible.builtin.meta: flush_handlers + + - name: Wait for the service port + ansible.builtin.wait_for: + port: "{{ example_service_port | default(8080) }}" + timeout: 60 + when: not ansible_check_mode + + - name: Check the health endpoint + ansible.builtin.uri: + url: "{{ rollout_health_url }}" + status_code: 200 + register: rollout_health + until: rollout_health.status == 200 + retries: 5 + delay: 5 + when: not ansible_check_mode + + rescue: + - name: Report the failure for this host + ansible.builtin.debug: + msg: "Update failed on {{ inventory_hostname }}; restore the previous version/config before re-adding it." + + # Without this the rescue would clear the failure and the batch gate (max_fail_percentage) would not trip. + - name: Keep the host failed so the batch gate sees it + ansible.builtin.fail: + msg: "Update failed on {{ inventory_hostname }}" + + post_tasks: + # Hosts that failed are removed from the play, so they are NOT put back into the load balancer. + - name: Put the host back into the load balancer + ansible.builtin.uri: + url: "{{ rollout_lb_drain_url }}" + method: DELETE + status_code: [200, 204] + delegate_to: localhost + become: false + when: + - rollout_lb_drain_url | length > 0 + - not ansible_check_mode diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/testing-lint-molecule.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/testing-lint-molecule.md new file mode 100644 index 0000000..8605000 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/testing-lint-molecule.md @@ -0,0 +1,91 @@ +--- +name: testing-lint-molecule +description: Testing Ansible content - ansible-lint config and profiles, syntax check, Molecule scenarios (ansible-native and legacy), the idempotence step, a CI test ladder, and ansible-test for collections +when-to-use: Adding CI to a role or playbook repo, choosing lint strictness, writing a Molecule scenario, proving idempotency +keywords: ansible-lint, profile, production, skip_list, warn_list, enable_list, molecule, scenario, test_sequence, idempotence, converge, verify, ansible-native, ansible-test +priority: medium +related: idempotency-handlers-errors.md, execution-environments.md, versions-support.md +--- + +# Testing: ansible-lint, Molecule, ansible-test + +**Load when:** setting up or reviewing tests for roles, playbooks or collections. Templates: [ansible-lint.yml](templates/ansible-lint.yml), [molecule/](templates/molecule/molecule.yml). Versions: ansible-lint 26.11.0, molecule 26.11.0 ([versions-support.md](versions-support.md)). + +## Test ladder (cheapest first) + +1. `yamllint` style is part of ansible-lint (`yaml` rule; needs the `yamllint` library). +2. `ansible-playbook --syntax-check` (ansible-lint's `syntax-check` rule runs it). +3. `ansible-lint` with a profile. +4. Molecule on a disposable target: converge, **idempotence**, verify. +5. `--check --diff` against staging, then canary, then production ([safe-run-protocol.md](safe-run-protocol.md)). + +## ansible-lint + +- Run from the project root (required for file discovery and dependency resolution). Config file: `.ansible-lint`, `.ansible-lint.yml`, `.ansible-lint.yaml`, `.config/ansible-lint.yml` or `.config/ansible-lint.yaml`; or `-c `. CLI scalars override the file; list values (`skip_list`, `warn_list`, `exclude_paths`) are **appended**, not replaced. +- Profiles, each extending the previous: `min` (can Ansible load it) -> `basic` -> `moderate` -> `safety` -> `shared` -> `production`. Rules per profile include: + - `basic`: `command-instead-of-module`, `command-instead-of-shell`, `deprecated-module`, `inline-env-var`, `key-order`, `literal-compare`, `jinja`, `no-jinja-when`, `no-tabs`, `partial-become`, `role-name`, `schema`, `name`, `var-naming`, `yaml` + - `safety`: `avoid-implicit`, `latest`, `package-latest`, `risky-file-permissions`, `risky-octal`, `risky-shell-pipe` + - `shared`: `galaxy`, `ignore-errors`, `meta-incorrect`, `meta-runtime`, `no-changed-when`, `no-handler`, `no-relative-paths` + - `production`: `fqcn` (plus not-yet-implemented placeholders such as `sanity`, `single-entry-point`) +- Aim: **`production`** for new code that you own; start at `basic`/`moderate` on a legacy repo and ratchet up. Prefer `.ansible-lint-ignore` entries (file + rule, remain visible) over `skip_list` (hides violations). `warn_list` keeps non-fatal findings visible. Opt-in rules (`no-log-password`, `no-same-owner`, `name[prefix]`, `galaxy-version-incorrect`, `empty-string-compare`, `args`) are switched on in `enable_list`. +- `offline: true` disables installing `requirements.yml` and schema refresh (air-gapped CI). `mock_modules` / `mock_roles` let syntax-check pass for content not installed. `ansible-lint --fix` rewrites YAML and applies rule transforms (`write_list` limits them): review the diff, never run it on production repos unreviewed. +- Output can be written as SARIF (`sarif_file`) for code-scanning UIs. + +## Molecule + +Molecule tests roles, playbooks and collections; it supports only the **latest two major Ansible versions (N / N-1)**. Called as `molecule ...` or `python3 -m molecule ...`. + +### Scenario sequences + +A scenario is a directory (`molecule/default/`) holding everything for one way of testing. Default `test_sequence`: `dependency, cleanup, destroy, syntax, create, prepare, converge, idempotence, side_effect, verify, cleanup, destroy` (also `create_sequence`, `converge_sequence`, `check_sequence`, `destroy_sequence`). Override any sequence under `scenario:`. `idempotence` re-runs converge and fails if anything reports `changed` - this is the automated form of the idempotency contract ([idempotency-handlers-errors.md](idempotency-handlers-errors.md)). `side_effect` can run extra playbooks (reboot, config change) between converge and verify; `verify` can take test names. + +### Configuration styles + +- **Ansible-native** (current): top-level `ansible:` (executor backend `ansible-playbook` or `ansible-navigator`, `env`, `cfg`, `playbooks:` mapping create/destroy/converge/prepare/verify/cleanup/side_effect), `scenario:`, `shared_state`, `verifier: {name: ansible}`; targets are plain Ansible inventory (static, dynamic plugin, constructed). Driver/platforms/provisioner constructs are not needed. +- **Legacy**: `driver`, `platforms`, `provisioner`, `verifier` sections; the default driver is `delegated` (you write create/destroy playbooks). Marked "pre ansible-native" in the docs. +- `dependency: {name: galaxy}` installs roles/collections from `requirements.yml`; `prerun` (default on) installs project-level requirements into `.cache`. Disable with `prerun: false`. +- Scenario template here: [molecule.yml](templates/molecule/molecule.yml), [converge.yml](templates/molecule/converge.yml), [verify.yml](templates/molecule/verify.yml). The inventory points at a lab target you own - never at production. + +### Running + +```bash +molecule test # full default sequence (destroys at the end) +molecule converge && molecule idempotence && molecule verify # iterate while developing +``` + +Molecule `create`/`destroy` act on the test targets only; a scenario that points `inventory` at real hosts makes `converge` a real run: apply [safe-run-protocol.md](safe-run-protocol.md). + +## ansible-test (collections) + +`ansible-test` ships with ansible-core and runs sanity, unit and integration tests for collection authors (it must be run from inside `ansible_collections///`). For playbook/role repos use lint + Molecule instead. + +## CI sketch + +```yaml +# any CI: run inside an EE or a pinned venv +- ansible-galaxy collection install -r collections/requirements.yml +- ansible-lint +- ansible-playbook -i inventories/staging playbooks/site.yml --syntax-check +- molecule test # per changed role +- ansible-playbook -i inventories/staging playbooks/site.yml --check --diff # optional, staging only +``` + +Pipeline authoring itself belongs to `devops-cicd`. + +## Sources (read 2026-11-10) + +- https://docs.ansible.com/projects/lint/configuring/ +- https://docs.ansible.com/projects/lint/profiles/ +- https://docs.ansible.com/projects/molecule/ +- https://docs.ansible.com/projects/molecule/configuration/ +- https://docs.ansible.com/projects/molecule/ansible-native/ +- https://pypi.org/pypi/ansible-lint/json , https://pypi.org/pypi/molecule/json +- Context7 `/ansible/ansible-lint`, `/ansible/molecule` (resolved; query not run beyond resolution) + +## Not verified + +- `molecule` subcommand names and flags (`test`, `converge`, `idempotence`, `verify`): derived from the sequence names; command help not read. +- Whether `idempotence` is available as a sequence step in ansible-native mode (it is listed in the shared `scenario:` defaults; the ansible-native `playbooks:` list has no idempotence key). +- `ansible-test` invocation details and the per-collection test layout (not read today). +- Rule behaviour of individual lint rules beyond their profile membership. +- Nothing here was executed: ansible, ansible-lint and molecule are not installed on this machine. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/vault-secrets.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/vault-secrets.md new file mode 100644 index 0000000..b2782d9 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/vault-secrets.md @@ -0,0 +1,103 @@ +--- +name: vault-secrets +description: Ansible Vault - encrypting variables and files, vault ids, supplying passwords, rekeying, the vault_ indirection pattern, no_log and diff false, become passwords, and the boundary with external secret stores +when-to-use: Any secret in an Ansible project - encrypting, rotating, passing the vault password to a run, or deciding vault versus a secret manager +keywords: ansible-vault, encrypt_string, vault-id, vault-password-file, ask-vault-pass, !vault, rekey, no_log, diff false, become password +priority: high +related: safe-run-protocol.md, inventory-variables.md +--- + +# Vault and Secret Handling + +**Load when:** a task, variable or file contains a credential, key or token. Pointer: external stores (HashiCorp Vault, SOPS, cloud secret managers) are the scope of `devops-secrets`; this file covers what Ansible itself does. Template: [group_vars-all.yml](templates/group_vars-all.yml). + +**Never print, log or paste a secret value in a report, a diff or a commit.** Report names and keys only. + +## Two ways to encrypt + +| | Encrypted variables (`encrypt_string`) | Encrypted files (`encrypt`, `create`, `edit`) | +|---|----------------------------------------|-----------------------------------------------| +| Result | a `!vault` block scalar inside a normal YAML file | the whole file is ciphertext | +| Visible | variable names stay readable in the repo and diffs | names and structure hidden | +| Rotation | per value | `ansible-vault rekey` on many files at once | + +Ciphertext starts with `$ANSIBLE_VAULT;1.2;AES256;` when a vault id was used (`1.1` without one); the label is optional metadata saying which password decrypts it. + +```bash +# value: read it from stdin so it never enters shell history or argv +ansible-vault encrypt_string --vault-id prod@prompt --stdin-name 'db_password' +# file level +ansible-vault create --vault-id prod@prompt inventories/production/group_vars/all/vault.yml +ansible-vault edit --vault-id prod@prompt inventories/production/group_vars/all/vault.yml +ansible-vault encrypt --vault-id prod@prompt secrets.yml +``` + +(`encrypt_string 'plain' --name x` also works, but puts the plaintext on the command line; the docs show it for tests only.) + +## Supplying the password at run time + +```bash +ansible-playbook site.yml --ask-vault-pass # single password, prompt +ansible-playbook site.yml --vault-password-file /path/file # file or executable client script +ansible-playbook site.yml --vault-id prod@prompt # labelled +ansible-playbook site.yml --vault-id dev@dev-password --vault-id prod@prompt # several ids +``` + +- `--vault-password-file` may point to an **executable script** that prints the password from a keyring or secret manager (`--vault-id prod@my-vault-password-client.py`); this is how a store plugs in without a plaintext file. +- `DEFAULT_VAULT_IDENTITY_LIST` in config sets the ids you use most often. +- Keep the password file outside the repository, mode `0600`, listed in `.gitignore`; never commit it. In CI inject it from the CI secret store into a tmpfs file or a client script. +- Rekey: `ansible-vault rekey --vault-id old@src --new-vault-id new@prompt files...` (see the ACTION block below). + +## Indirection pattern ("keep vaulted variables visible") + +```yaml +# group_vars/all/main.yml (plaintext, reviewable) +app_db_password: "{{ vault_app_db_password }}" +# group_vars/all/vault.yml (encrypted file or !vault values) +# vault_app_db_password: !vault | ... +``` + +Reviewers and `grep` see which variables exist and where they are used; only the value is secret. + +## Secrets inside tasks + +- `no_log: true` on any task whose arguments or result hold a credential. ansible-lint's opt-in rule `no-log-password` (enable it in `enable_list`) catches `user` password tasks without it. +- `diff: false` on tasks that template or edit secret-bearing files, so `--diff` does not print them. +- `ansible_become_password`, `ansible_ssh_pass` and similar connection variables belong in vault or are prompted (`--ask-become-pass`), never in plain `group_vars`. +- Secrets in `-e` land in shell history and process lists; use `-e @vars.yml` with a vaulted file. +- Do not `debug:` a secret variable; the docs' own example uses `debug` only on throwaway data. + +## When to leave Vault + +Ansible Vault is a single symmetric key per id: no per-secret ACL, no audit, no dynamic credentials. Move to an external store (lookup plugins for the store, or SOPS) when more than a small team needs access, rotation must be automated or audit is required: hand off to `devops-secrets`. + +## Destructive / sensitive operations + +``` +ACTION ansible-vault rekey --vault-id prod@prompt --new-vault-id prod2@prompt inventories/production/group_vars/all/vault.yml +EFFECT rewrites the file with a new key; the old password no longer decrypts it; every pipeline and teammate using the old key breaks until updated +PROOF list of files that carry the old vault id (grep of the header only); new password stored in the agreed store; rollback = git revert + old password +WAIT-FOR-GO +``` + +``` +ACTION ansible-vault decrypt (or encrypt_string output pasted into a ticket) +EFFECT writes plaintext secrets to disk / output; they can end up in git history, logs or the transcript +PROOF none acceptable for routine work: use `ansible-vault view` in a terminal the user controls instead +WAIT-FOR-GO +``` + +## Sources (read 2026-11-10) + +- https://docs.ansible.com/projects/ansible/latest/vault_guide/vault_encrypting_content.html +- https://docs.ansible.com/projects/ansible/latest/vault_guide/vault_using_encrypted_content.html +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_privilege_escalation.html (`ansible_become_password`) +- https://docs.ansible.com/projects/lint/configuring/ (`no-log-password` opt-in rule) +- https://docs.ansible.com/projects/ansible/latest/playbook_guide/playbooks_checkmode.html (`diff: false`) + +## Not verified + +- The `no_log` keyword page itself and its interaction with `-vvv` output (not read today). +- `ansible-vault view` flags and `--ask-become-pass` (standard CLI, not on a page read today). +- Per-version behaviour of vault id `1.1` versus `1.2` headers beyond what the format section states. +- Cryptographic parameters (docs name the cipher AES256 in the header; key derivation details not read). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/versions-support.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/versions-support.md new file mode 100644 index 0000000..e122502 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-ansible/references/versions-support.md @@ -0,0 +1,105 @@ +--- +name: versions-support +description: ansible-core versus the ansible community package, current versions, support windows, control node and managed node Python requirements, tooling versions and key collection versions +when-to-use: Before installing or upgrading Ansible, choosing a Python for the control node, or deciding whether a managed host is too old +keywords: ansible-core, ansible package, 2.22, 2.21, 14.6.0, python 3.13, support matrix, EOL, pip, pipx, community.general +priority: high +related: execution-environments.md, testing-lint-molecule.md, playbooks-roles-collections.md +--- + +# Versions and Support + +**Load when:** deciding which Ansible to install or whether a host can be managed. Facts read 2026-11-10. + +## Two packages + +| | `ansible-core` | `ansible` (community package) | +|---|----------------|-------------------------------| +| Content | language, runtime, `ansible.builtin` plugins | core + curated collections (85+ at last count on the docs page) | +| Versioning | `2.Y.Z`, major in May and November, patch every 4 weeks | semantic (`14.6.0`), one maintained major at a time | +| Maintained | latest + two older lines | only the latest major | + +`ansible --version` prints the ansible-core version; `ansible-community --version` prints the package version. + +## Current releases + +| Item | Version | Constraint read today | +|------|---------|-----------------------| +| ansible-core | **2.22.0** | PyPI: `requires_python >=3.13`; classifiers Python 3.13, 3.14, 3.15 | +| ansible (package) | **14.6.0** | PyPI: `ansible-core~=2.21.6`, `requires_python >=3.12` | +| ansible 15.0.0 | in development | docs table: depends on ansible-core 2.22 | +| ansible 13.x | EOL June 2026 | docs table | + +ansible-core support windows (docs matrix, which at read time lists 2.21 as newest row - 2.22 was not yet in the table): + +| Core | GA | Critical fixes until | Security fixes until | EOL | Control node Python | Managed node Python | +|------|----|---------------------|----------------------|-----|---------------------|---------------------| +| 2.21 | May 2026 | Nov 2026 | May 2027 | Nov 2027 | 3.12 - 3.14 | 3.9 - 3.14, PowerShell 5.1 - 7 | +| 2.20 | 03 Nov 2025 | 18 May 2026 | 02 Nov 2026 | May 2027 | 3.12 - 3.14 | 3.9 - 3.14 | +| 2.19 | 21 Jul 2025 | 03 Nov 2025 | 18 May 2026 | Nov 2026 | 3.11 - 3.13 | 3.8 - 3.13 | + +Rules from the same page: +- Control node: each release supports the three most recently released Python versions (consistent with 2.22 on 3.13 - 3.15). +- Managed node: the 6 most recent Python versions, and 7 for every 6th release (2.16, 2.22) - so 2.22 reaches back to 3.9 with 3.15 as newest (derived from the rule; the 2.22 row itself was not in the table). +- Python 2.7 targets: only ansible-core 2.16 and earlier. +- Windows managed nodes: Windows PowerShell 5.1; PowerShell 7 LTS from 2.21. +- Network modules need no Python on the device. The managed node needs Python plus an SSH user with a POSIX shell. + +Practical rule: pin the control node to a line that is still maintained and whose Python you can install (2.22 needs 3.13; 2.21 and 2.20 run on 3.12). Do not run an EOL line: the docs warn about unfixed CVEs. + +## Install + +```bash +python3 -m pip install --user ansible # full package (inside a venv in practice) +python3 -m pip install --user ansible-core # minimal; then ansible-galaxy collection install -r ... +pipx install --include-deps ansible # when the OS blocks pip +``` + +Windows is not a supported control node (use WSL). For containers use an execution environment ([execution-environments.md](execution-environments.md)). + +## Tooling versions (PyPI, 2026-11-10) + +| Tool | Version | Python | Notes | +|------|---------|--------|-------| +| ansible-lint | 26.11.0 | >=3.10 | depends `ansible-core !=2.17.*, >=2.16.19` | +| molecule | 26.11.0 | >=3.11 | docs: supports only the latest two major Ansible versions (N / N-1) | +| ansible-navigator | 26.11.0 | >=3.10 | | +| ansible-builder | 3.2.0 | >=3.11 | EE schema version 3 | +| ansible-dev-tools | 26.11.0 | >=3.11 | bundle of the tools above | + +## Collections often needed (Galaxy API, 2026-11-10) + +| Collection | Latest | Used for | +|------------|--------|----------| +| community.general | 14.0.1 | many utility modules and plugins | +| ansible.posix | 2.2.2 | `authorized_key`, `firewalld`, `mount`, `sysctl`, `synchronize` family | +| community.proxmox | 2.1.0 | Proxmox VE modules and inventory (see devops-proxmox) | +| openstack.cloud | 2.6.0 | OpenStack modules (see devops-openstack) | + +Check the collection's own `meta/runtime.yml` / docs for the ansible-core range it supports before pinning it next to a new core. + +## Upgrading + +1. Read the porting guide of the target package/core (docs list porting guides up to Ansible 14). +2. Upgrade in a venv or EE on a copy of the project; run `ansible-lint`, `--syntax-check`, then `--check --diff` against staging. +3. Roll to production like any change ([safe-run-protocol.md](safe-run-protocol.md)). +4. ansible-core deprecations normally run across 4 feature releases; removals can appear without warning for undocumented behaviour. + +Removed in 2.21: `interpreter_python` values `auto_legacy` and `auto_legacy_silent`; use `auto` or `auto_silent`. + +## Sources (read 2026-11-10) + +- https://docs.ansible.com/projects/ansible/latest/reference_appendices/release_and_maintenance.html +- https://docs.ansible.com/projects/ansible/latest/installation_guide/intro_installation.html +- https://docs.ansible.com/projects/ansible/latest/reference_appendices/interpreter_discovery.html +- https://docs.ansible.com/projects/ansible/latest/porting_guides/porting_guides.html +- https://pypi.org/pypi/ansible-core/json , https://pypi.org/pypi/ansible/json +- https://pypi.org/pypi/ansible-lint/json , molecule , ansible-navigator , ansible-builder , ansible-dev-tools (same URL pattern) +- https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/index///?format=json + +## Not verified + +- The 2.22 row of the support matrix (docs page lagged); the 3.9 - 3.15 managed range is derived from the stated rule and the research notes, not from a table row. +- Release date 2026-11-02 of 2.22.0 (from `docs/research-devops-plugin.md`; PyPI JSON read today confirms the version, not the date). +- Which collection versions the 14.6.0 package bundles (see `ansible-build-data` changelog; not read). +- Minimum ansible-core supported by each collection listed above. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/SKILL.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/SKILL.md new file mode 100644 index 0000000..151ae5b --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/SKILL.md @@ -0,0 +1,170 @@ +--- +name: devops-cicd +description: Use when writing, reviewing or debugging CI/CD pipelines for self-hosted infrastructure - GitHub Actions (workflows, reusable workflows, composite actions, matrix, concurrency, caching, environments with required reviewers, OIDC instead of stored secrets, pinning actions by SHA or lockfile, permissions least privilege, self-hosted runners), GitLab CI (rules, needs, includes and components, environments, resource_group, protected variables, runners and executors), pointers for Jenkins / Forgejo-Gitea Actions / Woodpecker, pipeline design (build once promote many, quality gates, speed), image builds in CI (BuildKit cache, multi-arch, provenance/SBOM/signing), deploy strategies and rollback, release automation and pipeline troubleshooting. Not for GitOps controllers, Helm or Kustomize (devops-gitops-packaging), Dockerfile and registry craft (devops-docker), secret stores (devops-secrets), language-specific CI gates (framework experts), repo commit/release flow (commit agent) or a supply-chain audit (fuse-security:security-expert). +versions: + github-actions-runner: "2.338.2 (actions/runner release 2026-10-26)" + github-hosted-runner-labels: "ubuntu-latest = ubuntu-24.04; ubuntu-26.04 and ubuntu-24.04-arm available" + gitlab: "19.5 released (19.6 upcoming); GitLab Runner 19.5.3 (2026-11-06)" + actions-pinned-in-templates: "checkout v7.0.1, upload-artifact v7.0.2, setup-node v7.1.0, docker/build-push-action v7.4.0, actions/attest v4.2.2" +user-invocable: true +references: + - references/github-actions-workflows.md + - references/github-actions-security.md + - references/gitlab-ci.md + - references/other-ci-systems.md + - references/pipeline-design.md + - references/image-build-ci.md + - references/deploy-strategies-release.md + - references/pipeline-secrets.md + - references/destructive-pipeline-actions.md + - references/troubleshooting.md + - references/templates/github-ci.yml + - references/templates/github-reusable-workflow.yml + - references/templates/github-composite-action.yml + - references/templates/github-oidc-deploy.yml + - references/templates/github-image-build-push.yml + - references/templates/github-release.yml + - references/templates/gitlab-ci.yml + - references/templates/gitlab-component.yml +related-skills: + - devops-gitops-packaging + - devops-docker + - devops-secrets + - devops-kubernetes + - devops-iac + - devops-ansible + - devops-observability + - devops-sre-ops +--- + + +Covers designing, writing and troubleshooting CI/CD pipelines: GitHub Actions and +GitLab CI in depth, pointers for Jenkins, Forgejo/Gitea Actions and Woodpecker, +build-once-promote-many design, container image builds in CI, deployment +strategies, release automation, rollback and pipeline secrets. Does not cover +GitOps controllers and packaging (devops-gitops-packaging), Dockerfile and +registry practice (devops-docker), secret stores (devops-secrets), language CI +gates (framework experts) or supply-chain audits (fuse-security:security-expert). + + +# DevOps CI/CD + +## Agent Workflow (MANDATORY) + +Before ANY pipeline change, spawn in parallel, one `Task` call each: + +1. **fuse-ai-pilot:explore-codebase** - find existing workflows (`.github/workflows`, `.gitlab-ci.yml`, `Jenkinsfile`, `.woodpecker`), scripts they call, runners/labels in use, environments, how deploys reach the target +2. **fuse-ai-pilot:research-expert** - verify the keyword, action version and runner image for the platform version in use at docs.github.com / docs.gitlab.com +3. **mcp__context7__query-docs** - confirm action inputs and CLI flags + +Then read the platform state read-only (`gh run list`, `gh workflow list`, `gh api .../environments`, CI Lint, runner status). After writing files, run **fuse-ai-pilot:sniper**; run `actionlint` on GitHub workflows when installed. + +## Versions (checked 2026-11-10) + +| Item | Version | Source | +|------|---------|--------| +| actions/runner | 2.338.2 (2026-10-26) | `gh api repos/actions/runner/releases/latest` | +| GitHub-hosted labels | `ubuntu-latest` = 24.04; `ubuntu-24.04`, `ubuntu-22.04`, `ubuntu-26.04`, `ubuntu-slim`, `*-arm` | https://docs.github.com/en/actions/reference/runners/github-hosted-runners | +| actions/checkout | v7.0.1 (2026-07-20) | `gh api` | +| upload / download-artifact | v7.0.2 / v8.0.2 (2026-10-07) | `gh api` | +| actions/cache | v6.1.0 (2026-06-26) | `gh api` | +| actions/setup-node | v7.1.0 (2026-10-08) | `gh api` | +| docker/{setup-buildx, build-push, login, metadata, setup-qemu} | v4.4.1 / v7.4.0 / v4.8.0 / v6.2.0 / v4.4.0 | `gh api` | +| actions/attest | v4.2.2 (2026-08-04); `attest-build-provenance` v4 is a wrapper | `gh api`, README | +| sigstore/cosign-installer | v4.1.3 (2026-11-04) | `gh api` | +| actionlint | v1.7.12 (2026-03-30), not installed locally | `gh api` | +| GitLab | 19.5 (19.6 upcoming); Runner 19.5.3 | docs release notes, GitLab.com releases API | +| Jenkins / Forgejo / Woodpecker | 2.590 (LTS 2.580.2) / 17.0.1 (LTS 15.0.11) / 3.20.1 | [other-ci-systems.md](references/other-ci-systems.md) | + +Docs examples often lag the latest action majors: use the SHA/tag from the release, and re-check with `gh api repos///releases/latest` and `gh api repos///commits/ --jq .sha` before pinning. + +## Overview + +| Area | Decision in one line | +|------|----------------------| +| **Artifact** | Build once, promote the same immutable artifact (image digest) through every environment | +| **Permissions** | `permissions: {}` at the top, widen per job; `GITHUB_TOKEN` read-only by default | +| **Actions** | Pin third-party actions by full SHA or lockfile; Dependabot raises bumps | +| **Credentials** | OIDC federation, bound to repo + environment claims; no long-lived cloud/SSH keys in CI | +| **Production** | Environment with required reviewers (GitHub) / protected environment + manual job (GitLab), serialised, never auto-cancelled | +| **Runners** | Ephemeral or private-only; never self-hosted on public repos; separate runner group for deploys | +| **Speed** | Parallel jobs, DAG (`needs`), lockfile-keyed cache, external BuildKit cache, cancel redundant PR runs | +| **Deploy model** | CI hands off to GitOps when a controller owns the cluster; otherwise push with a short-lived credential | + +## Critical Rules + +1. **Read before writing** - inspect existing workflows, runners, environments and what the deploy step actually touches +2. **Pin and constrain** - SHA/lockfile pins, `permissions`, `timeout-minutes` on every job +3. **No untrusted `${{ }}` in `run:`** - pass through `env:`; never check out PR code under `pull_request_target`/`workflow_run` +4. **Production and destructive pipeline actions are proposed, not run** - deploy, forced re-run, rollback, deleting environments/releases/tags/artifacts, unregistering runners: ACTION / EFFECT / PROOF / WAIT-FOR-GO in [destructive-pipeline-actions.md](references/destructive-pipeline-actions.md) +5. **Never print a secret value** - names only; stores and rotation are `devops-secrets` +6. **Never invent a keyword, input or version** - cite the docs page or release; unknown = "not verified" +7. **Troubleshoot in order** - run, class, log, reproduce, one change, prove ([troubleshooting.md](references/troubleshooting.md)) + +## Architecture + +``` +repo/ +├── .github/ +│ ├── workflows/ci.yml # PR + main: lint, test matrix, build, upload artifact +│ ├── workflows/image.yml # build/push by digest, SBOM, provenance, sign +│ ├── workflows/deploy.yml # workflow_dispatch, environment gate, OIDC +│ ├── workflows/release.yml # tag -> draft release +│ └── actions/setup-project/ # composite action +└── .gitlab-ci.yml # or: stages, rules, needs, environments, components +``` + +## Reference Guide + +### Concepts + +| Topic | Reference | When to Consult | +|-------|-----------|-----------------| +| **GitHub workflows** | [github-actions-workflows.md](references/github-actions-workflows.md) | Triggers, matrix, concurrency, reusable workflows, cache, environments | +| **GitHub security** | [github-actions-security.md](references/github-actions-security.md) | permissions, pinning, OIDC, injection, self-hosted runners | +| **GitLab CI** | [gitlab-ci.md](references/gitlab-ci.md) | rules, needs, components/inputs, environments, runners | +| **Other CI** | [other-ci-systems.md](references/other-ci-systems.md) | Jenkins, Forgejo/Gitea Actions, Woodpecker | +| **Pipeline design** | [pipeline-design.md](references/pipeline-design.md) | Stages, gates, branching, speed, flaky tests | +| **Image build** | [image-build-ci.md](references/image-build-ci.md) | BuildKit cache, multi-arch, provenance, SBOM, signing | +| **Deploy and release** | [deploy-strategies-release.md](references/deploy-strategies-release.md) | Rolling/blue-green/canary, versioning, rollback, migrations | +| **Pipeline secrets** | [pipeline-secrets.md](references/pipeline-secrets.md) | Scoping, masking, leaks | +| **Destructive actions** | [destructive-pipeline-actions.md](references/destructive-pipeline-actions.md) | Any prod deploy, re-run, rollback, delete, unregister | +| **Troubleshooting** | [troubleshooting.md](references/troubleshooting.md) | Red, queued, skipped or slow pipelines | + +### Templates + +| Template | When to Use | +|----------|-------------| +| [github-ci.yml](references/templates/github-ci.yml) | PR/main CI: lint, matrix tests, artifact | +| [github-reusable-workflow.yml](references/templates/github-reusable-workflow.yml) | Shared `workflow_call` job with typed inputs and secrets | +| [github-composite-action.yml](references/templates/github-composite-action.yml) | Shared steps as `action.yml` | +| [github-oidc-deploy.yml](references/templates/github-oidc-deploy.yml) | Gated production deploy with OIDC to OpenBao/Vault | +| [github-image-build-push.yml](references/templates/github-image-build-push.yml) | Multi-arch build, GHCR push, attestation, cosign | +| [github-release.yml](references/templates/github-release.yml) | Tag push to draft GitHub Release | +| [gitlab-ci.yml](references/templates/gitlab-ci.yml) | rules + needs + environments + protected prod job | +| [gitlab-component.yml](references/templates/gitlab-component.yml) | Versioned component with `spec:inputs` | + +## Quick Reference + +```bash +gh workflow list; gh run list --limit 10 # read-only state +gh run view --log-failed # why it failed +gh api repos///environments --jq '.environments[].name' +gh api repos/actions/checkout/commits/v7.0.1 --jq .sha # SHA to pin +actionlint .github/workflows/*.yml # when installed +gh workflow run deploy.yml -f image-digest=sha256:... # PRODUCTION: only after the user's go +``` + +## Best Practices + +### DO +- Give every workflow `permissions`, `concurrency` and a `timeout-minutes` per job +- Hand deploys to GitOps (`devops-gitops-packaging`) where a controller exists +- Keep CI steps as scripts/make targets that run locally +- Record the previous artifact digest before each production deploy + +### DON'T +- Rebuild per environment, deploy `latest`, or store cloud/SSH keys as long-lived CI secrets +- Run self-hosted persistent runners on public repositories +- Use `pull_request_target` with a PR checkout +- Set `cancel-in-progress: true` on deploy jobs diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/deploy-strategies-release.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/deploy-strategies-release.md new file mode 100644 index 0000000..18bf574 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/deploy-strategies-release.md @@ -0,0 +1,63 @@ +--- +name: deploy-strategies-release +description: Deployment strategies driven from CI (rolling, blue-green, canary, progressive delivery pointer), GitOps handoff, release automation and versioning, rollback design and database migration compatibility +when-to-use: Designing the deploy/promote stage of a pipeline, choosing how to ship a risky change, automating releases and version tags, or planning rollback +keywords: rolling update, blue-green, canary, progressive delivery, feature flag, semver, release notes, tag, rollback, expand contract, migration, GitOps handoff +priority: high +related: destructive-pipeline-actions.md, pipeline-design.md, image-build-ci.md, github-actions-workflows.md, gitlab-ci.md +--- + +# Deployment Strategies and Releases + +**Load when:** the pipeline's last stages are being designed or a release process is being automated. Every command here that changes production is carded in [destructive-pipeline-actions.md](destructive-pipeline-actions.md). + +## Who performs the deploy + +| Model | How | When | +|-------|-----|------| +| **Push from CI** | job runs the platform command (`kubectl`, `helm`, `docker compose`, SSH, Ansible) with a short-lived credential (OIDC) | VMs, Compose hosts, small clusters without a GitOps controller | +| **GitOps handoff** | CI builds the artifact, then commits the new digest/version to the config repo (or opens a PR); the in-cluster controller applies it | Kubernetes with Argo CD or Flux. Mechanics: `devops-gitops-packaging`; do not also `kubectl apply` from CI against a GitOps-managed namespace | + +CI should hold no standing production credentials. Prefer OIDC federation ([github-actions-security.md](github-actions-security.md), [gitlab-ci.md](gitlab-ci.md)) or the pull model. + +## Strategies + +| Strategy | Mechanism | Rollback | Costs | +|----------|-----------|----------|-------| +| **Rolling** | replace instances gradually (Kubernetes Deployment `RollingUpdate`, Compose/Ansible batches) | roll back to previous revision/artifact | old and new versions run together: API and schema must be compatible | +| **Blue-green** | two full environments, switch traffic (router, Gateway weight, DNS) after checks | switch back (fast) | double capacity; data/migration coordination | +| **Canary** | send a small traffic share to the new version, widen on healthy metrics | send weight back to 0 | needs traffic splitting and automated metric checks | +| **Recreate** | stop old, start new | redeploy old | downtime; only for singletons | +| **Feature flag** | ship dark, enable per cohort | disable the flag | flag debt; flag service as dependency | + +Kubernetes specifics (Deployment `strategy`, probes, PDB) live in `devops-kubernetes`; weighted routing with Gateway API HTTPRoute `backendRefs` weights is in its `services-gateway-api.md`. Progressive delivery controllers (Argo Rollouts, Flagger) automate canary/blue-green with metric analysis: pointer only, see `devops-gitops-packaging` and `devops-observability`; not evaluated here. + +Gate every strategy with: readiness/health endpoint, a smoke test, and a metric check (error rate, latency) over a defined window; a failed gate triggers rollback automatically for canary and by decision for the rest. + +## Release automation and versioning + +- **Semantic Versioning** (`MAJOR.MINOR.PATCH`) for anything with consumers; tags `vX.Y.Z` on the default branch commit that CI already built. The tag is created by a human or by the project's commit/release agent after review; the tag push triggers the release workflow ([templates/github-release.yml](templates/github-release.yml): `gh release create --verify-tag --generate-notes --draft`, human publishes). +- Version is derived from the tag at build time and stamped into the artifact (image label `org.opencontainers.image.version` from `docker/metadata-action`), never edited into code by the pipeline. +- Changelog: conventional-commit history or GitHub generated release notes (`--generate-notes`, `--notes-start-tag`). GitLab: the `release:` job keyword (`tag_name`, `name`, `description`; runs after `script`, image `registry.gitlab.com/gitlab-org/cli`) creates a release on tag push. +- Release candidates: pre-release tags (`v1.4.0-rc.1`) deploy to staging/pre-production only. +- Draft first, publish after smoke test; publishing is the gated step. + +## Rollback design + +1. Write down the previous artifact reference (digest) before every production deploy; pipelines should print it and store it with the deployment record. +2. Rollback = deploy the previous artifact through the **same** pipeline and gates; never hand-edit production. +3. **Database changes are the rollback killer.** Use expand/contract: release N adds nullable columns/tables (backward compatible), release N+1 starts using them, release N+2 drops the old ones. Then code can roll back without a down-migration. Run migrations as a separate, reviewed step with a backup first (`devops-databases`, `devops-storage-backup`). +4. GitLab: rollback by re-running an earlier deploy job from the Environments page; "Prevent outdated deployment jobs" otherwise blocks old deploy jobs, with retries for rollback deployments allowed by default. +5. Time-box: if a deploy does not pass its gate within the window, roll back first, diagnose second. + +## Sources (read 2026-11-10) + +- https://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/control-deployments +- https://docs.gitlab.com/ci/environments/deployment_safety/ , https://docs.gitlab.com/ci/yaml/ (`release`, `environment`) +- `gh release create --help` (gh 2.103.3); README of docker/metadata-action + +## Not verified + +- Strategy mechanics (rolling, blue-green, canary, expand/contract, SemVer) are general practice, not read from vendor pages today. +- Argo Rollouts / Flagger capabilities and versions, release-automation tools (release-please, semantic-release and similar) were not evaluated. +- Whether the `release:` keyword still requires `registry.gitlab.com/gitlab-org/cli` in every GitLab version (read from the 19.x docs example only). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/destructive-pipeline-actions.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/destructive-pipeline-actions.md new file mode 100644 index 0000000..a6bc7f9 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/destructive-pipeline-actions.md @@ -0,0 +1,115 @@ +--- +name: destructive-pipeline-actions +description: Catalogue of CI/CD actions that change or destroy real state (production deploy, forced re-run of a deploy, rollback, deleting environments/releases/tags/artifacts, runner unregistration) in ACTION / EFFECT / PROOF / WAIT-FOR-GO format +when-to-use: Before writing a pipeline step that touches production, and before running ANY gh/glab/API command that deploys, re-runs, deletes or unregisters +keywords: deploy, production, rerun, rollback, delete environment, delete release, delete tag, delete artifact, unregister runner, gh run, glab, WAIT-FOR-GO +priority: critical +related: deploy-strategies-release.md, github-actions-security.md, gitlab-ci.md, troubleshooting.md +--- + +# Destructive Pipeline Actions + +**Load when:** a step or command in this file's list is on the table. Reading state (`gh run list`, `gh run view`, `glab ci status`, environment pages) needs no go. Every block below is proposed to the user with the real object names filled in, and runs only after an explicit go naming the object (repo, environment, tag, runner). + +Pipeline-side rule: a job that deploys production is gated by a protected environment (GitHub: required reviewers; GitLab: protected environment + `when: manual`) and a concurrency lock (`concurrency` with `cancel-in-progress: false`; `resource_group`). The gate is configuration in the platform, not a comment in YAML. + +## Production deploy (first run of a new deploy job, or any run) + +``` +ACTION gh workflow run deploy.yml --repo / -f image-digest=sha256: +EFFECT starts the production deploy job; with required reviewers it pauses at the environment gate, then + changes the live service to ; GitHub records a deployment on environment `production` +PROOF digest was built by CI on the commit being released and already ran on staging; `gh run view ` after; + health endpoint and the rollout status of the platform (not exit code 0) confirm it +ROLLBACK redeploy the previous digest (see Rollback below); the digest of the previous release is written down first +WAIT-FOR-GO +``` + +``` +ACTION GitLab: play the manual job `deploy_production` on pipeline (UI Play, or `glab ci run` / API play) +EFFECT same as above on environment `production`; resource_group `production` serialises concurrent deploys +PROOF protected environment limits who may play it; the pipeline ran on the tag being released; staging job green +ROLLBACK re-run the previous successful deploy job from the Environments page (rollback deployment) +WAIT-FOR-GO +``` + +## Forced re-run of a deploy job + +``` +ACTION gh run rerun [--failed] | GitLab: retry the deploy job +EFFECT executes the deploy again with the SAME commit and inputs of the original run, even if main moved on; + a half-applied deploy may be applied twice; secrets/variables are read as they are now, not as they were +PROOF `gh run view --log-failed` shows why it failed; deploy step is idempotent (verified, not assumed); + no newer deploy has run since (an old run overwriting a newer release is the main hazard) +WAIT-FOR-GO +``` + +GitLab can block this: the project setting "Prevent outdated deployment jobs" fails an older deployment job when a newer deployment exists; job retries for rollback deployments stay allowed by default (deployment_safety page). + +## Rollback + +``` +ACTION redeploy the previous immutable artifact (image digest / tag) through the SAME pipeline and gates +EFFECT replaces the running release; schema migrations that already ran are NOT undone by redeploying old code +PROOF previous digest recorded before the deploy; migrations in the failed release are backward compatible + (expand/contract), or a tested down-migration exists; smoke test after +WAIT-FOR-GO +``` + +Never "fix forward" by editing the running system by hand; the next pipeline run will overwrite it. + +## Deleting environments, releases, tags, artifacts, caches + +``` +ACTION gh release delete [--cleanup-tag] | git push origin --delete +EFFECT removes the published release (and with --cleanup-tag the git tag); consumers pinned to the tag break; + an image/package published from that tag is NOT removed by this +PROOF `gh release view ` read first; nothing deploys from the tag (grep workflows and GitOps repo); release assets copied +WAIT-FOR-GO +``` + +``` +ACTION gh api -X DELETE repos///environments/ +EFFECT deletes the environment with its protection rules, secrets and variables; the deploy gate for it disappears +PROOF environment settings screenshotted/exported (reviewers, branch policy, secret NAMES); no workflow still references it +WAIT-FOR-GO +``` + +``` +ACTION gh run delete | gh cache delete | delete artifact via API +EFFECT run logs / caches / artifacts are gone; a deploy that consumed the artifact can no longer be audited or re-promoted +PROOF artifact is not the release candidate of an unreleased version; retention policy already covers the case +WAIT-FOR-GO +``` + +GitLab equivalents (stop or delete an environment, delete a release or tag, delete job artifacts) follow the same card: an environment stop runs its `on_stop` job, which may tear the deployment down. + +## Runner unregistration and removal + +``` +ACTION GitHub: ./config.sh remove --token (on the runner host) + GitLab: gitlab-runner unregister --name | delete the runner in the UI/API +EFFECT the runner stops receiving jobs; jobs queued only for its labels/tags wait forever; a GitLab runner authentication + token is revoked and cannot be reused +PROOF no job currently running on it (`gh api` runners list shows `busy`; GitLab runner page); another runner carries the same + labels/tags; token never printed in logs +WAIT-FOR-GO +``` + +## Changing who can deploy + +Changing environment reviewers, branch policies, protected environment allow-lists, or the default `GITHUB_TOKEN` permissions changes the security boundary of every future deploy. Propose the exact diff of the setting and wait for the go. + +## Sources (read 2026-11-10) + +- https://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/manage-environments +- https://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/control-deployments +- https://docs.gitlab.com/ci/environments/deployment_safety/ +- https://docs.gitlab.com/ci/environments/protected_environments/ +- https://docs.gitlab.com/ci/resource_groups/ +- `gh run rerun|delete|view`, `gh release create|delete`, `gh cache delete`, `gh workflow run --help` (gh 2.103.3, run locally 2026-11-10) + +## Not verified + +- `gh api -X DELETE .../environments/`, GitHub `config.sh remove`, `gitlab-runner unregister` and any `glab` command (glab is not installed here) were not checked today; confirm with `--help` or the docs before running. +- Whether deleting an environment also removes past deployment records was not checked. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/github-actions-security.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/github-actions-security.md new file mode 100644 index 0000000..a8ce575 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/github-actions-security.md @@ -0,0 +1,82 @@ +--- +name: github-actions-security +description: Writing safe GitHub Actions pipelines - GITHUB_TOKEN permissions least privilege, pinning actions by SHA or lockfile, OIDC federation instead of stored secrets, script injection, pull_request_target, self-hosted runner risks and ephemeral runners +when-to-use: Writing any workflow that uses secrets, deploys, runs on self-hosted runners, or triggers on external pull requests +keywords: permissions, GITHUB_TOKEN, pin SHA, actions.lock, gh actions-lock, OIDC, id-token, sub claim, pull_request_target, script injection, self-hosted runner, ephemeral, JIT +priority: critical +related: github-actions-workflows.md, pipeline-secrets.md, destructive-pipeline-actions.md, templates/github-oidc-deploy.yml +--- + +# GitHub Actions Security (pipeline author's view) + +**Load when:** a workflow touches secrets, deploys, uses third-party actions, or runs on self-hosted runners. This is how to write safe workflows; auditing an existing repo's supply chain belongs to `fuse-security:dependency-audit` / `security-expert`. + +## 1. `permissions`: least privilege + +- Set the default to read-only for repository contents (`permissions: {}` or `contents: read` at workflow level), then raise per job. Any user with write access to the repo can read all repository secrets, so credentials used in workflows must be minimal. +- Common job-level grants: `contents: write` (create release), `packages: write` (GHCR), `id-token: write` (mint OIDC token, nothing else), `attestations: write` + `artifact-metadata: write` (build attestations), `pull-requests: write`. +- Without `id-token: write` the OIDC JWT cannot be requested. +- Reusable workflow: the callee's `GITHUB_TOKEN` permissions can be kept or reduced by the caller, not raised (see [Not verified](#not-verified)). + +## 2. Third-party actions: pin them + +Order of preference: +1. **Workflow dependency lockfile** (`.github/workflows/actions.lock`, generated by the `gh actions-lock` CLI extension, never hand-edited): pins every action including transitive ones, records owner/repo IDs, GitHub states it supersedes SHA pinning for repos that adopt it. It does not pin reusable workflow calls (`jobs..uses`) - pin those by full SHA. +2. **Full-length commit SHA** with the tag in a trailing comment (as in all templates here). Verify the SHA belongs to the action's own repo, not a fork. +3. Tag only for actions you trust (verified creator), accepting that tags can be moved. + +Org policy "Require actions to be pinned to a full-length commit SHA" exists; a lockfile satisfies it for onboarded workflows. Let Dependabot raise the SHA bumps. Audit the source of any action that handles secrets. + +## 3. OIDC instead of stored credentials + +The job asks GitHub's OIDC provider (`https://token.actions.githubusercontent.com`) for a JWT; the target system validates the claims and returns a short-lived credential valid for the job. + +- Needs `permissions: id-token: write`. Request: `curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" "$ACTIONS_ID_TOKEN_REQUEST_URL&audience="` (JSON field `value`). Default `aud` is the repository owner URL; set your own audience per consumer. +- The trust rule on the target side must bind **claims**, not just "any GitHub token": `sub` (for example `repo:/:environment:production`), `repository`, `ref`, `job_workflow_ref`, `environment`. A role that trusts only the issuer lets every repository on GitHub in. +- Repositories created after 2026-07-15 use an immutable `sub` format that includes owner and repository IDs; read the OIDC reference before copying an old `sub` string. +- Self-hosted fit: OpenBao/Vault JWT auth (`POST /v1/auth/jwt/login` with `{"jwt","role"}`, token in `auth.client_token`), see [templates/github-oidc-deploy.yml](templates/github-oidc-deploy.yml) and `devops-secrets`. +- Mask every derived value (`::add-mask::`); register generated secrets; never pass structured JSON/YAML as one secret (redaction needs exact matches). + +## 4. Script injection + +`${{ ... }}` is substituted into the script text before the shell runs. Never inline untrusted fields (`github.event.pull_request.title`, branch names, issue bodies, `inputs.*`) in `run:`. + +```yaml +- env: + TITLE: ${{ github.event.pull_request.title }} + run: ./check.sh "$TITLE" # value stays data, quoted +``` + +Or pass it as an input to an action. Quote shell variables. + +## 5. Privileged triggers + +`pull_request_target` and `workflow_run` run with secrets and write token in the base context. Combined with a checkout of the PR head they hand your secrets to the PR author. Rule: never check out or execute PR code in these; use plain `pull_request` for tests. Also keep `persist-credentials: false` on `actions/checkout` unless the job pushes. + +## 6. Self-hosted runners + +- Persistent self-hosted runners do not give a clean instance per job: a prior job's files, credentials and processes remain. **Do not use self-hosted runners on public repositories** (anyone can open a PR); be cautious on private ones. +- Prefer **ephemeral** runners (one job, then gone; GitHub assigns only one job to such a runner) with automated provisioning, or the runner-scale-set / Kubernetes controller; GitHub recommends autoscaling with ephemeral runners only, and forwarding runner logs off-host. +- Isolate by trust: separate runner groups for production-deploy jobs; no shared Docker socket; no long-lived credentials on the host; network egress filtered; the host patched (you own OS updates; only the runner application updates itself). +- Registering/removing runners: see [destructive-pipeline-actions.md](destructive-pipeline-actions.md). + +## 7. Checklist + +`permissions` set at top and per job; actions pinned (lockfile or SHA); no `${{ }}` of untrusted data in `run`; OIDC claims bound to repo+environment; deploy jobs behind an environment with reviewers; `timeout-minutes` everywhere; no secrets in caches or artifacts; runners ephemeral or private-only. + +## Sources (read 2026-11-10) + +- https://docs.github.com/en/actions/reference/security/secure-use +- https://docs.github.com/en/actions/concepts/security/openid-connect +- https://docs.github.com/en/actions/reference/security/oidc +- https://docs.github.com/en/actions/concepts/security/workflow-dependency-lockfiles +- https://docs.github.com/en/actions/concepts/runners/self-hosted-runners +- https://docs.github.com/en/actions/reference/runners/self-hosted-runners +- https://openbao.org/docs/auth/jwt/ + +## Not verified + +- Whether the callee `GITHUB_TOKEN` can only be reduced (not raised) by the caller in every case: standard documented behaviour, not re-read on the pages above today. +- `gh actions-lock` command syntax and the lockfile feature's plan/GHES availability were read as a concept page only; the how-to (`.../lock-workflow-dependencies`) was not opened. +- Runner scale set / ARC (Actions Runner Controller) setup details were not read. +- The exact immutable `sub` string format for repos created after 2026-07-15 was not read. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/github-actions-workflows.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/github-actions-workflows.md new file mode 100644 index 0000000..233fc15 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/github-actions-workflows.md @@ -0,0 +1,100 @@ +--- +name: github-actions-workflows +description: GitHub Actions workflow design - triggers, jobs, matrix, concurrency, timeouts, reusable workflows, composite actions, artifacts, caching and cache-mode, environments and required reviewers +when-to-use: Writing or reviewing a .github/workflows file, splitting shared CI into reusable workflows or composite actions, tuning cache/artifact/concurrency behaviour +keywords: workflow_call, workflow_dispatch, matrix, fail-fast, concurrency, cancel-in-progress, timeout-minutes, composite, artifacts, actions/cache, cache-mode, environment, required reviewers +priority: high +related: github-actions-security.md, pipeline-design.md, destructive-pipeline-actions.md, templates/github-ci.yml +--- + +# GitHub Actions Workflows + +**Load when:** authoring or reviewing `.github/workflows/*.yml`. Security settings (permissions, pinning, OIDC, runners) are in [github-actions-security.md](github-actions-security.md). + +## Skeleton rules + +- Workflow files live in `.github/workflows/` (subdirectories are not supported for reusable workflows); extension `.yml` or `.yaml`. +- `on:` events: list each event with its filters; once any event has filters, give all events a trailing colon (`page_build:`). `branches` and `branches-ignore` cannot both be used for the same event. +- Top of every workflow: `permissions: {}` or `contents: read`, then widen per job ([github-actions-security.md](github-actions-security.md)). +- Every job gets `timeout-minutes` (maximum 360 on GitHub-hosted and self-hosted runners; integers only). +- A workflow run holds at most 256 jobs from a matrix. +- Template: [templates/github-ci.yml](templates/github-ci.yml). + +## Matrix + +```yaml +strategy: + fail-fast: false # one failing cell must not hide the others + max-parallel: 4 # optional cap for shared self-hosted runners + matrix: + os: [ubuntu-24.04, ubuntu-24.04-arm] + node: ["22", "24"] +``` + +Use `fail-fast: false` for compatibility matrices and the default (true) only when any failure makes the rest worthless. + +## Concurrency + +```yaml +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} # expression allowed +``` + +- At most one running and one pending run per group; a new pending run replaces the old pending one. `cancel-in-progress: true` also cancels the running one. +- Group names are case insensitive. Optional `queue: max` (up to 100 pending) cannot be combined with `cancel-in-progress: true` (validation error). +- Deploys: `cancel-in-progress: false`, one group per environment, so a deploy is never killed halfway ([templates/github-oidc-deploy.yml](templates/github-oidc-deploy.yml)). + +## Reusable workflows vs composite actions + +| Need | Use | +|------|-----| +| Share whole jobs (own runner, own permissions, own environment) | Reusable workflow: `on: workflow_call`, called by `jobs..uses` | +| Share a few steps inside a job | Composite action: `runs: using: composite` ([template](templates/github-composite-action.yml)) | + +- Caller syntax: `owner/repo/.github/workflows/file.yml@`; `ref` can be SHA, tag or branch; a full commit SHA is the safe choice for workflows you do not own. Same repo: `./.github/workflows/file.yml` (runs from the caller's commit). +- Inputs are typed (`string`, `number`, `boolean`); secrets are passed with `secrets:` or `secrets: inherit` (same organization or enterprise only). +- A job that calls a reusable workflow cannot use `environment:`; put `environment` on the job inside the callee. If the caller does not pass an environment secret, it resolves to an **empty string without error**: mark it `required: true` in `workflow_call.secrets`. +- Template: [templates/github-reusable-workflow.yml](templates/github-reusable-workflow.yml). Every `run` step in a composite action needs an explicit `shell`. + +## Artifacts vs cache + +| | Artifact (`upload-artifact` / `download-artifact`) | Cache (`actions/cache`) | +|---|---|---| +| Purpose | pass a build result between jobs; keep a release candidate | speed up dependency download | +| Contract | exact content you uploaded | best effort, may be evicted | +| Key | name | `key` (max 512 chars) + `restore-keys` fallbacks | + +- Cache entries are immutable: to change content, change the key (`hashFiles('**/package-lock.json')`). An exact `key` match is a hit; otherwise `restore-keys` are tried in order, and a new cache is saved only when the job succeeds. +- **Never put credentials in a cache path**: anyone with read access can open a PR and read a cache; forks can read base-branch caches. +- Cache access control (new, see [Not verified](#not-verified)): `cache-mode` at workflow or job level takes `read`, `write`, `write-only`, `none`; unset, it defaults by trigger. Only `push`, `workflow_dispatch`, `repository_dispatch`, `delete`, `registry_package`, `page_build` and `schedule` may create or overwrite caches in the default branch scope; other triggers that resolve to the default branch get read-only access. Do not set `cache-mode: write` on low-trust triggers (cache poisoning). + +## Environments + +`environment:` on a job ties it to protection rules and environment secrets; the job does not start, and cannot read the environment's secrets, until the rules pass. + +- Required reviewers: up to 6 people or teams, any one approval suffices; optionally prevent self-review. Wait timer, branch/tag deployment policy, custom protection rules (GitHub Apps). +- Environments on private repositories need GitHub Team or Pro (Free: public repos only); protection rules silently stop applying when a repo goes from public to private on Free. +- `environment: { name, deployment: false }` uses an environment's secrets and approval without creating a deployment record. +- Configure reviewers in repository settings (needs admin); the YAML only names the environment. Disable "Allow administrators to bypass configured protection rules" for production. + +## Triggers worth knowing + +- `workflow_dispatch` with typed `inputs` for manual runs (deploy, rollback). +- `pull_request` (no secrets for forks, safe) vs `pull_request_target` and `workflow_run` (privileged; never check out untrusted PR code there, see [github-actions-security.md](github-actions-security.md)). +- `schedule` (cron, UTC, default branch only), `push` with `tags:` filters for releases. + +## Sources (read 2026-11-10) + +- https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax +- https://docs.github.com/en/actions/how-tos/reuse-automations/reuse-workflows +- https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency +- https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching +- https://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/manage-environments +- https://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/control-deployments + +## Not verified + +- `cache-mode` is documented on the pages above but its rollout (hosted vs GHES, plan limits) was not checked; test on a throwaway workflow before relying on the default. +- `schedule` UTC/default-branch rule and the `max-parallel` semantics are standard behaviour but were not re-read today. +- The actions/cache major version in current docs examples is `v4` while the latest release is `v6.1.0`; the pinned SHAs in templates are the latest releases read via `gh api` on 2026-11-10. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/gitlab-ci.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/gitlab-ci.md new file mode 100644 index 0000000..8da4c6c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/gitlab-ci.md @@ -0,0 +1,80 @@ +--- +name: gitlab-ci +description: GitLab CI/CD pipeline design - stages, rules and workflow, needs/DAG, include and components with inputs, environments and resource_group, variables (masked, protected), ID tokens, runners and executors +when-to-use: Writing or reviewing .gitlab-ci.yml, building a CI component, wiring deploy environments, choosing a runner executor +keywords: .gitlab-ci.yml, rules, workflow, needs, include, component, spec inputs, environment, resource_group, protected variable, id_tokens, runner, executor, dind, cache, artifacts +priority: high +related: pipeline-design.md, pipeline-secrets.md, destructive-pipeline-actions.md, templates/gitlab-ci.yml, templates/gitlab-component.yml +--- + +# GitLab CI/CD + +**Load when:** the repo has (or should have) a `.gitlab-ci.yml`. Templates: [gitlab-ci.yml](templates/gitlab-ci.yml), [gitlab-component.yml](templates/gitlab-component.yml). Validate any edit with CI Lint (UI "Pipeline editor", or the lint API) before pushing. + +## Versions (checked 2026-11-10) + +GitLab 19.5 is the latest released minor in the release-notes index (19.6 listed as upcoming); GitLab Runner 19.5.3 (released 2026-11-06, GitLab.com releases API). Keyword history in this file: CI/CD inputs GA in 17.0; `manual_confirmation` 17.1 (support for environment stop jobs 18.3); `sub` claim configurable fields in ID tokens 18.7. + +## Pipeline shape + +- `stages:` give coarse order; `needs:` turns the pipeline into a DAG so a job starts when its dependencies finish. `needs: []` starts a job immediately (lint, scanners). Mix staged jobs and `needs` freely. +- `workflow:rules` decides which pipeline types exist; use it to avoid duplicate branch + merge-request pipelines (template). `workflow:auto_cancel:on_new_commit` (`conservative` default, `interruptible`, `none`) plus per-job `interruptible: true` cancels redundant runs; deploy jobs set `interruptible: false`. +- `default:` sets image, tags, timeout, retry, cache defaults per job; a job keyword replaces the default entirely (no merge). + +## `rules` + +- Rules are evaluated in order; first match wins; no match = job not created. `when: never` excludes. A final bare `when:` rule can create both a branch and a merge-request pipeline: add the `workflow:rules` guard. +- Useful predicates: `$CI_PIPELINE_SOURCE == "merge_request_event"`, `$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH`, `$CI_COMMIT_TAG`, `changes:` (with `compare_to:` to compare against another ref), `exists:`. +- Rules run before any job: variables created in job scripts (dotenv) cannot be used in them. + +## Reuse: include, components, inputs + +- `include:` types: `local`, `project` (+ `ref`, `file`), `remote`, `template`, `component`. Included files merge into the pipeline; same-named keys in the main file override. +- **Components**: a project with `templates/.yml` (or `templates//template.yml`), a `README.md`, released with a version tag; max 100 components per project. Consume with `include: - component: $CI_SERVER_FQDN//@` and `inputs:`. Only components on the same GitLab instance can be referenced. Pin a version; never track a moving ref for production pipelines. +- **Inputs**: `spec: inputs:` header (then `---`), interpolated as `$[[ inputs. ]]`; typed, with `default`, `description`, `options`, `regex`. Inputs are fixed at pipeline creation; use CI/CD variables for runtime values. Inputs are only visible in the file that defines them. + +## Environments and deploy safety + +- `environment: { name, url, on_stop, auto_stop_in, action }`: `action: start` (default), `prepare`, `stop`. Static (`staging`) vs dynamic (`review/$CI_COMMIT_REF_SLUG`) environments; stopping runs the `on_stop` job. +- `resource_group: ` serialises deploy jobs across pipelines (modes `unordered`, `oldest_first`, ...). Project setting "Prevent outdated deployment jobs" fails an older deploy that would overwrite a newer one. Rollback = re-run an earlier deploy job from the Environments page; job retries for rollback deployments are on by default. +- **Protected environments** (Premium/Ultimate): only roles/users/groups in "Allowed to deploy" may run the job; optional approvers. Environments are otherwise writable by any Developer. +- Manual gate: `when: manual` in rules plus job-level `manual_confirmation:` message. A manual job in `rules:` with `allow_failure: true` does not block the pipeline. + +## Variables and secrets + +- Values in `.gitlab-ci.yml` are readable by everyone with repo access: non-sensitive config only. Sensitive values go in Settings > CI/CD > Variables: **Masked** or **Masked and hidden** visibility, **Protect variable** (only pipelines on protected branches/tags), environment scope, type `File` for certificates. Always quote variable values in YAML (an unquoted `012345` parses as octal). +- Fork pipelines cannot read the parent's variables unless a merge request pipeline runs in the parent project. +- `id_tokens:` (OIDC JWT per job, custom `aud`, claims include `project_path`, `ref`, and for jobs with an environment `environment_protected` / `deployment_tier` in the `sub`) replace stored cloud/secret-store credentials; the `secrets:` keyword uses ID tokens. Bind the receiving role to claims. Pointer: `devops-secrets`. + +## Cache vs artifacts + +Cache = dependencies, best effort, per runner (or S3 for distributed runners), not shared across projects, branches do not share by default; `cache:key:files` ties the key to a lockfile; up to four caches per job; `fallback_keys`. Artifacts = results passed between stages, stored in GitLab, expire after 30 days by default; `dependencies:` limits which jobs' artifacts are fetched. + +## Runners and executors + +| Executor | Fit | Note | +|----------|-----|------| +| Docker | clean container per job on a host | most common self-hosted choice; also works with Podman | +| Kubernetes | one pod per job in an existing cluster | cloud-native, scales with the cluster | +| Docker Autoscaler / Instance | on-demand VMs via fleeting plugins | the actively developed autoscaling executors | +| Shell, SSH, VirtualBox, Parallels, Custom | maintenance mode (security fixes only) | Shell has no isolation: avoid for untrusted code | + +- Runner scope: instance, group, project. Register with a **runner authentication token** (created in the UI/API); the registration-token flow is legacy. Tag runners and tag jobs to steer sensitive deploy jobs to a dedicated, protected runner. +- Docker-in-Docker (`docker:dind`) needs privileged mode: that is host-root-equivalent; keep it on isolated runners, or build with rootless/daemonless tooling (pointer: `devops-docker`). +- Unregistering a runner: [destructive-pipeline-actions.md](destructive-pipeline-actions.md). + +## Sources (read 2026-11-10) + +- https://docs.gitlab.com/ci/yaml/ , https://docs.gitlab.com/ci/yaml/needs/ , https://docs.gitlab.com/ci/yaml/includes/ +- https://docs.gitlab.com/ci/jobs/job_rules/ , https://docs.gitlab.com/ci/inputs/ , https://docs.gitlab.com/ci/components/ +- https://docs.gitlab.com/ci/environments/ , .../environments/deployment_safety/ , .../environments/protected_environments/ , https://docs.gitlab.com/ci/resource_groups/ +- https://docs.gitlab.com/ci/variables/ , https://docs.gitlab.com/ci/secrets/id_token_authentication/ , https://docs.gitlab.com/ci/caching/ +- https://docs.gitlab.com/runner/executors/ , https://docs.gitlab.com/ci/runners/runners_scope/ +- https://docs.gitlab.com/releases/19/ ; https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab-runner/releases + +## Not verified + +- The latest GitLab self-managed patch within 19.5 (only the minor and the runner version were confirmed). +- `docker:cli` image shipping the `buildx` plugin (used in the component template) and dind TLS variable details. +- Merge-train / merge-request-pipeline behaviour, child/parent pipelines, `trigger:` and `parallel:matrix` limits were not read in depth. +- Exact `include:rules` and `include:inputs` syntax beyond the inputs page summary. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/image-build-ci.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/image-build-ci.md new file mode 100644 index 0000000..b2e59a5 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/image-build-ci.md @@ -0,0 +1,66 @@ +--- +name: image-build-ci +description: Building container images in CI - BuildKit cache backends, multi-platform builds, push by digest, tags, provenance and SBOM attestations, signing pointers; GitHub Actions and GitLab flavours +when-to-use: A pipeline builds and pushes an image, a build is slow, a multi-arch image is needed, or SBOM/provenance/signing is requested +keywords: buildx, build-push-action, cache-from, cache-to, type=gha, type=registry, multi-platform, qemu, digest, provenance, SBOM, cosign, attest, dind +priority: high +related: pipeline-design.md, deploy-strategies-release.md, pipeline-secrets.md, templates/github-image-build-push.yml, templates/gitlab-component.yml +--- + +# Container Image Build in CI + +**Load when:** the pipeline produces an image. Dockerfile craft, registries, SBOM tooling and signing policy live in `devops-docker`; this file is the CI wiring. Template: [github-image-build-push.yml](templates/github-image-build-push.yml) (GitLab: [gitlab-component.yml](templates/gitlab-component.yml)). + +## Principles + +1. **Build once.** The image built from a commit is the artifact promoted through environments; never rebuild per environment ([pipeline-design.md](pipeline-design.md)). +2. **Reference by digest.** Tags move; the digest output of the push step (`steps.push.outputs.digest` in `docker/build-push-action`, outputs `imageid`, `digest`, `metadata`) is what deploy jobs receive. +3. **PRs build but do not push or sign** (`push: ${{ github.event_name != 'pull_request' }}`); credentials for pushing exist only on trusted refs. +4. **Secrets never via `ARG`/`COPY`**: use build secret mounts (`--secret`). Provenance on public repos records build-arg values, so a secret passed as build arg leaks into the attestation. + +## Cache + +External cache is "almost essential" in CI because runners keep nothing between runs. Backends: `inline` (min mode only), `registry` (separate cache image, supports `mode=max`), `local`, `gha` (GitHub Actions cache; beta per docs; only valid inside a workflow; only cache service API v2 since 2025-04-15), `s3`/`azblob` (marked unreleased in the backends page). + +```yaml +cache-from: type=gha +cache-to: type=gha,mode=max +# or, runner-independent: +# cache-from: type=registry,ref=registry.example.com/app:buildcache +# cache-to: type=registry,ref=registry.example.com/app:buildcache,mode=max +``` + +Each cache location is overwritten by the next write: use a distinct `ref`/`scope` per branch when you want per-branch caches. With a GitHub cache, mind the repo cache-size and eviction limits ([github-actions-workflows.md](github-actions-workflows.md)). + +## Multi-platform + +- `platforms: linux/amd64,linux/arm64` with `docker/setup-qemu-action` (emulation, slow) and `docker/setup-buildx-action`. The default runner Docker setup can push multi-platform images but cannot `load` them locally (needs the containerd image store). +- Faster: native runners per platform (`ubuntu-24.04-arm` exists as a GitHub-hosted label) merging manifests, or Docker's "GitHub Builder" reusable workflows that compute the matrix and create the manifest. + +## Tags + +`docker/metadata-action` turns refs into tags and OCI labels (`type=ref,event=branch|pr`, `type=semver,pattern={{version}}`, `type=sha`, `type=raw`). Avoid deploying by `latest`; the deploy input is the digest. + +## Provenance, SBOM, signing (pointers) + +- `docker/build-push-action` v4+: `provenance` (public repos get `mode=max` by default, private `mode=min`; set `mode=max` explicitly) and `sbom: true`. Attestations require pushing to a registry (no `load`). +- GitHub artifact attestations: `actions/attest` with `permissions: id-token: write, attestations: write, artifact-metadata: write`; inputs `subject-name`, `subject-digest`, `push-to-registry`. `actions/attest-build-provenance` v4 is only a wrapper; new work uses `actions/attest`. Available for public repos, and private/internal on plans that include it; not on GHES. +- Keyless signing with `sigstore/cosign-installer` + `cosign sign --yes @` signs by OIDC identity. Verification policy at deploy time (admission control, `cosign verify` with identity and issuer) belongs to `devops-docker` / `devops-kubernetes`. +- GitLab: attach the same flags to `docker buildx build`; sign with cosign using the job's ID token (`id_tokens`). Not verified here. + +## Runner and daemon constraints + +- Docker-in-Docker needs a privileged container; prefer buildx with a remote/containerised BuildKit builder or daemonless builders on isolated runners. A mounted host Docker socket gives the job root on the host: never on shared or untrusted-PR runners. +- Registry login in CI uses the platform token (`GITHUB_TOKEN` with `packages: write`, `CI_REGISTRY_USER`/`CI_REGISTRY_PASSWORD`), not a personal access token. + +## Sources (read 2026-11-10) + +- https://docs.docker.com/build/ci/github-actions/cache/ , .../multi-platform/ , .../attestations/ ; https://docs.docker.com/build/cache/backends/ +- README of docker/build-push-action, docker/metadata-action, actions/attest, sigstore/cosign-installer (via `gh api .../readme`) +- Versions via `gh api repos//releases/latest` on 2026-11-10: build-push-action v7.4.0, setup-buildx-action v4.4.1, setup-qemu-action v4.4.0, login-action v4.8.0, metadata-action v6.2.0, actions/attest v4.2.2, cosign-installer v4.1.3 + +## Not verified + +- Which cosign major `sigstore/cosign-installer` v4.1.3 installs by default and whether `cosign sign --yes` is unchanged in it. +- GitLab-side cosign keyless signing and `docker:cli` shipping buildx. +- Current GHCR retention/storage limits and GitHub cache size limit. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/other-ci-systems.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/other-ci-systems.md new file mode 100644 index 0000000..76cc05c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/other-ci-systems.md @@ -0,0 +1,62 @@ +--- +name: other-ci-systems +description: Pointer-level notes on Jenkins, Forgejo/Gitea Actions and Woodpecker CI for self-hosted shops - what each is, current versions, when it fits, how its concepts map to GitHub Actions/GitLab CI +when-to-use: The repo or the user runs Jenkins (Jenkinsfile), Forgejo/Gitea Actions (.forgejo/workflows) or Woodpecker (.woodpecker/), or a team is choosing a self-hosted CI +keywords: Jenkins, Jenkinsfile, Forgejo Actions, forgejo-runner, Gitea act_runner, Woodpecker, .woodpecker, self-hosted CI +priority: medium +related: github-actions-workflows.md, gitlab-ci.md, pipeline-design.md +--- + +# Other CI Systems (pointers only) + +**Load when:** the CI is not GitHub Actions or GitLab CI. This file does not teach the products; it says what they are, where the docs are, and which rules from the sibling references still apply. Read the official docs for the installed version before writing config. + +## Versions (checked 2026-11-10) + +| System | Version | Source | +|--------|---------|--------| +| Jenkins | 2.590 weekly, 2.580.2 LTS (LTS baseline every 12 weeks, stable updates every 4) | https://www.jenkins.io/download/ | +| Forgejo | v17.0.1 latest, v15.0.11 LTS | https://forgejo.org/docs/latest/ (nav header) | +| Forgejo Runner | v17.0.1 tag (Codeberg releases API, 2026-10-26) | https://codeberg.org/forgejo/runner | +| Woodpecker CI | v3.20.1 (GitHub release 2026-10-29); docs 3.20.x | https://woodpecker-ci.org/docs/intro | + +## Jenkins + +- Pipeline as code in a `Jenkinsfile` checked into the repo (Declarative or Scripted); controller + agents; shared libraries for reuse; credentials are handled through the credentials store, not string interpolation. +- The Jenkinsfile handbook has sections on "Interpolation of sensitive environment variables" and "Injection via interpolation": read them before passing credentials or user input into `sh` steps (page body beyond its table of contents was not read). +- Fit: existing estates, heavy plugin dependence. Costs: controller and plugin patching is yours; treat the controller as a high-value target (it holds credentials). New greenfield self-hosted work usually lands on GitLab CI, Forgejo/Gitea Actions or Woodpecker. +- Same rules still apply (pipeline-design.md): build once, immutable artifacts, no deploy from a developer laptop, human gate before production. + +## Forgejo / Gitea Actions + +- Forgejo Actions: workflow files are YAML modelled on GitHub Actions and executed by `forgejo-runner` registered against the instance. Admin guide pages: Runner Configuration, Runner Registration, "Securing Forgejo Actions Deployments", "Utilizing Docker within Actions". +- Syntax and the action ecosystem are similar to, but not identical with, GitHub Actions: test every workflow you copy from GitHub, and do not assume `permissions`, OIDC, environments, reusable-workflow or cache features exist (not verified). +- Runner security is the GitHub self-hosted-runner story again: untrusted PRs + a runner with Docker access = host compromise. Keep runners isolated, ephemeral where possible, and read the "Securing" page before enabling Actions for outside contributors. + +## Woodpecker CI + +- Container-native: each step runs in an image, steps share a workspace volume and run serially unless `depends_on` is used; `[CI SKIP]` or `[SKIP CI]` in a commit message skips a pipeline. +- Plugins are container images; secrets and registries are configured per project/org; a local `exec`-style run and a linter exist (see the docs index). +- Fit: small teams on Gitea/Forgejo wanting simple, container-only pipelines. Review its "Security boundaries" doc page before giving it privileged runners. + +## Concept map + +| Concept | GitHub Actions | GitLab CI | Woodpecker | Jenkins | +|---------|----------------|-----------|------------|---------| +| Pipeline file | `.github/workflows/*.yml` | `.gitlab-ci.yml` | `.woodpecker/*.yaml` | `Jenkinsfile` | +| Unit of work | job + steps | job (+ stage) | workflow + step | stage + step | +| Reuse | reusable workflow, composite action | include, component | plugins, shared YAML | shared library | +| Prod gate | environment reviewers | protected environment + manual | project settings (not verified) | `input` step / folder permissions | + +## Sources (read 2026-11-10) + +- https://www.jenkins.io/download/ , https://www.jenkins.io/doc/book/pipeline/jenkinsfile/ +- https://forgejo.org/docs/latest/ (navigation and version header only; the Actions overview page body did not extract) +- https://woodpecker-ci.org/docs/intro , https://woodpecker-ci.org/docs/usage/workflow-syntax +- `gh api repos/woodpecker-ci/woodpecker/releases/latest`, `gh api repos/jenkinsci/jenkins/releases/latest`, https://codeberg.org/api/v1/repos/forgejo/runner/releases + +## Not verified + +- Forgejo Actions feature parity with GitHub Actions (permissions, OIDC, environments, reusable workflows, cache) and the workflow directory name: the overview page body was not readable. +- Gitea `act_runner` version and status were not checked. +- Woodpecker workflow file location (`.woodpecker/` directory vs single file; the concept-map cell is from memory), secrets/promotion gating details, and Jenkins LTS line beyond the download page. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/pipeline-design.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/pipeline-design.md new file mode 100644 index 0000000..331bd78 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/pipeline-design.md @@ -0,0 +1,81 @@ +--- +name: pipeline-design +description: CI/CD pipeline design principles - build once promote many, trunk-based flow, quality gates, test stage layout, pipeline speed (parallelism, caching, DAG, path filters), flaky tests, pipeline-as-code hygiene +when-to-use: Designing a new pipeline, reviewing a slow or flaky one, deciding what runs on PR vs main vs tag +keywords: build once, promote, trunk-based, quality gate, test pyramid, parallel, needs, path filter, flaky, pipeline speed, immutable artifact +priority: high +related: github-actions-workflows.md, gitlab-ci.md, image-build-ci.md, deploy-strategies-release.md +--- + +# Pipeline Design + +**Load when:** shaping what a pipeline does and in what order, independent of the CI product. + +## Core rules + +1. **Build once, promote many.** One commit -> one immutable artifact (image digest, package version, binary checksum). Every environment receives that exact artifact; only configuration differs. A rebuild for production means production runs something nobody tested. +2. **Pipeline as code, reviewed like code.** Pipeline files change through PRs; the deploy job on the default branch is protected by the same review rules as the application. +3. **Fast feedback first.** Cheap checks (format, lint, type check, unit tests) run first and in parallel; slow checks (integration, end-to-end, image scan) after them; deploy last. +4. **Fail closed, name the gate.** Each gate has an owner and an exit condition: lint clean, tests green, image built and scanned, staging smoke test green, human approval for production. +5. **Idempotent deploy jobs.** Running the same deploy twice yields the same state (a re-run after a partial failure must be safe). +6. **No pipeline-only knowledge.** Every CI step is a script or make target that works locally (`make test`), CI only orchestrates. + +## Branching and triggers + +| Event | Runs | Never | +|-------|------|-------| +| Pull / merge request | lint, tests, build (no push), preview deploy if any | production secrets, signing, publish | +| Push to default branch | full CI, build+push image, deploy to staging | direct production deploy without gate | +| Version tag | release build/promote, GitHub/GitLab release, production deploy behind approval | tags created by CI without review | +| Schedule | dependency/image rescans, nightly slow tests | deploys | + +Trunk-based: short-lived branches (days), merge to the default branch often, feature flags instead of long branches; release from a tag on the default branch. Keep release branches only when you must patch old versions. + +## Stage layout + +``` +lint/format ─┐ +unit tests ──┼─> build artifact ─> image scan ─> deploy staging ─> smoke/e2e ─> [approval] ─> deploy production ─> smoke +type check ──┘ +``` + +- Unit and static checks do not need the artifact: run them in parallel with the build (GitLab `needs: []`; GitHub jobs without `needs`). +- Integration/e2e run against the deployed staging artifact, not a rebuilt one. +- Quality gates: coverage thresholds only if the team owns them; static analysis and secret scanning on every PR (pointer: `fuse-security:*`, `devops-secrets`); dependency/image vulnerability scan with an explicit policy for severities. + +## Speed + +- **Parallelise** independent work (matrix, `parallel:matrix`, test sharding); **use the DAG** (`needs`) instead of stage barriers. +- **Skip irrelevant work**: path filters (`on.push.paths`, GitLab `rules:changes`), but never skip required status checks in a way that leaves a required check forever "pending". +- **Cache dependencies by lockfile hash**; cache is best effort, correctness must not depend on it. Use external BuildKit cache for images ([image-build-ci.md](image-build-ci.md)). +- **Cancel redundant runs** on PRs (`concurrency` / `interruptible`), but never cancel deploys. +- **Right-size runners**; measure queue time separately from run time. A slow pipeline is first measured (job durations, critical path), then fixed. +- **Small images for CI** (pinned, slim tool images) cut pull time; pre-bake tools into a runner image rather than installing on every run. + +## Flaky tests + +Quarantine a flaky test with a ticket and an owner; automatic retries (`retry:` in GitLab, rerun-failed) hide the problem and lengthen the pipeline, so cap them and report the retry rate. Retries are for infrastructure faults (runner lost, network), not for assertions. + +## Environments and configuration + +One definition of the deploy, parametrised per environment (variables, values files); environment-specific secrets scoped to that environment. Promotion = same artifact + next environment's config. Pointer: GitOps promotion by pull request belongs to `devops-gitops-packaging`. + +## Anti-patterns + +- Rebuilding the artifact in the deploy job. +- Deploy steps that `ssh`/`kubectl` into production with a long-lived key stored as a CI secret (use OIDC or a GitOps pull model). +- `latest` as a deployment reference. +- A pipeline that only works on one runner because of leftover state (non-ephemeral self-hosted runners). +- One giant job: no parallelism, no partial reruns, unreadable failures. +- Required checks that are skipped by path filters and block merges. + +## Sources (read 2026-11-10) + +- https://docs.gitlab.com/ci/pipelines/pipeline_efficiency/ , https://docs.gitlab.com/ci/yaml/needs/ +- https://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/control-deployments +- https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency + +## Not verified + +- The principles (build once promote many, trunk-based, quality gates, flaky-test handling) are established practice, not claims from the pages above; only the platform behaviours cited in the speed section were read. +- GitHub path-filter interaction with required status checks was not re-read today. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/pipeline-secrets.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/pipeline-secrets.md new file mode 100644 index 0000000..50a168d --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/pipeline-secrets.md @@ -0,0 +1,52 @@ +--- +name: pipeline-secrets +description: Handling credentials inside CI/CD pipelines - what never goes in YAML, scoping and masking, environment-bound secrets, OIDC instead of stored keys, leak response; pointer to devops-secrets for the stores themselves +when-to-use: A pipeline needs a token, key, certificate or password, or a secret may have leaked into logs, caches, artifacts or image layers +keywords: secrets, masked, protected variable, environment secret, OIDC, add-mask, leaked secret, rotate, build secret, artifact leak +priority: high +related: github-actions-security.md, gitlab-ci.md, image-build-ci.md, destructive-pipeline-actions.md +--- + +# Secrets in Pipelines + +**Load when:** a pipeline needs a credential. Secret stores (SOPS, Vault/OpenBao, External Secrets), rotation design and Kubernetes Secrets belong to `devops-secrets`; scanning a repo for leaks belongs to `fuse-security:security-expert`. This file covers the CI side only. Never print a secret value in output or chat: names and keys only. + +## Order of preference + +1. **No secret**: OIDC federation. GitHub: `permissions: id-token: write` and a trust rule on the target bound to repository + environment claims. GitLab: `id_tokens:` with a custom `aud`. The job gets a short-lived credential valid for that job. +2. **Secret manager fetch at run time** (OpenBao/Vault via JWT login after OIDC), scoped to a role per environment. +3. **Platform secret**, scoped as narrowly as possible: GitHub environment secret (behind required reviewers) over repository secret over organization secret; GitLab project/group variable that is **masked** (or masked and hidden), **protected**, and **environment-scoped**. + +## Rules + +- Never put secrets in the pipeline file, in `.env` files in the repo, in `docker build --build-arg`, in the cache, or in artifacts (anyone with read access can read caches and artifacts; provenance records build args on public repos). +- Any user with write access to a GitHub repo can read its repository secrets via a workflow they can push: protect branches and use environments with reviewers for anything sensitive. +- Redaction is best effort: transformed values (Base64, URL-encoded, JWTs derived from a secret) are not masked unless registered (`::add-mask::` on GitHub). Do not store structured JSON/YAML as one secret. +- Secrets are exposed only to the steps that need them: set `env:` on the one step, not the job. Avoid passing secrets as command-line arguments (visible in process lists); use stdin or files with restrictive permissions. +- Fork and external PRs get no secrets (GitHub `pull_request` from forks; GitLab fork pipelines cannot read parent variables). Do not "fix" this with `pull_request_target` + PR checkout. +- GitLab YAML values: quote them; `.gitlab-ci.yml` variables are readable by everyone with repo access. +- Production deploy credentials exist only in the production environment/role, never in a repository-wide variable. +- Self-hosted runners: credentials must not sit on the host; use ephemeral runners ([github-actions-security.md](github-actions-security.md)). + +## If a secret leaked (pipeline log, artifact, cache, image layer) + +``` +ACTION rotate the credential at its issuer FIRST, then purge the copies (log, artifact, cache, image tag) +EFFECT everything using the old credential breaks until updated; purging logs/artifacts removes audit evidence +PROOF old credential rejected by the issuer; new one stored in the secret store; pipeline green with the new one +WAIT-FOR-GO +``` + +Deleting the log does not make the secret safe; the provider's guidance is to delete the log **and** rotate. For deleting runs/artifacts/caches see [destructive-pipeline-actions.md](destructive-pipeline-actions.md). + +## Sources (read 2026-11-10) + +- https://docs.github.com/en/actions/reference/security/secure-use +- https://docs.github.com/en/actions/concepts/security/openid-connect +- https://docs.gitlab.com/ci/variables/ , https://docs.gitlab.com/ci/secrets/id_token_authentication/ +- https://docs.docker.com/build/ci/github-actions/attestations/ (build-arg exposure in provenance) + +## Not verified + +- GitLab "Masked and hidden" visibility details and GitLab Secrets Manager / `secrets:` keyword providers were not read beyond the variables and ID-token pages. +- "Avoid secrets on the command line" is general hardening advice, not from the pages above. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-ci.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-ci.yml new file mode 100644 index 0000000..ddec797 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-ci.yml @@ -0,0 +1,70 @@ +# CI workflow: lint + test matrix + build artifact, least privilege, SHA-pinned. +# Replace the `make` targets with the project's own commands. Pin refs are the +# commit SHAs of the tags named in the trailing comments (read 2026-11-10). +name: ci + +on: + pull_request: + push: + branches: [main] + +# Deny everything by default; each job adds only what it needs. +permissions: {} + +# One run per ref; a newer push cancels the older run (safe for CI, never for deploys). +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + lint: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: make lint + + test: + needs: lint + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-24.04-arm] + node: ["22", "24"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.node }} + cache: npm + - run: npm ci + - run: npm test + + build: + needs: test + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: make build + # Build once, promote many: this artifact is what later jobs deploy. + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: dist-${{ github.sha }} + path: dist/ + if-no-files-found: error + retention-days: 14 diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-composite-action.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-composite-action.yml new file mode 100644 index 0000000..496003c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-composite-action.yml @@ -0,0 +1,30 @@ +# Composite action: save as .github/actions/setup-project/action.yml (the file +# name MUST be action.yml or action.yaml). Use it with: +# - uses: ./.github/actions/setup-project +# with: +# node-version: "24" +name: Setup project +description: Install Node and project dependencies with a lockfile-keyed cache + +inputs: + node-version: + description: Node.js major version + required: false + default: "24" + +outputs: + cache-hit: + description: Whether the npm cache was restored + value: ${{ steps.node.outputs.cache-hit }} + +runs: + using: composite + steps: + - id: node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ inputs.node-version }} + cache: npm + - name: Install dependencies + shell: bash # `shell` is required on every run step of a composite action + run: npm ci diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-image-build-push.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-image-build-push.yml new file mode 100644 index 0000000..712926e --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-image-build-push.yml @@ -0,0 +1,86 @@ +# Build a multi-arch image once, push by digest to GHCR, attach SBOM + provenance, +# sign the digest. Immutable digest is the artifact that gets promoted. +name: image + +on: + push: + branches: [main] + tags: ["v*.*.*"] + pull_request: # builds but does NOT push or sign on PRs + +permissions: {} + +concurrency: + group: image-${{ github.ref }} + cancel-in-progress: true + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: read + packages: write # push to GHCR + id-token: write # keyless signing + attestation (OIDC) + attestations: write + artifact-metadata: write + outputs: + digest: ${{ steps.push.outputs.digest }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - uses: docker/login-action@8aa5831fb8d48d8f76594c7ddde61ae58019f550 # v4.8.0 + if: github.event_name != 'pull_request' + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: meta + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=sha + + - id: push + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: mode=max + sbom: true + + - name: Attest build provenance + if: github.event_name != 'pull_request' + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true + + - uses: sigstore/cosign-installer@756fe2d284c7292c4e9b62c8a28f1e5629c59532 # v4.1.3 + if: github.event_name != 'pull_request' + + - name: Sign the image digest (keyless) + if: github.event_name != 'pull_request' + env: + REF: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.push.outputs.digest }} + run: cosign sign --yes "$REF" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-oidc-deploy.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-oidc-deploy.yml new file mode 100644 index 0000000..4883c0c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-oidc-deploy.yml @@ -0,0 +1,65 @@ +# Deploy with a short-lived OIDC token instead of a stored credential. +# The target here is OpenBao/Vault JWT auth (self-hosted); the same pattern +# fits any OIDC-aware system. PRODUCTION DEPLOY: every run is gated by the +# `production` environment (required reviewers + deployment branch rule set in +# the repository settings, not in this file). See +# references/destructive-pipeline-actions.md before editing the deploy step. +name: deploy + +on: + workflow_dispatch: + inputs: + image-digest: + description: "Immutable image reference to deploy (sha256 digest)" + required: true + type: string + +permissions: {} + +# Never cancel a deploy mid-flight: queue instead. +concurrency: + group: deploy-production + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + environment: + name: production + url: https://app.example.com + permissions: + contents: read + id-token: write # lets the job request the OIDC JWT; grants nothing else + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Get OIDC token and exchange it for a short-lived Vault/OpenBao token + env: + BAO_ADDR: ${{ vars.BAO_ADDR }} # https URL, a repository/environment variable + BAO_ROLE: ci-deploy-production # role bound to repo + environment claims + run: | + set -euo pipefail + jwt=$(curl -sS -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=${BAO_ADDR}" | jq -r .value) + echo "::add-mask::${jwt}" + token=$(curl -sS --request POST \ + --data "$(jq -n --arg role "$BAO_ROLE" --arg jwt "$jwt" '{role:$role,jwt:$jwt}')" \ + "${BAO_ADDR}/v1/auth/jwt/login" | jq -r .auth.client_token) + echo "::add-mask::${token}" + echo "BAO_TOKEN=${token}" >> "$GITHUB_ENV" + + - name: Deploy (GitOps handoff - see devops-gitops-packaging) + env: + IMAGE_DIGEST: ${{ inputs.image-digest }} + run: | + set -euo pipefail + # Do NOT run kubectl/ssh against production from here when a GitOps + # controller owns the cluster: commit the new digest to the config + # repo instead. Replace with the project's real, reviewed command. + ./scripts/promote.sh "$IMAGE_DIGEST" + + - name: Smoke test + run: curl -fsS --retry 5 --retry-delay 5 https://app.example.com/healthz diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-release.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-release.yml new file mode 100644 index 0000000..291409d --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-release.yml @@ -0,0 +1,35 @@ +# Release automation: a pushed version tag creates a GitHub Release with +# generated notes. Tag creation itself stays a human (or commit-agent) act. +# PUBLISHES a release: see references/destructive-pipeline-actions.md. +name: release + +on: + push: + tags: ["v*.*.*"] + +permissions: {} + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +jobs: + release: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + environment: release # add required reviewers here for a human gate + permissions: + contents: write # create the release; nothing else + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Create draft release with generated notes + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + # --verify-tag aborts if the tag is missing on the remote; + # --draft lets a human review before publishing. + gh release create "$TAG" --verify-tag --generate-notes --draft diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-reusable-workflow.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-reusable-workflow.yml new file mode 100644 index 0000000..2eb1784 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/github-reusable-workflow.yml @@ -0,0 +1,56 @@ +# Reusable workflow (callee). Lives in .github/workflows/ of a shared repo. +# Caller example (pin by full SHA for any workflow you do not own): +# jobs: +# build: +# uses: my-org/ci-shared/.github/workflows/build-test.yml@ +# with: +# make-target: test +# secrets: +# registry_token: ${{ secrets.REGISTRY_TOKEN }} +name: build-test (reusable) + +on: + workflow_call: + inputs: + make-target: + description: Make target to run + required: false + type: string + default: test + runner: + description: Runner label (use a pinned image label, not -latest) + required: false + type: string + default: ubuntu-24.04 + secrets: + registry_token: + description: Token to pull private dependencies + required: false + outputs: + sha: + description: Commit that was built + value: ${{ jobs.run.outputs.sha }} + +permissions: {} + +jobs: + run: + runs-on: ${{ inputs.runner }} + timeout-minutes: 20 + # The caller can only reduce these, never raise them. + permissions: + contents: read + outputs: + sha: ${{ steps.meta.outputs.sha }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - id: meta + run: echo "sha=$GITHUB_SHA" >> "$GITHUB_OUTPUT" + - name: Run target + env: + # Untrusted/templated values go through env, never inline in run:. + TARGET: ${{ inputs.make-target }} + REGISTRY_TOKEN: ${{ secrets.registry_token }} + run: make -- "$TARGET" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/gitlab-ci.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/gitlab-ci.yml new file mode 100644 index 0000000..fc30299 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/gitlab-ci.yml @@ -0,0 +1,114 @@ +# .gitlab-ci.yml: rules, needs (DAG), environments, resource_group, protected +# production deploy. Keywords verified at docs.gitlab.com/ci/yaml on 2026-11-10. +# Adapt `image`, `script` lines and the runner `tags` to the project. +stages: + - build + - test + - deploy + +workflow: + # Run branch pipelines only when no merge request is open (avoids duplicates). + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS + when: never + - if: $CI_COMMIT_BRANCH + - if: $CI_COMMIT_TAG + auto_cancel: + on_new_commit: interruptible + +default: + image: node:24-bookworm-slim + interruptible: true # newer pipeline may cancel this job... + timeout: 20m + tags: [docker] # ...on runners carrying this tag + +variables: + # Non-sensitive config only. Secrets live in Settings > CI/CD > Variables + # (masked, protected), never in this file. + NPM_CONFIG_CACHE: "$CI_PROJECT_DIR/.npm" + +lint: + stage: test + needs: [] # starts immediately, does not wait for build + script: + - npm ci + - npm run lint + cache: + key: + files: [package-lock.json] + paths: [.npm/] + +build: + stage: build + script: + - npm ci --cache .npm + - npm run build + artifacts: + paths: [dist/] + expire_in: 14 days + cache: + key: + files: [package-lock.json] + paths: [.npm/] + policy: pull-push + +unit: + stage: test + needs: [build] # DAG: runs as soon as build finishes + parallel: + matrix: + - NODE: ["22", "24"] + image: node:${NODE}-bookworm-slim + script: + - npm ci + - npm test + +deploy_staging: + stage: deploy + needs: [build, unit] + resource_group: staging # one deploy at a time per environment + environment: + name: staging + url: https://staging.example.com + on_stop: stop_staging + auto_stop_in: 7 days + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + interruptible: false # never cancel a deploy mid-flight + script: + - ./scripts/promote.sh staging "$CI_COMMIT_SHA" + +stop_staging: + stage: deploy + needs: [] + environment: + name: staging + action: stop + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: manual + interruptible: false + script: + - ./scripts/teardown.sh staging + +# PRODUCTION DEPLOY. Needs a protected environment (Settings > CI/CD > +# Protected environments, Premium/Ultimate) so only approved roles can run it. +# See references/destructive-pipeline-actions.md before editing. +deploy_production: + stage: deploy + needs: [deploy_staging] + resource_group: production + environment: + name: production + url: https://app.example.com + rules: + - if: $CI_COMMIT_TAG + when: manual + manual_confirmation: "Deploy $CI_COMMIT_TAG to PRODUCTION?" + interruptible: false + id_tokens: + BAO_ID_TOKEN: + aud: https://bao.example.com # audience the secret store expects + script: + - ./scripts/promote.sh production "$CI_COMMIT_SHA" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/gitlab-component.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/gitlab-component.yml new file mode 100644 index 0000000..4ff1f4d --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/templates/gitlab-component.yml @@ -0,0 +1,44 @@ +# GitLab CI/CD component. Save as templates/build-image.yml inside a +# component project (a project whose repo has templates/, README.md and a +# release tag). A `spec:` header, then `---`, then the jobs. +# +# Consumer (same GitLab instance only; pin to a released version, never ~latest +# for production): +# include: +# - component: $CI_SERVER_FQDN/my-org/ci-components/build-image@1.0.0 +# inputs: +# stage: build +# image-name: registry.example.com/my-org/app +spec: + inputs: + stage: + default: build + description: Pipeline stage the job runs in + image-name: + description: Full image repository to push + dockerfile: + default: Dockerfile + platforms: + default: linux/amd64 + options: [linux/amd64, "linux/amd64,linux/arm64"] +--- +build-image: + stage: $[[ inputs.stage ]] + image: docker:cli + services: + - docker:dind + variables: + DOCKER_TLS_CERTDIR: "/certs" + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_COMMIT_TAG + script: + - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY" + - docker buildx create --use + - > + docker buildx build + --platform "$[[ inputs.platforms ]]" + --file "$[[ inputs.dockerfile ]]" + --tag "$[[ inputs.image-name ]]:$CI_COMMIT_SHA" + --provenance=mode=max --sbom=true + --push . diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/troubleshooting.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/troubleshooting.md new file mode 100644 index 0000000..9a42948 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-cicd/references/troubleshooting.md @@ -0,0 +1,67 @@ +--- +name: troubleshooting +description: Fixed-order troubleshooting for failing CI/CD pipelines on GitHub Actions and GitLab CI - find the failing step, classify the failure, read the right log, reproduce locally, then change +when-to-use: A pipeline is red, stuck queued, unexpectedly skipped, slow, or a deploy job behaves differently from last time +keywords: failed run, queued, skipped, no runner, permission denied, 403, OIDC failure, cache miss, flaky, gh run view, log-failed, CI Lint, rules not matching +priority: high +related: destructive-pipeline-actions.md, github-actions-workflows.md, gitlab-ci.md, pipeline-design.md +--- + +# Pipeline Troubleshooting + +**Load when:** a pipeline fails, hangs or behaves unexpectedly. Evidence before fixes; reading is free, re-running a deploy is not ([destructive-pipeline-actions.md](destructive-pipeline-actions.md)). + +## Order + +1. **Which run, which commit, which job.** `gh run list --workflow --limit 10`, `gh run view ` (jobs and steps), `gh run view --log-failed`. GitLab: pipeline page, failed job log, "Job dependencies" graph. +2. **Classify** the failure: + +| Class | Signal | Next | +|-------|--------|------| +| Config invalid | run never started / "invalid workflow file" / pipeline "yaml invalid" | GitHub: the Actions tab error and `actionlint` locally; GitLab: CI Lint | +| Not scheduled | no run, no pipeline | trigger filters (`on:` branches/paths), `workflow:rules`, `[CI SKIP]`, disabled workflow, wrong default branch | +| Waiting | job `queued` / "pending" / "stuck" | no runner with the labels/tags; runner offline/busy; concurrency group or `resource_group` held; environment awaiting approval | +| Script failure | non-zero exit in a step | reproduce locally with the same command and image | +| Permission | 403/401, "Resource not accessible by integration" | job `permissions:`, protected variable on unprotected ref, fork PR without secrets, environment scope | +| Identity (OIDC) | token exchange refused | claims vs trust rule: `sub`, `aud`, environment name, ref; `id-token: write` missing | +| Environment drift | "worked yesterday" | runner image/`-latest` label moved, unpinned action or base image, expired cache, changed variable | +| Infra | timeouts, runner lost, 5xx from registry | retry once (infra only), check runner/registry health | + +3. **Reproduce** outside CI with the exact command, image and env; GitHub debug logging via the "Enable debug logging" re-run option; GitLab: run the job's image locally with the same script. +4. **Change one thing**, push to a branch (not the default), compare. Never stack a second fix on an unexplained first one. +5. **Prove**: the previously failing job is green on a fresh run of the same commit class; for deploy jobs, the health check and the platform's rollout status, not the job's exit code alone. + +## Frequent causes + +- **Stuck "queued"**: `runs-on` label that no runner has (GitHub), job `tags:` with no matching online runner (GitLab), or the concurrency lock held by a hung older run. +- **Skipped job**: a `needs:` dependency was skipped or failed; GitLab `rules` matched nothing; GitHub `if:` evaluates false (`success()` default); path filters. +- **Different result on re-run**: a re-run reuses the original commit and event payload, but reads secrets/variables as they are now. +- **Cache not restored**: key changed (lockfile hash), entry evicted, or `cache-mode` read-only for the trigger (GitHub); GitLab cache is per runner unless distributed, and not shared between protected and unprotected branches by default. +- **OIDC 403 at the target**: trust rule expects `environment:production` in `sub` but the job has no `environment:`; custom `aud` mismatch; for repos created after 2026-07-15 the default `sub` format changed. +- **`secrets` empty in a reusable workflow**: caller did not pass it (empty string, no error) - mark `required: true`. +- **Duplicate pipelines in GitLab**: branch and merge-request pipelines both created; add `workflow:rules`. +- **Self-hosted runner flaky**: leftover files from earlier jobs (non-ephemeral), full disk, runner app auto-updated mid-job. +- **Slow**: look at queue time vs run time, then the critical path (GitLab "needs" view), then dependency/image download, then test duration. + +## Local checks (read-only) + +```bash +gh run list --limit 10 +gh run view --log-failed +actionlint .github/workflows/*.yml # if installed; static check of workflow syntax and expressions +gh api repos///actions/runners --jq '.runners[] | {name,status,busy,labels:[.labels[].name]}' +``` + +GitLab: CI Lint in the Pipeline editor, the pipeline's "Jobs" and "Job dependencies" views, and the runner page for status and tags. + +## Sources (read 2026-11-10) + +- https://docs.github.com/en/actions/reference/security/oidc , .../workflows-and-actions/workflow-syntax , .../how-tos/reuse-automations/reuse-workflows +- https://docs.gitlab.com/ci/jobs/job_rules/ , https://docs.gitlab.com/ci/caching/ , https://docs.gitlab.com/ci/yaml/ +- `gh run view|list --help` (gh 2.103.3) + +## Not verified + +- `actionlint` is not installed in this environment (latest release v1.7.12, 2026-03-30 via `gh api`); command shown from its standard usage. +- The GitHub debug-logging re-run option and the runners list API field names were not re-read today. +- GitLab "pending/stuck" job messages and the lint API endpoint were not read. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/SKILL.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/SKILL.md new file mode 100644 index 0000000..9333695 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/SKILL.md @@ -0,0 +1,162 @@ +--- +name: devops-databases +description: Use when operating (not designing) databases - PostgreSQL installation and tuning (postgresql.conf, pg_hba.conf, roles), PgBouncer pooling, backups and point-in-time recovery (pg_dump, pg_basebackup, pgBackRest, restore tests), streaming and logical replication, Patroni high availability and failover/switchover, major version upgrades (pg_upgrade, logical route), vacuum/bloat/transaction ID wraparound, monitoring queries, lock-safe schema changes in production, PostgreSQL on Kubernetes with CloudNativePG; plus MySQL/MariaDB (GTID replication, backup tools), Redis/Valkey (RDB/AOF, Sentinel vs Cluster) and MongoDB pointers. Not for application schema or ORM modelling (framework experts), generic backup/storage strategy (devops-storage-backup), exporters and dashboards (devops-observability), secret stores (devops-secrets) or managed cloud databases. +versions: + postgresql: "18.6 / 17.11 / 16.15 / 15.19 supported; 14.24 final release 2026-11-12; 19 RC1" + pgbackrest: "2.60.0" + patroni: "4.1.5" + pgbouncer: "1.26.1" + cloudnativepg: "1.30.1" + mysql: "8.4 LTS, 9.7 LTS, 26.7 / 26.10 innovation" + mariadb: "12.3 LTS, 13.0 rolling" + redis: "8.10 (8.12 RC2)" + valkey: "9.1.3 / 9.0.7 / 8.1.11 / 8.0.12 / 7.2.15" +user-invocable: true +references: + - references/postgres-versions-config.md + - references/postgres-roles-auth.md + - references/connection-pooling-pgbouncer.md + - references/backup-pitr-pgbackrest.md + - references/backup-logical-physical.md + - references/replication-streaming-logical.md + - references/ha-patroni.md + - references/major-upgrades.md + - references/vacuum-bloat-wraparound.md + - references/monitoring-queries.md + - references/schema-changes-safe.md + - references/postgres-on-kubernetes-cnpg.md + - references/destructive-actions.md + - references/mysql-mariadb-ops.md + - references/redis-valkey-ops.md + - references/mongodb-pointers.md + - references/templates/postgresql-tuned.conf + - references/templates/pg_hba.conf.example + - references/templates/pgbouncer.ini + - references/templates/pgbackrest.conf + - references/templates/pgbackrest-commands.md + - references/templates/patroni.yml + - references/templates/cnpg-cluster.yaml + - references/templates/cnpg-scheduledbackup.yaml + - references/templates/monitoring-queries.sql + - references/templates/lock-safe-migration-checklist.md + - references/templates/restore-test-pitr-runbook.md +related-skills: + - devops-storage-backup + - devops-observability + - devops-kubernetes + - devops-k8s-cluster + - devops-secrets + - devops-linux + - devops-networking + - devops-sre-ops +--- + + +Covers running databases in production as an operator: PostgreSQL in depth +(config, auth, pooling, backup and PITR, replication, Patroni HA, upgrades, +vacuum, monitoring, safe migrations, CloudNativePG), then MySQL/MariaDB and +Redis/Valkey essentials and MongoDB pointers. Does not cover application schema +or ORMs (framework experts), generic backup and storage design +(devops-storage-backup), exporters and dashboards (devops-observability) or +managed cloud databases. + + +# DevOps Databases + +## Agent Workflow (MANDATORY) + +Before ANY change on a database host or cluster, spawn in parallel, one `Task` call each: + +1. **fuse-ai-pilot:explore-codebase** - find existing configs, migrations, operator manifests, backup jobs +2. **fuse-ai-pilot:research-expert** - verify the parameter or command for the server's major version at postgresql.org/docs, pgbackrest.org, patroni.readthedocs.io, cloudnative-pg.io +3. **mcp__context7__query-docs** - `/websites/postgresql`, `/pgbackrest/pgbackrest`, `/patroni/patroni`, `/pgbouncer/pgbouncer`, `/cloudnative-pg/cloudnative-pg` + +Then read the target read-only (`SELECT version()`, `SHOW ALL`, backup `info`, replication state) BEFORE proposing anything. After writing files, run **fuse-ai-pilot:sniper**. + +## Versions (checked 2026-11-10) + +PostgreSQL supports a major version for 5 years; 18.6, 17.11, 16.15, 15.19 are supported and **14.24 gets its final release on 2026-11-12** - treat 14 as end of life this week; 13 and older are unsupported; 19 is at RC1 (https://www.postgresql.org/support/versioning/). pgBackRest 2.60.0 (2026-10-18), Patroni 4.1.5 (2026-08-12), PgBouncer 1.26.1 (2026-10-14, fixes three CVEs: run 1.26.1 or later), CloudNativePG 1.30.1 (1.30.x supports Kubernetes 1.34-1.36 and PostgreSQL 14-18). Details and sources inside each reference. + +## Overview + +| Area | Decision in one line | +|------|----------------------| +| **Version** | New builds on the newest supported major with the current minor; plan a move off any major within 12 months of its final release | +| **Pooling** | PgBouncer in `transaction` mode with `max_prepared_statements` > 0; session features break in that mode | +| **Backup** | Physical (pgBackRest or CNPG Barman Cloud plugin) + WAL archive for PITR; `pg_dump` is a complement, never the only copy; a backup counts only after a restore test | +| **HA** | Patroni + a 3-node DCS (etcd) on VMs; CloudNativePG on Kubernetes; failover automatic, switchover planned | +| **Upgrade** | `pg_upgrade --check` first; logical replication route for near-zero downtime; take a new base backup after | +| **Maintenance** | Keep autovacuum on; watch `age(datfrozenxid)`; never `VACUUM FULL` on a live table without a window | +| **Migrations** | `lock_timeout` on every DDL, `CREATE INDEX CONCURRENTLY`, `NOT VALID` then `VALIDATE` | +| **Other engines** | MySQL/MariaDB: GTID replication, official dump/backup tools; Redis/Valkey: persistence is a choice, Sentinel for HA | + +## Critical Rules + +1. **Version first** - `SELECT version();` before any parameter or command; parameters appear and disappear between majors +2. **Backup proof before any risky step** - a listed backup is not a proof; the proof is a completed restore test ([restore-test-pitr-runbook.md](references/templates/restore-test-pitr-runbook.md)) +3. **Destructive actions are proposed, not run** - ACTION / EFFECT / PROOF / WAIT-FOR-GO, naming what is unrecoverable ([destructive-actions.md](references/destructive-actions.md)) +4. **Read-only first** - discovery queries and `info`/`status` commands; the agent never runs a statement against a real database that it has not shown +5. **No credential on a command line or in a file** - `CHANGE_ME` placeholders in templates; real secrets from `devops-secrets`, `PGPASSFILE` or `.pgpass` +6. **One writer** - never promote or restore while another node may still accept writes (split brain) +7. **Never invent a parameter, flag or CRD field** - cite the doc page; unknown = "not verified" + +## Reference Guide + +### Concepts + +| Topic | Reference | When to Consult | +|-------|-----------|-----------------| +| **Versions and config** | [postgres-versions-config.md](references/postgres-versions-config.md) | Install, `postgresql.conf` memory/WAL/checkpoints | +| **Roles and auth** | [postgres-roles-auth.md](references/postgres-roles-auth.md) | `pg_hba.conf`, SCRAM, least-privilege roles | +| **Pooling** | [connection-pooling-pgbouncer.md](references/connection-pooling-pgbouncer.md) | Too many connections, transaction mode pitfalls | +| **Backup and PITR** | [backup-pitr-pgbackrest.md](references/backup-pitr-pgbackrest.md) | pgBackRest stanza, WAL archive, restore to a point in time | +| **Dump and base backup** | [backup-logical-physical.md](references/backup-logical-physical.md) | `pg_dump`, `pg_basebackup`, incremental, Barman pointer | +| **Replication** | [replication-streaming-logical.md](references/replication-streaming-logical.md) | Replicas, slots, lag, logical replication | +| **High availability** | [ha-patroni.md](references/ha-patroni.md) | Patroni, etcd, failover vs switchover, split brain | +| **Major upgrades** | [major-upgrades.md](references/major-upgrades.md) | `pg_upgrade`, logical route, extensions | +| **Vacuum** | [vacuum-bloat-wraparound.md](references/vacuum-bloat-wraparound.md) | Bloat, autovacuum tuning, XID wraparound | +| **Monitoring** | [monitoring-queries.md](references/monitoring-queries.md) | Activity, locks, slow queries, lag | +| **Safe migrations** | [schema-changes-safe.md](references/schema-changes-safe.md) | DDL on a busy table | +| **Kubernetes** | [postgres-on-kubernetes-cnpg.md](references/postgres-on-kubernetes-cnpg.md) | CloudNativePG cluster, backup, upgrade, alternatives | +| **Destructive catalogue** | [destructive-actions.md](references/destructive-actions.md) | Any irreversible command | +| **MySQL / MariaDB** | [mysql-mariadb-ops.md](references/mysql-mariadb-ops.md) | Backup, GTID replication, divergence | +| **Redis / Valkey** | [redis-valkey-ops.md](references/redis-valkey-ops.md) | Persistence, Sentinel vs Cluster, eviction | +| **MongoDB** | [mongodb-pointers.md](references/mongodb-pointers.md) | One-line pointers only | + +### Templates + +| Template | When to Use | +|----------|-------------| +| [postgresql-tuned.conf](references/templates/postgresql-tuned.conf) | Starting parameters with the reasoning per line | +| [pg_hba.conf.example](references/templates/pg_hba.conf.example) | SCRAM, TLS, narrow CIDRs, replication entries | +| [pgbouncer.ini](references/templates/pgbouncer.ini) | Transaction pooling with prepared statements | +| [pgbackrest.conf](references/templates/pgbackrest.conf) | Repository, retention, archive settings | +| [pgbackrest-commands.md](references/templates/pgbackrest-commands.md) | Backup, info, verify, restore command sheet | +| [patroni.yml](references/templates/patroni.yml) | One Patroni node with etcd3 | +| [cnpg-cluster.yaml](references/templates/cnpg-cluster.yaml) | CloudNativePG Cluster, ObjectStore, Pooler | +| [cnpg-scheduledbackup.yaml](references/templates/cnpg-scheduledbackup.yaml) | Scheduled base backups via the plugin | +| [monitoring-queries.sql](references/templates/monitoring-queries.sql) | Read-only health queries | +| [lock-safe-migration-checklist.md](references/templates/lock-safe-migration-checklist.md) | Before any production DDL | +| [restore-test-pitr-runbook.md](references/templates/restore-test-pitr-runbook.md) | Proving a backup and a PITR | + +## Quick Reference + +```bash +psql "host=db1 dbname=postgres user=ops" -Atc "select version(), pg_is_in_recovery()" # password from PGPASSFILE +pgbackrest --stanza=main info # backups and WAL range, read-only +patronictl -c /etc/patroni/patroni.yml list # topology and lag, read-only +kubectl --context "$CTX" -n "$NS" get cluster.postgresql.cnpg.io # CNPG status, read-only +``` + +## Best Practices + +### DO +- Keep WAL archiving on in production and alert on `pg_stat_archiver.failed_count` +- Test restores on a schedule and record the date, the target time and the row count compared +- Run PgBouncer, backups and monitoring with their own least-privilege roles +- Put retention, `max_slot_wal_keep_size` and disk alerts in place before a replica or slot is added + +### DON'T +- Use `pg_dump` alone as disaster recovery, or run a restore over a live instance +- Delete a replication slot, a Cluster or a PVC to "clean up" without the proof in [destructive-actions.md](references/destructive-actions.md) +- Edit `postgresql.conf` on a Patroni or CloudNativePG member by hand: the manager owns it diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/backup-logical-physical.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/backup-logical-physical.md new file mode 100644 index 0000000..cf8db77 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/backup-logical-physical.md @@ -0,0 +1,63 @@ +--- +name: backup-logical-physical +description: pg_dump, pg_dumpall and pg_restore (logical) versus pg_basebackup, incremental backups and pg_combinebackup (physical), when each is the right tool, and what a dump does not capture +when-to-use: Copying one database to another host, taking a pre-migration safety dump, seeding a replica, scripting a base backup without pgBackRest, or deciding logical vs physical +keywords: pg_dump, pg_restore, pg_dumpall, pg_basebackup, pg_combinebackup, pg_verifybackup, summarize_wal, custom format, directory format, logical backup +priority: high +related: backup-pitr-pgbackrest.md, major-upgrades.md, replication-streaming-logical.md, destructive-actions.md +--- + +# Logical and Physical Backup Tools + +**Load when:** choosing how to copy PostgreSQL data. For disaster recovery with PITR use [backup-pitr-pgbackrest.md](backup-pitr-pgbackrest.md); this file covers the built-in tools around it. Where backups are stored and the 3-2-1 rule: `devops-storage-backup`. + +## Which tool + +| Need | Tool | Why | +|------|------|-----| +| Whole-cluster disaster recovery, PITR | physical + WAL archive (pgBackRest, CNPG plugin) | Restores the exact files and replays WAL | +| Seed a streaming replica | `pg_basebackup` | Official tool, `-R` writes the standby setup | +| Move one database, a few tables, or cross-major | `pg_dump` + `pg_restore` | Logical; independent of file layout and major version | +| Safety net before a risky migration | `pg_dump -Fc` of the affected schema, plus a pgBackRest restore point | Fast to restore a few tables | +| Roles and tablespaces | `pg_dumpall --globals-only` | `pg_dump` dumps one database and does not include cluster-wide roles | + +`pg_dump` takes a consistent snapshot without blocking readers or writers, but the PostgreSQL manual itself says that, except in simple cases, it is generally not the right choice for taking regular backups of production systems. A dump has no PITR, restores slowly on large data, and omits roles unless you dump globals. + +## pg_dump / pg_restore + +- Formats: plain SQL (`-Fp`), **custom** (`-Fc`: compressed, selectable, supports parallel restore) and **directory** (`-Fd`: the only format that supports **parallel dump** with `-j`). Prefer `-Fd -j N` for large databases and `-Fc` for small ones. +- Run `pg_dump` with the **newest** client version available when targeting a newer server; restore with `pg_restore -j N` into an empty, correctly versioned database. +- `pg_restore --clean --if-exists` drops objects before recreating them: that is an overwrite of the target database - use it only on a database you intend to replace ([destructive-actions.md](destructive-actions.md)). `--single-transaction` makes the restore all-or-nothing (incompatible with parallel jobs). +- Always restore into a **new database name** first, check counts and constraints, then switch the application. +- Extensions must exist (same or compatible version) on the target; check `\dx` on both sides. +- Dump credentials come from `PGPASSFILE` or a service file, not from the command line. + +```bash +pg_dump "host=db1 dbname=appdb user=backup" -Fd -j 4 -f /backup/appdb.dir # read-only on the source +pg_dumpall "host=db1 user=backup" --globals-only -f /backup/globals.sql # roles and tablespaces +pg_restore -l /backup/appdb.dir | head # list contents, writes nothing +createdb -h db2 appdb_restore_test # change: new empty database +pg_restore -h db2 -d appdb_restore_test -j 4 /backup/appdb.dir # change: loads into the NEW database +``` + +## pg_basebackup + +Takes a physical copy of a running cluster over the replication protocol. Needs a role with `REPLICATION` and a matching `pg_hba.conf` line ([postgres-roles-auth.md](postgres-roles-auth.md)). Useful options: `-D` target directory, `-X stream` (include the WAL needed for consistency), `-c fast` (immediate checkpoint instead of a spread one, otherwise the tool looks idle for a while), `-R` (`--write-recovery-conf`: creates `standby.signal` and appends connection settings to `postgresql.auto.conf`, for building a replica), `-Ft` (tar). The target directory must be empty. `pg_verifybackup` checks a backup against its manifest. + +**Incremental (PostgreSQL 17+)**: set `summarize_wal = on` on the source (default off; runs the WAL summarizer, works on a primary or a standby), take a full base backup, then `pg_basebackup --incremental=`. Relation files become smaller incremental files; non-relation files are copied whole. To restore, `pg_combinebackup` reconstructs a synthetic full backup from the chain: list the backups oldest to newest and write to a new directory with `-o`. Every backup in the chain must be kept. Without a WAL archive this is not PITR; for PITR use pgBackRest. + +## Barman + +Pointer: Barman is a separate backup server for PostgreSQL ([backup-pitr-pgbackrest.md](backup-pitr-pgbackrest.md) explains the choice). Version and CLI not verified today. + +## Sources + +- https://www.postgresql.org/docs/18/app-pgdump.html , app-pgrestore.html , app-pg-dumpall.html , app-pgbasebackup.html , app-pgcombinebackup.html (read 2026-11-10; formats, `--clean`, `--globals-only`, `--write-recovery-conf`, `pg_verifybackup`) +- https://www.postgresql.org/docs/18/runtime-config-wal.html (`summarize_wal`, default off) +- Context7 `/websites/postgresql`: "Making an Incremental Backup" (`pg_basebackup --incremental`), `pg_combinebackup` description + +## Not verified + +- The incremental-backup feature is described for the version 18 manual; that it first appeared in 17 is from general knowledge (check the 17 release notes). +- Exact `pg_basebackup -X`/`-c` option wording was not re-read on the page today (the extract did not return those paragraphs); they are long-standing options. +- Parallel-dump behaviour with `--single-transaction` and snapshot options (`--snapshot`) were not read. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/backup-pitr-pgbackrest.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/backup-pitr-pgbackrest.md new file mode 100644 index 0000000..e5b6189 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/backup-pitr-pgbackrest.md @@ -0,0 +1,74 @@ +--- +name: backup-pitr-pgbackrest +description: PostgreSQL continuous backup and point-in-time recovery with pgBackRest - stanza, WAL archiving, full/diff/incr backups, retention, verify, restore and PITR, and the restore test that proves it +when-to-use: Setting up or reviewing PostgreSQL backups, planning RPO/RTO, recovering to a point in time, or before any change that could lose data +keywords: pgbackrest, stanza, archive-push, archive_command, repo1-retention-full, restore --delta, --type=time, target-action, expire, stanza-delete, PITR, verify, check +priority: critical +related: backup-logical-physical.md, destructive-actions.md, major-upgrades.md, templates/pgbackrest.conf, templates/pgbackrest-commands.md, templates/restore-test-pitr-runbook.md +--- + +# Backup and PITR with pgBackRest + +**Load when:** PostgreSQL data must survive a mistake, a disk loss or a site loss. Generic 3-2-1 strategy, repository storage and DR drills belong to `devops-storage-backup`; this file is the PostgreSQL-specific part. Version: pgBackRest **2.60.0** (2026-10-18). + +## Model + +A backup = a **base backup** (full, differential or incremental) + the **WAL archive** from the base backup's start onward. Base backups alone restore to the end of the backup; the WAL archive gives any point after it (PITR). A WAL archive without a base backup is useless; a base backup without a healthy WAL archive may not even be consistent. pgBackRest waits for the WAL needed by a backup to reach the repository before it finishes. + +Features that matter to an operator: parallel processes (`process-max`), lz4/zstd compression, block-level incremental (`repo1-block`) and file bundling (`repo1-bundle`), multiple repositories (local fast restore + remote or object store long retention), backup from a standby (`backup-standby`), page checksum validation, optional repository encryption (`repo1-cipher-type`). Repository kinds include POSIX, S3, Azure and GCS. + +## Setup sequence (each step is a change: show it, get the go) + +1. Repository host or path with its own disk/bucket and credentials; on the database host `pgbackrest.conf` ([templates/pgbackrest.conf](templates/pgbackrest.conf)). +2. `postgresql.conf`: `archive_mode = on` (restart), `archive_command = 'pgbackrest --stanza=main archive-push %p'`, `wal_level = replica` or higher. +3. `pgbackrest --stanza=main stanza-create` (on the host that owns the repository config), then `pgbackrest --stanza=main check` - it validates configuration and `archive_command` end to end. Do not proceed if `check` fails. +4. First `backup --type=full`, then `info`. +5. Schedule: weekly full, daily differential or incremental, via cron or systemd timer (`devops-linux`); `archive-async=y` plus a `spool-path` on busy systems. +6. **Restore test on a schedule** ([templates/restore-test-pitr-runbook.md](templates/restore-test-pitr-runbook.md)) and alerts: `pg_stat_archiver.failed_count` rising, `last_archived_time` stale, `pgbackrest info` newest backup older than the policy. + +Command sheet with read-only vs write markers: [templates/pgbackrest-commands.md](templates/pgbackrest-commands.md). + +## Retention + +`repo1-retention-full` (count, or time with `repo1-retention-full-type`) keeps that many full backups; differentials are count-based and expire with their full; incrementals expire with their full or differential. WAL is retained for backups that have not expired (per-repo `retention-archive*` can change this; not recommended). `expire` runs automatically after each successful backup and can be run by hand; the archive retention is re-checked every run. Since 2.60.0 `expire` refuses to run if the oldest retained backup would contain page checksum errors (`checksum-page-error`, default on for `expire` and `restore`, off for `backup`): a backup that finishes with checksum warnings is a corruption signal to investigate, not noise. + +## Restore and PITR + +```bash +# Read-only first: what exists and what WAL range is covered +pgbackrest --stanza=main info +# On the TARGET host, PostgreSQL stopped, target data directory empty or --delta (see destructive-actions.md): +pgbackrest --stanza=main --type=time --target="2026-11-10 09:41:00+00" \ + --target-action=pause --delta restore +# Start PostgreSQL; it replays WAL and PAUSES at the target. Inspect read-only, then decide: +# psql -Atc "select pg_is_in_recovery(), now()" +# psql -Atc "select pg_promote()" -- accept this point: becomes a primary on a new timeline +# or restore again with another --target -- wrong point +``` + +- Recovery targets for `restore --type`: `time`, `lsn`, `xid`, `name` (a marker made by `SELECT pg_create_restore_point('before_migration');`), `immediate` (end of the backup), `standby` (add recovery config for a replica), `default`, `preserve`, `none`. +- `--target-action` supports `pause` (default), `promote`, `shutdown`. Use `pause` for incident PITR so the result can be checked before it becomes a primary; use `promote` in automated restore tests. +- `--delta` restores in place by reusing files that already match and deleting the rest. It is destructive for the existing data directory: only on a directory you intend to replace and only after the proof listed in [destructive-actions.md](destructive-actions.md). Never on a live instance. +- Restore to a **different host or path** first whenever the goal is to recover a few rows or inspect: then copy the data out with `pg_dump`/`COPY`. Overwriting production is the last resort. +- A restore with an invalid page checksum now stops with an error and leaves the cluster unstartable; re-run the restore (a different backup set with `--set`) rather than starting it. `--no-checksum-page-error` downgrades to a warning only with an explicit decision. +- After promotion take a new full backup at once; recovery created a new timeline. + +## Dangerous commands + +`expire` (deletes backups and WAL), `stanza-delete` (permanently removes all backups and archives of a stanza; needs the cluster shut down or `--force`) and `restore --delta` over a live path are in [destructive-actions.md](destructive-actions.md). Do not delete files from the repository by hand. + +## Barman + +Barman is the other widely used PostgreSQL backup manager (backup server model, `barman-cloud-*` tools; CloudNativePG uses the Barman Cloud plugin). Its version and current CLI were not verified today: read https://docs.pgbarman.org before use. Choose one tool per cluster; do not run two archivers on the same `archive_command`. + +## Sources + +- https://pgbackrest.org/ (2.60.0 current stable, features), https://pgbackrest.org/release.html (2.60.0 notes of 2026-10-18), https://pgbackrest.org/command.html (`expire`, `stanza-delete`, `check`, `--target-action`), https://pgbackrest.org/configuration.html , https://pgbackrest.org/user-guide.html (all read 2026-11-10) +- Context7 `/pgbackrest/pgbackrest`: `archive_command`/`archive_mode` snippet, `restore --type=time` example, `restore --type` values, config option table +- https://www.postgresql.org/docs/18/functions-admin.html (`pg_promote`, `pg_wal_replay_resume`, `pg_create_restore_point`), https://www.postgresql.org/docs/18/runtime-config-wal.html (`recovery_target_action` default `pause`) + +## Not verified + +- `process-max` sizing, `repo1-bundle`/`repo1-block` behaviour on your storage, and S3/Azure/GCS option names beyond the configuration page were not tested. +- Barman's current version, CLI and support status. +- The remediation steps of the 2.59.3 "weak encryption subkeys and salts" notice (2026-10-04) were read only up to its impact paragraph: encrypted repositories created before 2.59.3 may hold weak subkeys in `archive.info`/`backup.info` that an upgrade does not replace (unencrypted repositories are not affected); read the full notice at https://pgbackrest.org/news.html#weak-encryption before trusting an old encrypted stanza. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/connection-pooling-pgbouncer.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/connection-pooling-pgbouncer.md new file mode 100644 index 0000000..454792e --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/connection-pooling-pgbouncer.md @@ -0,0 +1,70 @@ +--- +name: connection-pooling-pgbouncer +description: PgBouncer pool modes, sizing, prepared statements in transaction mode, authentication (auth_file, auth_query), features that break under transaction pooling, and the 2026 CVE fixes +when-to-use: Too many PostgreSQL connections, connection storms from many app instances, deciding session vs transaction pooling, debugging "prepared statement does not exist" or lost SET/LISTEN behaviour +keywords: pgbouncer, pool_mode, transaction pooling, default_pool_size, max_client_conn, max_prepared_statements, auth_query, auth_type, server_reset_query, SHOW POOLS +priority: high +related: postgres-roles-auth.md, postgres-versions-config.md, postgres-on-kubernetes-cnpg.md, templates/pgbouncer.ini +--- + +# Connection Pooling with PgBouncer + +**Load when:** connection counts, latency from connection setup, or an application misbehaves behind a pooler. Each PostgreSQL backend is a process with memory; a pooler multiplexes many client connections onto few server connections. + +## Version and security + +Run **PgBouncer 1.26.1 or later** (2026-10-14). 1.26.1 fixes CVE-2026-107350 (a second StartupMessage during SCRAM auth lets a client who knows one user's password log in as another SCRAM user, 1.17.0-1.26.0), CVE-2026-107349 (CPU spin on an empty startup option name when `ignore_startup_parameters` is non-empty, 1.20.0-1.26.0) and CVE-2026-107348 (infinite loop with an oversized `max_packet_size`, non-default only). 1.26.0 fixed CVE-2026-19888, CVE-2026-6668, CVE-2026-6669. 1.26.0 also tracks `search_path` (PostgreSQL 18+) and `default_transaction_read_only` by default; `ignore_startup_parameters = search_path` restores the 1.25 behaviour, and later releases are expected to fix `search_path` tracking bugs. Check the changelog before upgrading. + +## Pool modes (`pool_mode`) + +| Mode | Server connection released | Use | +|------|---------------------------|-----| +| `session` (default) | when the client disconnects | Apps that rely on session state; least multiplexing | +| `transaction` | when the transaction ends | Default choice for web apps; the application must not use session state | +| `statement` | after each statement; multi-statement transactions refused | Rare, autocommit-only workloads | + +Features by mode (PgBouncer "SQL feature map"): under **transaction** pooling these never work - `SET`/`RESET` (session-level), `LISTEN`, `WITH HOLD` cursors, SQL-level `PREPARE`/`DEALLOCATE`, `PRESERVE`/`DELETE ROWS` temp tables, `LOAD`, session-level advisory locks. Use `SET LOCAL` inside a transaction instead of `SET`. Startup parameters (`client_encoding`, `DateStyle`, `IntervalStyle`, `TimeZone`, `standard_conforming_strings`, `application_name`) are tracked and work in both modes. Migration tools that take session advisory locks (many do) must connect directly to PostgreSQL or through a session-mode pool. + +## Prepared statements + +Protocol-level named prepared statements work in transaction and statement mode only when `max_prepared_statements` is non-zero (PgBouncer 1.21.0+); the value is the LRU cache size per server connection, larger than the application's commonly used statements (the release note suggests 100 as reasonable). SQL-level `PREPARE`/`EXECUTE` are not tracked. If a prepared statement's result type changes you may see `cached plan must not change result type`. Many drivers need a setting to use protocol-level prepares; check the driver docs. + +## Sizing + +- `default_pool_size` (default 20) is per user/database pair. Server connections ~= pairs x pool size; keep their sum below `max_connections` minus `superuser_reserved_connections` (default 3) minus replication and admin sessions. +- `max_client_conn` (default 100): raise to the real client count; raise the file descriptor limit of the service (`LimitNOFILE` in the unit - `devops-linux`). +- `reserve_pool_size` (default 0) with `reserve_pool_timeout` (default 5s) absorbs bursts; `max_db_connections` / `max_user_connections` cap per database or user. +- `query_wait_timeout` (default 120s) disconnects clients that wait for a server connection; `server_idle_timeout` default 600s; `server_lifetime` default 3600s. +- PgBouncer is single-threaded: for more CPU, several instances can listen on the same port with `so_reuseport`, or run one per application node. +- Pool in front of every replica role separately (read/write vs read-only endpoints); a pool in front of Patroni follows the leader through HAProxy or a VIP, not by editing PgBouncer ([ha-patroni.md](ha-patroni.md)). With CloudNativePG use the `Pooler` resource ([postgres-on-kubernetes-cnpg.md](postgres-on-kubernetes-cnpg.md)). + +## Authentication + +- `auth_type`: default is `md5` (a user with a SCRAM secret is authenticated with SCRAM automatically). Set `scram-sha-256` explicitly; `plain` is deprecated; `trust` skips the check. +- `auth_file` lists users (`"user" "SCRAM-SHA-256$..."`). For many users use `auth_user` + `auth_query`: PgBouncer reads credentials from the database. Direct access to `pg_authid` needs superuser; use a non-superuser role that calls a `SECURITY DEFINER` function, installed in every database (the query runs inside the target database). +- TLS toward clients (`client_tls_sslmode`) and servers (`server_tls_sslmode`); certificates via `devops-networking`. +- Admin console: connect to the virtual `pgbouncer` database as an `admin_users` member; `stats_users` read-only. Read-only commands: `SHOW POOLS;`, `SHOW STATS;`, `SHOW CLIENTS;`, `SHOW SERVERS;`. `RELOAD;` re-reads the config; `PAUSE`/`RESUME` quiesce a pool (planned switchover helper, still a change: ask first). + +## Troubleshooting + +| Symptom | Likely cause | Check | +|---------|--------------|-------| +| `no more connections allowed (max_client_conn)` | client count above the limit | `SHOW CLIENTS;` count, raise limit and fd limit | +| Clients stall, `cl_waiting` > 0 in `SHOW POOLS` | pool exhausted by slow or idle-in-transaction sessions | `pg_stat_activity` `state`, [monitoring-queries.md](monitoring-queries.md) | +| `prepared statement "..." does not exist` | transaction mode with `max_prepared_statements = 0` | set it > 0 and use protocol-level prepares | +| Setting lost between queries | session `SET` under transaction pooling | `SET LOCAL` or session mode | +| `LISTEN` never fires | transaction mode | direct connection for listeners | + +Template: [templates/pgbouncer.ini](templates/pgbouncer.ini). + +## Sources + +- https://www.pgbouncer.org/changelog.html (1.26.1 and 1.26.0 entries, read 2026-11-10) +- https://github.com/pgbouncer/pgbouncer/blob/master/doc/config.md (defaults for `pool_mode`, `default_pool_size`, `max_client_conn`, `auth_type`, `auth_query`, `auth_user`, timeouts, `max_prepared_statements`, read via raw file 2026-11-10 and Context7 `/pgbouncer/pgbouncer`) +- https://www.pgbouncer.org/features.html ("SQL feature map for pooling modes", read 2026-11-10) + +## Not verified + +- Admin-console command list (`SHOW ...`, `RELOAD`, `PAUSE`, `RESUME`) is from general knowledge of the console; check https://www.pgbouncer.org/usage.html before relying on exact column names. +- `so_reuseport` is cited from the `max_prepared_statements` doc text; its platform support was not checked. +- Exact fix status of the `search_path` tracking bugs after 1.26.1 was not checked. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/destructive-actions.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/destructive-actions.md new file mode 100644 index 0000000..e2e8788 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/destructive-actions.md @@ -0,0 +1,104 @@ +--- +name: destructive-actions +description: Catalogue of PostgreSQL, Patroni, pgBackRest, CloudNativePG and Redis/Valkey commands that destroy data or availability, each in ACTION / EFFECT / PROOF / WAIT-FOR-GO form with what is unrecoverable +when-to-use: Before proposing or running ANY command that deletes, overwrites, promotes, resets, rebuilds or takes a database offline; whenever the user asks to "clean up", "reset", "fix by restoring" or "force" something +keywords: DROP DATABASE, DROP TABLE, TRUNCATE, DELETE without WHERE, pg_resetwal, pg_ctl promote, patronictl failover, reinit, remove, drop replication slot, VACUUM FULL, pg_upgrade --link, pgbackrest expire, stanza-delete, restore --delta, CloudNativePG delete, FLUSHALL +priority: critical +related: backup-pitr-pgbackrest.md, ha-patroni.md, major-upgrades.md, replication-streaming-logical.md, vacuum-bloat-wraparound.md, templates/restore-test-pitr-runbook.md +--- + +# Destructive Database Actions + +**Load when:** a command can lose data for good or take a database down. Same discipline as the Operations Safety Protocol of the agent and the sibling `devops-proxmox` safe-change protocol: the agent **proposes**, shows the PROOF, and executes only after the user's explicit go **naming the object** (host, cluster, database, table, slot, stanza). One go covers one proposal. + +## Proposal format (one block per action) + +``` +ACTION +EFFECT +PROOF +UNRECOVERABLE +WAIT-FOR-GO waiting for your go on: +``` + +## The standing PROOF (every action below needs it first) + +1. Target confirmed read-only: `SELECT current_database(), inet_server_addr(), pg_is_in_recovery();` / `patronictl list` / `kubectl --context C -n NS get cluster.postgresql.cnpg.io` - right environment, right node, right role. +2. Backup that covers it: `pgbackrest --stanza=S info` (newest full/diff/incr, WAL range includes now), plus the date of the last **restore test** ([templates/restore-test-pitr-runbook.md](templates/restore-test-pitr-runbook.md)). A backup never restored is "not proven". +3. Dependants listed: sessions (`pg_stat_activity`), replicas and slots (`pg_stat_replication`, `pg_replication_slots`), applications, cron/backup jobs. +4. Rollback or recovery path written (restore to a new host and copy data back; not "undo"). + +## SQL data loss + +| Action | Effect | Unrecoverable without a backup | +|--------|--------|-------------------------------| +| `DROP DATABASE name;` (`WITH (FORCE)` terminates other sessions) | Removes the catalog entries and deletes the data directory of the database; cannot run while connected to it; fails if others are connected unless `FORCE` | Everything in it, immediately. "DROP DATABASE cannot be undone" (manual) | +| `DROP TABLE t [CASCADE]` | Removes table, indexes and (with `CASCADE`) dependent views/constraints | Rows and dependants. Inside an open transaction it can still be rolled back: do not commit before the user confirms | +| `TRUNCATE t [CASCADE] [RESTART IDENTITY]` | Empties tables with `ACCESS EXCLUSIVE`; same effect as unqualified `DELETE` but faster; `CASCADE` empties referencing tables; not MVCC-safe for concurrent snapshots | All rows. Rollback only inside its own open transaction | +| `DELETE`/`UPDATE` without `WHERE` | Touches every row; WAL-heavy; replicas replay it | Old values after commit. Run first as `BEGIN; ... ; SELECT count(*) ...; ROLLBACK;` with the `WHERE` shown; require a `WHERE` or the user's explicit "all rows" | +| `DROP SUBSCRIPTION` | Removes the subscription; if tied to a remote slot it also drops the slot (cannot run in a transaction block then); unset the slot first with `ALTER SUBSCRIPTION ... SET (slot_name = NONE)` to keep it | Replication position; subscriber must be re-seeded | +| `pg_restore --clean` / `psql -f` of a dump with `DROP` statements | Drops then recreates objects in the target | The target's current contents. Restore into a NEW database first | + +Block example: +``` +ACTION psql -h db1 -d postgres -c "DROP DATABASE appdb_old;" (host db1, role primary) +EFFECT deletes the appdb_old data directory; 3 sessions from app-batch would have to be terminated +PROOF pg_stat_activity shows 0 sessions in 24h; pgbackrest info: full 2026-11-09, WAL to 09:58; restore test 2026-11-02 OK; \dt shows 0 rows touched since 2026-08 +UNRECOVERABLE writes after the last archived WAL segment +WAIT-FOR-GO waiting for your go on: db1 / appdb_old +``` + +## Cluster and server state + +| Action | Effect | Unrecoverable / risk | +|--------|--------|----------------------| +| `pg_resetwal` (`-f` on an unclean/corrupt state) | Clears WAL and resets `pg_control`; the manual: last resort when the server will not start because of corruption; on anything but a cleanly shut down sound cluster it is "potentially dangerous" and can leave the database inconsistent (corrupt or lost transactions) | Consistency itself. Take a file-level copy of the data directory first, prefer restore from backup, then `pg_dump` everything out and rebuild; never leave a reset cluster in service | +| `pg_ctl promote` / `SELECT pg_promote();` | Ends recovery; the standby becomes a primary on a new timeline | Split brain if the old primary is still writable; the old primary needs `pg_rewind` or a rebuild. Under Patroni/CNPG never do it by hand | +| `patronictl failover --candidate X` | Forced promotion of X; documented as able to lose data depending on candidate lag | Transactions the candidate had not received. Check `patronictl list` lag first; prefer `switchover` when a leader is healthy | +| `patronictl reinit ` | Rebuilds that replica from the leader | The replica's local data directory (fine for a healthy leader; never target the leader) | +| `patronictl remove ` | Deletes the cluster's state from the DCS | Patroni's view of the cluster (members, leader lock, dynamic config); databases keep running unmanaged until Patroni is restarted | +| `patronictl pause` left on | No automatic failover | Availability: record and `resume` | +| `SELECT pg_drop_replication_slot('s');` | Removes the slot; the primary may then recycle WAL the consumer needs | The consumer (standby, subscriber, Debezium...) cannot resume and must be rebuilt/re-snapshotted. Check `active`, `wal_status`, owner first. Never drop slots managed by Patroni or CNPG | +| `VACUUM FULL`, `CLUSTER`, rewriting `ALTER TABLE`, `REINDEX` (non-concurrent), `LOCK TABLE` | `ACCESS EXCLUSIVE` for the whole run; needs about the table size of free disk | Availability of that table for the duration; no data loss. Prefer plain `VACUUM`, `REINDEX ... CONCURRENTLY`, pg_repack | +| `pg_terminate_backend(pid)` | Ends the session and rolls back its transaction | Work in flight; never a backup or migration session without checking what it is | +| `pg_upgrade --link` / `--swap` | Reuses or moves the old files | After the new cluster starts (`--link`) or once transfer begins (`--swap`) the old cluster is unusable: backup is the only way back ([major-upgrades.md](major-upgrades.md)) | +| `delete_old_cluster.sh`, `rm -rf` on a data/WAL directory, deleting files from `pg_wal` | Permanent file deletion; removing WAL by hand can make the server unrecoverable | Never delete `pg_wal` files; fix the archiver or slot instead | +| `systemctl stop`/restart of a primary | Drops all connections; crash recovery if not clean | Availability; use `pg_ctl stop -m fast` via the unit, not `kill -9` | + +## Backup system + +| Action | Effect | Unrecoverable / risk | +|--------|--------|----------------------| +| `pgbackrest --stanza=S expire` (also runs automatically after a backup) | Deletes backups beyond retention and their WAL; archive retention is checked on every run; `--set`, `--repo`, `--oldest` narrow it; in 2.60.0 refuses if the oldest retained backup has checksum errors | The expired restore points. Change `repo1-retention-*` only after reading `info`, with the new policy shown; never lower retention to "free space" without the user's go | +| `pgbackrest --stanza=S stanza-delete [--force]` | Permanently removes all backups and archives of the stanza from the repository; requires the cluster to be shut down or `--force`, and a prior `stop` | Every backup of that database. Needs the user's go naming the stanza AND repository path | +| `pgbackrest --delta restore` (or restore into a non-empty directory) | Replaces the target data directory contents to match the backup, deleting files that do not belong | The current state of that directory. Only on the intended replacement host or on a copy; never on a live instance; PostgreSQL stopped; `postmaster.pid` absent | +| Restoring over a live primary | Overwrites production with older data; in HA, risks a second writer | All writes since the restore point. Restore elsewhere first, extract, then decide | +| Deleting repository files, buckets or lifecycle rules by hand | Corrupts the repository metadata | Backups. Use `expire` only | + +## Kubernetes (CloudNativePG) and key-value stores + +| Action | Effect | Unrecoverable / risk | +|--------|--------|----------------------| +| `kubectl delete cluster.postgresql.cnpg.io NAME` | Removes the Cluster and its instances; whether the PVCs and PVs survive depends on owner references and the StorageClass `reclaimPolicy` (`Delete` removes the volume and its data) | Treat as deleting the data. Read-only first: `kubectl get pvc,pv`, the PV `persistentVolumeReclaimPolicy`, last backup in the object store, `ScheduledBackup` status. See `devops-kubernetes` storage-csi | +| `kubectl delete pvc ...` of an instance | Destroys that instance's data; the operator may re-clone a replica | Data of a primary PVC if no replica is in sync; never the primary's PVC | +| `kubectl delete namespace` | Deletes every Cluster, Pooler, Backup and Secret inside | Everything not backed up off-cluster | +| Major-version image change on a Cluster | Offline `pg_upgrade --link`; all pods shut down, replica PVCs destroyed and re-cloned | Availability for the duration; pre-upgrade PITR range | +| Deleting the `ObjectStore` or its bucket/credentials | Backups and WAL archive become unreachable | The whole recovery chain | +| `FLUSHALL` / `FLUSHDB` (Redis, Valkey) | Removes all keys of all (or the current) databases; replicas replay it | All data unless a persistence file from before still exists: with AOF, if no rewrite happened, stop the server, remove the last command from the AOF and restart (Valkey persistence docs); with RDB only the last snapshot. Never on an instance whose role is unconfirmed | + +## What the agent never does without being asked and cleared + +Run any statement it has not shown first; loop a destructive command over many databases or hosts; add `--force`, `--cascade`, `-f`, `WITH (FORCE)` or `--delta` on its own initiative; edit retention to a smaller value; print credentials; or "clean up" an object it did not create. + +## Sources + +- https://www.postgresql.org/docs/18/sql-dropdatabase.html , sql-truncate.html , sql-dropsubscription.html , app-pgresetwal.html (effects and warnings, read 2026-11-10); https://www.postgresql.org/docs/18/pgupgrade.html (reverting); https://www.postgresql.org/docs/18/functions-admin.html (`pg_promote`) +- Patroni docs `patronictl.rst`, `faq.rst` via raw GitHub (failover warning, remove, reinit, pause; read 2026-11-10) +- https://pgbackrest.org/command.html (`expire`, `stanza-delete` warning and prerequisites; read 2026-11-10) +- https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/docs/src/postgres_upgrades.md (in-place upgrade destroys replica PVCs); https://valkey.io/topics/persistence/ (FLUSHALL recovery via AOF) + +## Not verified + +- CloudNativePG behaviour on `Cluster` deletion (PVC ownership/retention): the project FAQ entry "Does CloudNativePG wipe the data directory when a cluster is deleted?" is a TODO in the docs read today; the table states the safe assumption. +- `pg_drop_replication_slot` was cited from the replication docs context; its page was not fetched. +- `pg_ctl stop -m fast` guidance and `DROP SUBSCRIPTION ... slot_name = NONE` follow the manual text read only in part. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/ha-patroni.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/ha-patroni.md new file mode 100644 index 0000000..ebc5f1d --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/ha-patroni.md @@ -0,0 +1,71 @@ +--- +name: ha-patroni +description: PostgreSQL high availability with Patroni and an etcd DCS - architecture, leader race and TTL rule, failover versus switchover, split-brain protections (leader lock, watchdog, failsafe, synchronous mode), patronictl operations and which are destructive +when-to-use: Building or operating a Patroni cluster on VMs, planning maintenance on a node, handling a failed primary, or deciding between Patroni and CloudNativePG +keywords: patroni, patronictl, etcd3, DCS, leader lock, ttl, loop_wait, retry_timeout, switchover, failover, reinit, pause, watchdog, failsafe_mode, synchronous_mode, split brain +priority: critical +related: replication-streaming-logical.md, backup-pitr-pgbackrest.md, postgres-on-kubernetes-cnpg.md, destructive-actions.md, templates/patroni.yml +--- + +# High Availability with Patroni + +**Load when:** a PostgreSQL cluster on VMs or bare metal must fail over automatically. On Kubernetes prefer an operator ([postgres-on-kubernetes-cnpg.md](postgres-on-kubernetes-cnpg.md)); do not run Patroni and CloudNativePG on the same data. Version: Patroni **4.1.5** (2026-08-12), tested against PostgreSQL 14.24 to 18.6. + +## Architecture + +Each node runs Patroni, which starts and manages PostgreSQL. The cluster state lives in a **DCS** (distributed configuration store): etcd (v3 API, config section `etcd3`), Consul, ZooKeeper/Exhibitor or the Kubernetes API. The **leader lock** in the DCS names the one member allowed to run as primary. Clients reach the primary through HAProxy or a VIP that health-checks Patroni's REST API (`/primary`, `/replica`, `/health`; `/metrics` for Prometheus - exporter side: `devops-observability`), or through a libpq multi-host string. Backups and PITR still come from pgBackRest ([backup-pitr-pgbackrest.md](backup-pitr-pgbackrest.md)); Patroni can bootstrap replicas from it with custom methods. + +The DCS needs its own odd-sized cluster (3 members minimum for fault tolerance) on separate failure domains and fast disks. etcd operation: `devops-k8s-cluster` covers etcd lifecycle; its version and the Patroni-specific etcd options were not re-verified here. + +## Timing rule + +`ttl` (default 30, min 20) is the time before an automatic failover starts when the leader stops renewing; `loop_wait` (default 10); `retry_timeout` (default 10, min 3) is how long DCS or network trouble is tolerated before the leader demotes itself. Constraint: **`loop_wait + 2 * retry_timeout <= ttl`**. Change them with `patronictl edit-config` (dynamic, stored in the DCS), never per node by hand, and test the failover time afterwards. `maximum_lag_on_failover` (default 1048576 bytes) excludes replicas lagging more than that from the leader race. + +## Failover vs switchover + +| | switchover | failover | +|---|-----------|----------| +| Use | Healthy cluster, planned (maintenance, upgrade) | Cluster unhealthy: no leader, or no synchronous standby available | +| Effect | Orderly demotion of the old leader, chosen replica promoted | Forced promotion of a candidate; docs warn it **can lose data** depending on how up to date the candidate is | +| Command | `patronictl -c /etc/patroni/patroni.yml switchover --leader --candidate ` | `patronictl -c ... failover --candidate ` | +| Gate | Proposal + go, after `patronictl list` shows all members streaming with ~0 lag | ACTION/EFFECT/PROOF/WAIT-FOR-GO ([destructive-actions.md](destructive-actions.md)) | + +Automatic failover is the **leader race**: when the leader lock expires every node checks whether it is the best candidate (including REST calls to peers); the first to take the lock promotes itself and the others follow. Tags per node: `nofailover: true` (or `failover_priority: 0`) keeps a node out of the race (use for a delayed or reporting replica); nodes with more receive/replay LSN win regardless of priority. + +## Split-brain protections + +1. **Leader lock**: a node may run as primary only if it can update the lock in the DCS; if the update fails it demotes. Hence DCS health is part of database availability. +2. **Watchdog** (`watchdog.mode`: `off`, `automatic`, `required`; device `/dev/watchdog`): with `required` a node does not become leader unless the watchdog is armed, so a frozen Patroni process cannot leave a stale primary. `safety_margin` sits between watchdog trigger and lock expiry. Needs kernel watchdog support (`devops-linux`). +3. **DCS failsafe mode** (`failsafe_mode`, dynamic config, default false): if the DCS is down but all known members acknowledge the leader through `POST /failsafe`, the primary keeps running instead of demoting; if one member does not answer, it demotes. Trade-off: availability during a DCS outage versus a stricter safety check; decide consciously. +4. **Synchronous mode** (`synchronous_mode: true` or `quorum`, with `synchronous_node_count` default 1, `synchronous_mode_strict`): commits wait for a standby so a failover to it loses no committed data; strict mode blocks writes when no synchronous standby exists. `primary_stop_timeout` interacts with it (SIGKILL after the timeout). +5. **pg_rewind**: `use_pg_rewind: true` lets the old leader rejoin as a replica without a full copy; requires data checksums and/or `wal_log_hints = on`. (initdb in 18 enables checksums by default.) +6. **Permanent slots** (`slots` in the dynamic config) are preserved across failover; replication slots with `wal_status=lost` are dropped by Patroni since 4.1.4. + +Never promote PostgreSQL by hand (`pg_ctl promote`, `pg_promote()`) on a Patroni member while the DCS shows another leader: that creates two primaries. Never stop Patroni "to restart Postgres": bouncing PostgreSQL without Patroni can cause failovers; use `patronictl restart`. + +## Operations map + +| Command | Reads/writes | Note | +|---------|--------------|------| +| `patronictl list`, `history`, `show-config` | read-only | Always first | +| `patronictl edit-config --set key=value` | writes the DCS config for the whole cluster | Show the diff; some parameters restart members (`pending restart` column) | +| `patronictl restart ` | restarts PostgreSQL | One member at a time, replicas first | +| `patronictl pause` / `resume` | disables/enables automatic failover | A paused cluster does not fail over: always `resume` and record who paused it | +| `patronictl switchover` | planned role change | Brief write interruption | +| `patronictl failover` | forced promotion | Destructive: possible data loss | +| `patronictl reinit ` | rebuilds a replica from the leader | Discards that replica's data; never on the leader | +| `patronictl remove ` | deletes the cluster's state from the DCS | Interactive; destroys Patroni's cluster information | + +Rolling OS or minor upgrades: `pause`-free approach is preferred (restart replicas, switchover, restart the old leader); major upgrades: [major-upgrades.md](major-upgrades.md). Template: [templates/patroni.yml](templates/patroni.yml). + +## Sources + +- https://patroni.readthedocs.io/en/latest/releases.html (4.1.5 notes, compatibility with PostgreSQL 14.24-18.6; read 2026-11-10) +- Patroni docs via raw GitHub (read 2026-11-10): `docs/dynamic_configuration.rst` (ttl, loop_wait, retry_timeout, rule, `failsafe_mode`, `synchronous_node_count`, `slots`, `primary_stop_timeout`), `docs/yaml_configuration.rst` (etcd3 section, `watchdog`, `nofailover`, `failover_priority`, `use_pg_rewind`, bin_dir), `docs/faq.rst` (leader race, pause, REST endpoints), `docs/dcs_failsafe_mode.rst`, `docs/patronictl.rst` (failover/switchover/reinit/remove/pause descriptions) +- Context7 `/patroni/patroni`: patronictl examples for switchover, failover, reinit, remove, `edit-config` + +## Not verified + +- Which etcd versions Patroni 4.1.5 supports and etcd cluster sizing/latency guidance (Patroni docs name `etcd3`; etcd operation itself was not fetched). +- HAProxy/VIP configuration and REST endpoint semantics (`/primary`, `/replica`) beyond the FAQ's mention of `/metrics` and `/patroni`. +- Exact `patronictl reinit` data-directory handling beyond "rebuilds a standby"; and Debian/RHEL packaging (Patroni via PGDG or pip). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/major-upgrades.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/major-upgrades.md new file mode 100644 index 0000000..3b49539 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/major-upgrades.md @@ -0,0 +1,64 @@ +--- +name: major-upgrades +description: PostgreSQL major version upgrade paths - pg_upgrade (copy, --link, --clone, --swap), the logical-replication near-zero-downtime route, dump/restore, extensions, replicas, rollback limits and post-upgrade steps +when-to-use: Moving off an end-of-life major (14 today), moving to 18, choosing the upgrade method, or validating an upgrade plan before the maintenance window +keywords: pg_upgrade, --check, --link, --clone, --swap, upgrade, logical replication, extensions, update_extensions.sql, vacuumdb, analyze-in-stages, rollback, pg_dump +priority: critical +related: replication-streaming-logical.md, backup-pitr-pgbackrest.md, ha-patroni.md, postgres-on-kubernetes-cnpg.md, destructive-actions.md +--- + +# Major Version Upgrades + +**Load when:** the PostgreSQL major version changes. Minor releases need only a binary swap and restart. Version context: 14 gets its last release 2026-11-12; 18.6 is the target for new work ([postgres-versions-config.md](postgres-versions-config.md)). + +## Choose the method + +| Method | Downtime | Rollback | Use when | +|--------|----------|----------|----------| +| `pg_upgrade` (copy, default) | minutes to hours (copies data files) | old cluster untouched: restart it | Enough disk for a second copy; simplest safe path | +| `pg_upgrade --link` | minutes | **none** once the new cluster has started | Large data, tested plan, verified backup | +| `pg_upgrade --clone` | minutes | old cluster stays usable | Btrfs/XFS(reflink) on Linux 4.5+, APFS | +| `pg_upgrade --swap` | fastest on many relations | old cluster **destroyed** once file transfer begins | Only with a tested restore path | +| Logical replication to a new-major cluster | seconds to minutes (cutover) | old primary kept intact until cutover | Near-zero downtime, DDL freeze acceptable ([replication-streaming-logical.md](replication-streaming-logical.md)) | +| `pg_dump`/`pg_restore` | long (size-bound) | old cluster untouched | Small data, cross-platform, cleanup of bloat | + +`pg_upgrade` supports upgrades from 9.2.X to the current major without passing through intermediate majors; read the release notes of every major in between. It lets the destination execute code chosen by the source superusers: only upgrade clusters whose superusers you trust. + +## pg_upgrade procedure (summary of the manual, each step a change after the go) + +1. **Proof first**: a fresh backup with a completed restore test ([templates/restore-test-pitr-runbook.md](templates/restore-test-pitr-runbook.md)). Link mode and swap mode make the backup the only way back. +2. Install the new binaries beside the old ones (version-specific directories) and the **extension shared objects** for the new major; do not run `CREATE EXTENSION` in the new cluster. Copy custom full-text search files. +3. `initdb` the new cluster with options compatible with the old one (checksums: 18 enables them by default; `pg_upgrade` checks `pg_controldata` compatibility). No need to start it. +4. Set `pg_hba.conf` to `peer` or use `.pgpass` for the install user; stop writers and **stop both servers** (standbys must be running during the old primary's shutdown so they receive everything). +5. **`pg_upgrade --check`** with the same options (`-b -B -d -D`, plus `--link` if chosen): it changes nothing, can run while the old server is running, and lists manual adjustments. Repeat until clean. +6. Run `pg_upgrade` for real (`-j N` for parallelism; `--link`/`--clone`/`--swap` per the table). It writes logs in the current directory (keep them; `--retain` keeps SQL and log files). +7. Start the new cluster; restore the original `pg_hba.conf` and carry over `postgresql.conf` differences by hand. +8. Run the `update_extensions.sql` script `pg_upgrade` produced if extension updates were reported. +9. **Statistics**: 18 transfers optimizer statistics (opt out with `--no-statistics`); still run `vacuumdb --all --analyze-in-stages --missing-stats-only`, then `vacuumdb --all --analyze-only` so autovacuum has current statistics. +10. **New base backup immediately**: WAL and backups from before the upgrade cannot be used for PITR across the major boundary. Point pgBackRest at the new `pg-path` and run `pgbackrest --stanza=main stanza-upgrade` (documented as required immediately after a major upgrade), then take the new full backup. +11. Only when satisfied (days, not hours) run the generated `delete_old_cluster.sh` ([destructive-actions.md](destructive-actions.md)). + +**Rollback limits**: with `--check`, copy mode or `--clone` the old cluster is unmodified and can be restarted. With `--link`: if aborted before linking, unmodified; if linking started but the new cluster never started, remove the `.old` suffix from `$PGDATA/global/pg_control` to reuse the old cluster; **if the new cluster was started, the old one is unsafe and must be restored from backup**. With `--swap` the old cluster may be destroyed once transfer begins. + +**Replicas**: after `pg_upgrade --link` the manual describes an `rsync --archive --delete --hard-links --size-only --no-inc-recursive` procedure run from the primary to bring standbys across quickly; the simpler alternative is to re-create the standbys from the new primary after the upgrade. Under Patroni, follow its own procedure (pause, upgrade the leader, re-initialise replicas) and rehearse it; CloudNativePG does an offline in-place upgrade when the image's major changes (below). + +## Logical route (online) + +Build a new-major cluster, copy the schema (`pg_dump --schema-only`), create publications on the old and subscriptions on the new, let it catch up, freeze DDL, copy sequence values, stop writes briefly, verify row counts and lag zero, repoint clients, keep the old cluster until the new one has run through a full business cycle. Sequences and DDL are not replicated; every table needs a replica identity. The manual's logical replication upgrade chapter covers upgrading logical replication clusters themselves. + +## CloudNativePG + +Three supported methods: logical dump/restore (blue/green, offline), native logical replication (blue/green, online) and `pg_upgrade --link` in place, which CloudNativePG runs when a new image with a higher major is declared: it shuts down all pods (the whole cluster, replicas included, is unavailable), records the previous image in `.status.pgDataImageInfo`, runs the upgrade job, then **destroys the replicas' PVCs and re-clones them**. Images must be the same OS distribution. A bug in 17.0-17.5 breaks the upgrade when `max_slot_wal_keep_size` is not `-1`; use 17.6+ or set `-1` temporarily. Take a new base backup right after ([postgres-on-kubernetes-cnpg.md](postgres-on-kubernetes-cnpg.md)). + +## Sources + +- https://www.postgresql.org/docs/18/pgupgrade.html (options, usage steps, reverting, standby rsync, post-upgrade; read 2026-11-10) +- https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/docs/src/postgres_upgrades.md (read 2026-11-10) +- https://www.postgresql.org/support/versioning/ (upgrade statement) + +## Not verified + +- pgBackRest `stanza-upgrade` was confirmed on the command page (set `pg-path` to the new location, then run it, `--no-online` if the database is offline); its effect on existing backups' PITR range was not read. +- Behaviour of specific extensions (PostGIS, TimescaleDB, pgvector...) across majors: check each extension's upgrade notes; none were fetched. +- Patroni major-upgrade procedure was not fetched; treat the summary as a hint to read the Patroni documentation. +- The 18 manual's statement that statistics are transferred was read as the `--no-statistics` option description only. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/mongodb-pointers.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/mongodb-pointers.md new file mode 100644 index 0000000..1d89bda --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/mongodb-pointers.md @@ -0,0 +1,32 @@ +--- +name: mongodb-pointers +description: One-line operational pointers for MongoDB (backup, replica set, upgrades, destructive commands) with no in-depth coverage, and the rule for what to do when a MongoDB task arrives +when-to-use: A task involves MongoDB and the agent needs the safe entry points and the official sources, not a full runbook +keywords: MongoDB, mongodump, mongorestore, replica set, rs.status, oplog, WiredTiger, sharding, dropDatabase +priority: low +related: destructive-actions.md, redis-valkey-ops.md, mysql-mariadb-ops.md +--- + +# MongoDB: Pointers Only + +**Load when:** MongoDB appears in the task. This skill does not cover MongoDB operations in depth; none of the statements below were verified against the MongoDB documentation today. Read the official manual for the server's exact version (https://www.mongodb.com/docs/manual/) before proposing anything, and state "not verified" for what you did not read. + +| Topic | Pointer (to verify at the manual) | +|-------|-----------------------------------| +| Backup | `mongodump` / `mongorestore` (logical, small data); filesystem snapshots with journaling for larger data; the manual's "Backup Methods" page lists the supported ones. A backup counts only after a restore test | +| HA | A replica set (primary + secondaries, election on failure); read `rs.status()` and `rs.conf()` first (read-only). Odd number of voting members | +| Sharding | Separate chapter in the manual; treat as a specialist operation | +| Upgrades | Version-by-version; read the release notes and the feature compatibility version rules of the manual before moving | +| Destructive | `db.dropDatabase()`, `db.collection.drop()`, `deleteMany({})`, `rs.reconfig(..., {force: true})`, removing replica set members, `mongorestore --drop`: all go through ACTION / EFFECT / PROOF / WAIT-FOR-GO in [destructive-actions.md](destructive-actions.md) | +| Licence | Check the server's licence for your use (the licence changed in the past); not verified here | +| Kubernetes | Operators exist from several vendors; none evaluated here | + +If MongoDB is the main task, tell the user this skill covers it only as pointers and proceed read-only, source-cited, with a restore-tested backup before any change. + +## Sources + +- https://www.mongodb.com/docs/manual/ (named as the starting point; not fetched in this session) + +## Not verified + +- Everything in the table above is general knowledge; no MongoDB page was fetched. Versions, licence, supported backup methods and operator status are unknown. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/monitoring-queries.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/monitoring-queries.md new file mode 100644 index 0000000..8c2982b --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/monitoring-queries.md @@ -0,0 +1,65 @@ +--- +name: monitoring-queries +description: Read-only PostgreSQL health queries an operator runs - sessions and wait events, blocking locks, slow statements with pg_stat_statements, cache and checkpoint health, archiver, replication lag, XID age - and what to alert on +when-to-use: Diagnosing slowness, locks, connection pileups or lag; deciding alert rules; before and after any change as the comparison baseline +keywords: pg_stat_activity, pg_stat_statements, pg_locks, pg_blocking_pids, pg_stat_replication, pg_stat_archiver, pg_stat_database, pg_stat_checkpointer, wait_event, idle in transaction, pg_cancel_backend +priority: high +related: vacuum-bloat-wraparound.md, replication-streaming-logical.md, schema-changes-safe.md, templates/monitoring-queries.sql +--- + +# PostgreSQL Monitoring Queries + +**Load when:** a PostgreSQL instance behaves badly or an alert is being designed. All queries in [templates/monitoring-queries.sql](templates/monitoring-queries.sql) are **read-only**. Metrics collection and dashboards (postgres_exporter, Prometheus, Grafana, alert rules): `devops-observability`; this file defines what to look at, that skill wires it. + +## Setup + +- Monitoring role: `GRANT pg_monitor TO monitor_user;` (read access to the statistics views and settings without superuser; [postgres-roles-auth.md](postgres-roles-auth.md)). +- `pg_stat_statements`: add it to `shared_preload_libraries` (**restart required**), ensure query ID calculation is on (`compute_query_id` is `auto` by default when the module is loaded), then `CREATE EXTENSION pg_stat_statements;` in the database you query. It tracks all databases of the server. `pg_stat_statements_reset()` is a write on statistics: ask before using it. +- `log_min_duration_statement`, `log_lock_waits`, `log_checkpoints`, `log_autovacuum_min_duration` complement the views ([postgres-versions-config.md](postgres-versions-config.md)). + +## Triage order + +1. **Is it up and what role?** `SELECT version(), pg_is_in_recovery(), now() - pg_postmaster_start_time();` +2. **Sessions**: group `pg_stat_activity` by `state` and `wait_event_type`. Many `idle in transaction` = application bug holding locks and the xmin horizon. Many `active` with `Lock` waits = a blocker exists. +3. **Blockers**: `pg_blocking_pids(pid)` gives, for a waiting session, the PIDs blocking it. Find the root of the chain (a pid that blocks others and is not blocked), see what it is doing and for how long. +4. **Slow statements**: `pg_stat_statements` ordered by `total_exec_time` (what costs the most overall) and `mean_exec_time` with `calls` (what is slow per call); compare `shared_blks_read` to `shared_blks_hit` and `temp_blks_written` (spills: `work_mem`). +5. **Database counters**: `pg_stat_database` - `xact_rollback`/`xact_commit`, `deadlocks`, `temp_bytes`, `conflicts` (standby), cache hit ratio from `blks_hit`/`blks_read`. +6. **Checkpoints and WAL**: checkpoints requested vs timed (`pg_stat_checkpointer` in 17+; `pg_stat_bgwriter` earlier), `log_checkpoints` lines. +7. **Archiving**: `pg_stat_archiver` - `failed_count`, `last_failed_time`, `last_archived_time`. A failing `archive_command` makes WAL pile up in `pg_wal` until the disk fills. +8. **Replication**: `pg_stat_replication` (primary), `pg_replication_slots.wal_status` and retained WAL, `pg_stat_wal_receiver` on a standby ([replication-streaming-logical.md](replication-streaming-logical.md)). +9. **Vacuum and XID**: `pg_stat_user_tables` (`n_dead_tup`, `last_autovacuum`), `pg_stat_progress_vacuum`, `age(datfrozenxid)` ([vacuum-bloat-wraparound.md](vacuum-bloat-wraparound.md)). + +## Reading a slow query + +Run `EXPLAIN (ANALYZE, BUFFERS)` only on a copy, a replica or a statement that is safe to execute: `ANALYZE` **executes** the statement (an `UPDATE`/`DELETE` really changes data; wrap in `BEGIN; ... ROLLBACK;`). Look for estimated vs actual rows far apart (stale statistics: `ANALYZE`), sequential scans on big tables with selective filters (missing index: build it with `CREATE INDEX CONCURRENTLY`, [schema-changes-safe.md](schema-changes-safe.md)), sort/hash spills, nested loops over large inputs. Deeper query tuning is outside this skill. + +## Intervening on sessions (writes: proposal, then go) + +`pg_cancel_backend(pid)` cancels the current query (SIGINT); `pg_terminate_backend(pid)` ends the session (SIGTERM) and rolls back its transaction. Roles other than superuser need the right role membership (`pg_signal_backend`). Terminating a migration or a backup session can leave work half done: check what it is first. Prefer cancelling the blocked DDL's `lock_timeout` approach over killing production sessions. + +## Alerts worth having + +| Signal | Condition (starting point) | +|--------|---------------------------| +| Archiver failing | `failed_count` increases, or `last_archived_time` older than a few `archive_timeout` periods | +| Replication lag | replay lag over the RPO budget; replica `streaming` state lost | +| Slot retention | `wal_status` `extended`/`unreserved`/`lost`, or retained WAL over X GB; inactive slot | +| Disk | data and WAL volumes under 20% free | +| XID age | `age(datfrozenxid)` above 500 million | +| Connections | sessions above 80% of `max_connections` (or PgBouncer `cl_waiting` > 0) | +| Long transactions | `idle in transaction` over a few minutes; oldest `xact_start` over an hour | +| Backups | newest successful backup older than policy; restore test overdue | +| Deadlocks | `deadlocks` rate above baseline | + +Thresholds are starting points to tune to the workload. + +## Sources + +- https://www.postgresql.org/docs/18/monitoring-stats.html (`pg_stat_activity`, `pg_stat_database`, `pg_stat_archiver`, `pg_stat_checkpointer`, `pg_stat_progress_vacuum`, `pg_stat_wal_receiver`; names confirmed present, read 2026-11-10) +- https://www.postgresql.org/docs/18/pgstatstatements.html (columns, `shared_preload_libraries`, `compute_query_id`), https://www.postgresql.org/docs/18/functions-info.html (`pg_blocking_pids`), https://www.postgresql.org/docs/18/functions-admin.html (`pg_cancel_backend`, `pg_terminate_backend`) + +## Not verified + +- Column-level details of each view were only checked by name presence on the pages; run `\d+ pg_stat_activity` on the target major before scripting against columns. +- Cache-hit-ratio targets and the alert thresholds above are practice, not manual values. +- `pg_stat_checkpointer` appears in 18's page; the claim that it exists since 17 is from general knowledge. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/mysql-mariadb-ops.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/mysql-mariadb-ops.md new file mode 100644 index 0000000..49295d7 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/mysql-mariadb-ops.md @@ -0,0 +1,62 @@ +--- +name: mysql-mariadb-ops +description: MySQL and MariaDB operations essentials - release lines and support in 2026 (MySQL 8.4/9.7 LTS and year-based versions, MariaDB 12.3 LTS), backup tools, GTID replication and why MySQL and MariaDB GTIDs are incompatible, upgrade rules, destructive actions +when-to-use: Operating, backing up, replicating or upgrading a MySQL or MariaDB server, choosing between the two, or reading a MySQL version number like 26.7 +keywords: MySQL, MariaDB, mysqldump, MySQL Shell dumpInstance, mariadb-backup, clone plugin, GTID, CHANGE REPLICATION SOURCE, master_use_gtid, LTS, innovation, Galera, binlog +priority: medium +related: destructive-actions.md, backup-logical-physical.md, redis-valkey-ops.md +--- + +# MySQL and MariaDB Operations Essentials + +**Load when:** the server is MySQL or MariaDB. Generic backup storage and drills: `devops-storage-backup`. Application schema: framework experts. PostgreSQL is the deep part of this skill; this file is the short, verified-core version. + +## Release lines (checked 2026-11-10) + +**MySQL**: two tracks, LTS and Innovation. LTS series follow Oracle Lifetime Support (5 years premier + 3 years extended), contain only necessary fixes and allow in-place upgrades and downgrades inside the series; Innovation releases are supported until the next Innovation release. Current manual lines: **8.4 LTS**, **9.7 LTS** (9.7 is the last release line using sequential numbers), **26.7** and **26.10** (Innovation, calendar versions `YY.M`: 26.7 is the July 2026 release; an LTS keeps its `YY.M` fixed for its life). **MySQL 8.0 moved to Oracle Sustaining Support on 2026-04-21**: upgrade to 8.4 or 9.7. LTS to the next LTS is supported (8.4 to 9.7), skipping an LTS series is not. Do not read "26.7" as a patch of 9.x. Pick the track once; the official repositories select it (`mysql-9.7-lts-community` vs `mysql-innovation-community` for yum). Solaris and EL7 binaries are discontinued. + +**MariaDB**: Community Server series listed: 13.2 (development), 13.1, 13.0 (current rolling), **12.3 (latest long-term stable, maintained three years)**, 11.8, 11.4, 10.11, 10.6. Per-series end-of-life dates are on the MariaDB Foundation maintenance policy and the platform deprecation page; they were not extracted today. + +## MySQL vs MariaDB: they have diverged + +Different GTID implementations that are **not compatible**: MariaDB can be a replica of a MySQL primary, MySQL cannot replicate from a MariaDB primary (MariaDB documentation). Different version numbering, tooling names (`mariadb-backup` vs MySQL Enterprise Backup / clone plugin) and feature sets (Galera Cluster is MariaDB's multi-primary option; MySQL has InnoDB Cluster / Group Replication). Treat a move between them as a migration, not an upgrade: dump/restore or logical replication with testing, never in-place file copy. Compatibility pages: the MariaDB documentation "Compatibility and Differences". + +## Backup + +| Need | MySQL | MariaDB | +|------|-------|---------| +| Logical, small/medium | `mysqldump`; or **MySQL Shell** `util.dumpInstance()` / `dumpSchemas()` / `dumpTables()` with `util.loadDump()` (parallel threads, compression, progress, dry run, DDL + tab-separated data files) | `mariadb-dump` (the `mysqldump`-compatible tool; name not re-checked) | +| Physical, hot | MySQL Enterprise Backup (commercial); the **clone plugin** for replica provisioning; Percona XtraBackup is a third-party alternative (not verified today) | **`mariadb-backup`** (renamed from `mariabackup`): physical hot backup of InnoDB, Aria, MyISAM; forked from Percona XtraBackup 2.3.8; uses `BACKUP STAGE` from 10.11.8 | +| PITR | physical or logical base backup + binary logs replayed to a point | same | + +Notes: `mysqlpump` is deprecated since MySQL 8.0.34 (manual points to `mysqldump` and MySQL Shell); the 9.7 manual has no `mysqlpump` page (the request for it returned the 8.0 page), so do not build on it. MySQL Shell dumps are the recommended modern logical path. Restore tests apply exactly as for PostgreSQL ([templates/restore-test-pitr-runbook.md](templates/restore-test-pitr-runbook.md) is the model: restore to a new host, compare counts, replay binlogs to a target). Enable the binary log (`log_bin`) on any server that needs PITR or replication, with a retention that exceeds the backup interval. + +## GTID replication + +**MySQL (8.4 manual)**: both servers need unique `server_id`, binary logging, `gtid_mode=ON` and `enforce-gtid-consistency=ON`. Cold-start sequence: make servers read-only if already replicating, wait for the replica to catch up, restart with GTID options, then on the replica `CHANGE REPLICATION SOURCE TO SOURCE_HOST=..., SOURCE_AUTO_POSITION=1, ...` (replication user and password from the secret store, never typed into history) and `START REPLICA;`, then take a **new backup** - binary logs without GTIDs cannot be used on servers where GTIDs are enabled, so old backups plus old binlogs are no longer a PITR chain. Online mode change is a separate documented procedure. + +**MariaDB**: GTID is on by default for new setups; the replica tracks `gtid_slave_pos` (crash-safe in an InnoDB table). `CHANGE MASTER TO master_use_gtid = slave_pos | current_pos | no`: use **`slave_pos`**; `current_pos` is documented as not recommended because extra transactions on the replica can corrupt the position. Moving a replica to another primary is simple with GTID. + +Both: use semi-sync or Group Replication / Galera only with a documented failure model; automatic failover tools (Orchestrator, MaxScale, ProxySQL, InnoDB Cluster/Router) were not evaluated. Never run two writable primaries without a conflict-resolution design. + +## Upgrades + +MySQL: in-place within an LTS; LTS to next LTS supported; use the clone plugin or logical routes for others; read the upgrade checker output (MySQL Shell `util.checkForServerUpgrade()` - from general knowledge). MariaDB: read the release notes of every series jumped; `mariadb-upgrade` after binaries change (general knowledge). Always: backup proof, replicas first, one major at a time. + +## Destructive actions + +`DROP DATABASE`/`DROP TABLE`/`TRUNCATE`, `DELETE`/`UPDATE` without `WHERE`, `RESET MASTER`/`RESET REPLICA` (deletes binary log files / replication state), `PURGE BINARY LOGS`, `mysqldump --add-drop-database` restores, and restoring over a live server: use the ACTION / EFFECT / PROOF / WAIT-FOR-GO format of [destructive-actions.md](destructive-actions.md), PROOF including the last restore test and the binlog range. MySQL sessions set `sql_safe_updates=1` for interactive operator sessions. + +## Sources + +- https://dev.mysql.com/doc/refman/9.7/en/mysql-releases.html (LTS/Innovation model, numbering, support terms, upgrade rules; read 2026-11-10) +- https://dev.mysql.com/doc/refman/26.10/en/ and 9.7 index (version selector), https://www.mysql.com/support/eol-notice.html (8.0 Sustaining Support 2026-04-21; read 2026-11-10) +- https://dev.mysql.com/doc/mysql-shell/26.10/en/mysql-shell-utilities-dump-instance-schema.html (dump utilities); https://dev.mysql.com/doc/refman/8.0/en/mysqlpump.html (deprecation); https://dev.mysql.com/doc/refman/8.4/en/replication-gtids-howto.html (GTID setup) +- https://mariadb.com/docs/release-notes/community-server (series list, 12.3 LTS), .../about/maintenance-policy.md , .../server/ha-and-performance/standard-replication/gtid.md (incompatibility, `master_use_gtid`), .../mariadb-backup/mariadb-backup-overview.md (read 2026-11-10) + +## Not verified + +- MariaDB per-series end-of-life dates, and the MariaDB dump tool name (`mariadb-dump`). +- Percona XtraBackup, Orchestrator, MaxScale, ProxySQL, InnoDB Cluster status and versions. +- `util.checkForServerUpgrade()`, `mariadb-upgrade`, `sql_safe_updates`, `RESET MASTER`/`PURGE BINARY LOGS` semantics: general knowledge, not fetched. +- Galera Cluster operations (listed as a topic in `docs/research-devops-plugin.md`) are not covered here. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/postgres-on-kubernetes-cnpg.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/postgres-on-kubernetes-cnpg.md new file mode 100644 index 0000000..e35a4a4 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/postgres-on-kubernetes-cnpg.md @@ -0,0 +1,75 @@ +--- +name: postgres-on-kubernetes-cnpg +description: Running PostgreSQL on Kubernetes with CloudNativePG - versions and support window, Cluster, Pooler, backups and PITR through the Barman Cloud plugin, rolling and major upgrades, fencing, and pointers to other operators +when-to-use: Deploying or operating PostgreSQL on Kubernetes, choosing an operator, setting up scheduled backups and a recovery Cluster, upgrading a CNPG-managed database +keywords: CloudNativePG, CNPG, Cluster, ScheduledBackup, ObjectStore, barman-cloud plugin, Pooler, bootstrap recovery, targetTime, imageName, instances, volume snapshot, switchover +priority: high +related: ha-patroni.md, backup-pitr-pgbackrest.md, major-upgrades.md, connection-pooling-pgbouncer.md, destructive-actions.md, templates/cnpg-cluster.yaml, templates/cnpg-scheduledbackup.yaml +--- + +# PostgreSQL on Kubernetes: CloudNativePG + +**Load when:** the database runs as pods. Cluster lifecycle, CSI, StorageClasses and etcd: `devops-k8s-cluster` and `devops-kubernetes` (storage-csi); packaging and GitOps of these manifests: `devops-gitops-packaging`; secrets: `devops-secrets`. Do not mix this with Patroni on the same data ([ha-patroni.md](ha-patroni.md)). + +## Why CloudNativePG, and the version status + +CloudNativePG (CNPG) is an operator that manages PostgreSQL primary/standby clusters directly through the Kubernetes API, without Patroni, repmgr or Stolon: it elects the new primary on failure, scales replicas, updates services and does rolling updates (replicas first, then a switchover). It is listed on the CNCF Landscape; PostgreSQL only, Kubernetes only, vanilla PostgreSQL only (no forks). + +Checked 2026-11-10: latest release **v1.30.1**. Support table: **1.30.x** (released 2026-06-29, end of life ~March 2027) supports Kubernetes 1.34, 1.35, 1.36 (1.37 tested, not supported) and PostgreSQL 14-18; 1.29.x supported until 2026-09-29 (passed: upgrade). A minor is supported until 3 months after the next minor ships; plan a new minor roughly quarterly. Default operand image in 1.30.1 is PostgreSQL 18.6 (`18.6-system-trixie`). The operator's own image moved to Debian 13. + +## Core resources + +| Resource | Purpose | +|----------|---------| +| `Cluster` (`postgresql.cnpg.io/v1`) | The PostgreSQL cluster: `instances`, `imageName` (or an image catalog), `storage`, `postgresql.parameters`, `bootstrap`, `plugins` | +| `Pooler` | PgBouncer deployment in front of a Cluster (`type: rw` or `ro`, `pgbouncer.poolMode`, `parameters`); the name must differ from any Cluster name; `spec.cluster` is immutable ([connection-pooling-pgbouncer.md](connection-pooling-pgbouncer.md)) | +| `Backup` / `ScheduledBackup` | On-demand and cron backups (`schedule` uses six fields with seconds: `"0 0 0 * * *"`; `method: plugin`; `backupOwnerReference`) | +| `ObjectStore` (`barmancloud.cnpg.io/v1`, plugin) | Object store config (destinationPath, credentials, `retentionPolicy`) | +| `ImageCatalog`, `ClusterImageCatalog`, `Database`, `Publication`, `Subscription` | Declarative images, databases and logical replication | + +Operations: `kubectl cnpg status ` and `kubectl cnpg backup` come from the `cnpg` kubectl plugin; `kubectl cnpg backup ... --dry-run` prints the Backup that would be created (1.30.1). Instance manager handles `postgresql.conf`/`pg_hba.conf` from the Cluster spec: do not edit them in the pod. + +## Backups: use the Barman Cloud plugin + +From 1.26 the native Barman object store support is **deprecated**; 1.31 is the last minor with it and it is **removed from 1.32.0**. The official path is the Barman Cloud Plugin (CNPG-I): install it in the operator's namespace, define an `ObjectStore`, reference it from `Cluster.spec.plugins` with `isWALArchiver: true`, back up with `method: plugin`. Do not set both `spec.backup.barmanObjectStore` and a WAL-archiver plugin (rejected). The `serverName` in the `ObjectStore` stays empty; use the `serverName` plugin parameter. Volume snapshots (CSI) are the other method: they support hot and cold backups and incremental/differential copies depending on storage, do not need WAL archiving (still recommended) and have no retention policy. + +Facts to design with: +- Object store backups always require WAL archiving, are hot backups only, no incremental/differential, and support retention (`ObjectStore.spec.retentionPolicy: "30d"` = recovery window; obsolete backups are deleted after the next backup completes). +- With a WAL archive CNPG states an RPO of at most 5 minutes (`archive_timeout` bound); "a WAL archive alone is useless: without a base backup you cannot restore". +- Backups can run from a standby. +- `pg_dump` is not a CNPG backup and is "not suitable for business continuity". + +Manifests: [templates/cnpg-cluster.yaml](templates/cnpg-cluster.yaml), [templates/cnpg-scheduledbackup.yaml](templates/cnpg-scheduledbackup.yaml). + +## Recovery and PITR + +Recovery creates a **new** Cluster (never in place): `bootstrap.recovery.source` points to an `externalClusters` entry with the plugin and `barmanObjectName` + `serverName` (the source cluster's name); add `recoveryTarget.targetTime` (the base backup is auto-selected as the closest complete one before the target). A snapshot recovery needs the snapshot's end time before the target (and the PGWAL snapshot if `walStorage` is used). The restored Cluster does **not** archive WAL unless `spec.plugins` is also set; give it a different bucket path or serverName than the source to avoid mixing timelines. If tablespaces were added or removed since the last base backup, WAL replay fails: take a base backup after the change. Prove restores on a schedule with a throwaway namespace ([templates/restore-test-pitr-runbook.md](templates/restore-test-pitr-runbook.md)). + +## Upgrades + +- **Minor/image change**: edit `imageName` (or the catalog): rolling update, replicas first, then switchover or restart of the primary. Use `primaryUpdateStrategy` deliberately (`unsupervised` automates the primary step; `supervised` waits for a manual switchover). +- **Operator**: read the release notes of each minor; stay inside the support table's Kubernetes range. +- **Major**: offline in-place `pg_upgrade --link` when the image's major rises (whole cluster down, replica PVCs destroyed and re-cloned), or blue/green with logical dump/restore or logical replication ([major-upgrades.md](major-upgrades.md)). A bug in PostgreSQL 17.0-17.5 breaks the in-place path unless `max_slot_wal_keep_size` is `-1`. +- Node maintenance: drain with PodDisruptionBudgets respected (`devops-kubernetes`); for fast node moves CNPG offers `nodeMaintenanceWindow` with `reusePVC`. + +## Failover behaviour to know (1.30.1 fixes) + +Demotion of an unreachable old primary during failover no longer waits on checkpoints; failover and pod recreation no longer stall while an instance is fenced; the primary now holds a `Lease` before PostgreSQL starts, closing a window where a restarting primary could write. Fencing an instance (annotation-based) is a deliberate availability reduction: record who fenced and unfence it. + +## Other operators (pointers, not evaluated) + +Zalando Postgres Operator (Patroni and Spilo based, PgBouncer, backups, web UI; seen in the Context7 library index), Crunchy PGO, StackGres, Percona Operator for PostgreSQL. Their current versions and support status were **not** checked; verify on each project's site before recommending. CNPG's README states it does not support other databases (for MariaDB, MySQL or Redis operators see those chapters). + +## Sources + +- https://github.com/cloudnative-pg/cloudnative-pg/releases (v1.30.1 notes, read 2026-11-10) +- https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/main/docs/src/supported_releases.md , backup.md , postgres_upgrades.md , connection_pooling.md , README.md (read 2026-11-10) +- https://raw.githubusercontent.com/cloudnative-pg/plugin-barman-cloud/main/web/docs/usage.md , retention.md (read 2026-11-10) +- Context7 `/cloudnative-pg/cloudnative-pg`: ScheduledBackup, PITR recovery snippets, `isWALArchiver` field comment + +## Not verified + +- Whether deleting a `Cluster` deletes its PVCs (the project FAQ entry is a TODO) - see [destructive-actions.md](destructive-actions.md). +- CNCF maturity level, exact `kubectl cnpg` subcommand set and flags beyond `backup --dry-run`, `nodeMaintenanceWindow` semantics in 1.30. +- The Barman Cloud plugin's own version and its Kubernetes/CNPG compatibility matrix. +- Other operators' current status. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/postgres-roles-auth.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/postgres-roles-auth.md new file mode 100644 index 0000000..3c1ab55 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/postgres-roles-auth.md @@ -0,0 +1,61 @@ +--- +name: postgres-roles-auth +description: pg_hba.conf rules and ordering, SCRAM-SHA-256 and the MD5 deprecation, TLS-only entries, least-privilege roles, predefined monitoring roles, public schema hardening +when-to-use: Opening access to a PostgreSQL server, adding an application, backup, replication or monitoring role, reviewing who can connect from where +keywords: pg_hba.conf, scram-sha-256, md5, hostssl, password_encryption, pg_monitor, pg_read_all_stats, REVOKE, roles, replication role +priority: high +related: postgres-versions-config.md, connection-pooling-pgbouncer.md, replication-streaming-logical.md, templates/pg_hba.conf.example +--- + +# PostgreSQL Roles and Authentication + +**Load when:** a new client, replica, backup agent or exporter needs to connect, or access is being reviewed. Network placement (firewall, VPN) is `devops-networking`; secrets handling is `devops-secrets`. + +## pg_hba.conf + +One line per rule: `TYPE DATABASE USER ADDRESS METHOD [OPTIONS]`. Types include `local`, `host`, `hostssl`, `hostnossl`, `hostgssenc`, `hostnogssenc`. Rules are matched top to bottom and the first match decides, so order narrow rules before broad ones and put a final explicit `reject` for anything else. Reload with `SELECT pg_reload_conf();` (no restart); check what the server parsed with `SELECT * FROM pg_hba_file_rules;` (shows line numbers and parse errors, read-only). + +- **Method**: `scram-sha-256`. `password_encryption` defaults to `scram-sha-256`; MD5 password storage is deprecated and will be removed in a future release; `password` sends clear text. To migrate: ensure every client library supports SCRAM, set `password_encryption = 'scram-sha-256'`, have users set new passwords, then change the `pg_hba.conf` methods. +- **TLS**: use `hostssl` for any non-local address and a `hostnossl ... reject` line after it. Certificates and cipher policy: `devops-networking`. +- **Narrow CIDRs**: application subnets only. Never `0.0.0.0/0` with a password method on a routable network. +- **Replication**: a separate line with database `replication`, a dedicated role with `REPLICATION` and no other rights, restricted to the replica addresses. +- **Behind PgBouncer**: allow PgBouncer's address, not every client ([connection-pooling-pgbouncer.md](connection-pooling-pgbouncer.md)). +- **Under Patroni** the `pg_hba` list in the Patroni configuration generates the file (ignored if `hba_file` is non-default); per-role lists `pg_hba_primary` / `pg_hba_replica` replace, not merge ([ha-patroni.md](ha-patroni.md)). + +Template with comments: [templates/pg_hba.conf.example](templates/pg_hba.conf.example). + +## Roles: least privilege + +```sql +-- Placeholders only; real passwords come from the secret store, set with \password (never typed into a shared history) +CREATE ROLE app_owner NOLOGIN; -- owns objects, nobody logs in as it +CREATE ROLE app_rw LOGIN; -- application runtime role +GRANT CONNECT ON DATABASE appdb TO app_rw; +GRANT USAGE ON SCHEMA app TO app_rw; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA app TO app_rw; +ALTER DEFAULT PRIVILEGES FOR ROLE app_owner IN SCHEMA app + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO app_rw; +``` + +- `CREATE ROLE` has no `LOGIN` by default; `CREATE USER` is `CREATE ROLE ... LOGIN`. +- Separate roles: migration/owner (DDL), application (DML), read-only analytics, `replicator` (`REPLICATION`), backup agent, monitoring, PgBouncer's auth role. No application connects as a superuser. +- **Monitoring**: grant `pg_monitor` (member of `pg_read_all_settings`, `pg_read_all_stats`, `pg_stat_scan_tables`) instead of superuser. +- **Other predefined roles to grant knowingly**: `pg_signal_backend` (cancel or terminate other sessions, not superuser sessions), `pg_read_all_data` / `pg_write_all_data`. +- **Public schema**: from PostgreSQL 15 new databases no longer give everyone `CREATE` on `public`; on older majors and on upgraded databases run `REVOKE CREATE ON SCHEMA public FROM PUBLIC;`. +- Per-role limits: `ALTER ROLE app_rw SET statement_timeout = '30s';`, `CONNECTION LIMIT`, `idle_in_transaction_session_timeout`. + +Auditing who can do what (read-only): `\du+`, `\dn+`, `\dp schema.*`, and `SELECT rolname, rolsuper, rolreplication, rolcanlogin FROM pg_roles;`. + +## Sources + +- https://www.postgresql.org/docs/18/auth-password.html (SCRAM, MD5 deprecation, migration steps, read 2026-11-10) +- https://www.postgresql.org/docs/18/auth-pg-hba-conf.html (record formats, read 2026-11-10) +- https://www.postgresql.org/docs/18/predefined-roles.html (read 2026-11-10) +- Context7 `/websites/postgresql`: `password_encryption` default, `REVOKE CREATE ON SCHEMA public`, `CREATE ROLE`/`CREATE USER` +- https://patroni.readthedocs.io (dynamic configuration: `pg_hba`, role-specific lists) + +## Not verified + +- The "first match decides" ordering wording was not re-read on the 18 page today (the fetched text did not return that sentence); it is long-standing documented behaviour. +- `pg_hba_file_rules` view and the PostgreSQL 15 default for the public schema were not fetched today. +- The SQL block is an illustrative pattern, not run against any server. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/postgres-versions-config.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/postgres-versions-config.md new file mode 100644 index 0000000..2c71725 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/postgres-versions-config.md @@ -0,0 +1,70 @@ +--- +name: postgres-versions-config +description: Supported PostgreSQL majors and the 5-year EOL policy, install choices, initdb defaults, and the postgresql.conf parameters an operator sets first (memory, WAL, checkpoints, connections, logging) +when-to-use: Choosing a major version, building a new instance, tuning or reviewing postgresql.conf, answering "is version X still supported" +keywords: postgresql.conf, shared_buffers, work_mem, max_wal_size, checkpoint_timeout, wal_level, initdb, data checksums, EOL, minor release +priority: high +related: postgres-roles-auth.md, replication-streaming-logical.md, major-upgrades.md, templates/postgresql-tuned.conf +--- + +# PostgreSQL Versions and Configuration + +**Load when:** creating, sizing or reviewing a PostgreSQL instance. Read `SELECT version();` first: every default below is from the 18 manual. + +## Versions and end of life (checked 2026-11-10) + +The project supports a major for 5 years after its first release, then ships a final minor and the major is unsupported. Always run the latest minor of your major: minors are bug, security and corruption fixes only and need a restart, not a dump/restore. + +| Major | Current minor | Final release (EOL) | Status | +|-------|---------------|---------------------|--------| +| 19 | RC1 (2026-10-15) | - | Not for production | +| 18 | 18.6 | 2030-11-14 | Default choice for new builds | +| 17 | 17.11 | 2029-11-08 | Supported | +| 16 | 16.15 | 2028-11-09 | Supported | +| 15 | 15.19 | 2027-11-11 | Supported, plan the move | +| 14 | 14.24 | **2026-11-12** | Last minor ships in two days: migrate now | +| 13 and older | - | passed | Unsupported: no fixes | + +## Install + +Use the distribution's PostgreSQL project repository (PGDG) or the OS package of a supported major; keep the binaries version-specific (`/usr/lib/postgresql/18`, `/usr/pgsql-18`) so two majors can coexist during an upgrade ([major-upgrades.md](major-upgrades.md)). `initdb` enables data checksums by default in 18 (`--no-data-checksums` turns them off); a cluster to be upgraded with `pg_upgrade` must be initialised with settings compatible with the old one. The data directory, WAL and backups belong on separate volumes when the platform allows it (filesystem and kernel tuning: `devops-linux`). + +## postgresql.conf: what to set first + +Defaults are conservative. Values below are starting points, not final: measure with [monitoring-queries.md](monitoring-queries.md). A ready fragment with reasoning per line is in [templates/postgresql-tuned.conf](templates/postgresql-tuned.conf). + +| Parameter | Default (18) | Starting rule | Restart? | +|-----------|--------------|---------------|----------| +| `shared_buffers` | 128MB | ~25% of RAM on a dedicated server; more than 40% rarely helps | yes | +| `effective_cache_size` | 4GB | Planner hint only (buffers + OS cache): ~50-75% of RAM; reserves nothing | no | +| `work_mem` | 4MB | Per sort/hash node per query: multiply by concurrent nodes before raising; raise per role or per session first | no | +| `maintenance_work_mem` | 64MB | 512MB-2GB; autovacuum may use `autovacuum_max_workers` times this unless `autovacuum_work_mem` is set | no | +| `max_connections` | 100 | Keep low and pool with PgBouncer ([connection-pooling-pgbouncer.md](connection-pooling-pgbouncer.md)) | yes | +| `wal_level` | replica | `replica` for streaming and archiving; `logical` only when logical replication or decoding is needed | yes | +| `max_wal_size` | 1GB (soft limit) | 4-16GB: fewer forced checkpoints; larger values lengthen crash recovery | no | +| `checkpoint_timeout` | 5min | 15min on write-heavy systems, with `max_wal_size` sized so time, not size, triggers checkpoints | no | +| `checkpoint_completion_target` | 0.9 | Leave at 0.9 | no | +| `wal_compression` | off | `on` (or `lz4`/`zstd` if built in) when CPU is spare and WAL volume is high | no | +| `log_checkpoints` | on | Keep on; check "requested" vs "timed" counts | no | +| `log_min_duration_statement` | -1 (off) | e.g. 500ms to find slow queries; `log_lock_waits = on` (default off) | no | +| `log_autovacuum_min_duration` | 10min | Lower to 0-1min while tuning vacuum ([vacuum-bloat-wraparound.md](vacuum-bloat-wraparound.md)) | no | +| `max_slot_wal_keep_size` | -1 (unlimited) | Set a cap: an abandoned slot otherwise fills the WAL disk ([replication-streaming-logical.md](replication-streaming-logical.md)) | no | +| `archive_mode` / `archive_command` | off | On in production, pointing at pgBackRest ([backup-pitr-pgbackrest.md](backup-pitr-pgbackrest.md)) | `archive_mode` yes | + +Rules: +- Change one thing, reload (`SELECT pg_reload_conf();`) or restart as the table says, and compare. `SELECT name, setting, pending_restart FROM pg_settings WHERE pending_restart;` lists values waiting for a restart. +- `ALTER SYSTEM` writes `postgresql.auto.conf`, which overrides `postgresql.conf`: grep both before concluding where a value comes from. On Patroni and CloudNativePG the manager owns these files; change the manager's config instead ([ha-patroni.md](ha-patroni.md), [postgres-on-kubernetes-cnpg.md](postgres-on-kubernetes-cnpg.md)). +- A restart drops every connection and replays WAL since the last checkpoint: schedule it, and on an HA cluster restart replicas first. + +## Sources + +- https://www.postgresql.org/support/versioning/ (read 2026-11-10: table of versions, current minors, final release dates, 19 RC1) +- https://www.postgresql.org/docs/18/runtime-config-resource.html , runtime-config-wal.html , runtime-config-connection.html , runtime-config-logging.html , runtime-config-vacuum.html , runtime-config-replication.html (defaults, read 2026-11-10) +- https://www.postgresql.org/docs/18/app-initdb.html (checksums enabled by default) +- Context7 `/websites/postgresql`: `shared_buffers` 25% / 40% guidance, `effective_cache_size`, `max_wal_size`, `checkpoint_completion_target` + +## Not verified + +- The numeric starting values in the "Starting rule" column other than the 25%/40% `shared_buffers` guidance are operator practice, not manual statements. +- PGDG repository setup steps and package names per distribution were not fetched; follow postgresql.org/download for the OS. +- `wal_compression` accepted algorithms depend on the build (not checked). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/redis-valkey-ops.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/redis-valkey-ops.md new file mode 100644 index 0000000..57d62e7 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/redis-valkey-ops.md @@ -0,0 +1,70 @@ +--- +name: redis-valkey-ops +description: Redis and the Valkey fork in 2026 - licence and version status, choosing between them, persistence (RDB, AOF, none) and backing up, replication, Sentinel versus Cluster, memory and eviction policy, exposure and ACLs, FLUSHALL risk +when-to-use: Deploying or operating Redis or Valkey as cache, queue or store; deciding Redis vs Valkey; choosing persistence and HA; recovering after an accidental flush +keywords: Redis, Valkey, RDB, AOF, appendfsync, Sentinel, Cluster, maxmemory, maxmemory-policy, noeviction, allkeys-lru, replication, AGPLv3, FLUSHALL, protected mode, ACL +priority: medium +related: destructive-actions.md, mysql-mariadb-ops.md, mongodb-pointers.md, postgres-on-kubernetes-cnpg.md +--- + +# Redis and Valkey Operations + +**Load when:** an in-memory key-value store is in the picture. First decide whether it must be durable at all: most Redis/Valkey uses are caches where loss is acceptable; stores that hold the only copy of data need a persistence and backup plan like any database. + +## Status (checked 2026-11-10) + +| | Redis Open Source | Valkey | +|---|-------------------|--------| +| Current | **8.10** on redis.io downloads; 8.12 is at RC2 (not for production) | Supported lines: **9.1** (9.1.3), **9.0** (9.0.7), **8.1** (8.1.11), **8.0** (8.0.12), **7.2** (7.2.15); 9.2.0-rc1 is a release candidate | +| Licence | Moved to SSPL/RSAL in March 2024; from **Redis 8** the OSI-approved **AGPLv3** is offered as an additional licence option (Redis blog, 2025-05-01) | Community-governed fork (ValkeyConf 2026 post title: "Nobody Owns Valkey"); foundation hosting and licence text were not re-read today | +| Notes | Redis 8 integrates former Redis Stack modules (JSON, time series, probabilistic types, query engine) and vector sets; AGPL has network-copyleft obligations: have the licence reviewed for your use | Valkey 9.x ships release notes per line on valkey.io; Kubernetes operator is young (blog: "road to 1.0") | + +Decision: for a new self-hosted deployment where licence terms matter, Valkey is the conservative choice; use Redis 8 where its module features are required and the AGPL/commercial terms are accepted. Clients using the RESP protocol work with both, but module commands are Redis-specific. Do not mix versions across a replication group. Pin images by tag and digest (`devops-docker`). + +## Persistence + +| Mode | Behaviour | Loss window | Use | +|------|-----------|-------------|-----| +| **None** | pure cache | everything on restart | cache only | +| **RDB** | point-in-time snapshots at `save` intervals (e.g. `save 60 1000`); compact file; `fork()` can stall large datasets for ms to ~1 s | minutes | backup-friendly caches and stores that tolerate minutes of loss | +| **AOF** (`appendonly yes`) | logs every write; fsync policy `always`, `everysec` (default; lose at most ~1 s), `no`; background rewrite keeps it small; `valkey-check-aof` repairs a half-written tail | ~1 s with `everysec` | stores that need durability | +| **RDB + AOF** | both | ~1 s | common production choice | + +Backups: an RDB file is never modified once produced (written under a temporary name and renamed atomically), so **copying it while the server runs is safe**; the documented practice is hourly snapshots in one directory and daily in another, pruned by `find`, copied off the host to a second site. Test restores: start a throwaway instance with the file and check `DBSIZE` and sample keys. Storage and offsite policy: `devops-storage-backup`. Replication is not a backup (a `FLUSHALL` replicates). + +## Replication and HA + +Replication is asynchronous leader-follower: replicas reconnect and partially resync, full resync creates a snapshot on the primary (memory and I/O spike). Clients can request acknowledgement with `WAIT`/`WAITAOF`, but that does **not** make it strongly consistent: acknowledged writes can still be lost in a failover. + +| | Sentinel | Cluster | +|---|----------|---------| +| Purpose | HA for a **single** primary dataset: monitoring, notification, automatic failover, client discovery | Sharding across 16384 hash slots **plus** failover | +| Minimum | at least **three** Sentinel instances on independent failure domains; quorum set per monitored primary (e.g. `monitor mymaster ... 2`) | at least **three primaries** (recommended six nodes: three primaries + three replicas) | +| Choose when | dataset fits one node's memory, multi-key operations across all keys needed | dataset or throughput exceeds one node; keys can be partitioned (multi-key operations restricted to one slot) | +| Caveats | clients must be Sentinel-aware; Sentinel does not fix lost writes during partitions | can lose acknowledged writes (asynchronous replication, partitions); resharding is an operation to rehearse | + +Failover drills belong in `devops-sre-ops`. On Kubernetes use the project's operator or a vetted chart; their maturity was not evaluated. + +## Memory and eviction + +Set `maxmemory` below the container/host limit with headroom for fork copy-on-write, replication buffers and fragmentation. `maxmemory-policy`: `noeviction` (writes refused at the limit; safest for stores, errors for caches), `allkeys-lru`/`allkeys-lfu` (cache), `volatile-*` (only keys with a TTL). Pick per role: a cache with `noeviction` fails under load; a store with `allkeys-lru` silently deletes data. Monitor `used_memory`, `evicted_keys`, `rdb_last_bgsave_status`, `aof_last_write_status`, replica offset lag (exporter wiring: `devops-observability`; both projects publish Prometheus guidance). + +## Exposure and access + +Never expose the port to untrusted networks. Without a password and with default binding, protected mode answers only loopback clients with an error; that is a safety net, not a design. Restrict by network, use TLS and ACL users with minimal command and key permissions, disable or rename dangerous commands via ACL (`FLUSHALL`, `FLUSHDB`, `CONFIG`, `DEBUG`, `KEYS`), keep passwords in `devops-secrets`. + +## Destructive: FLUSHALL / FLUSHDB + +They remove all keys (all databases / the current one) and replicate to replicas. ACTION / EFFECT / PROOF / WAIT-FOR-GO ([destructive-actions.md](destructive-actions.md)); PROOF includes `INFO replication` role, `DBSIZE`, the newest RDB timestamp. Recovery after an accident with AOF: if no AOF rewrite has run since, stop the server, remove the last (flush) command from the AOF file and restart; stop the AOF rewrite from running first. Otherwise only the last RDB. + +## Sources + +- https://redis.io/downloads/ (8.10), https://github.com/redis/redis/releases (8.12 RC2), https://redis.io/blog/agplv3/ (licence history, AGPLv3 from Redis 8), https://redis.io/docs/latest/operate/oss_and_stack/management/sentinel/ (read 2026-11-10) +- https://valkey.io/download/ (supported lines), https://valkey.io/topics/persistence/ , replication/ , sentinel/ , cluster-tutorial/ , lru-cache/ , security/ , https://valkey.io/blog/ (read 2026-11-10) + +## Not verified + +- Valkey licence terms and governance (not re-read); Redis 8 exact licence set beyond the blog's AGPL statement. +- Default value of `maxmemory-policy` in current versions and exact `save` defaults (check each version's shipped config). +- Redis-side persistence/Sentinel/Cluster behaviour is assumed identical to Valkey's documented behaviour except where stated; the Redis docs pages for them were not compared line by line. +- Kubernetes operators for either, Redis Software/Enterprise, Memurai. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/replication-streaming-logical.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/replication-streaming-logical.md new file mode 100644 index 0000000..676a4df --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/replication-streaming-logical.md @@ -0,0 +1,73 @@ +--- +name: replication-streaming-logical +description: PostgreSQL streaming (physical) replication, replication slots and their disk risk, synchronous replication, lag checks, and logical replication with its limits and failover behaviour +when-to-use: Adding or rebuilding a replica, a replica falling behind, WAL filling the disk because of a slot, planning synchronous commit, or choosing logical replication for migration or a partial copy +keywords: streaming replication, replication slot, max_slot_wal_keep_size, wal_status, synchronous_standby_names, hot_standby_feedback, primary_conninfo, standby.signal, logical replication, publication, subscription, pg_stat_replication +priority: high +related: ha-patroni.md, major-upgrades.md, monitoring-queries.md, destructive-actions.md +--- + +# Replication: Streaming and Logical + +**Load when:** replicas, lag, slots or logical replication are involved. Failover automation is [ha-patroni.md](ha-patroni.md); do not hand-promote a Patroni or CloudNativePG member. + +## Streaming (physical) replication + +A standby replays the primary's WAL byte for byte: same major version, same architecture, whole cluster. It is the base of HA, read scaling and the safe copy for major upgrades via promotion only within the same major. + +Build a standby (each step a change; the data directory must be empty): +1. On the primary: a `REPLICATION` role, a `pg_hba.conf` `replication` line for the standby address, `max_wal_senders` and `max_replication_slots` high enough (defaults 10 and 10 in 18), `wal_level` at least `replica` (default). +2. On the standby: `pg_basebackup -h primary -U replicator -D -X stream -R -c fast` (or a pgBackRest `restore --type=standby`), which creates `standby.signal` and `primary_conninfo` in `postgresql.auto.conf`. +3. Create the slot first if you use one (`SELECT pg_create_physical_replication_slot('standby1');`) and set `primary_slot_name`. +4. Start the standby and verify on the primary: `SELECT application_name, state, sync_state, replay_lag FROM pg_stat_replication;` and on the standby `SELECT pg_is_in_recovery();`. + +`hot_standby` is on by default, so a standby serves read queries. `hot_standby_feedback = on` prevents query cancellations caused by cleanup records but can bloat the primary: choose consciously. `primary_conninfo` takes a libpq connection string; keep the password in `.pgpass`, not in the string. + +## Slots: the disk-fill trap + +A replication slot makes the primary keep WAL until the consumer has it. A standby or subscriber that is down or abandoned makes `pg_wal` grow until the disk is full and the primary stops. Defences: +- `max_slot_wal_keep_size` (default -1 = unlimited). Set a finite cap: a slot that falls behind it loses WAL and the standby can no longer resume (it must be rebuilt). `pg_replication_slots.wal_status` reports `reserved`, `extended`, `unreserved` or `lost`. +- `idle_replication_slot_timeout` (PostgreSQL 18, default 0 = disabled) invalidates slots that stay inactive longer than the duration. +- Alert on slot lag and inactive slots ([monitoring-queries.md](monitoring-queries.md)) and on free space of the WAL volume. +- **Deleting a slot is irreversible for that consumer** and is in [destructive-actions.md](destructive-actions.md). Patroni and CloudNativePG manage their own slots: do not drop theirs. + +## Synchronous replication + +`synchronous_standby_names` lists the standbys that must confirm before commit returns (with `synchronous_commit` at its level). It trades availability for zero data loss: if the only synchronous standby is unreachable, writes hang. Use a quorum list (`ANY 1 (a, b, c)`) rather than a single name, and let Patroni (`synchronous_mode`) or CloudNativePG manage it in HA setups. Asynchronous replication (default) can lose the last transactions on failover; measure with `replay_lag` and WAL positions. + +## Lag checks (read-only) + +```sql +-- on the primary +SELECT application_name, state, sync_state, + pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn) AS replay_bytes, + write_lag, flush_lag, replay_lag +FROM pg_stat_replication; +-- on a standby +SELECT now() - pg_last_xact_replay_timestamp() AS replay_delay, pg_is_in_recovery(); +``` + +`replay_lag` is the time between flushing WAL locally and the standby reporting it applied. Standby delay measured from the last replayed transaction is misleading on an idle primary. + +## Logical replication + +Publications on the source, subscriptions on the target, row changes decoded from WAL. Needs `wal_level = logical` and enough `max_replication_slots` for subscriptions plus table synchronisation. Works across major versions and for a subset of tables. Limits to plan around: +- **Schema and DDL are not replicated**: create the schema on the subscriber first (`pg_dump --schema-only`) and apply every later DDL on both sides in a safe order. +- **Sequence values are not replicated**: the subscriber's sequences keep their start values; set them (`setval`) before switching writers to the subscriber. +- Large objects and some other objects are not replicated (see the restrictions page); every replicated table needs a replica identity (a primary key, or `REPLICA IDENTITY FULL` at higher cost). +- The subscription's slot lives on the publisher and holds WAL like any slot; a stalled subscriber fills the publisher's disk. `DROP SUBSCRIPTION` on a live subscription also drops the remote slot unless detached first; do that deliberately. +- **Failover**: with `failover = true` on the subscription and `sync_replication_slots = on` on the physical standby (default off), the logical slot is synchronised to the standby so subscribers resume from the new primary after promotion. Without it, a failover breaks logical replication. Patroni can also keep permanent slots through failover (`slots` in the dynamic configuration). +- `pg_createsubscriber` converts a physical standby into a logical replica (shown in the manual as `pg_createsubscriber -D -P "host=foo" -d hr -d finance`). + +Use for: online major upgrade ([major-upgrades.md](major-upgrades.md)), migrating a subset, feeding analytics. Do not use as a substitute for backups or HA. + +## Sources + +- https://www.postgresql.org/docs/18/runtime-config-replication.html (`max_slot_wal_keep_size`, `idle_replication_slot_timeout`, `sync_replication_slots`, `hot_standby_feedback`, `primary_conninfo`, `primary_slot_name`, `synchronous_standby_names`; read 2026-11-10) +- https://www.postgresql.org/docs/18/warm-standby.html , monitoring-stats.html (`replay_lag`), view-pg-replication-slots.html (`wal_status`), logical-replication-config.html (`wal_level = logical`), logical-replication-restrictions.html (schema, sequences), logical-replication-failover.html (read 2026-11-10) +- Context7 `/websites/postgresql`: `pg_createsubscriber` example, `max_slot_wal_keep_size`; Patroni dynamic configuration (`slots`) + +## Not verified + +- The behaviour of `DROP SUBSCRIPTION` regarding the remote slot and the exact replica-identity requirements were not re-read on the restrictions page today; check https://www.postgresql.org/docs/18/sql-dropsubscription.html before running it. +- That synchronous quorum syntax `ANY n (...)` is accepted in 18 was not shown in the extracted text (the page was read only up to the opening paragraph). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/schema-changes-safe.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/schema-changes-safe.md new file mode 100644 index 0000000..a4bedd4 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/schema-changes-safe.md @@ -0,0 +1,69 @@ +--- +name: schema-changes-safe +description: Lock-aware schema changes on a live PostgreSQL - lock queue behaviour, lock_timeout and retries, CREATE INDEX CONCURRENTLY, NOT VALID constraints, table rewrites to avoid, backfills, and a pre-flight checklist +when-to-use: Before running any ALTER, CREATE INDEX, DROP, constraint or type change on a production table, reviewing a migration for lock risk, or after a migration caused a stall +keywords: lock_timeout, statement_timeout, ACCESS EXCLUSIVE, CREATE INDEX CONCURRENTLY, DROP INDEX CONCURRENTLY, NOT VALID, VALIDATE CONSTRAINT, table rewrite, backfill, REINDEX CONCURRENTLY, migration, lock queue +priority: critical +related: monitoring-queries.md, vacuum-bloat-wraparound.md, destructive-actions.md, templates/lock-safe-migration-checklist.md +--- + +# Safe Schema Changes in Production + +**Load when:** a migration will touch a table that serves traffic. The application's migration tool and schema design belong to the framework experts; this is the operator's lock and downtime view. Checklist to attach to every change: [templates/lock-safe-migration-checklist.md](templates/lock-safe-migration-checklist.md). + +## The lock-queue trap + +A DDL statement that needs a strong lock (`ACCESS EXCLUSIVE`) waits behind every open transaction touching the table - and while it waits, **every new query on that table queues behind it**. A harmless `ALTER TABLE` stuck behind one forgotten `idle in transaction` session becomes an outage. Defence on every migration session: + +```sql +SET lock_timeout = '3s'; -- abort if the lock is not acquired quickly (applies to each lock wait) +SET statement_timeout = '15min'; -- cap runaway work; 0 for a known long concurrent build +-- run the DDL; on lock_timeout failure: find the blocker (monitoring-queries.md), then retry in a quiet moment +``` + +`lock_timeout` (default 0 = wait forever) aborts any statement that waits longer than the value for a table, index or row lock. Build a retry loop in the deploy tooling (backoff, max attempts, alert) rather than raising the timeout. Check for long transactions before starting: query 4 and 5 of [templates/monitoring-queries.sql](templates/monitoring-queries.sql). + +## Locks that matter + +| Operation | Lock / effect | Safe pattern | +|-----------|---------------|--------------| +| `CREATE INDEX` | blocks writes (reads continue) for the whole build | `CREATE INDEX CONCURRENTLY` | +| `CREATE INDEX CONCURRENTLY` | takes two table scans and waits for older transactions; cannot run inside a transaction block; not supported on partitioned tables (build per partition, then attach) | A failure leaves an **invalid** index that still costs write overhead: check `pg_index.indisvalid`, `DROP INDEX CONCURRENTLY` it, retry | +| `DROP INDEX` | `ACCESS EXCLUSIVE` | `DROP INDEX CONCURRENTLY` (one index; no `CASCADE`; not for constraint-backed indexes) | +| `REINDEX` | locks | `REINDEX INDEX CONCURRENTLY` | +| `ADD CONSTRAINT ... CHECK/FOREIGN KEY` | full-table validation scan under a strong lock | `ADD CONSTRAINT ... NOT VALID` first (no scan of existing rows, new rows checked), then `VALIDATE CONSTRAINT` (takes only `SHARE UPDATE EXCLUSIVE`, runs alongside traffic) | +| `ADD FOREIGN KEY` | `SHARE ROW EXCLUSIVE` on both tables | `NOT VALID` then `VALIDATE` | +| `SET NOT NULL` | scans the table, unless a valid `CHECK (col IS NOT NULL)` proves it | 18+: `ADD CONSTRAINT nn NOT NULL col NOT VALID`, then `VALIDATE CONSTRAINT nn`; older: add a CHECK `NOT VALID`, `VALIDATE`, then `SET NOT NULL`, then drop the check | +| `ADD COLUMN` | fast with no default or a non-volatile default | A volatile default, stored generated column, identity column or domain with constraints **rewrites the whole table and its indexes** | +| `ALTER COLUMN TYPE` | normally rewrites the table and indexes (a few binary-coercible changes do not) | add a new column, backfill in batches, switch, drop later (expand/contract) | +| `DETACH PARTITION` | strong lock | `DETACH PARTITION ... CONCURRENTLY` | +| `ALTER TABLE ... SET (autovacuum_...)` | brief metadata lock | still use `lock_timeout` | +| Rename column/table, drop column | instant, but breaks running application versions | expand/contract across two deploys | + +## Expand / contract + +1. **Expand**: add the new nullable column, table or index in a backward-compatible way. Old and new application versions both work. +2. **Backfill** in batches of a few thousand rows with a pause between batches, `COMMIT` per batch, ordered by primary key, so vacuum and replication keep up and locks stay short. Monitor replica lag and dead tuples while it runs. +3. **Switch** the application (feature flag), then add constraints with `NOT VALID` / `VALIDATE`. +4. **Contract** in a later release: drop the old column or table. Drops are irreversible: backup proof and ACTION/EFFECT/PROOF/WAIT-FOR-GO ([destructive-actions.md](destructive-actions.md)). + +## Rules of thumb + +- One DDL per transaction/migration step when each takes a strong lock; combining them extends the time the lock is held. +- Never run a data backfill and a schema change in the same transaction. +- Test the migration on a restored copy of production size, time it, and record the locks it takes (`pg_locks` while it runs). +- Prefer off-peak windows for any step that is not concurrent-safe, and tell the on-call. +- Migrations that need a session advisory lock must not go through a transaction-mode PgBouncer ([connection-pooling-pgbouncer.md](connection-pooling-pgbouncer.md)). +- `DROP TABLE` / `TRUNCATE` / `DELETE` without `WHERE` in a migration is a destructive action, whatever the tool. + +## Sources + +- https://www.postgresql.org/docs/18/sql-createindex.html (concurrent build steps, invalid index on failure, not in a transaction block, partitioned tables; read 2026-11-10) +- https://www.postgresql.org/docs/18/sql-dropindex.html (`CONCURRENTLY` caveats), https://www.postgresql.org/docs/18/sql-altertable.html (`NOT VALID`, `VALIDATE CONSTRAINT` lock, `SET NOT NULL` scan rule, rewrite cases, `ADD FOREIGN KEY` lock, `DETACH PARTITION CONCURRENTLY`), https://www.postgresql.org/docs/18/sql-reindex.html (`CONCURRENTLY`), https://www.postgresql.org/docs/18/runtime-config-client.html (`lock_timeout` semantics, default 0), https://www.postgresql.org/docs/18/explicit-locking.html (read 2026-11-10) + +## Not verified + +- The statement that every new query queues behind a waiting `ACCESS EXCLUSIVE` request is the documented lock-queue behaviour from general knowledge; the explicit-locking page was only checked for the lock names. +- That a non-volatile default does not rewrite is inferred from the manual's list of rewrite cases (volatile default etc. rewrite); the PostgreSQL 11 origin of the fast path was not fetched. +- Which `ALTER COLUMN TYPE` changes avoid a rewrite (the manual says "as an exception"; the list was truncated in the extract). +- Batch sizes and pause values are practice, not manual values. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/cnpg-cluster.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/cnpg-cluster.yaml new file mode 100644 index 0000000..bd1cd02 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/cnpg-cluster.yaml @@ -0,0 +1,112 @@ +# cnpg-cluster.yaml - CloudNativePG 1.30.x: Cluster + Barman Cloud plugin ObjectStore + Pooler. +# Prerequisites (separate, verified before applying): +# - CloudNativePG operator 1.30.x installed (supports Kubernetes 1.34-1.36) and the Barman Cloud plugin installed +# in the SAME namespace as the operator (https://cloudnative-pg.io/plugin-barman-cloud/). +# - Secrets created by your secret tooling (devops-secrets), never from this file: 'app-user-credentials' (basic-auth), +# 'object-store-credentials' (ACCESS_KEY_ID / ACCESS_SECRET_KEY). The values below are NOT in this file. +# Apply sequence: kubectl diff -f, kubectl apply --dry-run=server -f, apply after the user's go (devops-kubernetes). +# API groups: postgresql.cnpg.io/v1 (Cluster, Pooler), barmancloud.cnpg.io/v1 (ObjectStore). +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: pg-main-store + namespace: databases +spec: + # Recovery window: backups older than the first valid backup before the window are deleted after the next backup completes. + retentionPolicy: "30d" + configuration: + destinationPath: s3://CHANGE_ME-bucket/pg-main/ + endpointURL: https://CHANGE_ME-s3-endpoint + s3Credentials: + accessKeyId: + name: object-store-credentials + key: ACCESS_KEY_ID + secretAccessKey: + name: object-store-credentials + key: ACCESS_SECRET_KEY + wal: + compression: gzip + data: + compression: gzip +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: pg-main + namespace: databases +spec: + description: "Main application database" + # Pin the operand image (tag and, in production, digest). 18.6 is the current 18 minor (2026-11-10). + imageName: ghcr.io/cloudnative-pg/postgresql:18.6-system-trixie + instances: 3 # 1 primary + 2 replicas; the operator handles failover + primaryUpdateStrategy: unsupervised # rolling update switches the primary automatically; 'supervised' waits for a manual switchover + enableSuperuserAccess: false # no postgres superuser Secret exposed to apps + + postgresql: + parameters: + shared_buffers: 2GB + max_connections: "200" + work_mem: 16MB + maintenance_work_mem: 512MB + max_wal_size: 8GB + checkpoint_timeout: 15min + log_min_duration_statement: 500ms + pg_stat_statements.max: "10000" + pg_stat_statements.track: all + pg_hba: + - hostssl all all 10.0.0.0/8 scram-sha-256 # CHANGE_ME: your pod CIDR / app network + + bootstrap: + initdb: + database: app + owner: app + secret: + name: app-user-credentials # kubernetes.io/basic-auth with username and password keys + + storage: + storageClass: CHANGE_ME-storageclass + size: 100Gi + walStorage: # separate WAL volume: keeps WAL I/O and a filling WAL disk away from data + storageClass: CHANGE_ME-storageclass + size: 20Gi + + resources: + requests: + cpu: "2" + memory: 8Gi + limits: + memory: 8Gi # memory limit = request; CPU limit left unset on purpose (devops-kubernetes) + + affinity: + enablePodAntiAffinity: true + topologyKey: topology.kubernetes.io/zone + podAntiAffinityType: required # one instance per zone/node; use 'preferred' on small clusters + + # WAL archiving + base backups through the Barman Cloud plugin + plugins: + - name: barman-cloud.cloudnative-pg.io + isWALArchiver: true + parameters: + barmanObjectName: pg-main-store + + monitoring: + enablePodMonitor: false # deprecated in CNPG 1.30; create a PodMonitor yourself (devops-observability) +--- +# PgBouncer in front of the primary. The Pooler name must differ from every Cluster name in the namespace. +apiVersion: postgresql.cnpg.io/v1 +kind: Pooler +metadata: + name: pg-main-pooler-rw + namespace: databases +spec: + cluster: + name: pg-main # immutable after creation + instances: 2 + type: rw + pgbouncer: + poolMode: transaction # session state (SET, LISTEN, session advisory locks) does not work; see connection-pooling-pgbouncer.md + parameters: + max_client_conn: "1000" + default_pool_size: "20" + max_prepared_statements: "100" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/cnpg-scheduledbackup.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/cnpg-scheduledbackup.yaml new file mode 100644 index 0000000..6a83a98 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/cnpg-scheduledbackup.yaml @@ -0,0 +1,34 @@ +# cnpg-scheduledbackup.yaml - weekly base backup of the Cluster in cnpg-cluster.yaml through the Barman Cloud plugin. +# CloudNativePG docs: "In most cases a weekly base backup is sufficient"; WAL archiving (continuous) gives point-in-time recovery +# between base backups. The schedule field is a SIX-field cron expression WITH SECONDS: sec min hour day-of-month month day-of-week. +# A completed base backup is not a proof: restore it into a throwaway namespace on a schedule (restore-test-pitr-runbook.md). +--- +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: pg-main-weekly + namespace: databases +spec: + schedule: "0 30 2 * * 0" # Sundays 02:30:00 (time zone semantics not verified: check the first run in 'kubectl get backup') + immediate: true # also take one backup right after creation, so the first restore test has a backup + backupOwnerReference: self # self = the ScheduledBackup owns its Backup objects; cluster = the Cluster owns them; none = legacy. + # Kubernetes garbage collection follows ownership: decide before deleting either object + cluster: + name: pg-main # immutable after creation + method: plugin + pluginConfiguration: + name: barman-cloud.cloudnative-pg.io + # suspend: true # uncomment to pause scheduled backups temporarily (record who paused and when to resume) +--- +# On-demand backup (apply only when needed, e.g. right before a risky change; each is a change, ask first): +# apiVersion: postgresql.cnpg.io/v1 +# kind: Backup +# metadata: +# name: pg-main-pre-migration-20261110 +# namespace: databases +# spec: +# cluster: +# name: pg-main +# method: plugin +# pluginConfiguration: +# name: barman-cloud.cloudnative-pg.io diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/lock-safe-migration-checklist.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/lock-safe-migration-checklist.md new file mode 100644 index 0000000..2d77ccb --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/lock-safe-migration-checklist.md @@ -0,0 +1,44 @@ +# Lock-Safe Migration Checklist (PostgreSQL) + +Copy into the change ticket. Every box must be ticked or marked N/A with a reason. Background: `references/schema-changes-safe.md`. + +## Before + +- [ ] Target identified: host/cluster, database, schema, tables, row counts and sizes (`pg_total_relation_size`) +- [ ] Server major and minor recorded (`SELECT version();`); the DDL was checked against that major's manual +- [ ] Newest backup listed (`pgbackrest --stanza=main info`) AND the last restore test date recorded: ____ +- [ ] For data-destroying steps (drop column/table, delete, truncate): ACTION/EFFECT/PROOF/WAIT-FOR-GO written and the user's go received +- [ ] Migration ran on a restored copy of production size; duration: ____; locks observed (`pg_locks`): ____ +- [ ] No long transaction now: queries 4 and 5 of `monitoring-queries.sql` are clean (oldest `xact_age` < ____) +- [ ] No replica or slot already lagging (query 14, 15); WAL volume has headroom +- [ ] Application versions compatible with both the old and new schema (expand/contract) + +## Session settings (first statements of the migration session) + +- [ ] `SET lock_timeout = '3s';` +- [ ] `SET statement_timeout = '15min';` (or 0 for a concurrent index build, with a watcher) +- [ ] Not going through a transaction-mode pooler if the tool takes session advisory locks + +## The statements + +- [ ] Index: `CREATE INDEX CONCURRENTLY ...` (not in a transaction block); invalid-index check after (`pg_index.indisvalid`) +- [ ] Constraint: `ADD CONSTRAINT ... NOT VALID`, then `VALIDATE CONSTRAINT` in a separate step +- [ ] NOT NULL: valid CHECK proof first, then `SET NOT NULL` +- [ ] New column: no volatile default, no stored generated or identity column on a big table (they rewrite) +- [ ] Type change: expand/contract, not `ALTER COLUMN TYPE` on a big table +- [ ] Backfill in batches with commit per batch and a pause; replica lag watched +- [ ] Retry-on-lock_timeout logic with backoff and a maximum attempt count + +## During + +- [ ] Someone watches `pg_stat_activity` (waiting sessions), replica lag, and error rates +- [ ] Abort criteria written: ____ (e.g. blocked sessions above N for over 10 s: cancel with `pg_cancel_backend()`) + +## After + +- [ ] Schema matches intent (`\d+ table`); no invalid index (`SELECT ... FROM pg_index WHERE NOT indisvalid`) +- [ ] `ANALYZE` run on touched tables (new indexes and columns need statistics) +- [ ] Autovacuum healthy; replication lag back to baseline +- [ ] A new base backup taken if the change was large (a rewrite, a mass delete, a major upgrade) +- [ ] Contract step (drop of old columns/tables) scheduled for a later release, not today +- [ ] Result recorded in the ticket: duration, locks, surprises diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/monitoring-queries.sql b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/monitoring-queries.sql new file mode 100644 index 0000000..9f60ab7 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/monitoring-queries.sql @@ -0,0 +1,121 @@ +-- monitoring-queries.sql +-- READ-ONLY health queries for PostgreSQL 17/18 (checked against the 18 manual, 2026-11-10). +-- Run as a role that is a member of pg_monitor. Nothing here changes data or settings. +-- Lines that DO change something are commented out and marked WRITE: they need the user's go. +-- Usage: psql "host=db1 dbname=postgres user=monitor_user" -f monitoring-queries.sql (password from PGPASSFILE) + +\echo '== 1. Identity, role, uptime' +SELECT version(), pg_is_in_recovery() AS is_standby, + now() - pg_postmaster_start_time() AS uptime; + +\echo '== 2. Sessions by state and wait event' +SELECT state, wait_event_type, wait_event, count(*) +FROM pg_stat_activity +WHERE backend_type = 'client backend' +GROUP BY 1, 2, 3 ORDER BY 4 DESC; + +\echo '== 3. Connections vs limit' +SELECT count(*) AS sessions, + current_setting('max_connections')::int AS max_connections, + round(100.0 * count(*) / current_setting('max_connections')::int, 1) AS pct_used +FROM pg_stat_activity; + +\echo '== 4. Oldest transactions (xmin horizon holders)' +SELECT pid, usename, application_name, state, + now() - xact_start AS xact_age, now() - state_change AS in_state_for, + age(backend_xmin) AS xmin_age, left(query, 100) AS query +FROM pg_stat_activity +WHERE xact_start IS NOT NULL +ORDER BY xact_start LIMIT 15; + +\echo '== 5. Idle in transaction longer than 1 minute' +SELECT pid, usename, application_name, now() - state_change AS idle_for, left(query, 100) AS last_query +FROM pg_stat_activity +WHERE state = 'idle in transaction' AND now() - state_change > interval '1 minute' +ORDER BY idle_for DESC; + +\echo '== 6. Blocked sessions and who blocks them' +SELECT a.pid AS blocked_pid, a.usename, a.wait_event_type, a.wait_event, + now() - a.query_start AS waiting_for, + pg_blocking_pids(a.pid) AS blocked_by, left(a.query, 100) AS blocked_query +FROM pg_stat_activity a +WHERE cardinality(pg_blocking_pids(a.pid)) > 0 +ORDER BY waiting_for DESC; + +\echo '== 7. Ungranted locks' +SELECT locktype, relation::regclass AS relation, mode, pid, granted +FROM pg_locks WHERE NOT granted; + +\echo '== 8. Prepared transactions (hold the xmin horizon until resolved)' +SELECT gid, prepared, owner, database, age(transaction) AS xid_age FROM pg_prepared_xacts; + +\echo '== 9. Top statements by total time (needs pg_stat_statements in shared_preload_libraries + CREATE EXTENSION)' +SELECT queryid, calls, round(total_exec_time::numeric, 1) AS total_ms, + round(mean_exec_time::numeric, 2) AS mean_ms, rows, + shared_blks_hit, shared_blks_read, temp_blks_written, left(query, 100) AS query +FROM pg_stat_statements ORDER BY total_exec_time DESC LIMIT 15; +-- WRITE (ask first): SELECT pg_stat_statements_reset(); + +\echo '== 10. Database counters and cache hit ratio' +SELECT datname, xact_commit, xact_rollback, deadlocks, temp_bytes, conflicts, + round(100.0 * blks_hit / NULLIF(blks_hit + blks_read, 0), 2) AS cache_hit_pct +FROM pg_stat_database WHERE datname IS NOT NULL ORDER BY xact_commit DESC; + +\echo '== 11. Checkpoints (PostgreSQL 17+; on 16 and older use pg_stat_bgwriter)' +SELECT num_timed, num_requested, buffers_written FROM pg_stat_checkpointer; + +\echo '== 12. WAL archiver' +SELECT archived_count, last_archived_time, failed_count, last_failed_time FROM pg_stat_archiver; + +\echo '== 13. WAL directory size' +SELECT count(*) AS files, pg_size_pretty(sum(size)) AS size FROM pg_ls_waldir(); + +\echo '== 14. Replication (run on the primary)' +SELECT application_name, client_addr, state, sync_state, + pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn) AS replay_bytes_behind, + write_lag, flush_lag, replay_lag +FROM pg_stat_replication; + +\echo '== 15. Replication slots and retained WAL (primary)' +SELECT slot_name, slot_type, active, wal_status, safe_wal_size, + pg_size_pretty(pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)) AS retained_wal +FROM pg_replication_slots ORDER BY pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) DESC NULLS LAST; + +\echo '== 16. Standby delay (run on a standby; returns NULL on a primary)' +SELECT now() - pg_last_xact_replay_timestamp() AS replay_delay; + +\echo '== 17. Transaction ID age per database (alert > 500M)' +SELECT datname, age(datfrozenxid) AS xid_age FROM pg_database ORDER BY 2 DESC; + +\echo '== 18. Worst tables by XID age' +SELECT c.oid::regclass AS table_name, + greatest(age(c.relfrozenxid), age(t.relfrozenxid)) AS xid_age +FROM pg_class c LEFT JOIN pg_class t ON c.reltoastrelid = t.oid +WHERE c.relkind IN ('r', 'm') ORDER BY 2 DESC LIMIT 20; + +\echo '== 19. Dead tuples and last autovacuum (current database)' +SELECT relname, n_live_tup, n_dead_tup, + round(100.0 * n_dead_tup / NULLIF(n_live_tup + n_dead_tup, 0), 1) AS dead_pct, + last_autovacuum, last_autoanalyze, autovacuum_count +FROM pg_stat_user_tables ORDER BY n_dead_tup DESC LIMIT 20; + +\echo '== 20. Vacuums running now' +SELECT pid, datname, relid::regclass AS table_name, phase, heap_blks_total, heap_blks_scanned +FROM pg_stat_progress_vacuum; + +\echo '== 21. Biggest relations (current database)' +SELECT c.oid::regclass AS relation, pg_size_pretty(pg_total_relation_size(c.oid)) AS total_size +FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE c.relkind = 'r' AND n.nspname NOT IN ('pg_catalog', 'information_schema') +ORDER BY pg_total_relation_size(c.oid) DESC LIMIT 15; + +\echo '== 22. Invalid indexes left by a failed CONCURRENTLY build' +SELECT indexrelid::regclass AS index_name, indrelid::regclass AS table_name +FROM pg_index WHERE NOT indisvalid; + +\echo '== 23. Settings waiting for a restart' +SELECT name, setting, pending_restart FROM pg_settings WHERE pending_restart; + +-- WRITE examples, never run without the user's go and the PROOF in destructive-actions.md: +-- SELECT pg_cancel_backend(); -- cancels the current query of that session +-- SELECT pg_terminate_backend(); -- ends the session, rolls back its transaction diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/patroni.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/patroni.yml new file mode 100644 index 0000000..f1d7b78 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/patroni.yml @@ -0,0 +1,94 @@ +# patroni.yml - one node of a 3-node Patroni 4.1.x cluster with an etcd (v3 API) DCS. +# Install as /etc/patroni/patroni.yml on each node; change name, listen and connect_address per node. +# Keys checked against the Patroni docs (yaml_configuration.rst, dynamic_configuration.rst) on 2026-11-10. +# SECRETS: values below are placeholders. Real passwords are injected through the environment variables Patroni supports +# (PATRONI_SUPERUSER_PASSWORD, PATRONI_REPLICATION_PASSWORD, PATRONI_RESTAPI_PASSWORD) from the secret store (devops-secrets); +# never commit them. The file mode must be 0600, owner postgres. +# `bootstrap.dcs` is applied ONCE when the cluster is first initialised; afterwards change it with +# patronictl -c /etc/patroni/patroni.yml edit-config (dynamic configuration lives in the DCS). +# Validate before use: patroni --validate-config /etc/patroni/patroni.yml + +scope: pg-main # cluster name, identical on every node +name: CHANGE_ME_node1 # unique per node +namespace: /service/ # path in the DCS (default /service) + +restapi: + listen: 0.0.0.0:8008 + connect_address: CHANGE_ME_node1_ip:8008 # must be reachable from ALL nodes (used in the leader race) + authentication: + username: patroni_api + password: CHANGE_ME # use PATRONI_RESTAPI_PASSWORD instead of a literal + # TLS for the REST API (recommended): + # certfile: /etc/patroni/tls/node.crt + # keyfile: /etc/patroni/tls/node.key + # cafile: /etc/patroni/tls/ca.crt + +etcd3: + hosts: CHANGE_ME_etcd1:2379,CHANGE_ME_etcd2:2379,CHANGE_ME_etcd3:2379 + protocol: https + cacert: /etc/patroni/tls/etcd-ca.crt + cert: /etc/patroni/tls/etcd-client.crt + key: /etc/patroni/tls/etcd-client.key + +bootstrap: + dcs: + ttl: 30 # failover starts after the leader fails to renew for ttl seconds (min 20) + loop_wait: 10 # rule: loop_wait + 2 * retry_timeout <= ttl + retry_timeout: 10 # DCS/network trouble shorter than this does not demote the leader (min 3) + maximum_lag_on_failover: 1048576 # bytes; replicas lagging more than this cannot win the leader race + synchronous_mode: false # true or quorum = no committed-data loss on failover, writes can block (see ha-patroni.md) + failsafe_mode: false # true = a primary survives a DCS outage if all members acknowledge it + postgresql: + use_pg_rewind: true # needs data checksums (default in initdb 18) and/or wal_log_hints = on + parameters: + wal_level: replica + hot_standby: "on" + max_wal_senders: 10 + max_replication_slots: 10 + wal_log_hints: "on" + archive_mode: "on" + archive_command: pgbackrest --stanza=main archive-push %p + archive_timeout: 5min + max_slot_wal_keep_size: 50GB # cap it: see replication-streaming-logical.md + password_encryption: scram-sha-256 + shared_preload_libraries: pg_stat_statements + pg_hba: + - local all postgres peer + - hostssl replication replicator CHANGE_ME_db_subnet/24 scram-sha-256 + - hostssl all rewind_user CHANGE_ME_db_subnet/24 scram-sha-256 # pg_rewind from a rejoining member + - hostssl all all CHANGE_ME_app_subnet/24 scram-sha-256 + - host all all 0.0.0.0/0 reject + initdb: # options passed to initdb on the first node + - encoding: UTF8 + - data-checksums # default in 18; explicit for older majors + # Custom bootstrap from a pgBackRest backup (disaster recovery) replaces initdb: see the Patroni "custom bootstrap" docs. + +postgresql: + listen: 0.0.0.0:5432 + connect_address: CHANGE_ME_node1_ip:5432 + data_dir: /var/lib/postgresql/18/main + bin_dir: /usr/lib/postgresql/18/bin + pgpass: /var/lib/postgresql/.pgpass_patroni # Patroni writes this file; keep outside the data directory + authentication: + superuser: + username: postgres + password: CHANGE_ME # PATRONI_SUPERUSER_PASSWORD + replication: + username: replicator + password: CHANGE_ME # PATRONI_REPLICATION_PASSWORD + rewind: + username: rewind_user + password: CHANGE_ME # needed by pg_rewind + # Rebuild replicas from the backup repository instead of the leader (requires a pgBackRest restore wrapper): + # create_replica_methods: [pgbackrest, basebackup] + +watchdog: + mode: automatic # off | automatic | required (required = a node without a working watchdog never leads) + device: /dev/watchdog + safety_margin: 5 + +tags: + nofailover: false # true keeps this node out of the leader race (reporting/delayed replica) + noloadbalance: false + clonefrom: false + nosync: false diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/pg_hba.conf.example b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/pg_hba.conf.example new file mode 100644 index 0000000..b1635a2 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/pg_hba.conf.example @@ -0,0 +1,38 @@ +# pg_hba.conf.example - rules are matched top to bottom; the first matching line decides. +# Format: TYPE DATABASE USER ADDRESS METHOD [OPTIONS] +# Reload after editing: SELECT pg_reload_conf(); Check parsing: SELECT * FROM pg_hba_file_rules; +# Placeholders: CHANGE_ME_* are examples, replace with your own addresses/names. No password appears in this file. +# Under Patroni the 'pg_hba' list of the Patroni config generates this file; under CloudNativePG use spec.postgresql.pg_hba. + +# --- Local administration (OS user 'postgres' only, over the unix socket) ------------- +local all postgres peer + +# --- Local service accounts that use the socket (monitoring agent, backup agent on the same host) +local all monitor_user scram-sha-256 +local all backup_user scram-sha-256 + +# --- Replication: dedicated role, replica addresses only, TLS required ------------------ +hostssl replication replicator CHANGE_ME_replica1/32 scram-sha-256 +hostssl replication replicator CHANGE_ME_replica2/32 scram-sha-256 +# pg_rewind (Patroni use_pg_rewind): a normal, non-replication connection from a rejoining member +hostssl all rewind_user CHANGE_ME_replica1/32 scram-sha-256 +hostssl all rewind_user CHANGE_ME_replica2/32 scram-sha-256 +# pgBackRest and pg_basebackup from the backup host, if it connects over TCP: +hostssl replication backup_user CHANGE_ME_backup_host/32 scram-sha-256 + +# --- Cluster-internal admin (Patroni users, same private subnet) -------------------- +hostssl all patroni_admin CHANGE_ME_db_subnet/24 scram-sha-256 + +# --- Pooler (PgBouncer) connects as application roles through its own address ----------- +hostssl appdb app_rw CHANGE_ME_pgbouncer/32 scram-sha-256 +hostssl appdb app_ro CHANGE_ME_pgbouncer/32 scram-sha-256 +# PgBouncer's auth_query role, if used (non-superuser calling a SECURITY DEFINER function): +hostssl appdb pgbouncer_auth CHANGE_ME_pgbouncer/32 scram-sha-256 + +# --- Monitoring exporter on the observability host ---------------------------------------- +hostssl postgres monitor_user CHANGE_ME_monitoring/32 scram-sha-256 + +# --- Explicit denials, last: no unencrypted TCP, nothing else --------------------------- +hostnossl all all 0.0.0.0/0 reject +host all all 0.0.0.0/0 reject +host all all ::/0 reject diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/pgbackrest-commands.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/pgbackrest-commands.md new file mode 100644 index 0000000..d8ff4c0 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/pgbackrest-commands.md @@ -0,0 +1,74 @@ +# pgBackRest Command Sheet (2.60.0) + +Stanza `main`, config `/etc/pgbackrest/pgbackrest.conf` ([pgbackrest.conf](pgbackrest.conf)). Run as the `postgres` OS user. `R` = read-only, `W` = writes (needs the go), `D` = destructive (ACTION/EFFECT/PROOF/WAIT-FOR-GO, see `../destructive-actions.md`). + +## Inspect (R) + +```bash +pgbackrest --stanza=main info # R backups, sizes, WAL range per repository +pgbackrest --stanza=main info --output=json # R for scripts and alert checks +pgbackrest --stanza=main check # R validates config and that archive_command reaches the repository +pgbackrest --stanza=main verify # R checks repository files against checksums (can be long) +pgbackrest version # R +``` + +## Set up (W, once per cluster) + +```bash +# postgresql.conf: archive_mode = on (restart), archive_command = 'pgbackrest --stanza=main archive-push %p' +pgbackrest --stanza=main stanza-create # W creates the stanza in every repository +pgbackrest --stanza=main check # R must pass before the first backup +``` + +After a PostgreSQL **major upgrade**: set `pg1-path` to the new data directory, then `pgbackrest --stanza=main stanza-upgrade` (W; `--no-online` if the database is offline) and take a new full backup. + +## Back up (W, scheduled) + +```bash +pgbackrest --stanza=main --type=full backup # W weekly +pgbackrest --stanza=main --type=diff backup # W daily (changes since last full) +pgbackrest --stanza=main --type=incr backup # W hourly (changes since last backup); default type for 'backup' is incr +# `expire` runs automatically after a successful backup (see D below) +``` + +Alerts: non-zero exit; newest backup in `info` older than policy; `pg_stat_archiver.failed_count` rising; `check` failing. + +## Restore (W/D, to a NEW or intended-to-be-replaced directory, PostgreSQL stopped) + +```bash +# Latest state of the newest backup, into an empty directory (restore test or rebuild of a replica) +pgbackrest --stanza=main restore # W target dir must be empty (or use --delta: D) +# Point in time; pause at the target so the result can be inspected before it is promoted +pgbackrest --stanza=main --type=time --target="2026-11-10 09:41:00+00" --target-action=pause --archive-mode=off restore # W +# Named restore point created earlier with: SELECT pg_create_restore_point('before_migration'); +pgbackrest --stanza=main --type=name --target=before_migration --target-action=pause --archive-mode=off restore # W +# A specific backup set instead of the newest suitable one +pgbackrest --stanza=main --set=20261109-020001F --type=immediate --target-action=pause --archive-mode=off restore # W +# Build a replica from the backups +pgbackrest --stanza=main --type=standby restore # W +# In-place replacement of an existing directory: files not matching the backup are removed +pgbackrest --stanza=main --delta --type=time --target="..." --target-action=pause restore # D +``` + +Then start PostgreSQL, wait for `pg_is_in_recovery()` pause at the target, check data read-only, and either `SELECT pg_promote();` (accept) or restore again with another target. Full procedure with proof capture: [restore-test-pitr-runbook.md](restore-test-pitr-runbook.md). + +Notes: `--target-action` supports `pause` (default), `promote`, `shutdown`. Since 2.60.0 `restore` stops with an error if a restored file has an invalid page checksum (unless `--no-checksum-page-error`): re-run with another backup set rather than starting the cluster. + +## Retention and deletion (D) + +```bash +pgbackrest --stanza=main --repo=1 expire # D applies retention now: deletes backups beyond policy and their WAL +pgbackrest --stanza=main --repo=1 --set= expire # D expires one set +pgbackrest --stanza=main stop # W stops the stanza's operations (needed before stanza-delete) +pgbackrest --stanza=main stanza-delete # D PERMANENTLY removes all backups and archives of the stanza; + # cluster must be shut down or --force. Needs the go naming stanza + repo path +pgbackrest --stanza=main start # W re-enables operations after a stop +``` + +## Cron / systemd timer example (W, installed once, reviewed) + +```cron +# /etc/cron.d/pgbackrest (user postgres) +30 02 * * 0 postgres pgbackrest --stanza=main --type=full backup +30 02 * * 1-6 postgres pgbackrest --stanza=main --type=diff backup +``` diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/pgbackrest.conf b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/pgbackrest.conf new file mode 100644 index 0000000..2cb682c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/pgbackrest.conf @@ -0,0 +1,65 @@ +# pgbackrest.conf - /etc/pgbackrest/pgbackrest.conf (pgBackRest 2.60.0) +# Local POSIX repository on a dedicated volume + optional second repository on S3-compatible storage. +# Option names checked against https://pgbackrest.org/configuration.html (2026-11-10). +# No secret is stored here: the cipher passphrase and S3 keys come from files or environment injected by the secret store +# (devops-secrets); the values below are placeholders. Keep the file mode 0640, owner postgres. +# The repository volume must NOT be the database volume (devops-storage-backup covers 3-2-1 and offsite copies). + +[global] +# --- Repository 1: local, fast restores, short retention --- +repo1-path=/var/lib/pgbackrest +# Keep 2 full backups; differentials expire with their full, incrementals with their parent. +repo1-retention-full=2 +repo1-retention-diff=6 +# Block-level incremental saves space; bundling combines small files (fewer objects, helps on object stores). +# Both are repository options: read their notes on pgbackrest.org/configuration.html before enabling on an existing stanza. +repo1-block=y +repo1-bundle=y +# Optional encryption at rest of the repository. Use a passphrase from the secret store. Stanzas created with +# pgBackRest older than 2.59.3 may hold weak subkeys: read the 2026-10-04 notice on pgbackrest.org/news.html first. +#repo1-cipher-type=aes-256-cbc +#repo1-cipher-pass=CHANGE_ME_from_secret_store + +# --- Repository 2: object storage, longer retention (uncomment and fill) --- +#repo2-type=s3 +#repo2-path=/pgbackrest +#repo2-s3-bucket=CHANGE_ME_bucket +#repo2-s3-endpoint=CHANGE_ME_s3_endpoint +#repo2-s3-region=CHANGE_ME_region +#repo2-s3-key=CHANGE_ME_access_key +#repo2-s3-key-secret=CHANGE_ME_secret_key +#repo2-retention-full=8 + +# --- Behaviour --- +# Parallel processes for backup/restore/archive. Raise to the number of spare cores, watch I/O. +process-max=4 +# Force an immediate checkpoint at backup start instead of waiting for a spread one. +start-fast=y +# zst (Zstandard) and lz4 are faster than the gz default but "not available on all platforms": check pgbackrest version/build. +compress-type=zst +compress-level=3 +# Take backups from a standby when one is configured (reduces primary load); needs the standby stanza entries below. +#backup-standby=prefer +# Log level on console/file. +log-level-console=info +log-level-file=detail +log-path=/var/log/pgbackrest + +[global:archive-push] +# Asynchronous archiving: batches WAL and reuses connections. Needs a spool path on local disk. +archive-async=y +spool-path=/var/spool/pgbackrest + +[global:archive-get] +archive-async=y +spool-path=/var/spool/pgbackrest + +# One section per stanza. Name must match archive_command in postgresql.conf. +[main] +pg1-path=/var/lib/postgresql/18/main +pg1-port=5432 +pg1-socket-path=/var/run/postgresql +# Standby example (second host), used by backup-standby: +#pg2-host=CHANGE_ME_standby_host +#pg2-host-user=postgres +#pg2-path=/var/lib/postgresql/18/main diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/pgbouncer.ini b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/pgbouncer.ini new file mode 100644 index 0000000..5718ac6 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/pgbouncer.ini @@ -0,0 +1,71 @@ +; pgbouncer.ini - transaction pooling for web applications. PgBouncer 1.26.1 or later (CVE fixes, 2026-10-14). +; Comments are on their own lines on purpose (INI parsers keep inline comments inside the value). +; Placeholders: CHANGE_ME_*. No password is stored here: users live in auth_file / auth_query (secret store: devops-secrets). +; Reload: connect to the admin console and run RELOAD; (or systemctl reload) - read SHOW POOLS; before and after. + +[databases] +; One entry per database. host is the PRIMARY endpoint (HAProxy/VIP in front of Patroni, or the CNPG -rw service). +appdb = host=CHANGE_ME_primary_endpoint port=5432 dbname=appdb +; Read-only pool pointing at a replica endpoint (separate logical name): +appdb_ro = host=CHANGE_ME_replica_endpoint port=5432 dbname=appdb + +[pgbouncer] +listen_addr = CHANGE_ME_private_ip +listen_port = 6432 +unix_socket_dir = /var/run/postgresql + +; --- Pooling --- +; transaction: server connection released when the transaction ends. Session state (SET, LISTEN, session advisory locks, +; SQL PREPARE, WITH HOLD cursors) does NOT work in this mode. Use session mode for tools that need it. +pool_mode = transaction +; Per user/database pair (default 20). Sum over pairs must stay below PostgreSQL max_connections minus reserved sessions. +default_pool_size = 20 +; Extra server connections for bursts, used after reserve_pool_timeout seconds of waiting. +reserve_pool_size = 5 +reserve_pool_timeout = 5 +; Client connections allowed (default 100). Raise the service LimitNOFILE accordingly. +max_client_conn = 1000 +; Hard cap of server connections per database across users. +max_db_connections = 80 +; Protocol-level prepared statements in transaction mode (1.21.0+). 0 disables. Larger than the app's common statements. +max_prepared_statements = 100 +; Clients waiting for a server connection longer than this are disconnected (default 120). +query_wait_timeout = 60 +server_idle_timeout = 600 +server_lifetime = 3600 +; 1.26.0 tracks search_path by default on PostgreSQL 18; uncomment to restore the 1.25 behaviour if it causes trouble. +; ignore_startup_parameters = search_path + +; --- Authentication --- +; SCRAM everywhere. auth_type defaults to md5, set it explicitly. +auth_type = scram-sha-256 +auth_file = /etc/pgbouncer/userlist.txt +; Users not in auth_file are looked up in the database through a non-superuser SECURITY DEFINER function +; installed in EVERY database the pool serves. auth_user must exist in auth_file. +; The function must return two columns (user name, password hash) and no row for an unknown user. Pattern, run once +; per database by a superuser after review (a change, ask first): +; CREATE SCHEMA pgbouncer AUTHORIZATION pgbouncer_owner; +; CREATE FUNCTION pgbouncer.get_auth(p_usename text) RETURNS TABLE(usename name, passwd text) +; LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog +; AS $$ SELECT rolname, rolpassword FROM pg_authid WHERE rolname = p_usename AND rolcanlogin $$; +; REVOKE ALL ON FUNCTION pgbouncer.get_auth(text) FROM PUBLIC; +; GRANT EXECUTE ON FUNCTION pgbouncer.get_auth(text) TO pgbouncer_auth; +auth_user = pgbouncer_auth +auth_query = SELECT usename, passwd FROM pgbouncer.get_auth($1) + +; --- TLS --- +client_tls_sslmode = require +client_tls_key_file = /etc/pgbouncer/tls/server.key +client_tls_cert_file = /etc/pgbouncer/tls/server.crt +server_tls_sslmode = verify-full +server_tls_ca_file = /etc/pgbouncer/tls/ca.crt + +; --- Admin console (connect to the virtual database "pgbouncer") --- +admin_users = pgbouncer_admin +stats_users = monitor_user + +; --- Logging --- +logfile = /var/log/pgbouncer/pgbouncer.log +pidfile = /var/run/pgbouncer/pgbouncer.pid +log_connections = 0 +log_disconnections = 0 diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/postgresql-tuned.conf b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/postgresql-tuned.conf new file mode 100644 index 0000000..ea28d42 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/postgresql-tuned.conf @@ -0,0 +1,62 @@ +# postgresql-tuned.conf - starting fragment for PostgreSQL 17/18 on a DEDICATED host +# Assumed sizing for the numbers: 8 vCPU, 32 GB RAM, NVMe/SSD, WAL archived with pgBackRest. +# Put it in a file included from postgresql.conf (include_dir 'conf.d') so it is easy to diff. +# Do NOT use on a Patroni or CloudNativePG member: put these keys in the manager's config instead. +# Defaults quoted are the PostgreSQL 18 manual defaults (read 2026-11-10). Measure, then change one thing at a time. +# Restart needed: shared_buffers, max_connections, wal_level, archive_mode, max_wal_senders, +# max_replication_slots, shared_preload_libraries. Everything else: reload. + +# --- Connections ----------------------------------------------------------------- +listen_addresses = 'CHANGE_ME_private_ip' # never '*' on a routable network; pair with pg_hba.conf +max_connections = 200 # default 100. Keep LOW and pool with PgBouncer; each backend costs memory +superuser_reserved_connections = 5 # default 3. Headroom for emergency superuser logins + +# --- Memory ---------------------------------------------------------------------- +shared_buffers = 8GB # default 128MB. ~25% of RAM on a dedicated server; >40% rarely helps +effective_cache_size = 22GB # default 4GB. Planner hint only: buffers + OS cache (~70% of RAM). Reserves nothing +work_mem = 16MB # default 4MB. PER sort/hash NODE PER QUERY: 200 sessions x several nodes can exceed RAM. + # Raise per role/session for reporting jobs, not globally +maintenance_work_mem = 1GB # default 64MB. VACUUM, CREATE INDEX. Autovacuum may use workers x this +autovacuum_work_mem = 512MB # default -1 (use maintenance_work_mem). Caps each autovacuum worker explicitly + +# --- WAL, checkpoints -------------------------------------------------------------- +wal_level = replica # default replica. Set 'logical' ONLY if logical replication/decoding is needed +max_wal_size = 8GB # default 1GB (soft limit). Fewer forced checkpoints; larger = longer crash recovery +min_wal_size = 1GB # default 80MB. Keeps recycled WAL files to absorb bursts +checkpoint_timeout = 15min # default 5min. Make time, not WAL volume, trigger most checkpoints +checkpoint_completion_target = 0.9 # default 0.9. Spread checkpoint I/O; leave as is +wal_compression = lz4 # default off. Less WAL volume for CPU; value 'lz4' needs a build with LZ4 (use 'on' otherwise) +log_checkpoints = on # default on. Compare timed vs requested checkpoints + +# --- Archiving (point-in-time recovery) ------------------------------------------- +archive_mode = on # default off. RESTART. Without it there is no PITR +archive_command = 'pgbackrest --stanza=main archive-push %p' # stanza name must match pgbackrest.conf +archive_timeout = 5min # forces a WAL switch so an idle primary still ships WAL (bounds RPO); 0 disables + +# --- Replication -------------------------------------------------------------------- +max_wal_senders = 10 # default 10. Replicas + pg_basebackup + backup tools + headroom +max_replication_slots = 10 # default 10. Physical + logical slots +max_slot_wal_keep_size = 50GB # default -1 (unlimited!). CAP it: an abandoned slot otherwise fills the WAL disk. + # A slot beyond the cap is lost and its standby must be rebuilt. Alert before this +hot_standby = on # default on. Replicas serve reads +hot_standby_feedback = off # default off. 'on' prevents standby query cancels but can bloat the primary + +# --- Authentication / TLS --------------------------------------------------------- +password_encryption = 'scram-sha-256' # default scram-sha-256 in 18. md5 storage is deprecated +ssl = on # then set the files below; certificates come from your PKI (devops-networking) +ssl_cert_file = 'server.crt' +ssl_key_file = 'server.key' + +# --- Safety timeouts ---------------------------------------------------------------- +idle_in_transaction_session_timeout = '10min' # default 0 (off). Kills sessions that hold locks and the xmin horizon while idle. + # Better set per application role; keep exceptions explicit +# lock_timeout and statement_timeout: set PER ROLE or per migration session, not globally (see schema-changes-safe.md) + +# --- Logging --------------------------------------------------------------------------- +log_min_duration_statement = 500ms # default -1 (off). Find slow queries; tune to your baseline +log_lock_waits = on # default off. Logs waits longer than deadlock_timeout +log_autovacuum_min_duration = 1min # default 10min. Lower while tuning vacuum +log_line_prefix = '%m [%p] %q%u@%d ' # timestamp, pid, user@db: needed to correlate with pg_stat_activity + +# --- Extensions requiring a restart ------------------------------------------------- +shared_preload_libraries = 'pg_stat_statements' # RESTART. Then CREATE EXTENSION pg_stat_statements; in the target database diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/restore-test-pitr-runbook.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/restore-test-pitr-runbook.md new file mode 100644 index 0000000..3623c68 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/templates/restore-test-pitr-runbook.md @@ -0,0 +1,73 @@ +# Restore Test and PITR Runbook (PostgreSQL) + +A backup is proven only by a completed restore. Run this monthly per cluster, after every major change to the backup setup, and before any risky operation on data (upgrade, mass delete, migration with a rewrite). The same steps, with a real target time, recover from an incident. Tools: pgBackRest ([pgbackrest-commands.md](pgbackrest-commands.md)) or the CloudNativePG recovery Cluster (section C). Storage and DR drill cadence: `devops-storage-backup`, `devops-sre-ops`. + +**Rules**: restore to a NEW host, path or namespace, never over the live instance; the restored copy must not ship WAL into the production archive and must not connect to production replication; no real credentials in the log; every step has its result captured. + +## 0. Record (fill in) + +| Field | Value | +|-------|-------| +| Date, operator | | +| Source cluster / stanza | | +| Backup label restored (`info`) | | +| Recovery target (time / name / immediate) | | +| Target host / namespace | | +| RTO budget / measured | | +| RPO budget / measured (last row visible vs expected) | | + +## A. pgBackRest, bare metal or VM + +1. **Pick the target** (read-only on production): `pgbackrest --stanza=main info` - note the backup label, the WAL range (`wal start/stop`) and that the target time lies inside it. For a recovery of an incident, take a target just BEFORE the damage (check application logs and `log_min_duration_statement` output for the time). +2. **Prepare an isolated restore host** with the same major PostgreSQL binaries and extensions, a repository read-only credential (or read access to a copy), and a `pgbackrest.conf` with the same stanza; the restore below uses `--archive-mode=off` so the restored instance never pushes WAL. +3. **Restore** (PostgreSQL stopped, data directory empty): + ```bash + date -u +%FT%TZ > restore.start + pgbackrest --stanza=main --type=time --target="2026-11-10 09:41:00+00" --target-action=pause --archive-mode=off restore + ``` + For a plain backup test use `--type=immediate` instead. Exit code and log excerpt saved. If the restore reports an invalid page checksum it fails by design (2.60.0): record the file, restore another set with `--set`, and open an incident on the corrupted source. +4. **Start PostgreSQL** with `archive_mode = off` and `listen_addresses = 'localhost'` (or a test address), no pooler, no application. Watch the log until it reports it reached the recovery target and paused. +5. **Verify read-only**: + ```sql + SELECT pg_is_in_recovery(), now(); + SELECT count(*) FROM ; -- compare with the expected value + SELECT max() FROM ; -- last row before the target + ``` + Run the application smoke tests against it if they exist (read-only user). Run `amcheck`/`pg_checksums` style checks only if you have them scripted; a successful `pg_dump -s` of every database is a minimum. +6. **Accept the point**: `SELECT pg_promote();` (becomes a primary on a new timeline). Or discard it and restore again with another target. +7. **Measure**: `date -u` end minus `restore.start` = measured RTO. Last visible row vs expected = measured RPO. Compare with the budget; if over, fix before the next incident (parallelism `process-max`, restore from the nearer repository, smaller base backup interval). +8. **Clean up the test copy** (its own change: it is a destructive step on the test host only; name the directory and host): stop the instance, remove the test data directory, record the removal. Never `--delta` into a directory you have not named. + +## B. Recovering a few rows or one table from an incident + +1. Restore to the isolated host to a time before the damage (steps A.1-A.6 with `--target-action=pause`). +2. Extract: `pg_dump -t schema.table --data-only` or `COPY (SELECT ...) TO ...` on the restored copy. +3. Load into production **inside a transaction with a reviewed query** (staging table first, then `INSERT ... SELECT` / `UPDATE ... FROM` after a diff), after the user's go with the row counts. +4. Overwriting production with the restored cluster is the last resort: it needs ACTION/EFFECT/PROOF/WAIT-FOR-GO from `references/destructive-actions.md`, a fresh backup of the damaged state first, and a plan for the second-writer risk. + +## C. CloudNativePG recovery Cluster (Kubernetes) + +1. Read-only: `kubectl --context $CTX -n databases get cluster,backup,scheduledbackup,objectstore` and note the newest completed `Backup` and the object store. +2. Create a NEW Cluster in a throwaway namespace (or with a different name) that bootstraps from the object store; set `serverName` to the source cluster's name and a `recoveryTarget`: + ```yaml + spec: + bootstrap: + recovery: + source: source + recoveryTarget: + targetTime: "2026-11-10 09:41:00+00" + externalClusters: + - name: source + plugin: + name: barman-cloud.cloudnative-pg.io + parameters: + barmanObjectName: pg-main-store + serverName: pg-main + ``` + Do NOT add `spec.plugins` with `isWALArchiver: true` pointing at the production store: the restored cluster would write WAL into the production archive. If it needs archiving later, give it its own `ObjectStore` path. +3. Wait for the cluster to be ready (`kubectl cnpg status`), then verify as in A.5 through a `kubectl exec` psql session or a temporary client pod. +4. Record RTO/RPO; delete the throwaway namespace (a destructive step, named in the record) and its PVCs only after the evidence is saved. + +## D. Pass / fail + +Pass: restore completed, target reached, row counts and last-row timestamp match, smoke tests pass, measured RTO and RPO within budget, evidence stored in the ticket. Fail: any other result - the backup system is NOT considered working; fix, then re-run. Alert if the last passing test is older than 35 days. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/vacuum-bloat-wraparound.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/vacuum-bloat-wraparound.md new file mode 100644 index 0000000..31a93c0 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-databases/references/vacuum-bloat-wraparound.md @@ -0,0 +1,72 @@ +--- +name: vacuum-bloat-wraparound +description: Autovacuum behaviour and tuning, measuring bloat, what blocks vacuum, why VACUUM FULL is dangerous, and transaction ID wraparound prevention and emergency handling +when-to-use: Tables growing without data growth, autovacuum never finishing, "database must be vacuumed within N transactions" warnings, planning a bloat cleanup, or setting wraparound alerts +keywords: autovacuum, VACUUM, VACUUM FULL, bloat, dead tuples, wraparound, datfrozenxid, relfrozenxid, autovacuum_freeze_max_age, vacuum_failsafe_age, pg_repack, xmin horizon +priority: high +related: monitoring-queries.md, replication-streaming-logical.md, schema-changes-safe.md, destructive-actions.md, templates/monitoring-queries.sql +--- + +# Vacuum, Bloat and Transaction ID Wraparound + +**Load when:** table or index size, autovacuum activity or XID age is the question. Query pack: [templates/monitoring-queries.sql](templates/monitoring-queries.sql). + +## What vacuum does + +`UPDATE` and `DELETE` leave old row versions (MVCC); `VACUUM` reclaims them for reuse, updates planner statistics (via `ANALYZE`), maintains the visibility map and **freezes old rows to prevent XID wraparound**. Standard `VACUUM` runs alongside normal traffic (it blocks `ALTER TABLE`-type changes on that table) and does not return space to the OS except at the table's tail; `VACUUM FULL` rewrites the table and needs `ACCESS EXCLUSIVE`. The autovacuum daemon never issues `VACUUM FULL`. Disabling autovacuum is unwise except for an extremely predictable workload, and the system launches anti-wraparound autovacuums even when autovacuum is off. + +## Autovacuum triggers and tuning + +A table is vacuumed when dead tuples exceed `min(autovacuum_vacuum_max_threshold, autovacuum_vacuum_threshold + autovacuum_vacuum_scale_factor * reltuples)`; also when inserts since the last vacuum exceed the insert threshold. Defaults (18): threshold 50 tuples, scale factor 0.2 (20% of the table), `autovacuum_vacuum_max_threshold` 100,000,000 tuples, `autovacuum_vacuum_cost_delay` 2ms, `autovacuum_max_workers` 3, `autovacuum_naptime` 1min, `log_autovacuum_min_duration` 10min. + +Practical tuning, in order: +1. **Per-table settings for hot or huge tables** instead of global changes: `ALTER TABLE big SET (autovacuum_vacuum_scale_factor = 0.02, autovacuum_vacuum_threshold = 1000);` (a short metadata lock; use `lock_timeout`, see [schema-changes-safe.md](schema-changes-safe.md)). +2. Raise throughput when vacuum cannot keep up: more `autovacuum_max_workers` (needs a restart; memory = workers x `maintenance_work_mem` unless `autovacuum_work_mem` is set), lower `autovacuum_vacuum_cost_delay`, or higher cost limit. Watch I/O. +3. Log it: `log_autovacuum_min_duration = 1min` (or 0 while investigating). +4. Fix the cause before the knobs. Vacuum cannot remove rows still visible to the oldest snapshot ("xmin horizon"). Typical holders (the manual lists the same): long-running or `idle in transaction` sessions, **prepared transactions** (`pg_prepared_xacts`), **replication slots** with old `xmin`/`catalog_xmin` (`pg_replication_slots`), and `hot_standby_feedback` from a lagging standby. Set `idle_in_transaction_session_timeout` per role. + +## Measuring bloat + +- Quick: `n_dead_tup`, `last_autovacuum`, `autovacuum_count` in `pg_stat_user_tables`; table and index sizes with `pg_total_relation_size`. +- Accurate: the `pgstattuple` extension (reads the relation; run off-peak) - exact dead-tuple and free-space figures; estimation-only queries are cheaper but approximate. +- Rule: bloat that returns after cleanup is a vacuum-throughput or xmin-horizon problem, not a one-off. + +## Reclaiming space (all rewrite or exclusive-lock operations are proposals) + +| Tool | Lock | Note | +|------|------|------| +| `VACUUM` | none blocking reads/writes | Makes space reusable; first step | +| `VACUUM FULL`, `CLUSTER`, rewriting `ALTER TABLE` | `ACCESS EXCLUSIVE` for the whole run; needs free disk roughly equal to the table plus indexes | Outage for that table: ACTION/EFFECT/PROOF/WAIT-FOR-GO ([destructive-actions.md](destructive-actions.md)) | +| `REINDEX INDEX CONCURRENTLY` / `REINDEX TABLE CONCURRENTLY` | brief locks, runs alongside traffic | Failure can leave an invalid index to drop | +| `pg_repack` (extension) | short locks at start and end | Extra disk, needs a primary key or unique index; third-party, verify the current version and your PostgreSQL major | + +`TRUNCATE` empties a table instantly with an `ACCESS EXCLUSIVE` lock and no vacuum needed, but violates strict MVCC and is irreversible ([destructive-actions.md](destructive-actions.md)). + +## Transaction ID wraparound + +XIDs are 32-bit and compared modulo 2^32: every table in every database must be vacuumed (frozen) at least once every two billion transactions, or old rows appear to be "in the future" and become invisible (the manual calls it catastrophic data loss). Safeguards, with defaults: +- `autovacuum_freeze_max_age` 200 million (restart to change; per-table override): forces an anti-wraparound autovacuum even if autovacuum is off. +- `vacuum_freeze_min_age` 50 million; `vacuum_failsafe_age` 1.6 billion: at that age `VACUUM` skips cost delay, index vacuuming and buffer rings - last resort. +- When too close, the server logs `WARNING: database "x" must be vacuumed within N transactions` (the example shows about 40 million remaining) and eventually refuses new transaction IDs. + +**Monitor** (page the on-call well before the warnings): +```sql +SELECT datname, age(datfrozenxid) FROM pg_database ORDER BY 2 DESC; -- alert > 500M, page > 1B +SELECT c.oid::regclass, greatest(age(c.relfrozenxid), age(t.relfrozenxid)) AS age -- worst tables +FROM pg_class c LEFT JOIN pg_class t ON c.reltoastrelid = t.oid +WHERE c.relkind IN ('r','m') ORDER BY 2 DESC LIMIT 20; +``` +(the same queries appear in the manual's routine-vacuuming page; thresholds are operator practice.) + +**If the age is high**: first find and remove what holds the horizon - old transactions (`age(backend_xid)`/`age(backend_xmin)` in `pg_stat_activity`), prepared transactions, stale replication slots. Dropping a slot for a consumer that still exists breaks that consumer ([destructive-actions.md](destructive-actions.md)). Then run `VACUUM (VERBOSE, FREEZE)` on the worst tables as a superuser, largest benefit first. The manual says single-user-mode vacuuming is typically no longer necessary, should be avoided and disables the wraparound safeguards. + +## Sources + +- https://www.postgresql.org/docs/18/routine-vacuuming.html (vacuum variants, thresholds, wraparound text, warning example, xmin holders; read 2026-11-10) +- https://www.postgresql.org/docs/18/runtime-config-vacuum.html (defaults; read 2026-11-10) and Context7 `/websites/postgresql` (freeze parameters, XID age queries) + +## Not verified + +- Alert thresholds for XID age (500M/1B) and the cost-limit tuning advice are operator practice, not manual values. +- `pgstattuple` and `pg_repack` behaviour and compatibility with 18 were not fetched. +- The exact point at which the server refuses new transaction IDs (the manual's page was not read to that sentence). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/SKILL.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/SKILL.md new file mode 100644 index 0000000..bcce11e --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/SKILL.md @@ -0,0 +1,148 @@ +--- +name: devops-docker +description: Use when configuring Docker Engine (daemon.json, logging, storage, live-restore), writing Compose files, building images with BuildKit/Buildx (multi-stage, cache mounts, multi-platform, SBOM/provenance, Bake), publishing ports safely next to a host firewall (iptables, experimental nftables backend, ufw/firewalld), running a private registry, signing images with cosign, running daemonless containers with Podman/Quadlet, or troubleshooting/cleaning a container host. Not for Kubernetes (devops-kubernetes), CI pipelines (devops-cicd), dependency vulnerability audits (fuse-security:dependency-audit) or Swarm. +versions: + docker-engine: "29.9.3" + docker-compose: "5.7.0" + buildkit: "0.34.2" + buildx: "0.38.2" + podman: "6.1.4" + cosign: "3.1.4" + distribution-registry: "3.1.2" +user-invocable: true +references: + - references/engine-daemon-config.md + - references/compose-v5.md + - references/buildkit-dockerfile.md + - references/networking-firewall.md + - references/registries-supply-chain.md + - references/podman-quadlet.md + - references/runtime-troubleshooting.md + - references/safe-ops.md + - references/templates/daemon-json.md + - references/templates/dockerfile-hardened.md + - references/templates/compose-production.md + - references/templates/bake-and-sign.md + - references/templates/quadlet-units.md +related-skills: + - devops-kubernetes + - devops-cicd + - devops-linux + - devops-networking + - devops-secrets + - devops-observability + - devops-storage-backup + - devops-sre-ops + - fuse-security:dependency-audit +--- + + +Covers a container host end to end: Engine daemon config, Compose v5, BuildKit/Buildx +builds (multi-stage, multi-platform, attestations, Bake), networking versus the host +firewall, registries, cosign signing, Podman/Quadlet, troubleshooting and safe cleanup. +Not covered: Kubernetes (devops-kubernetes), CI pipelines (devops-cicd), host nftables +design (devops-networking), secret stores (devops-secrets), CVE audits +(fuse-security:dependency-audit), Swarm. + + +# Docker, Compose, BuildKit and Podman + +## Agent Workflow (MANDATORY) + +Before ANY container change, spawn in parallel, one `Task` call each: + +1. **fuse-ai-pilot:explore-codebase** - find `Dockerfile*`, `compose*.yaml`, `docker-bake.hcl`, `daemon.json`, Quadlet units +2. **fuse-ai-pilot:research-expert** - re-check the versions in the frontmatter at docs.docker.com and the GitHub releases +3. **mcp__context7__query-docs** (`/docker/docs`, `/docker/compose`) - confirm every flag or key you are about to write + +On a live host, inspect read-only first (`docker version`, `docker info`, `docker compose ls`, `docker system df`). After changes, run **fuse-ai-pilot:sniper**. + +--- + +## Overview + +| Area | What matters in 2026 | +|------|----------------------| +| **Engine 29** | containerd image store is the default on fresh installs only; cgroup v1 deprecated; Docker Content Trust removed from the CLI | +| **Compose v5** | internal builder removed, builds go through Bake; top-level `version:` is obsolete | +| **BuildKit / Buildx** | min-mode provenance added by default; SBOM is opt-in; Bake files drive multi-target builds | +| **Firewall** | iptables is the default backend; nftables backend (`firewall-backend`) is experimental, no `DOCKER-USER` chain, not usable in Swarm | +| **Supply chain** | cosign v3 stores signatures as OCI 1.1 referrers and uses the bundle format by default | +| **Podman 6 / Quadlet** | `.container`/`.network`/`.volume` units generated into systemd services | + +--- + +## Critical Rules + +1. **Never invent a flag or key** - `dockerd --validate --config-file=...` and `docker compose config -q` prove syntax before any restart or `up`. +2. **Published ports bypass ufw** - Docker's `nat` rules divert traffic before ufw's `INPUT` chain; bind to `127.0.0.1` or filter in `DOCKER-USER` (iptables) or your own nftables table. +3. **Do not edit Docker's own firewall tables or chains** - Docker owns them and rewrites them. +4. **Destructive commands are proposed, never run on your own** - prune, `volume rm`, `compose down -v`, `podman system reset`: list what goes, state the effect, wait for the user's explicit go ([safe-ops.md](references/safe-ops.md)). +5. **Secrets never in `ENV`, `ARG` or image layers** - use build secret mounts and Compose `secrets:`; `mode=max` provenance exposes build args. +6. **Sign and deploy by digest, not tag** - tags move; cosign signs `image@sha256:...`. +7. **Read the release notes before a minor upgrade** - live-restore only survives patch upgrades, not `YY.MM` jumps. + +--- + +## Architecture + +``` +/etc/docker/daemon.json # engine config (validate before restart) +project/{Dockerfile,.dockerignore,docker-bake.hcl,compose.yaml,secrets/} # secrets/ never committed +~/.config/containers/systemd/ # rootless Quadlet units (Podman) +``` + +--- + +## Reference Guide + +### Concepts + +| Topic | Reference | When to Consult | +|-------|-----------|-----------------| +| **Daemon config** | [engine-daemon-config.md](references/engine-daemon-config.md) | daemon.json, logging, storage, live-restore, rootless | +| **Compose v5** | [compose-v5.md](references/compose-v5.md) | healthchecks, depends_on, profiles, secrets, limits | +| **BuildKit** | [buildkit-dockerfile.md](references/buildkit-dockerfile.md) | multi-stage, cache mounts, multi-platform, SBOM, Bake | +| **Networking** | [networking-firewall.md](references/networking-firewall.md) | bridge/macvlan, published ports, ufw, nftables backend | +| **Registries** | [registries-supply-chain.md](references/registries-supply-chain.md) | registry:3, mirrors, cosign sign/verify, digests | +| **Podman** | [podman-quadlet.md](references/podman-quadlet.md) | rootless, Quadlet units, auto-update | +| **Troubleshooting** | [runtime-troubleshooting.md](references/runtime-troubleshooting.md) | OOM, unhealthy loops, disk usage, events | +| **Safe ops** | [safe-ops.md](references/safe-ops.md) | any prune/rm/down -v/reset | + +### Templates + +| Template | When to Use | +|----------|-------------| +| [daemon-json.md](references/templates/daemon-json.md) | Baseline `/etc/docker/daemon.json` for a server | +| [dockerfile-hardened.md](references/templates/dockerfile-hardened.md) | Multi-stage, non-root, cache-mounted Dockerfile | +| [compose-production.md](references/templates/compose-production.md) | Compose app with healthchecks, limits, secrets | +| [bake-and-sign.md](references/templates/bake-and-sign.md) | Multi-arch Bake build + attestations + cosign sign/verify | +| [quadlet-units.md](references/templates/quadlet-units.md) | Rootless Podman service as systemd units | + +--- + +## Quick Reference + +```bash +docker version && docker info -f '{{ .DriverStatus }}' # engine + storage backend +dockerd --validate --config-file=/etc/docker/daemon.json # before any restart +docker compose config -q && docker compose up -d --wait # validate, then start until healthy +docker buildx bake --print # resolved build plan, builds nothing +docker system df -v # what uses the disk (read-only) +``` + +--- + +## Best Practices + +### DO +- Pin base images by digest in production and rebuild on a schedule for patched layers +- Set `log-driver: local` (rotated by default) or `max-size`/`max-file` on `json-file` +- Give every long-running service a healthcheck and memory/pids limits +- Prefer named volumes or explicit bind mounts for data; anonymous volumes are not reattached by a new `up` + +### DON'T +- Expose the Docker socket or add users to the `docker` group casually - it grants root-level privileges +- Set `"iptables": false` to "fix" ufw - it breaks masquerading and exposes container ports to the LAN +- Run `docker system prune -a --volumes` on a host that holds data +- Assume Compose `depends_on` waits for readiness without `condition: service_healthy` diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/buildkit-dockerfile.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/buildkit-dockerfile.md new file mode 100644 index 0000000..e1dabe1 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/buildkit-dockerfile.md @@ -0,0 +1,114 @@ +--- +name: buildkit-dockerfile +description: BuildKit and Buildx - multi-stage Dockerfiles, cache and secret mounts, build checks, multi-platform, SBOM/provenance, Bake +when-to-use: Writing a Dockerfile, speeding up builds, building arm64+amd64 images, adding attestations, defining a docker-bake.hcl +keywords: BuildKit, buildx, Dockerfile, multi-stage, RUN --mount, cache, secret, --check, multi-platform, BUILDPLATFORM, SBOM, provenance, bake +priority: high +related: registries-supply-chain.md, templates/dockerfile-hardened.md, templates/bake-and-sign.md +--- + +# BuildKit, Dockerfile and Buildx + +**Load when:** writing or reviewing a Dockerfile, a build is slow or leaks a secret, or images must ship for several CPU architectures. + +Verified 2026-11-10: BuildKit **v0.34.2** (2026-11-03, shipped in Engine 29.9.3, https://github.com/moby/buildkit/releases/tag/v0.34.2) and Buildx **v0.38.2** (2026-11-03, https://github.com/docker/buildx/releases/tag/v0.38.2). Buildx is GitHub-only as a source. + +--- + +## Dockerfile rules + +1. Start with `# syntax=docker/dockerfile:1` to get the current stable frontend. +2. **Multi-stage**: compile in a full image, copy only the artifact into a minimal final stage; `docker build --target ` builds one stage (https://docs.docker.com/build/building/multi-stage/). +3. **Order for cache**: copy dependency manifests and install dependencies before copying the source tree; keep a `.dockerignore` (https://docs.docker.com/build/cache/optimize/). +4. **Cache mounts** keep package caches across builds without putting them in a layer: + +```dockerfile +RUN --mount=type=cache,target=/root/.cache/go-build go build ./... +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get --no-install-recommends install -y gcc +``` + +`sharing=locked` serialises parallel builds on apt's cache. `docker system prune` also deletes cache mounts ([safe-ops.md](safe-ops.md)). + +5. **Secrets** never go through `ARG`/`ENV`; mount them for one `RUN`: + +```dockerfile +RUN --mount=type=secret,id=API_KEY,env=API_KEY ./fetch-private-deps.sh +``` + +```bash +docker build --secret id=API_KEY . # from the env var of the same name +docker build --secret id=aws,src=$HOME/.aws/credentials . # from a file +``` + +Source: https://docs.docker.com/build/building/secrets/ , https://docs.docker.com/reference/dockerfile/#run---mounttypesecret + +6. `RUN --mount=type=ssh` forwards an SSH agent for private Git; `RUN --network=none` proves a step is offline. +7. `COPY --link` puts copied files on an independent layer that survives changes to earlier layers. +8. `HEALTHCHECK` options: `--interval` (30s), `--timeout` (30s), `--start-period` (0s), `--start-interval` (5s, Engine 25.0+), `--retries` (3). A distroless image has no shell or curl: the check must call your binary. + +--- + +## Build checks (lint) + +```bash +docker build --check . # run checks only, no build +``` + +Add `# check=error=true` under the syntax line to fail the build on any violation; `# check=skip=JSONArgsRecommended` skips a rule (https://docs.docker.com/build/checks/). + +--- + +## Multi-platform + +- One invocation builds several platforms: `docker buildx build --platform linux/amd64,linux/arm64 .` +- Requires the **containerd image store** (default on fresh Engine 29 installs) or a `docker-container` builder: `docker buildx create --name container-builder --driver docker-container --bootstrap --use`. Results of a `docker-container` builder are not loaded into the local image store automatically: push them (`--push`). +- Three strategies, fastest first: native nodes per architecture, **cross-compilation** (`FROM --platform=$BUILDPLATFORM ...` then use `TARGETOS`/`TARGETARCH`), QEMU emulation (bundled with BuildKit; much slower). If emulation fails: `docker run --privileged --rm tonistiigi/binfmt --install all`. + +Source: https://docs.docker.com/build/building/multi-platform/ + +--- + +## Attestations (SBOM and provenance) + +| Flag | Effect | +|------|--------| +| default | **min-mode provenance** attached to images | +| `--provenance=mode=max` | Full provenance; **exposes build-arg values** | +| `--provenance=false` or `BUILDX_NO_DEFAULT_ATTESTATIONS=1` | No provenance | +| `--sbom=true` (= `--attest type=sbom`) | SPDX SBOM of the final stage | +| `ARG BUILDKIT_SBOM_SCAN_STAGE=true` / `BUILDKIT_SBOM_SCAN_CONTEXT=true` | Scan build stages / build context too | + +Inspect what was attached: + +```bash +docker buildx imagetools inspect REG/IMG:TAG --format '{{ json .SBOM.SPDX }}' +docker buildx imagetools inspect REG/IMG:TAG --format '{{ json .Provenance.SLSA }}' +``` + +Sources: https://docs.docker.com/build/metadata/attestations/ , https://docs.docker.com/build/metadata/attestations/sbom/ , https://docs.docker.com/build/metadata/attestations/slsa-provenance/ + +--- + +## Bake + +`docker-bake.hcl` declares `target`, `group` and `variable` blocks with `platforms`, `tags`, `attest`, `cache-from`/`cache-to`, `secret`, `output`, `inherits`. `docker buildx bake --print ` shows the resolved plan without building. A target can also read `compose.yaml` build sections (https://docs.docker.com/build/bake/reference/). Compose v5 delegates its builds to Bake. + +Remote cache example: `--cache-to type=registry,ref=REG/app:buildcache,mode=max` and `--cache-from type=registry,ref=REG/app:buildcache`. + +--- + +## Verification + +```bash +docker build --check . +docker buildx bake --print +docker buildx imagetools inspect REG/IMG:TAG # platforms + attestation manifests +``` + +## Not verified + +- Dockerfile frontend minor version behind `docker/dockerfile:1` was not read today. + +→ Templates: [dockerfile-hardened.md](templates/dockerfile-hardened.md), [bake-and-sign.md](templates/bake-and-sign.md) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/compose-v5.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/compose-v5.md new file mode 100644 index 0000000..671b7f3 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/compose-v5.md @@ -0,0 +1,114 @@ +--- +name: compose-v5 +description: Docker Compose v5 - startup order with healthchecks, profiles, secrets, interpolation, resource limits +when-to-use: Writing or reviewing compose.yaml, debugging startup order, moving credentials out of environment variables +keywords: compose.yaml, depends_on, service_healthy, healthcheck, start_interval, profiles, secrets, mem_limit, deploy.resources, up --wait +priority: high +related: safe-ops.md, templates/compose-production.md +--- + +# Docker Compose v5 + +**Load when:** writing a `compose.yaml`, a service starts before its database is ready, or credentials sit in `environment:`. + +Verified 2026-11-10: Compose **v5.7.0** (2026-11-02), https://github.com/docker/compose/releases/tag/v5.7.0 (single source: GitHub releases; Context7 `/docker/compose` lists v5.x). + +--- + +## What changed with v5 + +- v5.0.0 (2025-12-02): the **internal builder was removed**; `build:` is delegated to Docker Bake, like `docker build`. Compose can also be used as a Go SDK. Versions 3 and 4 were skipped to avoid confusion with legacy file formats (https://github.com/docker/compose/releases/tag/v5.0.0). +- The top-level `version:` key is **obsolete**: informative only, it triggers a warning; Compose always validates against the latest schema (https://docs.docker.com/reference/compose-file/version-and-name/). Remove it. +- Canonical file name: `compose.yaml` (preferred) or `compose.yml`; `docker-compose.y(a)ml` still read for compatibility; `compose.yaml` wins if both exist. + +--- + +## Startup order: running is not ready + +Compose waits for a dependency to be **running**, not ready, unless you use a condition (https://docs.docker.com/compose/how-tos/startup-order/): + +| Condition | Meaning | +|-----------|---------| +| `service_started` | Container started (default behaviour) | +| `service_healthy` | Dependency's `healthcheck` passes | +| `service_completed_successfully` | One-shot job (migration) exited 0 | + +```yaml +services: + web: + depends_on: + db: + condition: service_healthy + restart: true # restart web when db is restarted by a Compose operation + db: + image: postgres:18 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 10s + timeout: 10s + retries: 5 + start_period: 30s +``` + +`$$` escapes interpolation so the variable is expanded inside the container. Healthcheck keys `interval`, `timeout`, `start_period`, `start_interval` are durations; `test` list starts with `CMD`, `CMD-SHELL` or `NONE`; `disable: true` turns off an image healthcheck (https://docs.docker.com/reference/compose-file/services/#healthcheck). The Dockerfile defaults are interval 30s, timeout 30s, start-period 0s, start-interval 5s, retries 3; `start_interval` needs Engine 25.0+ (https://docs.docker.com/reference/dockerfile/#healthcheck). + +Use `docker compose up -d --wait` (with `--wait-timeout`) in scripts: it waits for services to be running/healthy (https://docs.docker.com/reference/cli/docker/compose/up/). + +--- + +## Secrets instead of environment variables + +Environment variables are visible to every process and can end up in debug logs. Compose secrets are bind-mounted as a single file at `/run/secrets/`, granted per service (https://docs.docker.com/compose/how-tos/use-secrets/): + +Pattern: service `secrets: [db_password]` + `POSTGRES_PASSWORD_FILE: /run/secrets/db_password`, top-level `secrets: {db_password: {file: ./secrets/db_password.txt}}` (full example in [compose-production.md](templates/compose-production.md)). Linux containers only. Prefer images that support `*_FILE` variables. Where the secret comes from (SOPS, Vault/OpenBao) is `devops-secrets`. + +--- + +## Interpolation and profiles + +- `${VAR:-default}` uses a default; `${VAR:?message}` **fails** the command if unset or empty - use it for mandatory settings (https://docs.docker.com/reference/compose-file/interpolation/). +- `profiles: [debug]` keeps a service out of the default `up`; enable with `docker compose --profile debug up` or `COMPOSE_PROFILES=debug` (https://docs.docker.com/compose/how-tos/profiles/). +- Top-level `name:` sets the project name (`COMPOSE_PROJECT_NAME`), which prefixes containers, networks and volumes. + +--- + +## Limits and hardening keys + +| Key | Effect | Source | +|-----|--------|--------| +| `mem_limit` / `deploy.resources.limits.memory` | Hard memory cap (must be consistent if both set) | services, deploy reference | +| `deploy.resources.limits.cpus`, `.pids` | CPU and PID caps | deploy reference | +| `pids_limit` | PID cap (`-1` = unlimited) | services reference | +| `read_only: true` + `tmpfs:` | Read-only root filesystem, writable scratch paths | services reference | +| `cap_drop: [ALL]` (+ `cap_add` only what is needed) | Drop Linux capabilities; `ALL` is accepted (engine run reference) | services reference, https://docs.docker.com/engine/containers/run/ | +| `security_opt: [no-new-privileges:true]` | Block setuid escalation | services reference | +| `init: true` | Init as PID 1 reaps zombies, forwards signals | services reference | +| `stop_grace_period` | Wait before SIGKILL on stop | services reference | + +Sources: https://docs.docker.com/reference/compose-file/services/ , https://docs.docker.com/reference/compose-file/deploy/ + +## Verification + +```bash +docker compose config -q # validate only, prints nothing on success +docker compose config --services # resolved service list (profiles applied) +docker compose up -d --wait --wait-timeout 120 +docker compose ps # STATUS shows (healthy) +``` + +## Common Mistakes + +| Mistake | Fix | +|---------|-----| +| App crashes on first boot, works on restart | `condition: service_healthy` + real healthcheck | +| Data lost after `down`/`up` | Data was in an anonymous volume; use a named volume | +| `version: "3.8"` warning | Delete the key | +| Password set in `environment:` | Move to `secrets:` and `*_FILE` | + +`docker compose down -v` deletes named volumes: see [safe-ops.md](safe-ops.md) before running it. + +## Not verified + +- Compose v5.7.0 release content was not read beyond the version tag; check its notes before relying on a new feature. + +→ Template: [compose-production.md](templates/compose-production.md) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/engine-daemon-config.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/engine-daemon-config.md new file mode 100644 index 0000000..44f25db --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/engine-daemon-config.md @@ -0,0 +1,112 @@ +--- +name: engine-daemon-config +description: Docker Engine 29 daemon.json - logging, storage backend, live-restore, address pools, rootless +when-to-use: Creating or changing /etc/docker/daemon.json, choosing a log driver or storage backend, upgrading the engine +keywords: daemon.json, dockerd, live-restore, log-driver, local, json-file, containerd-snapshotter, overlay2, default-address-pools, rootless +priority: high +related: networking-firewall.md, templates/daemon-json.md +--- + +# Engine Daemon Configuration + +**Load when:** editing `/etc/docker/daemon.json`, planning an engine upgrade, or a host fills its disk with logs. + +Verified 2026-11-10: Docker Engine **29.9.3** (2026-11-03), https://docs.docker.com/engine/release-notes/29/ ; endoflife.date agrees and lists 28.x EOL on 2026-05-13. + +--- + +## How daemon.json works + +- Default path on Linux: `/etc/docker/daemon.json`; keys are the `dockerd` flag names (plural for repeatable flags, e.g. `labels`). +- A key set both in the file and as a flag makes the daemon **fail to start**, even with the same value. On systemd hosts `-H` is already set, so `hosts` in the file conflicts. +- Validate before restarting: + +```bash +sudo dockerd --validate --config-file=/etc/docker/daemon.json # prints "configuration OK", exit 0 +``` + +Source: https://docs.docker.com/reference/cli/dockerd/ (section "Daemon configuration file"). + +--- + +## Decision rules + +| Need | Key | Notes | +|------|-----|-------| +| Logs must not fill the disk | `"log-driver": "local"` | Rotates by default: 20m per file, 5 files, compressed | +| Keep `docker logs` + JSON files for a shipper | `"log-driver": "json-file"` + `"log-opts": {"max-size": "10m", "max-file": "3"}` | `json-file` has **no rotation by default** | +| Containers survive a daemon restart | `"live-restore": true` | Patch upgrades only; not across `YY.MM` jumps | +| Avoid subnet clashes with the LAN/VPN | `"default-address-pools"` | Each entry `{"base": "...", "size": N}` | +| Ports published to localhost by default | `"default-network-opts"` | See [networking-firewall.md](networking-firewall.md) | +| Containers cannot gain privileges | `"no-new-privileges": true` | Applies to new containers | +| Debug a startup issue | `"debug": true` | Turn off afterwards | + +Logging changes apply only to **containers created after the restart**; existing containers keep their old driver (https://docs.docker.com/engine/logging/configure/). Driver defaults: https://docs.docker.com/engine/logging/drivers/local/. + +`live-restore` can be activated with `systemctl reload docker` (SIGHUP) instead of a restart. It only restores containers if daemon options such as bridge IPs or the storage driver did not change (https://docs.docker.com/engine/daemon/live-restore/). + +--- + +## Storage backend (changed in Engine 29) + +- **Fresh installs of 29.0+** use the **containerd image store** (snapshotters). **Upgraded hosts keep overlay2** until you opt in. +- Opt in with `"features": {"containerd-snapshotter": true}`, restart, then check: + +```bash +docker info -f '{{ .DriverStatus }}' # [[driver-type io.containerd.snapshotter.v1]] +``` + +- Switching backend **hides** images and containers of the other backend (data stays on disk). Push or `docker save` what you need first. +- The containerd store uses more disk (compressed + unpacked layers) and keeps its data **outside** Docker's `data-root`: moving `data-root` does not move containerd's storage. +- Not available with `userns-remap`. +- Needed for local multi-platform images and attestations without an external builder. + +Source: https://docs.docker.com/engine/storage/containerd/ , https://docs.docker.com/engine/storage/drivers/select-storage-driver/ + +--- + +## Engine 29 breaking changes to check before upgrading + +From https://docs.docker.com/engine/release-notes/29/ (29.0.0, 2025-11-10): +- The daemon requires API **v1.44+** (clients from Docker 25.0+). +- **cgroup v1 deprecated** (support until at least May 2029): migrate hosts to cgroup v2. +- **Docker Content Trust removed** from the CLI: use cosign ([registries-supply-chain.md](registries-supply-chain.md)). +- Raspbian 32-bit packages dropped; Debian armhf targets ARMv7. +- Experimental nftables firewall backend added ([networking-firewall.md](networking-firewall.md)). + +29.9.3 itself is a security release (Go/zstd/RSA fixes, BuildKit 0.34.2, Swarm overlay encryption after a firewalld flush): patch promptly. + +--- + +## Rootless and the docker group + +- Members of the `docker` group get **root-level privileges** on the host (https://docs.docker.com/engine/install/linux-postinstall/). +- Rootless mode runs daemon and containers in a user namespace; needs `newuidmap`/`newgidmap` (`uidmap` package) and at least 65,536 subordinate IDs in `/etc/subuid` and `/etc/subgid`. Start at boot with `sudo loginctl enable-linger ` (https://docs.docker.com/engine/security/rootless/). +- For a daemonless, systemd-native alternative see [podman-quadlet.md](podman-quadlet.md). + +--- + +## Verification + +```bash +sudo dockerd --validate --config-file=/etc/docker/daemon.json +sudo systemctl restart docker && systemctl is-active docker +docker info --format '{{.LoggingDriver}} {{.Driver}} cgroup={{.CgroupVersion}} live-restore={{.LiveRestoreEnabled}}' +``` + +Field names come from the Engine API `SystemInfo` schema (https://raw.githubusercontent.com/moby/moby/master/api/swagger.yaml, read at `master`, not at the 29.9.3 tag). + +## Common Mistakes + +| Mistake | Fix | +|---------|-----| +| Daemon will not start after edit | Same option in a flag and the file; remove one | +| Disk full of `*-json.log` | `local` driver or `max-size`/`max-file`, then recreate containers | +| Images "vanished" after enabling containerd store | Backend switch hides them; switch back or re-pull | + +## Not verified + +- The `docker info` field names were read from the API schema at `master`, not at the 29.9.3 tag; if a template fails, inspect `docker info --format '{{json .}}'`. +- Rootless Docker limitations (port numbers below 1024, cgroup delegation) were not read today; see https://docs.docker.com/engine/security/rootless/troubleshoot/ before relying on rootless in production. + +→ Template: [daemon-json.md](templates/daemon-json.md) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/networking-firewall.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/networking-firewall.md new file mode 100644 index 0000000..7ec08d3 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/networking-firewall.md @@ -0,0 +1,110 @@ +--- +name: networking-firewall +description: Container networking and the host firewall - bridge vs macvlan, published ports, ufw/firewalld, DOCKER-USER, experimental nftables backend +when-to-use: Publishing ports on an internet-facing host, a firewall rule "does not work" for containers, migrating Docker to nftables +keywords: bridge, macvlan, publish, 127.0.0.1, ufw, firewalld, iptables, DOCKER-USER, nftables, firewall-backend, ip-forward, gateway_mode, bridge-accept-fwmark +priority: high +related: engine-daemon-config.md, templates/daemon-json.md +--- + +# Networking and the Host Firewall + +**Load when:** a container port must (or must not) be reachable from outside, ufw/firewalld rules seem ignored, or someone wants Docker on nftables. + +Verified 2026-11-10 against Engine 29.9.3 docs. Host-wide nftables design, routing and reverse proxies belong to `devops-networking`; this file covers only how Docker's own rules interact with them. + +--- + +## Choose the network driver + +| Need | Driver | Caveat | +|------|--------|--------| +| Services of one app talk to each other | **user-defined bridge** | Built-in DNS by service/container name; default `bridge` is legacy (IP only) | +| Container needs its own LAN IP/MAC | `macvlan` | Switch must accept several MACs per port; **host cannot reach its own macvlan containers** (kernel restriction) unless a host macvlan interface or an extra bridge is added | +| No isolation, max performance | `host` | No port mapping, no Docker firewall rules | + +Docker creates firewall rules only for **bridge** networks, none for `ipvlan`, `macvlan` or `host` (https://docs.docker.com/engine/network/packet-filtering-firewalls/). Sources: https://docs.docker.com/engine/network/drivers/bridge/ , https://docs.docker.com/engine/network/drivers/macvlan/ + +--- + +## Published ports are public by default + +`-p 8080:80` (or `ports: ["8080:80"]`) binds **all host interfaces, IPv4 and IPv6**. Decision rule: + +- Reached only through a local reverse proxy → `127.0.0.1:8080:80` (and `[::1]:8080:80`). +- Make localhost the default for user-defined bridges, daemon-wide: + +```json +{ "default-network-opts": { "bridge": { "com.docker.network.bridge.host_binding_ipv4": "127.0.0.1" } } } +``` + +- Before Engine **28.0.0**, hosts on the same L2 segment could reach ports published to localhost (moby#45610): upgrade. +- Swarm services ignore the default bind address and always publish on `0.0.0.0`. + +Source: https://docs.docker.com/engine/network/port-publishing/ + +--- + +## Why ufw rules are bypassed + +Docker DNATs published ports in the `nat` table, so packets are routed to the container **before** they reach the `INPUT`/`OUTPUT` chains ufw uses: ufw "deny" does not protect published ports (https://docs.docker.com/engine/network/packet-filtering-firewalls/). Fixes, in order of preference: bind to `127.0.0.1`; filter in `DOCKER-USER` (iptables backend); a dedicated nftables table (nftables backend). + +**Never set `"iptables": false`** to make ufw work: containers lose masquerading and every container port becomes reachable from the LAN. Docker cannot be fully prevented from creating rules. + +firewalld: Docker adds bridge interfaces to a `docker` zone (target `ACCEPT`) and a `docker-forwarding` policy. + +--- + +## iptables backend (default): DOCKER-USER + +Rules in `DOCKER-USER` run before Docker's rules; rules appended to `FORWARD` run after them. Packets are already DNATed, so match original addresses with conntrack (https://docs.docker.com/engine/network/firewall-iptables/): + +```bash +sudo iptables -I DOCKER-USER -i ext_if ! -s 192.0.2.0/24 -j DROP # only this subnet reaches containers +sudo iptables -I DOCKER-USER -p tcp -m conntrack --ctorigdst 198.51.100.2 --ctorigdstport 80 -j ACCEPT +``` + +When Docker enables IP forwarding it sets the `filter` `FORWARD` policy to `DROP` (so the host stops routing other traffic); `"ip-forward-no-drop": true` disables that for a host that must route. Direct-routed packets to a container IP are dropped in `raw PREROUTING` before `DOCKER-USER`. + +--- + +## nftables backend (new in 29.0, still experimental) + +Status 2026-11-10: **experimental**; options and behaviour may change; overlay rules are not migrated, so it **cannot be used in Swarm mode** (https://docs.docker.com/engine/network/firewall-nftables/). + +- Enable: `"firewall-backend": "nftables"` in daemon.json (or `--firewall-backend=nftables`). Docker creates tables `ip docker-bridges` and `ip6 docker-bridges`: **do not edit them**. +- Docker **does not enable IP forwarding** itself with nftables; startup fails if the default bridge needs it. Set `net.ipv4.ip_forward=1` and `net.ipv6.conf.all.forwarding=1` (sysctl.d) and block unwanted forwarding between non-Docker interfaces yourself, or use `"ip-forward": false` (some bridge features, including port forwarding, may then fail). +- **No `DOCKER-USER` chain.** Put your rules in your own table with base chains on the same hooks; order with chain priority. An nftables `accept` is not final: to override Docker's drop, mark packets and start the daemon with `--bridge-accept-fwmark` (mark set at priority `filter - 1` or lower). +- Migration traps: an existing iptables `FORWARD` policy `DROP` still drops packets accepted by Docker's nftables rules; the old jump to `DOCKER-USER` keeps working until reboot, then those rules are silently ignored. + +``` +table ip my-table { + chain my-filter-forward { + type filter hook forward priority filter; policy accept; + iifname "ext_if" oifname "br-*" ip saddr != 192.0.2.0/24 counter drop + } +} +``` + +Check the active backend: `docker info --format '{{json .FirewallBackend}}'` (API `SystemInfo.FirewallBackend`). + +--- + +## Gateway modes (bridge option) + +`com.docker.network.bridge.gateway_mode_ipv4|ipv6` = `nat` (default), `nat-unprotected` (legacy, unpublished ports reachable), `routed` (no NAT, only published ports reachable by direct routing), `isolated`. + +## Verification + +```bash +docker container inspect --format '{{json .NetworkSettings.Ports}}' # HostIp 127.0.0.1 expected +sudo ss -ltnp | grep docker-proxy # what listens on the host +sudo iptables -L DOCKER-USER -n -v # or: sudo nft list ruleset | less +``` + +Then test from **another host**, not from the Docker host itself. + +## Not verified + +- Whether the nftables backend is planned to become default, and ufw-specific workarounds beyond the above: not stated in the docs read today. +- `ss` output naming `docker-proxy` depends on `userland-proxy` (default true per `dockerd` reference); not tested here. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/podman-quadlet.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/podman-quadlet.md new file mode 100644 index 0000000..606607e --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/podman-quadlet.md @@ -0,0 +1,111 @@ +--- +name: podman-quadlet +description: Podman 6 as the daemonless alternative - rootless containers managed by systemd through Quadlet units, secrets, auto-update +when-to-use: A host should run containers without a root daemon, containers must be plain systemd services, or Docker is not wanted +keywords: podman, quadlet, rootless, systemd, .container, .network, .volume, podman-system-generator, auto-update, loginctl enable-linger, podman secret +priority: medium +related: engine-daemon-config.md, safe-ops.md, templates/quadlet-units.md +--- + +# Podman and Quadlet + +**Load when:** choosing between Docker and Podman on a server, or turning a container into a systemd service that starts at boot. + +Verified 2026-11-10: Podman **v6.1.4** (2026-10-12), https://github.com/podman-container-tools/podman/releases/tag/v6.1.4 ; endoflife.date agrees (6.0 EOL 2026-08-12). **Doc-link trap:** the repository moved from `containers/podman` to `podman-container-tools/podman` (old URLs redirect). Unit reference read from the v6.1.4 tag: `docs/source/markdown/podman-systemd.unit.5.md`. + +--- + +## Docker or Podman? + +| Situation | Choose | +|-----------|--------| +| Compose-heavy stack, Docker-specific tooling, Buildx/Bake pipelines | Docker Engine | +| No root daemon wanted; each container a systemd service with journald, restart and dependencies | Podman + Quadlet | +| RHEL/Fedora-family host where Podman is the vendor default | Podman | +| Need Kubernetes YAML locally | Podman `.kube` units (cluster work is `devops-kubernetes`) | + +--- + +## How Quadlet works + +A systemd **generator** reads unit files and generates `.service` units at boot and on `systemctl daemon-reload`. File types: `.container`, `.network`, `.volume`, `.pod`, `.kube`, `.image`, `.build`, `.artifact`. All normal systemd sections (`[Unit]`, `[Service]`, `[Install]`) pass through. + +Search paths: +- rootful: `/etc/containers/systemd/` (also `/run/...` for tests, `/usr/share/...` for distro units) +- rootless: `~/.config/containers/systemd/` (or `$XDG_CONFIG_HOME/containers/systemd/`), `/etc/containers/systemd/users/${UID}` + +Rules that bite: +- Generated services are **transient**: `systemctl enable` does not work. Put `WantedBy=default.target` in `[Install]`; the generator applies it (`Alias`, `WantedBy`, `RequiredBy`, `UpheldBy` only). +- Quadlet **requires cgroup v2** (`podman info --format {{.Host.CgroupsVersion}}`). +- Do not use systemd `User=`/`Group=`/`DynamicUser=` to go rootless: create the user and place units in its rootless path. +- Image pulls can exceed systemd's 90 s start timeout: pre-pull or set `TimeoutStartSec=900` in `[Service]`. +- Default `Type`: `notify` for `.container`/`.kube`, `oneshot` for `.network`/`.volume`/`.image`/`.build`. +- A rootless user's services start at boot only with lingering: `sudo loginctl enable-linger `. + +--- + +## Key `[Container]` options + +| Key | Purpose | +|-----|---------| +| `Image=` | Fully-qualified image (needed for `AutoUpdate=registry`) | +| `PublishPort=` | `ip:hostPort:containerPort`; empty IP or `0.0.0.0` = all IPv4 addresses | +| `Network=app.network` | Uses network `systemd-app`, adds a dependency on its service | +| `Volume=data.volume:/data` | Uses volume `systemd-data`, adds a dependency | +| `Secret=` | Podman secret as a file or env var (same syntax as `--secret`) | +| `HealthCmd=`, `HealthInterval=`, `HealthOnFailure=kill` | `kill` lets systemd restart an unhealthy container | +| `Notify=healthy` | Unit becomes active only once the healthcheck passes | +| `ReadOnly=true`, `NoNewPrivileges=true`, `DropCapability=all` | Hardening | +| `Memory=`, `PidsLimit=` | Limits | +| `UserNS=keep-id` | Map your UID into the container (rootless bind mounts) | +| `AutoUpdate=registry` | Pull and restart when the registry image changes | + +Dependencies between Quadlet units are written in `[Unit]` with the Quadlet file name (`Requires=db.container`, `After=db.container`) and translated automatically. + +--- + +## Secrets and auto-update + +```bash +printf '%s' "$DB_PASSWORD" | podman secret create db_password - # from stdin, value not on disk +podman auto-update --dry-run --format "{{.Image}} {{.Updated}}" # preview +sudo systemctl enable --now podman-auto-update.timer # shipped timer, daily at midnight +``` + +`podman auto-update` rolls back to the previous image if the restarted unit fails (`--rollback`, default true). Sources: `podman-secret-create.1.md`, `podman-auto-update.1.md.in` at the v6.1.4 tag. + +--- + +## Debugging generated units + +```bash +/usr/lib/systemd/system-generators/podman-system-generator --user --dryrun # shows generated units/errors +systemd-analyze --user --generators=true verify app.service # errors only +systemctl --user daemon-reload && systemctl --user start app.service +journalctl --user -u app.service -e +``` + +"Unit app.service not found" after daemon-reload = syntax error or a key from a newer Podman: run the generator dry run. + +--- + +## Destructive commands + +`podman system reset` removes **all** pods, containers, images, networks, volumes and machines and deletes `graphRoot`/`runRoot`. Never run it without the user's explicit go; see [safe-ops.md](safe-ops.md). + +## Verification + +```bash +podman info --format '{{.Host.CgroupsVersion}} rootless={{.Host.Security.Rootless}}' +systemctl --user status app.service +podman healthcheck run app && podman ps --format '{{.Names}} {{.Status}}' +``` + +## Not verified + +- The template path `{{.Host.Security.Rootless}}` is derived from the `host.security.rootless` key in the `podman info` man page example; confirm with `podman info --format json`. +- Enabling the auto-update timer as a rootless user (`systemctl --user`) is not stated in the man page read today. +- The generator path `/usr/lib/systemd/system-generators/` may differ per distribution (doc shows this path). +- Docker Compose compatibility of Podman (`podman compose`) was not researched. + +→ Template: [quadlet-units.md](templates/quadlet-units.md) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/registries-supply-chain.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/registries-supply-chain.md new file mode 100644 index 0000000..82c0adc --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/registries-supply-chain.md @@ -0,0 +1,111 @@ +--- +name: registries-supply-chain +description: Private registries (distribution registry:3, Harbor), Docker Hub pull-through mirror, digest pinning, cosign v3 signing and verification +when-to-use: Running a self-hosted registry, mirroring Docker Hub, pinning images, signing or verifying images and attestations +keywords: registry:3, distribution, htpasswd, TLS, garbage-collect, registry-mirrors, Harbor, digest, cosign, sign, verify, attest, keyless, OCI referrers +priority: high +related: buildkit-dockerfile.md, safe-ops.md, templates/bake-and-sign.md +--- + +# Registries and Image Supply Chain + +**Load when:** standing up a private registry, adding a Docker Hub mirror, or enforcing that only signed images run. + +Verified 2026-11-10 (GitHub releases): distribution **v3.1.2** (image `registry:3`, Docker Hub tags `3`, `3.1`, `3.1.2`), Harbor **v2.14.6**, cosign **v3.1.4**. Docker Content Trust was **removed from the CLI in Engine 29.0**: use cosign instead of `DOCKER_CONTENT_TRUST`. + +Dependency/CVE scanning of image contents is `fuse-security:dependency-audit`, not this file. + +--- + +## Choose a registry + +| Need | Choice | +|------|--------| +| Single team, simple push/pull | `registry:3` (CNCF distribution) behind TLS + htpasswd | +| UI, RBAC, projects, replication, scanning, quotas | Harbor | +| Cut Docker Hub pulls on many hosts | `registry:3` in proxy (pull-through) mode + `registry-mirrors` | + +--- + +## registry:3 essentials + +Source: https://distribution.github.io/distribution/about/deploying/ (read from the v3.1.2 docs source) + +- The plain `docker run -p 5000:5000 registry:3` is **test only**. Production needs TLS and access control. +- TLS: `REGISTRY_HTTP_ADDR=0.0.0.0:443`, `REGISTRY_HTTP_TLS_CERTIFICATE`, `REGISTRY_HTTP_TLS_KEY`. +- Auth: `REGISTRY_AUTH=htpasswd`, `REGISTRY_AUTH_HTPASSWD_REALM`, `REGISTRY_AUTH_HTPASSWD_PATH`; only **bcrypt** entries work (`htpasswd -B`), otherwise every login fails. +- Data in `/var/lib/registry`; config in v3 at `/etc/distribution/config.yml`. +- The default config sends OpenTelemetry traces to `localhost:4318`; add `-e OTEL_TRACES_EXPORTER=none` if you do not run a collector. + +**Garbage collection** is stop-the-world: run it with the registry stopped or read-only, otherwise layers of an image pushed during GC can be deleted (https://distribution.github.io/distribution/about/garbage-collection/): + +```bash +# the registry:3 image ships /bin/registry with ENTRYPOINT ["registry"] (v3.1.2 Dockerfile) +docker run --rm -v /srv/registry:/var/lib/registry registry:3 \ + garbage-collect --dry-run /etc/distribution/config.yml # read-only preview, registry stopped +``` + +The real run (without `--dry-run`, optionally `--delete-untagged`) deletes blobs: propose it, wait for the user's go ([safe-ops.md](safe-ops.md)). + +--- + +## Docker Hub pull-through mirror + +Set `proxy.remoteurl: https://registry-1.docker.io` in the mirror's config, then on each host: + +```json +{ "registry-mirrors": ["https://mirror.example.internal"] } +``` + +The Docker daemon only supports **mirrors of Docker Hub** through this key (https://distribution.github.io/distribution/recipes/mirror/). + +--- + +## Pin by digest + +Tags are mutable. Resolve the digest, then reference `image@sha256:...` in Dockerfiles, Compose and Quadlet: + +```bash +docker buildx imagetools inspect postgres:18 # shows the index digest and per-platform manifests +``` + +Compose accepts `image: name@sha256:` (https://docs.docker.com/reference/compose-file/services/#image). + +--- + +## cosign v3 + +v3.0 turned on by default: the **protobuf bundle format**, service URLs from the TUF signing config, and signatures stored as **OCI 1.1 referring artifacts** (https://github.com/sigstore/cosign/blob/main/CHANGELOG.md, v3.0.0). Always sign **by digest**. + +| Mode | Sign | Verify | +|------|------|--------| +| Key pair | `cosign generate-key-pair` then `cosign sign --key cosign.key IMG@sha256:...` | `cosign verify --key cosign.pub IMG@sha256:...` | +| KMS / Vault / OpenBao | `--key hashivault://KEY`, `--key openbao://KEY`, `--key awskms://...` | `cosign public-key --key > kms.pub` then verify with it | +| Keyless (OIDC, Fulcio + Rekor) | `cosign sign IMG@sha256:...` | `cosign verify IMG --certificate-identity=... --certificate-oidc-issuer=...` | + +Keyless verification **must** pin both identity (`--certificate-identity` or `--certificate-identity-regexp`) and issuer (`--certificate-oidc-issuer[-regexp]`); for GitHub Actions the issuer is `https://token.actions.githubusercontent.com`. Multi-arch: `cosign sign --recursive` signs the index and each platform manifest. + +Attestations: `cosign attest --predicate sbom.spdx.json --type spdxjson --key cosign.key IMG@sha256:...`, checked with `cosign verify-attestation --type spdxjson --key cosign.pub IMG`. Accepted `--type` values: `slsaprovenance`, `slsaprovenance02`, `slsaprovenance1`, `link`, `spdx`, `spdxjson`, `cyclonedx`, `vuln`, `openvex`, `custom` or a URI. + +`cosign tree IMG` lists attached signatures/attestations. `cosign clean IMG` **deletes** them (registry permitting): destructive, propose first. + +Sources: https://docs.sigstore.dev/cosign/signing/signing_with_containers/ , https://docs.sigstore.dev/cosign/verifying/verify/ , CLI docs at https://github.com/sigstore/cosign/tree/v3.1.4/doc (`cosign_sign.md`, `cosign_verify.md`, `cosign_attest.md`, `cosign_verify-attestation.md`). + +Admission enforcement in Kubernetes (policy-controller, Kyverno) is `devops-kubernetes`; key custody is `devops-secrets`. + +--- + +## Verification + +```bash +curl -fsS -u USER https://registry.example.internal/v2/_catalog # auth + TLS work (https://distribution.github.io/distribution/spec/api/) +cosign verify --key cosign.pub REG/app@sha256: # exit 0 = valid signature +cosign tree REG/app@sha256: +``` + +## Not verified + +- Harbor v2.14.6: single source (GitHub releases); Harbor install/config was not read today. +- cosign 3.1.x release notes after 3.0.5 were not read (the main CHANGELOG stops at 3.0.5); flags above come from the v3.1.4 tagged CLI docs. +- Notation (Notary Project) as an alternative signer: not researched. +- The GC example assumes the data volume is mounted at `/var/lib/registry` and the default config path; adapt to your deployment. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/runtime-troubleshooting.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/runtime-troubleshooting.md new file mode 100644 index 0000000..1546d49 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/runtime-troubleshooting.md @@ -0,0 +1,98 @@ +--- +name: runtime-troubleshooting +description: Read-only diagnosis of a container host - restarts, OOM kills, unhealthy loops, disk usage, events, logs +when-to-use: A container restarts or is unhealthy, the host runs out of disk or memory, a service is unreachable +keywords: docker inspect, OOMKilled, healthcheck, docker events, docker logs, docker stats, docker system df, restart loop, exit code +priority: high +related: safe-ops.md, networking-firewall.md, engine-daemon-config.md +--- + +# Runtime Troubleshooting + +**Load when:** "the container keeps restarting", "unhealthy", "disk full", "cannot reach the service". Every command here is **read-only**; cleanup is in [safe-ops.md](safe-ops.md). + +Metrics, dashboards and alerting on containers (cAdvisor, Prometheus) are `devops-observability`; host-level CPU/IO/disk tooling is `devops-linux`. + +--- + +## First five minutes + +```bash +docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Image}}' # Restarting / unhealthy / Exited (code) +docker compose ps # same, per Compose project +docker logs --since 30m -t -n 200 # last lines with timestamps +docker events --since 30m --filter container= # die, oom, health_status, restart +docker system df # images / containers / volumes / build cache +``` + +`docker logs` options: `--since` (timestamp or relative like `42m`), `-n/--tail`, `-t/--timestamps` (https://docs.docker.com/reference/cli/docker/container/logs/). `docker events` reports container events including `die`, `oom` and `health_status`, filterable with `--filter` and bounded with `--since/--until` (https://docs.docker.com/reference/cli/docker/system/events/). + +--- + +## Decision tree + +| Symptom | Check | Likely cause → action | +|---------|-------|-----------------------| +| `Exited (137)` or `oom` event | `docker inspect -f '{{.State.OOMKilled}} {{.State.ExitCode}}' ` | Memory limit too low or leak → raise `mem_limit` after measuring with `docker stats --no-stream` | +| `Exited (1)` right after start | `docker logs ` | App config/env error; dependency not ready → `depends_on: condition: service_healthy` | +| `Restarting` forever | restart policy + logs | Crash loop; stop the loop with `docker update --restart=no ` while debugging | +| `(unhealthy)` but app works | `docker inspect -f '{{json .State.Health}}' ` | Check command missing in image (no curl in distroless), wrong port, timeout too short, no `start_period` | +| Healthy, unreachable from outside | `docker container inspect --format '{{json .NetworkSettings.Ports}}'` | Bound to `127.0.0.1`, host firewall, or nftables backend forwarding off → [networking-firewall.md](networking-firewall.md) | +| Host disk full | `docker system df -v`, `du -sh /var/lib/docker/*` | Unrotated `json-file` logs, dangling images, build cache, volumes | +| Daemon down after config change | `journalctl -u docker -e` | Invalid or conflicting daemon.json key → `dockerd --validate` | + +`State.OOMKilled` and `State.Health.Status` (`none`, `starting`, `healthy`, `unhealthy`) are Engine API fields (https://raw.githubusercontent.com/moby/moby/master/api/swagger.yaml, `ContainerState`). + +Exit code 137 = 128 + 9 (SIGKILL, shell convention): OOM kill or a stop that exceeded the grace period. Confirm with `OOMKilled` before blaming memory. `docker update --restart` changes the policy of a running container (https://docs.docker.com/reference/cli/docker/container/update/). + +--- + +## Memory and OOM + +- No limit by default: a runaway container can push the **host** into the kernel OOM killer. Set `-m/--memory` (minimum 6m) or Compose `mem_limit`. +- `--memory-swap` only means something with `--memory`; equal values disable swap for the container. +- Never use `--oom-kill-disable` without `-m`: the host may then kill its own processes. + +Source: https://docs.docker.com/engine/containers/resource_constraints/ + +```bash +docker stats --no-stream --format 'table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}\t{{.PIDs}}' +``` + +--- + +## Healthcheck loops + +1. Read the last results: `docker inspect -f '{{json .State.Health}}' | jq '.Log[-3:]'`. +2. Run the probe yourself: `docker exec `. +3. Slow boot → raise `start_period` (failures inside it do not count) and keep `start_interval` short. +4. A check slower than `timeout` is killed with SIGKILL and counts as a failure. + +Healthcheck semantics: https://docs.docker.com/reference/dockerfile/#healthcheck + +--- + +## Disk usage + +```bash +docker system df -v # per image, container, volume, cache entry +docker buildx du --verbose # build cache detail (https://docs.docker.com/reference/cli/docker/buildx/du/) +sudo du -sh /var/lib/docker/containers/*/*-json.log 2>/dev/null | sort -h | tail +``` + +- The containerd image store keeps compressed **and** unpacked layers, so expect more disk than overlay2, and its data lives outside Docker's `data-root` (https://docs.docker.com/engine/storage/containerd/). +- Logs: switch to the `local` driver or rotate `json-file`, then **recreate** containers (logging config only applies to new containers). +- Long-term: BuildKit garbage collection policy in daemon.json (`builder.gc`), shown in the `dockerd` reference example. + +Freeing space = deleting things: build the list, then follow [safe-ops.md](safe-ops.md). + +--- + +## Verification + +After a fix, the symptom itself must be gone: `docker ps` shows `Up ... (healthy)` for at least one full `interval × retries` window, `docker events --since 10m --filter container=` shows no new `die`/`oom`, and the service answers from a remote host. + +## Not verified + +- `jq` is assumed installed; it is not part of Docker. +- `du` over `/var/lib/docker` assumes the default `data-root`; with the containerd store, image data is elsewhere (containerd's root, not read today). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/safe-ops.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/safe-ops.md new file mode 100644 index 0000000..91914b3 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/safe-ops.md @@ -0,0 +1,98 @@ +--- +name: safe-ops +description: Exact effect of every destructive Docker/Podman cleanup command and the propose-then-go protocol +when-to-use: Before any prune, volume removal, compose down -v, image removal, registry GC, cosign clean or podman system reset +keywords: docker system prune, docker volume rm, docker volume prune, docker compose down -v, docker image prune, docker builder prune, podman system reset, data loss +priority: critical +related: runtime-troubleshooting.md, registries-supply-chain.md, podman-quadlet.md +--- + +# Safe Operations on a Container Host + +**Load when:** about to delete anything on a Docker or Podman host, or a user asks to "clean up", "free disk" or "reset". + +Rule: **propose, then wait for the user's explicit go.** The agent lists exactly what will be removed, states the effect and whether it is reversible, and runs nothing destructive until the user answers. One go covers one listed command on one host; a new list needs a new go. Never add `-f/--force` to skip the prompt on a host you have not inventoried. + +Backups and restore tests of volume data are `devops-storage-backup`. + +--- + +## 1. Inventory (read-only, always first) + +```bash +docker system df -v +docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Mounts}}' +docker volume ls --format '{{.Name}}' | while read v; do + echo "$v -> $(docker ps -a --filter volume="$v" --format '{{.Names}}' | paste -sd, -)"; done +docker compose ls # projects, so `down` targets the right one +``` + +A volume with no container listed is "unused" for Docker - that does **not** mean its data is worthless (a stopped stack may have been removed with `docker rm`). + +--- + +## 2. Effect table (Engine 29 CLI reference, read 2026-11-10) + +| Command | Removes | Data risk | +|---------|---------|-----------| +| `docker system prune` | all stopped containers, networks not used by a container, **dangling** images, **all build cache** (including `RUN --mount=type=cache` caches) | Containers' writable layers lost; volumes kept | +| `docker system prune -a` | as above + **every image not used by a container** | Re-pull/rebuild needed; images without a registry copy are gone | +| `docker system prune --volumes` | as above + **anonymous** volumes | Data in anonymous volumes lost | +| `docker volume prune` | unused **anonymous** volumes | Data lost | +| `docker volume prune -a` | unused anonymous **and named** volumes | **Database data lost** | +| `docker volume rm NAME` | that volume (refused if a container uses it; `-f` forces) | Irreversible | +| `docker compose down` | project containers, project networks, default network | Volumes and images kept; external networks/volumes never removed | +| `docker compose down -v` | above + **named volumes declared in the file** + anonymous volumes attached | **Project data lost** | +| `docker compose down --rmi all` / `local` | above + images used by services (`local` = only images without a custom tag) | Rebuild/re-pull | +| `docker image prune` / `-a` | dangling images / all images not referenced by a container | Rebuild/re-pull | +| `docker builder prune` / `-a` | dangling / all unused build cache (`--keep-storage`, `--filter until=24h` narrow it) | Slower next build only | +| `podman system reset` | **all** pods, containers, images, networks, volumes, machines; deletes `graphRoot`/`runRoot` | Everything local lost | + +Sources: https://docs.docker.com/reference/cli/docker/system/prune/ , https://docs.docker.com/reference/cli/docker/volume/prune/ , https://docs.docker.com/reference/cli/docker/volume/rm/ , https://docs.docker.com/reference/cli/docker/compose/down/ , https://docs.docker.com/reference/cli/docker/image/prune/ , https://docs.docker.com/reference/cli/docker/builder/prune/ , `podman-system-reset.1.md` at Podman v6.1.4. + +Also destructive, covered elsewhere: registry `garbage-collect` without `--dry-run` and `cosign clean` ([registries-supply-chain.md](registries-supply-chain.md)); switching the storage backend hides (does not delete) images ([engine-daemon-config.md](engine-daemon-config.md)). + +--- + +## 3. Proposal format (what the agent shows before asking) + +``` +Host: Docker: Free on /var/lib/docker: +Command: docker image prune -a --filter "until=240h" +Will remove: 14 images, about 6.2 GB (list attached from `docker image ls`) +Will NOT touch: volumes, running containers, build cache +Reversible: no (images can be re-pulled from ; local-only images: none found) +Backup taken: n/a (no data) Waiting for your go. +``` + +For anything touching volumes: name each volume, its size, the containers/projects that used it, and the backup or snapshot taken (with its path and a restore check) before asking. + +--- + +## 4. Safer alternatives first + +| Goal | Least destructive option | +|------|-------------------------| +| Free build space | `docker builder prune --filter until=72h` or `--keep-storage` | +| Remove old images only | `docker image prune -a --filter "until=240h"` | +| Restart a stack cleanly | `docker compose down` (no `-v`) then `up -d --wait` | +| Logs filling the disk | Log rotation ([engine-daemon-config.md](engine-daemon-config.md)), then recreate containers | +| Recurring build cache growth | `builder.gc` policy in daemon.json instead of manual prunes | + +Prefer `--filter label=...` or `until=` to scope prunes; never `prune -a --volumes` on a host that runs databases. + +--- + +## 5. After the go + +```bash +docker system df # compare with the inventory: the expected space came back +docker compose ps # every expected service is Up (healthy) +``` + +Report what was removed (counts, names, reclaimed size) and what was kept. + +## Not verified + +- The `docker ps` reference describes the `volume` filter as matching "running containers"; that `-a` extends it to stopped containers was not confirmed - cross-check with `docker inspect` on stopped containers before calling a volume orphaned. +- `docker compose down --rmi` default when no value is given was not stated in the reference read today. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/bake-and-sign.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/bake-and-sign.md new file mode 100644 index 0000000..ee92802 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/bake-and-sign.md @@ -0,0 +1,122 @@ +--- +name: bake-and-sign +description: docker-bake.hcl for a multi-arch release with SBOM/provenance and registry cache, then cosign sign and verify by digest +keywords: docker buildx bake, docker-bake.hcl, platforms, attest, cache-to, metadata-file, containerimage.digest, cosign sign, cosign verify, cosign attest +--- + +# Template: Bake Release + cosign Sign/Verify + +## Usage + +One command builds `linux/amd64` and `linux/arm64`, attaches SBOM and max-mode provenance, pushes, and writes the pushed digest to a metadata file; cosign then signs that **digest** (never the tag) and the result is verified. Works on a workstation or inside any CI job; wiring it into a pipeline (OIDC permissions, runners) is `devops-cicd`. + +Verified 2026-11-10: Bake attributes and `--metadata-file` output at https://docs.docker.com/build/bake/reference/ and https://docs.docker.com/reference/cli/docker/buildx/bake/ ; cosign v3.1.4 CLI docs at https://github.com/sigstore/cosign/tree/v3.1.4/doc . + +--- + +## `docker-bake.hcl` + +```hcl +variable "REGISTRY" { + default = "registry.example.internal/team" +} + +variable "VERSION" { + default = "0.0.0-dev" +} + +group "default" { + targets = ["app"] +} + +target "app" { + context = "." + dockerfile = "Dockerfile" + target = "runtime" + platforms = ["linux/amd64", "linux/arm64"] + tags = ["${REGISTRY}/app:${VERSION}"] + args = { + VERSION = "${VERSION}" + } + attest = [ + { type = "provenance", mode = "max" }, # records build args: never pass secrets as args + { type = "sbom" } + ] + cache-from = [{ type = "registry", ref = "${REGISTRY}/app:buildcache" }] + cache-to = [{ type = "registry", ref = "${REGISTRY}/app:buildcache", mode = "max" }] +} +``` + +Variable defaults are overridden by environment variables of the same name: `VERSION=1.4.0 docker buildx bake` (Bake reference, "variable"). + +--- + +## Release sequence + +```bash +set -euo pipefail +export VERSION=1.4.0 +export REGISTRY=registry.example.internal/team + +docker buildx bake --print app # 1. review the resolved plan (builds nothing) +docker buildx bake --check app # 2. Dockerfile build checks only +docker buildx bake --push --metadata-file meta.json app # 3. build all platforms, push, record result + +DIGEST=$(jq -r '.app["containerimage.digest"]' meta.json) # 4. digest of the pushed index +IMAGE="${REGISTRY}/app@${DIGEST}" +docker buildx imagetools inspect "$IMAGE" # 5. both platforms + attestation manifests present +``` + +Multi-platform push needs the containerd image store or a `docker-container` builder ([buildkit-dockerfile.md](../buildkit-dockerfile.md)). + +--- + +## Sign: option A - key pair (air-gapped or self-hosted) + +```bash +cosign generate-key-pair # once; prompts for a password (or COSIGN_PASSWORD); writes cosign.key + cosign.pub +cosign sign --yes --key cosign.key --recursive "$IMAGE" # index + every platform manifest +cosign verify --key cosign.pub "$IMAGE" +``` + +Keep `cosign.key` out of the repository; store it in Vault/OpenBao or a KMS and sign with `--key hashivault://KEY` / `--key openbao://KEY` (custody: `devops-secrets`). + +## Sign: option B - keyless (OIDC identity, public Sigstore) + +```bash +cosign sign --yes --recursive "$IMAGE" # opens an OIDC flow, or uses the CI's OIDC token +cosign verify "$IMAGE" \ + --certificate-identity-regexp '^https://github.com/ORG/REPO/' \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com +``` + +Keyless binds a short-lived Fulcio certificate carrying the OIDC identity to the signature and logs the signing event in Rekor, the transparency log (public-good instance by default, https://docs.sigstore.dev/cosign/signing/overview/): do not use it when the identity or image name must stay private. + +--- + +## Optional: sign the SBOM as a cosign attestation + +BuildKit already attached an SBOM (step 3). To also bind a separately generated SBOM to the signer: + +```bash +cosign attest --yes --key cosign.key --type spdxjson --predicate sbom.spdx.json "$IMAGE" +cosign verify-attestation --key cosign.pub --type spdxjson "$IMAGE" > /dev/null && echo "attestation OK" +``` + +--- + +## Verify (what "done" means) + +```bash +cosign verify --key cosign.pub "$IMAGE" # exit 0 +cosign tree "$IMAGE" # signatures/attestations attached as OCI referrers +docker buildx imagetools inspect "$IMAGE" --format '{{ json .SBOM.SPDX }}' | head -c 300 +``` + +Deploy with the digest (`image: REG/app@sha256:...` in Compose, `Image=` in Quadlet) so what runs is what was signed. + +## Not verified + +- Behaviour of `cosign sign --recursive` with the attestation manifests BuildKit adds to the index was not tested; check `cosign tree` output after signing. +- The exact GitHub Actions identity URI format (`https://github.com/ORG/REPO/.github/workflows/...@refs/...`) was not read today; the regexp above only anchors on the repository. +- `jq` is assumed installed. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/compose-production.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/compose-production.md new file mode 100644 index 0000000..59cb039 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/compose-production.md @@ -0,0 +1,143 @@ +--- +name: compose-production +description: Production compose.yaml - healthchecks, ordered startup with a migration job, limits, secrets, internal network, localhost-only publishing +keywords: compose.yaml, healthcheck, depends_on, service_healthy, service_completed_successfully, secrets, mem_limit, pids_limit, read_only, internal network, postgres 18 +--- + +# Template: Production `compose.yaml` + +## Usage + +A web service behind a host reverse proxy, a one-shot migration job and PostgreSQL. Shows: readiness-gated startup, a migration that must succeed before the app starts, Compose secrets with `*_FILE` variables, memory/PID limits, read-only root filesystem, dropped capabilities, an internal-only database network and a port published on localhost only. Keys verified 2026-11-10 against https://docs.docker.com/reference/compose-file/services/ , https://docs.docker.com/reference/compose-file/networks/ and https://docs.docker.com/compose/how-tos/startup-order/. + +Replace `REG/app@sha256:...` with your image digest and adapt `healthcheck.test` to your app's real endpoint. + +--- + +## `compose.yaml` + +```yaml +name: shop + +x-hardening: &hardening + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + init: true + logging: + driver: local + options: { max-size: "20m", max-file: "5" } + +services: + db: + image: postgres:18 + restart: unless-stopped + environment: + POSTGRES_USER: shop + POSTGRES_DB: shop + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + volumes: + - db_data:/var/lib/postgresql # PostgreSQL 18+ image: VOLUME moved here (was .../data up to 17) + networks: [backend] + shm_size: 256m + mem_limit: 1g + pids_limit: 512 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + start_interval: 2s + stop_grace_period: 60s + + migrate: + <<: *hardening + image: REG/app@sha256:REPLACE_WITH_DIGEST + command: ["/app", "migrate"] + environment: + DATABASE_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + networks: [backend] + depends_on: + db: + condition: service_healthy + restart: "no" + + app: + <<: *hardening + image: REG/app@sha256:REPLACE_WITH_DIGEST + restart: unless-stopped + environment: + APP_ENV: production + APP_BASE_URL: ${APP_BASE_URL:?set APP_BASE_URL in .env} + DATABASE_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + ports: + - "127.0.0.1:8080:8080" # reverse proxy on the host; not reachable from the LAN + networks: [backend, frontend] + tmpfs: [/tmp] + mem_limit: 512m + pids_limit: 256 + cpus: 1.0 + depends_on: + db: + condition: service_healthy + restart: true + migrate: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "/app", "healthcheck"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 20s + start_interval: 2s + + adminer: + image: adminer + profiles: [debug] # only with: docker compose --profile debug up + ports: ["127.0.0.1:8081:8080"] + networks: [backend] + +networks: + backend: + internal: true # no external connectivity for db/migrate + frontend: {} + +volumes: + db_data: {} + +secrets: + db_password: + file: ./secrets/db_password.txt # chmod 600, never committed +``` + +The app is also on `frontend` so it keeps external connectivity (and a normal bridge for its published port); `db` and `migrate` stay on the externally isolated `backend` only. + +--- + +## Operate + +```bash +install -m 600 /dev/null secrets/db_password.txt && openssl rand -base64 32 > secrets/db_password.txt +docker compose config -q # validate (also fails if APP_BASE_URL is unset) +docker compose up -d --wait --wait-timeout 180 +docker compose ps # db, app (healthy); migrate Exited (0) +docker compose logs --since 10m app +``` + +Upgrade: change the digest, then `docker compose up -d --wait` (only changed services are recreated). Stop: `docker compose down` keeps `db_data`. **`docker compose down -v` deletes `db_data`**: propose it with its effect and wait for the user's go ([safe-ops.md](../safe-ops.md)). + +## Verify + +- `docker compose ps` shows `(healthy)` for `db` and `app`, `migrate` exited 0. +- `curl -fsS http://127.0.0.1:8080/` answers on the host; the same port is refused from another machine. +- `docker inspect shop-db-1 --format '{{range .Mounts}}{{.Destination}} {{end}}'` lists `/var/lib/postgresql` and no anonymous volume. + +## Not verified + +- Whether an `internal` network alone prevents port publishing was inferred from the doc's "externally isolated" wording, not tested; the template avoids the case by attaching `app` to `frontend`. +- The default container name pattern `shop-db-1` was not re-read today; use `docker compose ps -q db`. +- `adminer` tag not pinned; pin it if you keep the debug profile. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/daemon-json.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/daemon-json.md new file mode 100644 index 0000000..addd8a2 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/daemon-json.md @@ -0,0 +1,97 @@ +--- +name: daemon-json +description: Baseline /etc/docker/daemon.json for a Linux server running Docker Engine 29 +keywords: daemon.json, live-restore, log-driver local, default-address-pools, default-network-opts, no-new-privileges, builder gc +--- + +# Template: Server `daemon.json` (Engine 29) + +## Usage + +A conservative baseline for a single Linux host: rotated logs, containers kept alive across daemon restarts, localhost-by-default port publishing for user-defined bridges, private address pools that avoid common LAN ranges, and a build-cache GC policy. Every key appears in the full Linux example of https://docs.docker.com/reference/cli/dockerd/ (read 2026-11-10). Values are starting points: adapt the pools to your network and the GC sizes to your disk. + +Do not add a key here that is also passed as a `dockerd` flag (systemd unit or drop-in): the daemon refuses to start. + +--- + +## `/etc/docker/daemon.json` + +```json +{ + "live-restore": true, + "log-driver": "local", + "log-opts": { + "max-size": "20m", + "max-file": "5" + }, + "default-address-pools": [ + { "base": "172.30.0.0/16", "size": 24 }, + { "base": "172.31.0.0/16", "size": 24 } + ], + "default-network-opts": { + "bridge": { + "com.docker.network.bridge.host_binding_ipv4": "127.0.0.1" + } + }, + "no-new-privileges": true, + "builder": { + "gc": { + "enabled": true, + "defaultReservedSpace": "10GB", + "policy": [ + { "maxUsedSpace": "512MB", "keepDuration": "48h", "filter": ["type=source.local"] }, + { "reservedSpace": "10GB", "maxUsedSpace": "100GB", "keepDuration": "1440h" } + ] + } + } +} +``` + +--- + +## Optional keys (add only when the condition holds) + +| Condition | Key | Effect / caveat | +|-----------|-----|-----------------| +| Upgraded host, want multi-platform images and attestations locally | `"features": {"containerd-snapshotter": true}` | Hides existing overlay2 images; more disk; not with `userns-remap` | +| Host must keep routing non-Docker traffic | `"ip-forward-no-drop": true` | Docker no longer sets `FORWARD` policy to `DROP` (iptables backend) | +| Testing the nftables backend (not Swarm) | `"firewall-backend": "nftables"` | Experimental; enable IP forwarding yourself; no `DOCKER-USER` | +| Pull through a Docker Hub mirror | `"registry-mirrors": ["https://mirror.example.internal"]` | Docker Hub mirrors only | +| Logs shipped by a file-tailing agent | `"log-driver": "json-file"` with the same `log-opts` | `json-file` does not rotate without `max-size` | + +Details: [engine-daemon-config.md](../engine-daemon-config.md), [networking-firewall.md](../networking-firewall.md), [registries-supply-chain.md](../registries-supply-chain.md). + +--- + +## Apply + +```bash +sudo cp daemon.json /etc/docker/daemon.json.new +sudo dockerd --validate --config-file=/etc/docker/daemon.json.new # must print "configuration OK" +sudo cp /etc/docker/daemon.json /etc/docker/daemon.json.bak 2>/dev/null || true +sudo mv /etc/docker/daemon.json.new /etc/docker/daemon.json +sudo systemctl restart docker +``` + +A restart stops running containers unless `live-restore` was **already** active; on a production host, schedule it and announce it. Enabling only `live-restore` can be done with `sudo systemctl reload docker`. + +--- + +## Verify + +```bash +systemctl is-active docker +docker info --format '{{.LoggingDriver}} live-restore={{.LiveRestoreEnabled}}' +docker network create probe && docker network inspect probe --format '{{json .Options}}' && docker network rm probe +``` + +New logging options and `default-network-opts` apply to containers and networks **created after** the restart; recreate existing ones (`docker compose up -d --force-recreate`) when the change must cover them. + +## Notes + +- `default-network-opts` applies to user-defined bridge networks; it does not change Swarm services (port-publishing docs). + +## Not verified + +- `no-new-privileges` breaking images that rely on setuid binaries (for example `sudo` inside the container) is inferred from what the option blocks, not tested; try it on staging first. +- The `builder.gc` values are the ones from the `dockerd` reference example, not sized for a specific disk. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/dockerfile-hardened.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/dockerfile-hardened.md new file mode 100644 index 0000000..74d476d --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/dockerfile-hardened.md @@ -0,0 +1,108 @@ +--- +name: dockerfile-hardened +description: Multi-stage, cross-compiling, cache-mounted, non-root Dockerfile with build checks, plus .dockerignore +keywords: Dockerfile, multi-stage, BUILDPLATFORM, TARGETOS, TARGETARCH, cache mount, secret mount, distroless, nonroot, check=error=true +--- + +# Template: Hardened Multi-Stage Dockerfile + +## Usage + +Shown for a statically linked Go service because it exercises every pattern (cross-compilation, cache mounts, a minimal non-root runtime). For other stacks keep the structure - `deps` stage, `build` stage, minimal `runtime` stage - and swap the toolchain image and commands; the stack's own expert owns the build commands. + +Images checked 2026-11-10: `golang:1.27-alpine` (Docker Hub tag list) and `gcr.io/distroless/static-debian13` with tag `nonroot` (distroless README and registry tag list). In production **pin each `FROM` by digest** (`image:tag@sha256:...`, get it with `docker buildx imagetools inspect image:tag`). + +Patterns and flags: https://docs.docker.com/reference/dockerfile/ , https://docs.docker.com/build/building/multi-platform/ , https://docs.docker.com/build/checks/ , https://docs.docker.com/build/building/secrets/ + +--- + +## `Dockerfile` + +```dockerfile +# syntax=docker/dockerfile:1 +# check=error=true + +ARG GO_IMAGE=golang:1.27-alpine +ARG RUNTIME_IMAGE=gcr.io/distroless/static-debian13:nonroot + +# ---- deps: download modules once, cached across builds ----------------- +FROM --platform=$BUILDPLATFORM ${GO_IMAGE} AS deps +WORKDIR /src +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +# ---- build: cross-compile natively for each target platform ------------- +FROM deps AS build +ARG TARGETOS +ARG TARGETARCH +COPY . . +# Private modules: pass a token as a build secret, never as ARG/ENV. +# docker build --secret id=GIT_AUTH_TOKEN . +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=secret,id=GIT_AUTH_TOKEN,env=GIT_AUTH_TOKEN,required=false \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ + go build -trimpath -ldflags="-s -w" -o /out/app ./cmd/app + +# ---- runtime: no shell, no package manager, non-root -------------------- +FROM ${RUNTIME_IMAGE} AS runtime +COPY --link --from=build /out/app /app +EXPOSE 8080 +# distroless has no shell or curl: the binary must implement its own probe. +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --start-interval=2s --retries=3 \ + CMD ["/app", "healthcheck"] +ENTRYPOINT ["/app"] +``` + +--- + +## `.dockerignore` + +```gitignore +.git +.github +**/*_test.go +**/testdata +*.md +compose*.yaml +docker-bake.hcl +.env +secrets/ +``` + +Keeping `.env` and `secrets/` out of the context prevents them from landing in a layer through `COPY . .`. + +--- + +## Build and check + +```bash +docker build --check . # lint only +docker build -t app:dev . # local, current platform +docker buildx build --platform linux/amd64,linux/arm64 \ + --sbom=true --provenance=mode=max -t REG/app:1.0.0 --push . # multi-arch + attestations +``` + +`--provenance=mode=max` records build-arg values: nothing secret may travel as `--build-arg`. Multi-arch output needs the containerd image store or a `docker-container` builder ([buildkit-dockerfile.md](../buildkit-dockerfile.md)). + +## Verify + +```bash +docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges -p 127.0.0.1:8080:8080 app:dev +docker image inspect app:dev --format '{{.Config.User}} {{.Size}}' # non-empty user expected +docker buildx imagetools inspect REG/app:1.0.0 # both platforms + attestation manifests +``` + +## Adapting + +| Stack | Change | +|-------|--------| +| Needs libc (CGO, Python wheels) | runtime `gcr.io/distroless/cc-debian13:nonroot` or a slim distro image with a created user | +| Interpreted language | copy the venv / `node_modules` built in a `deps` stage; never install compilers in `runtime` | +| Needs a shell for debugging | use the `debug-nonroot` distroless tag only in a separate debug target, never in production | + +## Not verified + +- The user set by the distroless `nonroot` tag was not read in its README today; check it with `docker image inspect` before relying on a specific UID for volume permissions. (`required=false` is the documented default for secret mounts; it is written out for clarity.) +- The `/app healthcheck` sub-command is an application contract, not a Docker feature: implement it or remove the `HEALTHCHECK`. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/quadlet-units.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/quadlet-units.md new file mode 100644 index 0000000..0fd8525 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-docker/references/templates/quadlet-units.md @@ -0,0 +1,152 @@ +--- +name: quadlet-units +description: Rootless Podman Quadlet units - network, volume, database and app containers with healthcheck-gated startup, secrets and auto-update +keywords: quadlet, .container, .network, .volume, Notify=healthy, HealthOnFailure, Secret, AutoUpdate, loginctl enable-linger, rootless +--- + +# Template: Rootless Quadlet Service + +## Usage + +Runs an app and PostgreSQL as a dedicated unprivileged user, as plain systemd user services: the database unit becomes "started" only when healthy, the app starts after it, an unhealthy container is killed and restarted by systemd, and images auto-update from the registry. All keys come from `podman-systemd.unit.5.md` and `options/secret.md` at Podman **v6.1.4** (read 2026-11-10, https://github.com/podman-container-tools/podman/tree/v6.1.4/docs/source/markdown). + +Requirements: cgroup v2, Podman 6.x, a user with subordinate UIDs/GIDs. + +--- + +## 0. One-time host setup (as root) + +```bash +sudo useradd --create-home --shell /usr/sbin/nologin svc-shop +sudo loginctl enable-linger svc-shop # user services start at boot without a login +podman info --format '{{.Host.CgroupsVersion}}' # must print v2 +``` + +Run the following as `svc-shop` (e.g. `sudo machinectl shell svc-shop@` so `systemctl --user` has a session bus). + +```bash +mkdir -p ~/.config/containers/systemd +openssl rand -base64 32 | podman secret create db_password - +``` + +--- + +## 1. `~/.config/containers/systemd/shop.network` + +```ini +[Network] +NetworkName=shop +Internal=false +``` + +## 2. `~/.config/containers/systemd/db.volume` + +```ini +[Volume] +VolumeName=shop-db +``` + +## 3. `~/.config/containers/systemd/db.container` + +```ini +[Unit] +Description=Shop PostgreSQL + +[Container] +ContainerName=shop-db +Image=docker.io/library/postgres:18 +AutoUpdate=registry +Network=shop.network +NetworkAlias=db +# PostgreSQL 18+ image keeps data under /var/lib/postgresql (VOLUME moved in 18) +Volume=db.volume:/var/lib/postgresql +Environment=POSTGRES_USER=shop POSTGRES_DB=shop POSTGRES_PASSWORD_FILE=/run/secrets/db_password +Secret=db_password,type=mount +HealthCmd=pg_isready -U shop -d shop +HealthInterval=10s +HealthStartPeriod=30s +HealthOnFailure=kill +Notify=healthy +Memory=1g +PidsLimit=512 +NoNewPrivileges=true + +[Service] +Restart=always +TimeoutStartSec=900 + +[Install] +WantedBy=default.target +``` + +## 4. `~/.config/containers/systemd/app.container` + +```ini +[Unit] +Description=Shop app +Requires=db.container +After=db.container + +[Container] +ContainerName=shop-app +Image=registry.example.internal/team/app@sha256:REPLACE_WITH_DIGEST +Network=shop.network +PublishPort=127.0.0.1:8080:8080 +Environment=APP_ENV=production DATABASE_HOST=db DATABASE_PASSWORD_FILE=/run/secrets/db_password +Secret=db_password,type=mount +ReadOnly=true +DropCapability=all +NoNewPrivileges=true +Memory=512m +PidsLimit=256 +HealthCmd=/app healthcheck +HealthInterval=30s +HealthStartPeriod=20s +HealthOnFailure=kill +Notify=healthy + +[Service] +Restart=always +TimeoutStartSec=900 + +[Install] +WantedBy=default.target +``` + +A digest-pinned `Image=` cannot be auto-updated; use a fully-qualified tag with `AutoUpdate=registry` instead if you want automatic updates for the app too. + +--- + +## 5. Generate, start, verify + +```bash +/usr/lib/systemd/system-generators/podman-system-generator --user --dryrun # syntax check, shows generated units +systemctl --user daemon-reload +systemctl --user start app.service # pulls in db.service, network and volume services +systemctl --user status db.service app.service +podman ps --format '{{.Names}} {{.Status}}' # both "(healthy)" +curl -fsS http://127.0.0.1:8080/ +journalctl --user -u app.service -e +``` + +Do not run `systemctl --user enable`: Quadlet services are generated (transient); the `[Install]` section is applied by the generator. + +## 6. Auto-update + +```bash +podman auto-update --dry-run --format "{{.Image}} {{.Updated}}" +``` + +Updates roll back to the previous image if the restarted unit fails (`--rollback`, default true). The shipped `podman-auto-update.timer` runs daily at midnight. + +--- + +## Changing or removing + +Edit the file, then `systemctl --user daemon-reload && systemctl --user restart app.service`. Removing the unit file stops managing the container but keeps the `shop-db` volume. Deleting the volume (`podman volume rm shop-db`) or `podman system reset` destroys data: propose with its effect and wait for the user's go ([safe-ops.md](../safe-ops.md)). + +## Not verified + +- `machinectl shell` as the way to reach the user's systemd instance is a systemd convention, not read in the Podman docs today; any method giving the user a session bus works. +- Whether `Notify=healthy` on `db.container` delays `After=db.container` until healthy is inferred from systemd ordering semantics plus the `Notify=healthy` description; check with `systemctl --user show app.service -p ExecMainStartTimestamp` versus the db health transition. +- Space-separated multiple assignments on one `Environment=` line follow systemd syntax; not re-read in the Podman page. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/SKILL.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/SKILL.md new file mode 100644 index 0000000..085344d --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/SKILL.md @@ -0,0 +1,162 @@ +--- +name: devops-gitops-packaging +description: Use when packaging or delivering workloads to Kubernetes - Helm 4 charts (values, values.schema.json, dependencies, OCI registries, upgrade/rollback/test), Kustomize bases, overlays, patches and components, choosing Helm vs Kustomize vs both, Argo CD (Application, ApplicationSet, AppProject, sync policy, waves, hooks, app-of-apps), Flux (GitRepository, OCIRepository, Kustomization, HelmRelease, image automation), GitOps repository layout, promotion between environments, drift and rollback. Not for the Kubernetes resources themselves (devops-kubernetes), cluster install or upgrade (devops-k8s-cluster), CI pipelines that build images (devops-cicd) or secret stores (devops-secrets). +versions: + helm: "4.3.1 (helm.sh docs 'Version: 4.3.1')" + kustomize: "5.8.3" + argo-cd: "3.6.1 (released 2026-11-04)" + flux: "2.10.0 (released 2026-10-26)" +user-invocable: true +references: + - references/helm-4-changes.md + - references/helm-chart-authoring.md + - references/helm-release-ops.md + - references/kustomize.md + - references/helm-vs-kustomize.md + - references/argocd-applications.md + - references/argocd-applicationset-appofapps.md + - references/flux.md + - references/flux-image-automation.md + - references/repo-layout-promotion.md + - references/templates/chart/Chart.yaml + - references/templates/chart/values.yaml + - references/templates/chart/values.schema.json + - references/templates/chart/templates/_helpers.tpl + - references/templates/chart/templates/deployment.yaml + - references/templates/kustomize/base/kustomization.yaml + - references/templates/kustomize/base/deployment.yaml + - references/templates/kustomize/overlays/prod/kustomization.yaml + - references/templates/kustomize/components/ingress-tls/kustomization.yaml + - references/templates/argocd/appproject.yaml + - references/templates/argocd/application.yaml + - references/templates/argocd/applicationset.yaml + - references/templates/flux/gitrepository.yaml + - references/templates/flux/kustomization.yaml + - references/templates/flux/helmrelease.yaml + - references/templates/flux/image-automation.yaml + - references/templates/gitops-repo-layout.md +related-skills: + - devops-kubernetes + - devops-k8s-cluster + - devops-secrets + - devops-cicd + - devops-docker + - devops-observability + - devops-sre-ops +--- + + +Covers turning manifests into a versioned, reviewable, reversible delivery: +Helm 4 charts and releases, Kustomize overlays, the Helm vs Kustomize choice, +Argo CD and Flux controllers, repository layout, promotion, drift and rollback. +Does not cover the Kubernetes objects themselves (devops-kubernetes), cluster +build and upgrade (devops-k8s-cluster), the CI that builds and signs images +(devops-cicd, devops-docker) or secret stores (devops-secrets). + + +# DevOps GitOps and Packaging + +## Agent Workflow (MANDATORY) + +Before ANY chart, overlay, Application or Flux object, spawn in parallel, one `Task` call each: + +1. **fuse-ai-pilot:explore-codebase** - find existing `Chart.yaml`, `kustomization.yaml`, Argo CD or Flux manifests and the repo layout +2. **fuse-ai-pilot:research-expert** - verify fields and flags for the installed tool version at helm.sh, kubectl.docs.kubernetes.io, argo-cd.readthedocs.io, fluxcd.io +3. **mcp__context7__query-docs** - `/websites/helm_sh`, `/helm/helm`, `/kubernetes-sigs/kustomize`, `/websites/argo-cd_readthedocs_io_en_stable`, `/websites/fluxcd_io_flux` + +Then read the target read-only (`helm version`, `helm list -A`, `argocd version`, `flux check`, `kubectl get crd | grep -E 'argoproj|fluxcd'`). After changes, run **fuse-ai-pilot:sniper**. + +## Versions (checked 2026-11-10) + +Helm 4.3.1 (https://helm.sh/docs/overview/), Kustomize 5.8.3 (https://github.com/kubernetes-sigs/kustomize/releases), Argo CD 3.6.1 (https://github.com/argoproj/argo-cd/releases/tag/v3.6.1), Flux 2.10.0 (https://github.com/fluxcd/flux2/releases/tag/v2.10.0). API versions used here: `argoproj.io/v1alpha1`, `source.toolkit.fluxcd.io/v1`, `kustomize.toolkit.fluxcd.io/v1`, `helm.toolkit.fluxcd.io/v2`, `image.toolkit.fluxcd.io/v1`, `kustomize.config.k8s.io/v1beta1`. Read the installed version first (`helm version`, `argocd version`, `flux version`); the CRDs on the cluster win over this file. + +## Overview + +| Area | Decision in one line | +|------|----------------------| +| **Packaging** | Helm for third-party or parameterised apps; Kustomize for your own plain manifests; both only with a stated reason ([helm-vs-kustomize.md](references/helm-vs-kustomize.md)) | +| **Helm 4** | Server-side apply default for new releases, `--atomic` renamed `--rollback-on-failure`, `--force` renamed `--force-replace`, post-renderers are plugins ([helm-4-changes.md](references/helm-4-changes.md)) | +| **Controller** | Argo CD when a UI, multi-cluster and per-app RBAC matter; Flux when everything should be CRDs reconciled in-cluster; one controller per cluster, never both on the same objects | +| **Delivery** | Git is the source of truth; change Git, not the cluster; pin charts and images by version or digest | +| **Pruning** | Off by default or `confirm`; a prune is a delete (see Critical Rules) | +| **Promotion** | One artifact, environment-specific config only; promote by PR, not by rebuilding ([repo-layout-promotion.md](references/repo-layout-promotion.md)) | + +## Critical Rules + +1. **Version first** - read the tool versions and the CRDs on the cluster before writing a field; an apiVersion not in the cluster is not usable +2. **Render, then diff, then apply** - `helm template` / `kustomize build`, then `helm diff upgrade` or `kubectl diff`, `argocd app diff`, `flux diff kustomization`; never sync blind +3. **Destructive verbs are proposed, not run** - `helm uninstall`, `helm rollback`, Argo CD sync with prune, `argocd app delete --cascade`, `flux delete`, `flux suspend`, `kustomize build | kubectl delete -f -`: exact ACTION / EFFECT / PROOF, then WAIT-FOR-GO ([helm-release-ops.md](references/helm-release-ops.md)) +4. **Never hand-edit a GitOps-owned object** - the controller reverts it; the fix is a commit +5. **No secret value in Git or in output** - names and keys only, values via devops-secrets (SOPS, External Secrets); `helm template`/`--dry-run` print Secrets, so use `--hide-secret` +6. **Pin everything** - chart version, `targetRevision` as tag or SHA, image tag plus digest; no `HEAD`, `latest` or floating range in production +7. **Never invent a field or flag** - cite the official page; unknown = "not verified" + +## Architecture + +``` +gitops/ +├── charts// # Helm chart (Chart.yaml, values.yaml, values.schema.json, templates/) +├── apps//base/ # Kustomize base +├── apps//overlays/{staging,prod}/ +├── clusters/{staging,prod}/ # per-cluster entry: Flux Kustomizations or Argo CD Applications +└── infrastructure/ # controllers and shared config, reconciled before apps +``` + +## Reference Guide + +### Concepts + +| Topic | Reference | When to Consult | +|-------|-----------|-----------------| +| **Helm 4 changes** | [helm-4-changes.md](references/helm-4-changes.md) | Moving from Helm 3, CI flags, SSA, plugins | +| **Chart authoring** | [helm-chart-authoring.md](references/helm-chart-authoring.md) | Structure, values, schema, dependencies, hooks, tests | +| **Release operations** | [helm-release-ops.md](references/helm-release-ops.md) | OCI push/pull, diff, upgrade, rollback, uninstall | +| **Kustomize** | [kustomize.md](references/kustomize.md) | Bases, overlays, patches, generators, components | +| **Helm vs Kustomize** | [helm-vs-kustomize.md](references/helm-vs-kustomize.md) | Choosing, combining, `helmCharts` limits | +| **Argo CD** | [argocd-applications.md](references/argocd-applications.md) | Application, AppProject, sync policy, waves, hooks, deletion | +| **ApplicationSet** | [argocd-applicationset-appofapps.md](references/argocd-applicationset-appofapps.md) | Many apps or clusters, app-of-apps | +| **Flux** | [flux.md](references/flux.md) | Sources, Kustomization, HelmRelease, suspend/delete | +| **Flux image automation** | [flux-image-automation.md](references/flux-image-automation.md) | Tag scanning and commits back to Git | +| **Layout and promotion** | [repo-layout-promotion.md](references/repo-layout-promotion.md) | Environments, promotion, drift, rollback, secrets pointer | + +### Templates + +| Template | When to Use | +|----------|-------------| +| [chart/](references/templates/chart/Chart.yaml) | Minimal chart with `values.schema.json` (Helm templates are not parseable YAML) | +| [kustomize/](references/templates/kustomize/base/kustomization.yaml) | Base + prod overlay + component | +| [argocd/appproject.yaml](references/templates/argocd/appproject.yaml) | Project limiting repos, destinations, resource kinds | +| [argocd/application.yaml](references/templates/argocd/application.yaml) | Application with manual-prune safe sync policy | +| [argocd/applicationset.yaml](references/templates/argocd/applicationset.yaml) | One app per environment directory | +| [flux/gitrepository.yaml](references/templates/flux/gitrepository.yaml) | Git source | +| [flux/kustomization.yaml](references/templates/flux/kustomization.yaml) | Reconcile a path with ordering and health checks | +| [flux/helmrelease.yaml](references/templates/flux/helmrelease.yaml) | OCI chart, remediation, drift detection | +| [flux/image-automation.yaml](references/templates/flux/image-automation.yaml) | ImageRepository, ImagePolicy, ImageUpdateAutomation | +| [gitops-repo-layout.md](references/templates/gitops-repo-layout.md) | Repository tree and promotion flow | + +## Quick Reference + +```bash +CTX=prod-eu; NS=shop # always explicit +helm template web ./charts/web -f values-prod.yaml --hide-secret >/dev/null # render only +helm --kube-context "$CTX" -n "$NS" diff upgrade web ./charts/web -f values-prod.yaml # helm-diff plugin +helm --kube-context "$CTX" -n "$NS" upgrade web ./charts/web -f values-prod.yaml --dry-run=server --hide-secret +kustomize build apps/web/overlays/prod | kubectl --context "$CTX" diff -f - # exit 1 = differences +argocd app diff web --refresh # exit 1 = differences +flux diff kustomization web --path ./apps/web/overlays/prod # exit 1 = differences +flux reconcile kustomization web --with-source # after the user's go +``` + +## Best Practices + +### DO +- Add `values.schema.json` to every chart you own so bad values fail at render time +- Keep one Application or Kustomization per deployable unit, with `dependsOn` or sync waves for ordering +- Give Argo CD `AppProject` or Flux service accounts the narrowest repos, namespaces and kinds +- Record the rollback target (chart version, Git SHA, Helm revision) in the change proposal + +### DON'T +- Enable automated prune or `selfHeal` on a production Application without the owner's go +- Run `helm upgrade --reuse-values` across chart versions (stale values hide new defaults) +- Let Argo CD and Flux, or Helm CLI and a controller, manage the same release +- Template secrets into values files committed in clear text diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/argocd-applications.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/argocd-applications.md new file mode 100644 index 0000000..a3536b1 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/argocd-applications.md @@ -0,0 +1,101 @@ +--- +name: argocd-applications +description: Argo CD Application, AppProject, sync policy, prune and self-heal, waves and hooks, diffing, rollback and deletion +when-to-use: Writing or changing an Application or AppProject, deciding sync policy, ordering resources, syncing or deleting an app +keywords: Application, AppProject, syncPolicy, automated, prune, selfHeal, sync-wave, PreSync, Prune=confirm, argocd app sync, argocd app delete, finalizer +priority: critical +related: argocd-applicationset-appofapps.md, repo-layout-promotion.md, helm-vs-kustomize.md +--- + +# Argo CD Applications + +**Load when:** an `Application`, `AppProject` or `argocd` command is involved. Version 3.6.1 (released 2026-11-04). All Application and AppProject objects live in the Argo CD namespace (default `argocd`). Templates: [application.yaml](templates/argocd/application.yaml), [appproject.yaml](templates/argocd/appproject.yaml). + +## Core objects + +| Object | Key fields | +|--------|-----------| +| `Application` (`argoproj.io/v1alpha1`) | `spec.project`, `spec.source` (`repoURL`, `targetRevision`, `path` or `chart`, `helm`/`kustomize`/`directory`), `spec.destination` (`server` **or** `name`, `namespace`), `spec.syncPolicy`, `spec.ignoreDifferences` | +| `AppProject` | `sourceRepos`, `destinations`, `clusterResourceWhitelist`/`Blacklist`, `namespaceResourceWhitelist`/`Blacklist`, `roles`, `syncWindows`, `sourceNamespaces` | + +Git-hosted chart: use `path`; Helm repository chart: use `chart` (and `targetRevision` = chart version). A project whose `destinations` include the Argo CD namespace grants admin-level access: never give it to tenants. + +## Decision rules: sync policy + +| Setting | Meaning (sync policy docs) | Default stance | +|---------|----------------------------|----------------| +| `automated: {}` | Sync when the app is OutOfSync; one attempt per commit SHA plus parameters; no retry of a failed attempt on the same SHA | Staging yes, production only with the owner's go | +| `automated.prune: true` | Delete live resources no longer in Git | Off. Without it, a manual sync can still prune | +| `automated.allowEmpty` | Allow an app to end up with zero resources (guard against empty-render accidents) | Never true | +| `automated.selfHeal: true` | Re-sync when live state drifts (default delay 5 s) | On where manual `kubectl edit` must not survive; off while someone is debugging | +| `retry` (`limit`, `backoff.duration/factor/maxDuration`) | Exponential retry of failed syncs; `limit: -1` = unlimited | `limit: 5`, `maxDuration: 3m` | +| `syncOptions: CreateNamespace=true` | Create the destination namespace | Fine for non-shared namespaces | +| `Prune=false` / `Prune=confirm` (resource annotation `argocd.argoproj.io/sync-options` or app-level `syncOptions`) | Never prune / require confirmation before pruning | `Prune=confirm` on Namespaces, PVCs, CRDs | +| `Delete=false` / `Delete=confirm` | Keep / confirm before deleting the resource when the **app** is deleted | `Delete=false` on PVCs | +| `PruneLast=true` | Prune as a final implicit wave after other waves are healthy | On when ordering matters | +| `ServerSideApply=true` | Server-side apply | Not read in detail today; verify before use | +| `Replace=true` | `kubectl replace` instead of apply | Only for objects too big for `last-applied`; destructive-ish | +| `ApplyOutOfSyncOnly=true` | Apply only drifted resources | For apps with thousands of objects | + +Auto-sync facts: reconciliation interval is `timeout.reconciliation` in `argocd-cm` (default 120 s plus up to 60 s jitter); **rollback cannot be performed on an app with automated sync enabled**; for apps generated by an ApplicationSet, changing `syncPolicy.automated` on the child has no effect (change the ApplicationSet). + +## Ordering: phases, waves, hooks + +Hooks (`argocd.argoproj.io/hook`): `PreSync`, `Sync`, `Skip`, `PostSync`, `SyncFail`, `PreDelete`, `PostDelete`. A failed PreSync stops the sync. Waves: `argocd.argoproj.io/sync-wave: ""`, default 0, negative allowed, lowest first, 2 s delay between waves (`ARGOCD_SYNC_WAVE_DELAY`). Order of precedence: phase, wave, kind (Namespaces first), name. Pruning runs in reverse wave order. Cleanup: `argocd.argoproj.io/hook-delete-policy` (`HookSucceeded`, `HookFailed`, `BeforeHookCreation`, which is the default). Do not use `ttlSecondsAfterFinished` on hook Jobs. Hooks do not run on a selective sync. + +## Commands + +```bash +argocd app diff web --refresh # exit 0 none, 1 diff, 2 error; Secrets excluded from the diff +argocd app sync web --dry-run # preview apply +argocd app sync web --preview-changes # diff then wait for confirmation +argocd app sync web --prune --dry-run # preview what a prune would delete +argocd app sync web --revision # after the user's go +argocd app history web ; argocd app rollback web # history ids; rollback needs auto-sync off +``` + +## Destructive: propose, then the user's go + +| Command | Exact effect | Before proposing | +|---------|--------------|------------------| +| `argocd app sync --prune` or `automated.prune: true` | Deletes live resources absent from Git, in reverse wave order | `argocd app sync --prune --dry-run` list; PVC, Namespace, CRD in it? use `Prune=confirm` | +| `argocd app sync --force` / `--replace` | Force apply / `kubectl replace`: pods restart, fields reset | Prefer a normal sync | +| `argocd app delete ` (= `--cascade`) | Deletes the Application **and all its managed resources** (foreground by default) | Resource list (`argocd app resources`), `Delete=false` on data objects, backups; PreDelete hooks run | +| `argocd app delete --cascade=false` | Deletes only the Application; resources stay running, unmanaged | Say which resources are orphaned | +| `argocd app rollback ` | Syncs back to a previous history entry (`--prune` also deletes) | Auto-sync must be disabled; Git still holds the new version, so a later sync undoes it | +| `kubectl delete app ` | Cascade only if the app has the `resources-finalizer.argocd.argoproj.io` finalizer | Check `metadata.finalizers` first | + +``` +ACTION argocd app delete web --cascade (project shop, cluster prod-eu) +EFFECT deletes Deployment web, Service web, ConfigMap web-config, PVC data-web (reclaimPolicy Delete: data lost) in ns shop +PROOF argocd app resources web listing attached; backup id ; Delete=false not set on PVC +WAIT-FOR-GO yes +``` + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| `targetRevision: HEAD` in production | Every merge deploys; pin a tag or SHA and promote by editing it | +| Application without the finalizer | Deleting it leaves all resources running (orphaned) | +| App stays OutOfSync after sync | Mutating webhook, controller-written fields or `randAlphaNum`: use `ignoreDifferences` (`jsonPointers`, `jqPathExpressions`, `managedFieldsManagers`) deliberately | +| `ignoreDifferences` on `/spec/replicas` while an HPA exists | Required to stop the fight; omit it when no HPA | +| Two projects, same destination namespace | Overlapping ownership; one namespace, one owner | +| Helm hooks in a chart | Mapped to Argo CD hooks; delete-policy semantics follow Argo CD phases, not Helm events | + +## Verification + +`argocd app get web` shows `Synced` and `Healthy` at the intended revision; `kubectl rollout status`; the user-facing check. `Synced` does not prove the service works. + +## Sources (read 2026-11-10) + +- https://argo-cd.readthedocs.io/en/stable/operator-manual/declarative-setup/ , https://argo-cd.readthedocs.io/en/stable/operator-manual/application.yaml , https://argo-cd.readthedocs.io/en/stable/operator-manual/project-specification/ +- https://argo-cd.readthedocs.io/en/stable/user-guide/auto_sync/ , .../sync-options/ , .../sync-waves/ , .../app_deletion/ , .../diffing/ +- https://argo-cd.readthedocs.io/en/stable/user-guide/commands/argocd_app_sync/ , .../argocd_app_diff/ , .../argocd_app_rollback/ +- https://github.com/argoproj/argo-cd/releases/tag/v3.6.1 + +## Not verified + +- `ServerSideApply=true` option text (sync-options page truncated before it); `argocd app history`, `argocd app resources`, `argocd app get` pages not read. +- `Replace=true` consequences beyond the page excerpt; `--force` semantics of `argocd app sync` ("Use a force apply"). +- Argo CD 3.x upgrade notes (breaking changes between 3.3 and 3.6) were not read; check https://argo-cd.readthedocs.io/en/stable/operator-manual/upgrading/overview/ before upgrading Argo CD itself. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/argocd-applicationset-appofapps.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/argocd-applicationset-appofapps.md new file mode 100644 index 0000000..3384640 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/argocd-applicationset-appofapps.md @@ -0,0 +1,79 @@ +--- +name: argocd-applicationset-appofapps +description: Argo CD ApplicationSet generators, app-of-apps bootstrapping, cascading deletion and their security implications +when-to-use: One definition must produce many Applications (clusters, environments, directories), or a cluster must be bootstrapped from one root app +keywords: ApplicationSet, generators, list, cluster, git, matrix, OCI, goTemplate, app of apps, bootstrapping, cluster labels +priority: high +related: argocd-applications.md, repo-layout-promotion.md +--- + +# ApplicationSet and app-of-apps + +**Load when:** more than a handful of near-identical Applications exist, or you bootstrap a new cluster. Template: [applicationset.yaml](templates/argocd/applicationset.yaml). + +## ApplicationSet + +The controller ships with Argo CD since 2.3. A generator emits parameter sets; each set is substituted into `template` and becomes an `Application` in the Argo CD namespace. + +| Generator | Emits parameters from | Use for | +|-----------|----------------------|---------| +| `list` | A fixed `elements` list | A handful of known targets | +| `clusters` | Clusters registered in Argo CD (filter with `selector.matchLabels` on the cluster Secret) | One app on every cluster with a label (`env`, `region`, `type`) | +| `git` (`directories` or `files`) | Folders or JSON/YAML files in a repo | One app per folder (`apps/*/overlays/prod`) | +| `oci` | OCI artifacts (pre-rendered manifests) | Immutable, versioned manifest bundles | +| `matrix` | Combination of two generators | Directories x clusters | + +Always write `goTemplate: true` with `goTemplateOptions: ["missingkey=error"]` so a missing key fails instead of rendering an empty string. Templates then use `{{.cluster}}`, `{{.path.path}}`, `{{index .path.segments 1}}` and Sprig functions. + +Decision rules: + +| Question | Rule | +|----------|------| +| Environment per directory or per cluster label? | Directory generator when environments are folders in Git; cluster generator when the cluster Secret labels define the target | +| Who may create an ApplicationSet? | Admins only. Read https://argo-cd.readthedocs.io/en/stable/operator-manual/applicationset/Security/ first: a set can create Applications in any project it names | +| Auto-sync on generated apps? | Set it in the template's `syncPolicy`, not on the child (child edits have no effect) | +| Temporary stop of auto-sync | Follow "Controlling Resource Modification" (applicationset docs); not hand-editing children | + +## App of apps + +One parent `Application` whose source folder (a Helm chart or a directory) contains child `Application` manifests. Pros: simple, works with plain Argo CD. Cons: parent shows Synced while children are OutOfSync until synced; per-cluster variation needs templating in the parent. The cluster-bootstrapping guide recommends ApplicationSets and the cluster generator first, app-of-apps as the alternative. + +Rules: +- App of Apps is an **admin-only** tool: whoever can push to the parent repo can create Applications in any project; require PR review, check each child's `project`. +- Child `Application`s get `finalizers: [resources-finalizer.argocd.argoproj.io]` when deleting the parent must delete the workloads; omit it when it must not. +- Pin the parent's children to a commit SHA (`targetRevision`) so the parent's own commit is the release. +- Sync children by label: `argocd app sync -l app.kubernetes.io/instance=apps` (parent app named `apps`). +- To bootstrap in waves, use health assessment of Applications (operator-manual `health` page). + +## Destructive: propose, then the user's go + +| Action | Exact effect | Before proposing | +|--------|--------------|------------------| +| Delete an ApplicationSet | Its generated Applications are removed; with the finalizer on them, their resources go too | Apps list from the set, finalizers on the template, `Delete=false` on data objects | +| Delete the app-of-apps parent with cascading finalizers | Children and **all their resources** are deleted | Same, per child; app-deletion page lists the three UI propagation policies (foreground, background, non-cascading) | +| Remove a directory/cluster that a generator tracked | The generated Application disappears (and cascades per its finalizer) | `kubectl get applications -n argocd -l ...` before merging the PR | + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| Git generator on a branch HEAD | A merged folder deploys immediately everywhere | +| Missing `missingkey=error` | Empty `path`/`namespace` rendered, wrong target | +| Application names collide across sets | Names are unique per Argo CD installation | +| Cluster Secret labels edited | Apps appear or disappear on that cluster | + +## Verification + +`kubectl -n argocd get applicationset -o yaml` (status conditions), `kubectl -n argocd get applications` shows the expected generated names, `argocd app list -l ...` and one `argocd app diff` per new app before enabling auto-sync. + +## Sources (read 2026-11-10) + +- https://argo-cd.readthedocs.io/en/stable/operator-manual/applicationset/ +- https://argo-cd.readthedocs.io/en/stable/operator-manual/cluster-bootstrapping/ +- https://argo-cd.readthedocs.io/en/stable/user-guide/app_deletion/ +- https://argo-cd.readthedocs.io/en/stable/user-guide/auto_sync/ + +## Not verified + +- Individual generator pages (`Generators-Git`, `Generators-Cluster`) and the ApplicationSet `Security` and `Controlling-Resource-Modification` pages were not read; no field of them is asserted beyond the index page and the bootstrapping example. +- ApplicationSet deletion behaviour (which policy preserves children) was not read; the effects above follow from the finalizer rules only. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/flux-image-automation.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/flux-image-automation.md new file mode 100644 index 0000000..0c3bedb --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/flux-image-automation.md @@ -0,0 +1,65 @@ +--- +name: flux-image-automation +description: Flux image automation - ImageRepository, ImagePolicy, ImageUpdateAutomation, markers and commit-back to Git +when-to-use: Deploying new image tags automatically to staging, or proposing production bumps through a branch +keywords: ImageRepository, ImagePolicy, ImageUpdateAutomation, $imagepolicy marker, image-reflector-controller, image-automation-controller, semver +priority: medium +related: flux.md, repo-layout-promotion.md +--- + +# Flux Image Automation + +**Load when:** a new image tag should reach a cluster without a human editing a manifest. Template: [image-automation.yaml](templates/flux/image-automation.yaml). CI builds and pushes the image (devops-cicd); Flux scans the registry and writes the tag to Git. + +## Flow + +1. `ImageRepository` (`image.toolkit.fluxcd.io/v1`) scans a registry image; private images use `spec.secretRef`. +2. `ImagePolicy` selects a tag: `policy.semver.range`, or CalVer, regex, alphabetical. +3. A marker comment in the manifest names the policy: `image: ghcr.io/org/app:1.4.0 # {"$imagepolicy": "flux-system:app"}`. Variants: `:tag`, `:name`, `:digest` suffix (digest only when the policy sets `spec.digestReflectionPolicy` to `IfNotPresent` or `Always`; default `Never`). It also patches `HelmRelease.spec.values` and custom resources. +4. `ImageUpdateAutomation` checks out a branch, rewrites marked fields (`update.strategy: Setters`), commits and pushes. + +The controllers are **not installed by default**: bootstrap with `--components-extra=image-reflector-controller,image-automation-controller`, and give the deploy key write access (`--read-write-key`). An `ImageUpdateAutomation` applies all policies in its namespace. + +## Decision rules + +| Question | Rule | +|----------|------| +| Staging | Policy on a branch-SHA-timestamp tag or `>=1.0.0-0`; push to the same branch; fast loop | +| Production | Policy on stable semver only (`1.4.x`); push to a **separate branch** and open a PR so a human approves (repository-structure guide) | +| Tag filter | Exclude signatures and `latest`; the default excludes `^.*\.sig$` | +| Registry credentials in Git | Encrypted only (SOPS, Sealed Secrets), see devops-secrets | +| Commit author | A bot identity and a message template with `{{ .Changed.Summary }}` so history shows what moved | +| Rollback | Revert the bot's commit in Git; the automation will re-advance unless the policy range is tightened or the automation is suspended (`flux suspend image update `) | + +## Commands + +```bash +flux get image repository app ; flux get image policy app ; flux get images all -A +flux create image policy app --image-ref=app --select-semver='1.4.x' --export > policy.yaml # generates the manifest +flux suspend image update flux-system # stop commits; resume with flux resume image update flux-system +``` + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| Policy `latest` or a broad range in production | Untested images deployed by a bot | +| Automation pushes to `main` that other Kustomizations watch with `interval: 1m` | Deploys on every scan; separate environments by branch or path | +| Marker omitted or malformed | No update, no error in the app; check `flux get image update` | +| Image pinned by digest elsewhere (Kustomize `images`) | Two writers for the same field | + +## Verification + +`flux get image policy app` shows the expected latest tag; after a scan the bot commit appears in Git; the Kustomization applies it (`flux get kustomizations`); the pod runs that tag (`kubectl get deploy -o jsonpath` on the image). + +## Sources (read 2026-11-10) + +- https://fluxcd.io/flux/guides/image-update/ +- https://fluxcd.io/flux/guides/repository-structure/ (production approval through a pull request) + +## Not verified + +- ImagePolicy alternatives (CalVer, regex) and `sortable image tags` guide not read; only semver is shown. +- `flux suspend image update` exists in the CLI index; its page was not read. +- Behaviour when two automations target the same file. +- Nesting of `git.commit.author` in the template: the guide's rendered manifest showed `email`/`name` directly under `commit` in the fetched text (likely a rendering loss of `author:`); the Image update automation API reference v1 must be checked before applying. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/flux.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/flux.md new file mode 100644 index 0000000..7b20679 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/flux.md @@ -0,0 +1,93 @@ +--- +name: flux +description: Flux v2 - sources, Kustomization, HelmRelease, ordering, drift detection, diff, suspend, delete and bootstrap +when-to-use: Writing or changing GitRepository, OCIRepository, Kustomization or HelmRelease, or running flux suspend, resume, reconcile, delete +keywords: GitRepository, OCIRepository, Kustomization, HelmRelease, dependsOn, prune, deletionPolicy, driftDetection, flux diff, flux suspend, flux bootstrap +priority: critical +related: flux-image-automation.md, repo-layout-promotion.md, helm-vs-kustomize.md +--- + +# Flux + +**Load when:** Flux objects or the `flux` CLI are involved. Version 2.10.0 (released 2026-10-26); API versions: `source.toolkit.fluxcd.io/v1`, `kustomize.toolkit.fluxcd.io/v1`, `helm.toolkit.fluxcd.io/v2`, `notification.toolkit.fluxcd.io/v1`, `image.toolkit.fluxcd.io/v1`. Templates: [gitrepository.yaml](templates/flux/gitrepository.yaml), [kustomization.yaml](templates/flux/kustomization.yaml), [helmrelease.yaml](templates/flux/helmrelease.yaml). Flux objects live in `flux-system` by default and the CLI uses that namespace unless `-n` or `--ns-follows-kube-context` is set. + +## Model + +Source (`GitRepository`, `OCIRepository`, `Bucket`, `HelmRepository`, `HelmChart`, `ExternalArtifact` behind a feature gate) produces an Artifact. `Kustomization` builds and applies it; `HelmRelease` installs a chart from an Artifact. Controllers re-apply every `.spec.interval` (Kustomization: server-side apply dry-run to detect and correct drift). + +## Decision rules + +| Question | Rule | +|----------|------| +| Git or OCI as the delivery artifact? | OCI (`flux push artifact`, `OCIRepository`) when CI builds an immutable bundle; Git when humans review manifests in PRs | +| `prune` | Required boolean on `Kustomization`. `true` deletes objects missing from the source and, with the default `deletionPolicy: MirrorPrune`, deletes everything when the Kustomization is deleted. Protect single objects with label or annotation `kustomize.toolkit.fluxcd.io/prune: disabled` | +| `deletionPolicy` | `MirrorPrune` (default), `Delete`, `WaitForTermination`, `Orphan`; set `Orphan` when the namespace is removed by other means | +| Ordering | `spec.dependsOn: [{name: ...}]` between Kustomizations plus `wait: true` or `healthChecks`; split `infrastructure` (controllers, then configs) from `apps` | +| Health | `healthChecks` (Deployments, HelmReleases, kstatus CRs) or `healthCheckExprs` (CEL); `timeout` bounds them | +| Intervals | `interval` min 60 s; `retryInterval` for failures; Git source 1-5 min, apps 10 min-1 h, with webhook Receivers for instant updates | +| `HelmRelease` chart source | `spec.chart` (template creating an `OCIRepository` or `HelmChart`) atomically changes values and chart version; `spec.chartRef` reuses one source across releases but values and version can land out of order | +| Remediation | `install.remediation.retries`, `upgrade.remediation.retries`; failed releases retry up to N times, then stop | +| Drift detection | `spec.driftDetection.mode: enabled` (corrects drift) with `ignore` paths per target kind (e.g. `/spec/replicas` for HPA) | +| Helm tests | `spec.test.enable: true` runs `helm test` after install/upgrade | +| Changing the chart name | helm-controller uninstalls the previous release before installing, unless `.spec.upgrade.chartNameChangeStrategy: InPlaceUpdate`: data loss risk | +| Multi-tenancy | `serviceAccountName` per Kustomization/HelmRelease, controllers started with `--no-cross-namespace-refs=true` | + +## Commands + +```bash +flux check # controllers and CRDs healthy +flux get all -A # sources, kustomizations, helmreleases +flux build kustomization web --path ./apps/web/overlays/prod # render as the controller would +flux diff kustomization web --path ./apps/web/overlays/prod # server-side dry-run diff; exit 0 none, 1 diff, >1 error +flux reconcile source git flux-system ; flux reconcile kustomization web --with-source # after the user's go +flux events --for Kustomization/web ; flux logs --kind=Kustomization --name=web # evidence +flux bootstrap github --owner=ORG --repository=fleet --branch=main --path=clusters/prod --personal # needs GITHUB_TOKEN env; user-run +``` + +## Destructive: propose, then the user's go + +| Command | Exact effect | Before proposing | +|---------|--------------|------------------| +| `flux suspend kustomization ` / `helmrelease` / `source` | Stops reconciliation: new revisions are not applied **and drift is not corrected** until `flux resume`; `--all` suspends every resource of that kind in the namespace | State why and until when; a forgotten suspend is silent drift | +| `flux resume kustomization ` | Reconciles immediately: everything pending in Git (and every manual change reverted) lands at once | `flux diff kustomization` first | +| `flux delete kustomization ` | Deletes the Kustomization; with `prune: true` and `MirrorPrune` it **deletes all objects it applied** (asks for confirmation unless `--silent`) | `deletionPolicy`, `flux tree kustomization ` inventory, data objects, backups; consider `Orphan` | +| `flux delete helmrelease ` | Triggers uninstall of the Helm release | Same as `helm uninstall` effects | +| `flux uninstall` | Removes Flux from the cluster (the uninstall page was not read: effect on managed workloads and CRs is not verified; the Flux uninstall guide is at fluxcd.io/flux/installation/uninstall/) | Read that guide, then propose with the exact flags | + +``` +ACTION flux -n flux-system delete kustomization shop-prod (cluster prod-eu) +EFFECT prune=true, deletionPolicy default: 14 objects deleted incl. PVC data-web (reclaimPolicy Delete) +PROOF flux tree kustomization shop-prod output attached; set deletionPolicy: Orphan first to keep objects +WAIT-FOR-GO yes +``` + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| `HelmRelease` `chartRef` + values changed in the same commit | helm-controller may apply new values to the old chart first; use `spec.chart` for lockstep | +| `targetNamespace` that does not exist | Not created by Flux; ship the Namespace in the same Kustomization or an earlier one | +| `kubectl edit` on a Flux-owned object | Reverted at the next interval; suspend first only for a controlled experiment | +| Kustomization `path` points to a folder without `kustomization.yaml` | Flux generates one from all YAML there, including files you did not mean | +| Helm drift detection on a chart that writes status fields | Constant corrections; add `ignore` paths | + +## Verification + +`flux get kustomizations` / `helmreleases` show `Ready=True` at the expected revision; `kubectl get` of the workloads and the user-facing check. `Ready` is the controller's view: confirm the workload. + +## Sources (read 2026-11-10) + +- https://fluxcd.io/flux/components/kustomize/kustomizations/ , https://fluxcd.io/flux/components/helm/helmreleases/ +- https://fluxcd.io/flux/components/source/gitrepositories/ , .../ocirepositories/ (index only) +- https://fluxcd.io/flux/cmd/flux_suspend_kustomization/ , .../flux_delete_kustomization/ , .../flux_diff_kustomization/ +- https://fluxcd.io/flux/guides/repository-structure/ +- https://raw.githubusercontent.com/fluxcd/flux2-kustomize-helm-example/main/clusters/staging/infrastructure.yaml +- https://github.com/fluxcd/flux2/releases/tag/v2.10.0 + +## Not verified + +- `flux uninstall` flags and effects, `flux tree`, `flux events`, `flux logs`, `flux build kustomization` flags (only listed in the CLI index). +- `GitRepository` and `OCIRepository` field bodies beyond the examples (`interval`, `url`, `ref.branch`, `ref.semver`, `layerSelector`); spec index only. +- The example repo currently uses `ExternalArtifact` sources (feature-gated); this skill uses `GitRepository`. +- Exact `HelmRelease` remediation strategy fields (`strategy: rollback|uninstall`, `remediateLastFailure`): index names "Configuring failure handling", body not read. +- Flux 2.10 upgrade notes (breaking changes from 2.9) not read. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/helm-4-changes.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/helm-4-changes.md new file mode 100644 index 0000000..21294d3 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/helm-4-changes.md @@ -0,0 +1,64 @@ +--- +name: helm-4-changes +description: What changed between Helm 3 and Helm 4 - flags, server-side apply, wait strategies, plugins, OCI, charts v3 +when-to-use: Upgrading a pipeline or workstation from Helm 3, reading an old tutorial, or writing a flag in CI +keywords: helm 4, server-side apply, rollback-on-failure, force-replace, wait watcher, post-renderer plugin, registry login, chart v3 +priority: high +related: helm-release-ops.md, helm-chart-authoring.md +--- + +# Helm 4 versus Helm 3 + +**Load when:** a command, flag or doc page may date from Helm 3. Helm 4 keeps charts working ("maintaining backwards compatibility for charts") but breaks CLI and SDK details. + +## Version + +Helm docs show `Version: 4.3.1` (https://helm.sh/docs/overview/). Several docs pages still carry the banner "This page has not yet been updated for Helm 4" (charts, registries): treat their prose as Helm 3 text and confirm flags with `helm --help` on the installed binary. + +## Decision table + +| Area | Helm 4 behaviour | Action | +|------|------------------|--------| +| Apply method | New installs default to **server-side apply**; upgrade and rollback follow the previous release's method; Helm 3 releases stay client-side apply after the upgrade to 4. Override with `--server-side=true\|false\|auto` | Keep `auto`; set `--force-conflicts` only on purpose (it takes fields from other managers) | +| `--atomic` | Renamed `--rollback-on-failure` (old flag still works, prints a deprecation warning); it defaults `--wait` to `watcher` | Rename in CI scripts | +| `--force` | Renamed `--force-replace` (replacement of resources) | Rename; same risk as `kubectl replace --force` | +| `--wait` | Strategies `watcher` (kstatus), `hookOnly` (default when the flag is omitted), `legacy`; bare `--wait` = `watcher` | Use bare `--wait` or `--wait=watcher` in pipelines; add `--wait-for-jobs` if Jobs gate readiness | +| `--dry-run` | Takes a value: `none`, `client`, `server` | `--dry-run=server` validates against the API server and needs cluster access | +| Post-renderer | Implemented as a **plugin**; `--post-renderer` takes a plugin name, no longer an executable path | Wrap old scripts as a plugin | +| `helm registry login` | Takes a registry host only (`ghcr.io`, `localhost:5000`); a scheme or a path is rejected | Fix login steps (`ghcr.io/org` fails) | +| OCI install by digest | `oci://registry/charts/app@sha256:...`; a digest mismatch is not installed | Pin production installs by digest | +| Values | Multi-document values files; `--set-json`, `--set-literal`, `--set-file` | Prefer `-f` files in Git over `--set` | +| Schema | JSON Schema 2020 support listed in the v4 changelog (PR #13283) | Test schemas on Helm 4 before relying on 2020-only keywords | +| Plugins | New plugin system with optional Wasm runtime; three types: CLI, getter, post-renderer; existing plugins keep working | Re-test `helm-diff`, `helm-secrets`; `helm-diff` needs Helm 3.18+ per its README | +| `helm rollback` | New `--description` flag (max 256 chars) and a rollback-revision column in `helm history` (changelog) | Record the reason | +| `helm template` | `--hide-notes` and `--render-subchart-notes` are deprecated and removed in Helm 5 | Stop using them | +| Cache | Content-based chart cache at `~/.cache/helm/content` | Clear it when a pulled chart looks stale | +| Charts v3 | Experimental (`HELM_EXPERIMENTAL_CHART_V3=1`, `helm create --chart-api-version=v3`) | Do not use for production; v2 charts are unchanged | + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| Old CI uses `--atomic` or `--force` | Works with a warning today; the aliases can go in a later major | +| SSA on a release already managed by another controller | Field-ownership conflicts; resolve ownership, do not force | +| `--wait` omitted | Default `hookOnly`: the command can return before workloads are ready | +| Argo CD renders Helm itself | Its `helm.version` field accepts `v2` or `v3` in the application spec sample; whether Argo CD 3.6.1 renders with Helm 4 is not verified, so test chart rendering through the controller | + +## Verification + +`helm version` (client), `helm template` on a real chart with the production values file, then `helm upgrade --dry-run=server` against a non-production context. A passing `helm lint` is not proof of a valid upgrade. + +## Sources (read 2026-11-10) + +- https://helm.sh/docs/overview/ +- https://helm.sh/docs/changelog/ +- https://helm.sh/docs/helm/helm_upgrade/ , https://helm.sh/docs/helm/helm_rollback/ , https://helm.sh/docs/helm/helm_template/ +- https://helm.sh/docs/topics/registries/ +- https://helm.sh/blog/path-to-helm-v4/ (planned release at KubeCon North America 2025, 6 years after Helm 3) +- https://helm.sh/community/release_policy/ (patch monthly, minor every 4 months) + +## Not verified + +- Helm 3 end-of-life date and its security-fix window (no page found in this session). +- Helm 4.3.1 release date (2026-10-15 comes from a GitHub release lookup in the research doc, not re-read here). +- Whether Argo CD 3.6.1 and Flux 2.10.0 bundle Helm 4 or Helm 3 internally. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/helm-chart-authoring.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/helm-chart-authoring.md new file mode 100644 index 0000000..0f0cf2d --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/helm-chart-authoring.md @@ -0,0 +1,88 @@ +--- +name: helm-chart-authoring +description: Writing a Helm chart - structure, Chart.yaml, values and schema, dependencies, hooks, tests, CRDs +when-to-use: Creating or reviewing a chart you own, deciding where a setting lives, wiring subcharts +keywords: Chart.yaml, apiVersion v2, values.yaml, values.schema.json, dependencies, Chart.lock, hooks, helm test, crds +priority: high +related: helm-release-ops.md, helm-4-changes.md, templates/chart/Chart.yaml +--- + +# Helm Chart Authoring + +**Load when:** you write or review a chart. Cluster-side objects (probes, limits, securityContext) come from devops-kubernetes; this file covers the packaging around them. + +## Structure (helm.sh charts page) + +``` +web/ + Chart.yaml # required: apiVersion, name, version + values.yaml # default values + values.schema.json # optional JSON Schema over values + charts/ # dependencies (generated) + crds/ # CRDs (installed before templates, not templated) + templates/ # templates + NOTES.txt +``` + +Helm reserves `charts/`, `crds/`, `templates/`. A runnable skeleton: [templates/chart/](templates/chart/Chart.yaml) (Helm templates contain `{{ }}` and are not parseable as plain YAML). + +## Decision rules + +| Question | Rule | +|----------|------| +| `apiVersion` | `v2` (needs Helm 3+); `v1` charts still install. `v3` is experimental in Helm 4 | +| `version` vs `appVersion` | `version` = chart SemVer (bump on every chart change, it names the `.tgz`); `appVersion` = the app, informational, **quote it** (`"1.4.2"`) | +| Extra `Chart.yaml` keys | Not allowed since Helm 3.3.2: put custom metadata under `annotations` | +| `kubeVersion` | Set a SemVer range (`">=1.35.0-0"`) so installs fail on unsupported clusters | +| Values naming | camelCase, lower-case first letter, flat over nested, maps over lists (easier `--set`), quote strings, one comment per key starting with the key name | +| Keys with dashes | Read with `index .Values "gitlab-runner"`; avoid dashes in your own keys | +| Validation | Ship `values.schema.json` (`required`, `type`, `enum`, `additionalProperties: false` on the root); it is checked by install, upgrade, lint and template; `--skip-schema-validation` disables it, so ban it in CI | +| Dependencies | Declare in `Chart.yaml` `dependencies` (name, version range, repository, optional `condition`, `alias`, `import-values`); `helm dependency update` writes `Chart.lock`; commit `Chart.lock`, use `helm dependency build` in CI to reproduce it | +| Subchart values | Parent `values.yaml` under the subchart name (or `alias`); toggle with `condition: sub.enabled` | +| CRDs | Put in `crds/`: installed if absent, **not** upgraded or removed by Helm; ship CRD upgrades separately (own chart or controller) | +| Hooks | Annotation `helm.sh/hook: pre-install,pre-upgrade`, `helm.sh/hook-weight`, `helm.sh/hook-delete-policy`; hook resources are not tracked by the release, so `helm uninstall` does not remove them | +| Tests | Pod or Job templates annotated `helm.sh/hook: test`; run with `helm test ` after install | +| Releases ordering | Helm applies resources in its own kind order; use hooks for ordering inside a release, controller features (sync waves, `dependsOn`) across releases | + +## Chart.yaml dependency example + +```yaml +dependencies: + - name: postgresql + version: "16.x.x" # SemVer range, pinned by Chart.lock + repository: oci://registry.example.com/charts + condition: postgresql.enabled +``` + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| Unquoted `appVersion: 1.0` or a SHA like `1234e10` | Parsed as float or scientific notation | +| `randAlphaNum` in a template | A new value on every render: permanent drift in Argo CD and Flux diffs | +| Secrets generated in templates | Rotated on every upgrade, printed by `--dry-run`; reference an existing Secret instead (devops-secrets) | +| Dependency range `*` without `Chart.lock` | Different chart versions on two machines | +| Immutable fields (`Deployment.spec.selector`, Job template) changed in a template | Upgrade fails; plan a delete/recreate with the owner's go | + +## Verification + +```bash +helm lint ./web -f values-prod.yaml +helm template web ./web -f values-prod.yaml --hide-secret | kubectl apply --dry-run=server -f - +helm dependency build ./web # reproduces Chart.lock +helm package ./web # produces web-.tgz +``` + +Then a real `helm upgrade --install --dry-run=server` against a non-production context and `helm test` after the actual install. + +## Sources (read 2026-11-10) + +- https://helm.sh/docs/topics/charts/ (banner: not yet updated for Helm 4) +- https://helm.sh/docs/chart_best_practices/values/ +- https://helm.sh/docs/topics/charts_hooks/ +- https://helm.sh/docs/helm/helm_dependency_update/ , https://helm.sh/docs/helm/helm_test/ , https://helm.sh/docs/helm/helm_template/ + +## Not verified + +- `helm lint`, `helm package`, `helm dependency build` flags were not read individually (`lint` and `--skip-schema-validation` appear in the Helm 4 changelog and the template page). +- Recommended `kubeVersion` pre-release suffix (`-0`) is general Helm practice, not read at the source today. +- `crds/` upgrade/removal behaviour is from the Helm CRD docs page, not re-read today. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/helm-release-ops.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/helm-release-ops.md new file mode 100644 index 0000000..ae50247 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/helm-release-ops.md @@ -0,0 +1,89 @@ +--- +name: helm-release-ops +description: Operating Helm releases safely - OCI registries, digest pinning, diff and dry-run, upgrade, rollback, uninstall, test +when-to-use: Before any helm install, upgrade, rollback or uninstall, and when publishing or pulling a chart from a registry +keywords: helm push, oci, digest, helm diff, dry-run=server, upgrade, rollback, uninstall, keep-history, helm test, hide-secret +priority: critical +related: helm-4-changes.md, kustomize.md, repo-layout-promotion.md +--- + +# Helm Release Operations + +**Load when:** a Helm command will touch a cluster or a registry. Read-only (`helm template`, `helm list`, `helm history`, `helm diff`, `--dry-run`) needs no go; the rest follows this file. If a controller (Argo CD, Flux HelmRelease) owns the release, do not use the Helm CLI to change it: change Git. + +## OCI registry flow (Helm 4) + +```bash +helm registry login ghcr.io -u "$USER" # host only; no scheme, no path; password prompt, never on the command line +helm package ./web # web-.tgz +helm push web-1.4.0.tgz oci://ghcr.io/org/charts # no chart name, no tag: both come from the chart +helm pull oci://ghcr.io/org/charts/web --version 1.4.0 +helm template web oci://ghcr.io/org/charts/web --version 1.4.0 -f values-prod.yaml --hide-secret +helm install web oci://ghcr.io/org/charts/web@sha256: # immutable; a mismatch is refused +``` + +`push` takes only `.tgz` files made by `helm package`; a `.prov` file next to the chart is uploaded too. Some registries need the repository created first. Sign charts with cosign or the `helm-sigstore` plugin (supply chain details: devops-docker `registries-supply-chain.md`). Pin production to a digest recorded at publish time. + +## Safe upgrade sequence + +```bash +CTX=prod-eu; NS=shop; REL=web; CHART=oci://ghcr.io/org/charts/web; VER=1.4.0 +helm --kube-context "$CTX" -n "$NS" history "$REL" # note the current revision (rollback target) +helm --kube-context "$CTX" -n "$NS" get values "$REL" -o yaml > "$(mktemp)" # may hold secrets: keep it out of Git and out of the chat +helm --kube-context "$CTX" -n "$NS" diff upgrade "$REL" "$CHART" --version "$VER" -f values-prod.yaml # helm-diff plugin +helm --kube-context "$CTX" -n "$NS" upgrade "$REL" "$CHART" --version "$VER" -f values-prod.yaml --dry-run=server --hide-secret +# after the user's go: +helm --kube-context "$CTX" -n "$NS" upgrade "$REL" "$CHART" --version "$VER" -f values-prod.yaml \ + --wait --timeout 10m --rollback-on-failure --history-max 10 +helm --kube-context "$CTX" -n "$NS" test "$REL" --logs +``` + +- `--dry-run` prints rendered manifests **including Secrets**: add `--hide-secret`. +- `helm diff` is a plugin (`helm plugin install https://github.com/databus23/helm-diff`, Helm 3.18+ per its README). It diffs the deployed release against a `helm template` render. +- `--reuse-values` merges the previous values with new `--set`/`-f`; `--reset-then-reuse-values` resets to the chart defaults, applies the last release's values, then your overrides. For pipelines, pass the full values file every time and use neither. +- `--rollback-on-failure` rolls back to the last successful release when the upgrade fails; it is a rollback, so it carries the rollback effects below. + +## Destructive commands: propose, then the user's go + +| Command | Exact effect (helm docs) | Before proposing | +|---------|--------------------------|------------------| +| `helm rollback [rev]` | Re-applies the manifest of revision `rev` (default: the previous one) as a **new** revision; values and templates return, data and PVC contents do not; runs rollback hooks unless `--no-hooks`; `--force-replace` replaces resources; `--cleanup-on-fail` deletes resources created by a failed rollback | `helm history`, `helm diff rollback`, database migrations already run, GitOps controller will re-apply Git unless Git is reverted | +| `helm uninstall ` | Removes all resources of the last release **and the release history**; `--keep-history` keeps history and marks it deleted; `--cascade foreground --wait` waits for finalizers; hook resources are not removed | `helm get manifest`, list PVCs and CRs, backups, annotation `helm.sh/resource-policy: keep` on data objects; `--dry-run` first | +| `helm upgrade --force-replace` | Replaces resources (delete and recreate behaviour): pods restart, generated fields reset | Prefer a normal upgrade | +| `helm upgrade --force-conflicts` | Server-side apply takes over fields owned by other managers | Name the other manager first | + +Proposal format: + +``` +ACTION helm --kube-context prod-eu -n shop rollback web 7 --wait --timeout 10m +EFFECT web returns to the manifest of revision 7 (chart 1.3.2); revision 9 stays in history; DB migration of 1.4.0 is NOT reverted +PROOF helm history web (rev 7 = deployed ok), helm diff rollback web 7 attached, backup id +WAIT-FOR-GO yes +``` + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| `helm uninstall` without `--keep-history` | Rollback target is gone | +| `helm upgrade --install` in two pipelines | `another operation is in progress` / pending-upgrade release; inspect `helm history` before retrying | +| Same release name in two namespaces assumed identical | Releases are namespace-scoped; always pass `-n` and `--kube-context` | +| `helm repo add` with the same alias but different URL | Silent chart substitution; prefer OCI refs with a version or digest | +| Credentials in `--password` | Shell history leak; use `helm registry login` prompt or `--password-stdin` (flag not read today) | + +## Verification + +`helm status ` and `helm history ` show `deployed`; `kubectl rollout status` for each workload (devops-kubernetes); `helm test --logs` passes; a user-facing request succeeds. + +## Sources (read 2026-11-10) + +- https://helm.sh/docs/helm/helm_upgrade/ , https://helm.sh/docs/helm/helm_rollback/ , https://helm.sh/docs/helm/helm_uninstall/ +- https://helm.sh/docs/topics/registries/ , https://helm.sh/docs/helm/helm_test/ +- https://github.com/databus23/helm-diff + +## Not verified + +- `helm history`, `helm get values`, `helm status`, `helm get manifest`, `helm registry login --password-stdin` flags were not read as individual pages today. +- `helm diff rollback` subcommand exists in the plugin README (not read past the install section). +- "Rollback creates a new revision" is inferred from the `helm history` rollback column in the changelog and general Helm behaviour; the rollback page's synopsis does not say it. +- `--history-max` default 10 is from the upgrade page; rollback of CRs/hook Jobs side effects depend on the chart. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/helm-vs-kustomize.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/helm-vs-kustomize.md new file mode 100644 index 0000000..4979528 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/helm-vs-kustomize.md @@ -0,0 +1,63 @@ +--- +name: helm-vs-kustomize +description: Choosing Helm, Kustomize or both - decision rules, combination patterns, limits of helmCharts and of controller rendering +when-to-use: Starting a new app delivery, adopting a third-party chart, or an overlay fight with a chart +keywords: helm vs kustomize, helmCharts, post-render, HelmRelease patches, Argo CD kustomize helm, wrapper chart +priority: high +related: kustomize.md, helm-chart-authoring.md, argocd-applications.md, flux.md +--- + +# Helm vs Kustomize vs both + +**Load when:** you must decide how an app is packaged and customized per environment. + +## Decision table + +| Situation | Choice | Why | +|-----------|--------|-----| +| Third-party app with a maintained chart (ingress controller, database operator, monitoring) | Helm chart + values per environment | You consume upstream versions; values are the supported interface | +| Your own service, few environment differences | Kustomize base + overlays | Plain YAML stays reviewable; no templating language | +| Your own service distributed to others, or many parameters, conditional blocks, tests | Helm chart with `values.schema.json` | Parameters, validation, `helm test`, versioned artifact in OCI | +| Chart lacks one field you need | Patch the rendered output instead of forking | See combination patterns below | +| Same app in 20 clusters with small differences | Helm chart (values) or Kustomize overlays, driven by ApplicationSet or Flux Kustomization per cluster | Controller generates the per-cluster object | +| Need ordering, hooks, rollback history inside the release | Helm | Release object with revisions and hooks | +| Need to avoid a release object and Helm state | Kustomize, or render Helm to YAML and commit it | Git holds the final manifests | + +Rule of thumb: one packaging tool per app; add the second only to patch what the first cannot express. + +## Combination patterns + +| Pattern | How | Watch out | +|---------|-----|-----------| +| Flux `HelmRelease` + Kustomize overlay | Overlay patches the `HelmRelease` object (values) per environment: `patches` with `target.kind: HelmRelease` (pattern used by the Flux `flux2-kustomize-helm-example` repo) | You patch values, not chart output | +| Flux `HelmRelease` post-render | `.spec.postRenderers` supports Kustomize patches on the rendered chart (listed in the HelmRelease spec index; exact fields not read today) | Last resort; keeps drift between chart and patch | +| Argo CD Application with `source.kustomize` | Kustomize options in the Application (`images`, `patches`, `components`, `namePrefix`) | Overrides live in the Application, not in Git folders: keep them small | +| Argo CD Application with `source.helm` | `valueFiles` (relative to `source.path`), `valuesObject`, `parameters`, `releaseName`, `skipCrds`, `skipSchemaValidation` | Argo CD renders with `helm template`: Helm hooks map to Argo CD hooks, `lookup` and release history do not exist | +| Kustomize `helmCharts` | `kustomize build --enable-helm` inflates a chart inside a kustomization | Intentionally limited: no private repository or registry authentication, no post-renderer; a KRM function is planned (kustomize issue 4401) | +| Wrapper chart | Your chart depends on the upstream chart and sets values | Another chart version to maintain; use when you also add your own templates | + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| Helm release and Argo CD both think they own the release | Argo CD deploys with `helm template`; `helm list` shows nothing. Pick one | +| Chart upgraded through Renovate/Dependabot without a render diff | Silent default changes; CI must run `helm template` before and after and diff | +| Overlay patches a generated name that changes between chart versions | Patch breaks on upgrade; patch by `kind` and labels | +| Values duplicated across overlays | Keep a shared `values-common.yaml` and small per-environment files (`-f` order: last wins) | + +## Verification + +Render both before and after the change (`helm template` or `kustomize build`), diff the outputs, and run `kubectl apply --dry-run=server` on the final manifests. + +## Sources (read 2026-11-10) + +- https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/helmcharts/ +- https://argo-cd.readthedocs.io/en/stable/operator-manual/application.yaml +- https://fluxcd.io/flux/components/helm/helmreleases/ , https://fluxcd.io/flux/guides/repository-structure/ +- https://raw.githubusercontent.com/fluxcd/flux2-kustomize-helm-example/main/apps/staging/kustomization.yaml + +## Not verified + +- `HelmRelease.spec.postRenderers` field shape (named in the page index, body not read). +- The claim that Argo CD maps Helm hooks and has no `lookup` is general Argo CD behaviour; the sync-waves page confirms hook mapping only in passing. +- Wrapper chart pattern is general practice, not sourced. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/kustomize.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/kustomize.md new file mode 100644 index 0000000..432fb18 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/kustomize.md @@ -0,0 +1,88 @@ +--- +name: kustomize +description: Kustomize - bases and overlays, patches, images, generators, components, build and diff +when-to-use: Customizing your own plain manifests per environment, writing or reviewing a kustomization.yaml +keywords: kustomization.yaml, base, overlay, patches, images, configMapGenerator, components, kustomize build, kubectl apply -k +priority: high +related: helm-vs-kustomize.md, repo-layout-promotion.md, flux.md +--- + +# Kustomize + +**Load when:** manifests are plain YAML with per-environment differences. Kustomize is template-free: a `kustomization.yaml` lists resources and transformations; the output is plain manifests. + +## Model (kubectl.docs.kubernetes.io) + +`apiVersion: kustomize.config.k8s.io/v1beta1`, `kind: Kustomization`. Build order: `resources` first, then `generators` (add objects), then `transformers` (change them), then `validators`. A directory in `resources` is an **base**; the kustomization that refers to it is an **overlay**. Convenience fields (`patches`, `images`, `namePrefix`, `configMapGenerator`) are shorthand for built-in generators and transformers. + +Runnable layout: [templates/kustomize/](templates/kustomize/base/kustomization.yaml). + +## Decision rules + +| Need | Use | +|------|-----| +| Change one field per environment | `patches` with `target` (strategic merge or JSON6902, inline or `path`); `target` is required for JSON6902 | +| Change image tag or registry | `images:` (`name`, `newName`, `newTag`, `digest`); `kustomize edit set image app=registry/app:1.4.2` rewrites the file | +| Config from files or literals | `configMapGenerator` / `secretGenerator`: the name gets a content-hash suffix, so a config change rolls the pods; `behavior: create\|replace\|merge` lets an overlay change the base's map | +| Optional feature across overlays | `Component` (`apiVersion: kustomize.config.k8s.io/v1alpha1`, `kind: Component`) referenced from overlays under `components:` | +| Replica count, labels, namespace | `replicas`, `labels`, `namespace`, `namePrefix` | +| Inflate a Helm chart | `helmCharts` + `kustomize build --enable-helm`; limited support, see [helm-vs-kustomize.md](helm-vs-kustomize.md) | +| Patch a custom resource with a list merge key other than `name` | add an `openapi` field, or use JSON6902 | + +Patch `target` selects by `group`, `version`, `kind`, `name`, `namespace` (anchored regex), `labelSelector`, `annotationSelector`. Patches apply in the order listed. By default a patch cannot rename or change the kind of a resource (`allowNameChange`, `allowKindChange` options). + +## Commands + +```bash +kustomize build apps/web/overlays/prod # render only, no cluster +kubectl kustomize apps/web/overlays/prod # same, built into kubectl +kustomize build apps/web/overlays/prod | kubectl --context prod-eu diff -f - # exit 1 = differences +kustomize build apps/web/overlays/prod | kubectl --context prod-eu apply --dry-run=server -f - +kustomize build apps/web/overlays/prod | kubectl --context prod-eu apply --server-side -f - # after the user's go +``` + +`kubectl apply -k ` and `kubectl diff -k ` exist; `apply -k` is not verified today, prefer the explicit pipe. With a GitOps controller, never apply by hand: commit. + +## Destructive: propose, then the user's go + +| Command | Exact effect | Before proposing | +|---------|--------------|------------------| +| `kustomize build \| kubectl delete -f -` | Deletes **every** object the build renders (Namespaces, PVCs and CRDs included if they are in the build); generated ConfigMap names include the hash, so deleting a different revision's output leaves the real ones | `kustomize build \| kubectl get -f -` to list exactly what exists; backups; check shared bases | +| `kubectl apply --prune` with a kustomize build | Alpha in kubectl; do not use | Use the GitOps tool's reviewed prune | + +``` +ACTION kustomize build apps/web/overlays/staging | kubectl --context staging delete -f - +EFFECT deletes 9 objects in ns web-staging: Deployment, Service, ConfigMap web-config-7kd2t9, PVC data-web-0 (data lost if reclaimPolicy Delete) +PROOF build | kubectl get -f - output attached; snapshot id +WAIT-FOR-GO yes +``` + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| Two overlays patch the same field | Last patch listed wins silently; render and read the output | +| Remote bases (`github.com/org/repo//path?ref=...`) without a pinned ref | Moving base; pin `ref` to a tag or SHA | +| Hash-suffixed ConfigMap referenced by a literal name outside the build | Dangling reference; let kustomize rewrite references inside the build | +| `commonLabels`/selectors changed on an existing Deployment | `spec.selector` is immutable: the apply fails | +| Env-specific copies of the whole manifest instead of patches | Drift between environments | + +## Verification + +`kustomize build` exits 0 and the output contains the expected image, replicas and namespace (`| grep -E 'image:|replicas:'`); `kubectl diff` shows only intended changes; for components, build each overlay and compare. + +## Sources (read 2026-11-10) + +- https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/ +- https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/patches/ +- https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/configmapgenerator/ +- https://kubectl.docs.kubernetes.io/guides/config_management/components/ +- https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/helmcharts/ +- Context7 `/kubernetes-sigs/kustomize` (images field, `kustomize edit set image`) +- Version 5.8.3: https://github.com/kubernetes-sigs/kustomize/releases + +## Not verified + +- `kubectl apply -k`, `kubectl diff -k` flags; `replicas`, `labels`, `namespace` field details (used from general knowledge, also seen in the Argo CD application sample). +- The components page still shows `patchesStrategicMerge`/`patchesJson6902` in its example; the current docs use `patches`. Use `patches`. +- Pinning remote bases with `?ref=` is general Kustomize practice, not read at the source today. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/repo-layout-promotion.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/repo-layout-promotion.md new file mode 100644 index 0000000..040f138 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/repo-layout-promotion.md @@ -0,0 +1,84 @@ +--- +name: repo-layout-promotion +description: GitOps repository layout, environment promotion, drift handling, rollback and the pointer to secrets +when-to-use: Designing a GitOps repo, promoting a version from staging to production, reverting a bad release, or handling drift +keywords: monorepo, repo per environment, promotion, pull request, drift, rollback, revert, secrets in git, SOPS, ExternalSecret +priority: high +related: argocd-applications.md, flux.md, helm-release-ops.md, kustomize.md +--- + +# Repository Layout, Promotion, Drift, Rollback + +**Load when:** you decide where manifests live, how a change moves between environments, or how to undo it. Tree and flow: [gitops-repo-layout.md](templates/gitops-repo-layout.md). + +## Layout decision + +| Option | Use when | Cost | +|--------|----------|------| +| **Monorepo**, folders per environment (`apps/base`, `apps/staging`, `apps/production`, `infrastructure/...`, `clusters/`) | One team, trunk-based, few clusters (Flux guide's default) | Everyone can read production config; access control only through the Git host | +| **Repo per environment** | Production config needs restricted access and a small review scope | Promotion is a cross-repo PR, slower for infra changes | +| **Repo per team** (platform repo + dev team repos referenced by `GitRepository`/`Application`) | Platform team owns cluster add-ons, dev teams own app definitions | More objects; needs multi-tenancy lockdown (service accounts, AppProject limits) | +| **Repo per app** (manifests next to code, config repo holds a pointer) | App team owns deployment manifests | Two repos per change; pin the pointer to a tag | + +Separate `infrastructure/` (controllers, CRDs, then shared config) from `apps/` so the controller reconciles them in order (Flux `dependsOn`; Argo CD sync waves or separate Applications). Branches are short-lived; do not model environments as long-lived branches. + +## Promotion + +| Step | Rule | +|------|------| +| Build once | CI produces one image (tag plus digest) and one chart version; environments differ only in config (devops-cicd, devops-docker) | +| Staging | Automatic: Flux image automation or a CI commit to `staging` overlay | +| Production | A pull request that changes only the version or digest in the production overlay/values; reviewer sees the rendered diff (`helm diff`, `kustomize build` diff, `argocd app diff`, `flux diff`) attached by CI | +| Gate | Staging healthy (controller `Healthy`/`Ready`, smoke test passes) before the PR merges; progressive delivery (Flagger, Argo Rollouts) is out of scope here | +| Pin | `targetRevision`, chart `version`, image digest: tag or SHA, never `HEAD`/`latest` | +| Record | PR description lists previous version (rollback target), digest and evidence | + +## Drift + +| Source of drift | Handling | +|-----------------|----------| +| Manual `kubectl edit`/`patch` | Controller reverts (Argo CD `selfHeal`, Flux interval); decide per environment whether to auto-correct or alert only | +| Controller-written fields (HPA replicas, webhook `caBundle`) | Declare it: Argo CD `ignoreDifferences`, Flux HelmRelease `driftDetection.ignore` | +| Helm release modified with the CLI | A second owner; stop it, reconcile from Git | +| Suspended reconciliation | Drift accumulates silently; list suspended objects (`flux get all -A` shows SUSPENDED) in every review | + +## Rollback + +| Layer | Rollback | Note | +|-------|----------|------| +| Git (preferred) | `git revert ` through a PR; controller applies the old state | Works for every tool; the audit trail stays; propose, the commit agent commits | +| Argo CD | `argocd app rollback` needs auto-sync off, and Git still holds the new version | Use for an emergency, then revert in Git | +| Flux | Revert in Git; `flux suspend` to freeze while deciding | HelmRelease `upgrade.remediation` can roll back automatically on failure | +| Helm CLI | `helm rollback` (see helm-release-ops.md) | Only for releases not owned by a controller | +| Data | Not rolled back by any of the above | Migrations and PVC contents need backups (devops-storage-backup) | + +## Secrets pointer + +No secret value in a manifest, values file, Application parameter or Kustomize literal. Git holds either encrypted files (SOPS with age: Flux decrypts with `spec.decryption`; Argo CD needs a plugin or the repo-server setup) or references (`ExternalSecret`, existing `secretName` in a chart). Encryption, key custody and rotation: **devops-secrets**. Flux guides exist for SOPS and Sealed Secrets (https://fluxcd.io/flux/guides/mozilla-sops/ , .../sealed-secrets/). + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| Promotion by rebuilding the image per environment | Production runs something never tested | +| Environment branches that diverge | Merge conflicts and forgotten fixes; use folders | +| Auto-merge of version bumps to production | Removes the human gate that the layout exists for | +| App and infra in one Kustomization/Application | CRD not ready when the CR applies; split and order | +| Renovate/automation PR without rendered diff | Hidden default changes in a chart upgrade | + +## Verification + +For a promotion: PR shows a rendered diff limited to the intended version; after merge the controller reports the expected revision (`argocd app get`, `flux get kustomizations`), workloads roll out (`kubectl rollout status`), and the user-facing check passes. + +## Sources (read 2026-11-10) + +- https://fluxcd.io/flux/guides/repository-structure/ +- https://fluxcd.io/flux/guides/image-update/ (staging automatic, production through a pull request) +- https://argo-cd.readthedocs.io/en/stable/operator-manual/cluster-bootstrapping/ , .../user-guide/diffing/ , .../user-guide/auto_sync/ +- https://raw.githubusercontent.com/fluxcd/flux2-kustomize-helm-example/main/clusters/staging/infrastructure.yaml + +## Not verified + +- How to ignore controller-written fields in a Flux Kustomization (only HelmRelease `driftDetection.ignore` was read). +- Argo CD SOPS integration specifics, Flux `spec.decryption` field body (listed in the Kustomization index only), the two Flux secrets guides (not opened). +- Flagger and Argo Rollouts (named only as out of scope). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/argocd/application.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/argocd/application.yaml new file mode 100644 index 0000000..3787f29 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/argocd/application.yaml @@ -0,0 +1,34 @@ +# Production-safe default: manual sync, prune only on request, no HEAD. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: web-prod + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io # cascade-delete resources with the app; remove to orphan instead +spec: + project: shop + source: + repoURL: https://git.example.com/org/gitops.git + targetRevision: v1.4.2 # tag or commit SHA, never HEAD in production + path: apps/web/overlays/prod + destination: + server: https://kubernetes.default.svc + namespace: shop + syncPolicy: + # No `automated` block = manual sync. For staging use: + # automated: { prune: false, selfHeal: true } + syncOptions: + - PruneLast=true + - Prune=confirm # app-level default: pruning needs explicit confirmation + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + ignoreDifferences: + - group: apps + kind: Deployment + jsonPointers: + - /spec/replicas # only when an HPA owns replicas diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/argocd/applicationset.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/argocd/applicationset.yaml new file mode 100644 index 0000000..660836f --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/argocd/applicationset.yaml @@ -0,0 +1,31 @@ +# One Application per environment directory (apps/web/overlays/). +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: web + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: ["missingkey=error"] + generators: + - git: + repoURL: https://git.example.com/org/gitops.git + revision: v1.4.2 # pin: a merged folder must not deploy by itself + directories: + - path: apps/web/overlays/* + template: + metadata: + name: 'web-{{index .path.segments 3}}' + spec: + project: shop + source: + repoURL: https://git.example.com/org/gitops.git + targetRevision: v1.4.2 + path: '{{.path.path}}' + destination: + server: https://kubernetes.default.svc + namespace: 'shop-{{index .path.segments 3}}' # segments: [apps web overlays ] + syncPolicy: + syncOptions: + - CreateNamespace=true + # Automation is set here, not on the generated children. Production: leave it out. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/argocd/appproject.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/argocd/appproject.yaml new file mode 100644 index 0000000..50bd6d2 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/argocd/appproject.yaml @@ -0,0 +1,36 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: shop + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io # project cannot be deleted while applications reference it +spec: + description: Shop team applications + sourceRepos: + - https://git.example.com/org/gitops.git + destinations: + - server: https://kubernetes.default.svc + namespace: shop # never the argocd namespace: it grants admin access + clusterResourceWhitelist: [] # no cluster-scoped kinds (create Namespaces outside the project) + namespaceResourceBlacklist: + - group: "" + kind: ResourceQuota + - group: "" + kind: LimitRange + - group: networking.k8s.io + kind: NetworkPolicy + roles: + - name: read-only + description: Read-only access to the shop project + policies: + - p, proj:shop:read-only, applications, get, shop/*, allow + groups: + - shop-readers + syncWindows: + - kind: deny + schedule: "0 22 * * *" # no syncs 22:00-23:00 + duration: 1h + applications: + - "*-prod" + manualSync: false diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/Chart.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/Chart.yaml new file mode 100644 index 0000000..f337e44 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +name: web +description: Example stateless web service chart +type: application +version: 0.1.0 # chart SemVer: bump on every chart change +appVersion: "1.4.2" # quoted: informational, not related to version +kubeVersion: ">=1.35.0-0" +annotations: + example.com/owner: platform-team +# dependencies: +# - name: postgresql +# version: "16.x.x" +# repository: oci://registry.example.com/charts +# condition: postgresql.enabled diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/templates/_helpers.tpl b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/templates/_helpers.tpl new file mode 100644 index 0000000..a6d5372 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/templates/_helpers.tpl @@ -0,0 +1,25 @@ +{{/* Helm template (not parseable as plain YAML). Shared names and labels. */}} +{{- define "web.fullname" -}} +{{- printf "%s" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "web.labels" -}} +app.kubernetes.io/name: {{ .Chart.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | quote }} +{{- end -}} + +{{- define "web.selectorLabels" -}} +app.kubernetes.io/name: {{ .Chart.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "web.image" -}} +{{- if .Values.imageDigest -}} +{{ .Values.imageRepository }}@{{ .Values.imageDigest }} +{{- else -}} +{{ .Values.imageRepository }}:{{ .Values.imageTag }} +{{- end -}} +{{- end -}} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/templates/deployment.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/templates/deployment.yaml new file mode 100644 index 0000000..cb825b5 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/templates/deployment.yaml @@ -0,0 +1,32 @@ +# Helm template (contains {{ }}; exempt from plain YAML parsing). +# Pod hardening, probes and PDB: see devops-kubernetes templates/deployment-production.yaml. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "web.fullname" . }} + labels: + {{- include "web.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "web.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "web.selectorLabels" . | nindent 8 }} + spec: + automountServiceAccountToken: false + containers: + - name: web + image: {{ include "web.image" . | quote }} + ports: + - name: http + containerPort: {{ .Values.containerPort }} + {{- if .Values.existingSecret }} + envFrom: + - secretRef: + name: {{ .Values.existingSecret }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/values.schema.json b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/values.schema.json new file mode 100644 index 0000000..7e601d3 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/values.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "required": ["replicaCount", "imageRepository", "containerPort"], + "properties": { + "replicaCount": { "type": "integer", "minimum": 1 }, + "imageRepository": { "type": "string", "minLength": 1 }, + "imageTag": { "type": "string" }, + "imageDigest": { "type": "string", "pattern": "^(sha256:[a-f0-9]{64})?$" }, + "containerPort": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "existingSecret": { "type": "string" }, + "resources": { + "type": "object", + "properties": { + "requests": { + "type": "object", + "properties": { "cpu": { "type": "string" }, "memory": { "type": "string" } } + }, + "limits": { + "type": "object", + "properties": { "cpu": { "type": "string" }, "memory": { "type": "string" } } + } + } + } + } +} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/values.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/values.yaml new file mode 100644 index 0000000..5891a3a --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/chart/values.yaml @@ -0,0 +1,19 @@ +# replicaCount is the number of pods (ignored when autoscaling is managed elsewhere). +replicaCount: 2 +# imageRepository is the image without tag. +imageRepository: "registry.example.com/org/web" +# imageTag is the image tag; set imageDigest for production. +imageTag: "1.4.2" +# imageDigest, when set, wins over imageTag (format sha256:...). +imageDigest: "" +# containerPort is the port the app listens on. +containerPort: 8080 +# existingSecret names a pre-created Secret injected as env vars (never put values here). +existingSecret: "" +# resources are requests and limits for the web container. +resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + memory: "256Mi" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/flux/gitrepository.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/flux/gitrepository.yaml new file mode 100644 index 0000000..f2df7a5 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/flux/gitrepository.yaml @@ -0,0 +1,10 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: gitops + namespace: flux-system +spec: + interval: 5m + url: https://git.example.com/org/gitops.git + ref: + branch: main # production clusters: pin a tag or commit instead diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/flux/helmrelease.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/flux/helmrelease.yaml new file mode 100644 index 0000000..cb4da2d --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/flux/helmrelease.yaml @@ -0,0 +1,33 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: web + namespace: shop +spec: + interval: 15m + timeout: 5m + releaseName: web + chart: + kind: OCIRepository + spec: + interval: 5m + url: oci://registry.example.com/charts/web + ref: + semver: "1.4.x" # production: an exact version, raised by pull request + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + test: + enable: true + driftDetection: + mode: enabled + ignore: + - paths: ["/spec/replicas"] # only when an HPA owns replicas + target: + kind: Deployment + values: + replicaCount: 3 + existingSecret: web-env diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/flux/image-automation.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/flux/image-automation.yaml new file mode 100644 index 0000000..985818b --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/flux/image-automation.yaml @@ -0,0 +1,52 @@ +# Needs image-reflector-controller and image-automation-controller (flux bootstrap --components-extra=...). +# In the Deployment/HelmRelease, mark the field: image: registry.example.com/org/web:1.4.0 # {"$imagepolicy": "flux-system:web"} +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImageRepository +metadata: + name: web + namespace: flux-system +spec: + image: registry.example.com/org/web + interval: 5m + # secretRef: + # name: regcred # private registry; store the Secret encrypted (devops-secrets) +--- +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImagePolicy +metadata: + name: web + namespace: flux-system +spec: + imageRepositoryRef: + name: web + policy: + semver: + range: 1.4.x +--- +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImageUpdateAutomation +metadata: + name: flux-system + namespace: flux-system +spec: + interval: 30m + sourceRef: + kind: GitRepository + name: gitops + git: + checkout: + ref: + branch: main + commit: + author: + email: fluxcdbot@users.noreply.github.com + name: fluxcdbot + messageTemplate: |- + Update container images + + {{ .Changed.Summary }} + push: + branch: image-updates # production: push to a branch, review by pull request + update: + path: ./apps/web + strategy: Setters diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/flux/kustomization.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/flux/kustomization.yaml new file mode 100644 index 0000000..577deeb --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/flux/kustomization.yaml @@ -0,0 +1,40 @@ +# Two ordered Kustomizations: controllers first, then apps. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: infrastructure + namespace: flux-system +spec: + interval: 1h + retryInterval: 2m + timeout: 5m + sourceRef: + kind: GitRepository + name: gitops + path: ./infrastructure/production + prune: false # enable deliberately, after review: a prune deletes objects removed from Git + wait: true +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: web + namespace: flux-system +spec: + dependsOn: + - name: infrastructure + interval: 10m + retryInterval: 2m + timeout: 5m + sourceRef: + kind: GitRepository + name: gitops + path: ./apps/web/overlays/prod + targetNamespace: shop # the namespace must exist (ship it in infrastructure) + prune: true + deletionPolicy: Orphan # deleting this Kustomization leaves the workloads running + healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: web + namespace: shop diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/gitops-repo-layout.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/gitops-repo-layout.md new file mode 100644 index 0000000..7c2085f --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/gitops-repo-layout.md @@ -0,0 +1,48 @@ +# GitOps repository layout and promotion flow + +Based on the Flux monorepo structure (https://fluxcd.io/flux/guides/repository-structure/), usable with Argo CD by pointing Applications at the same folders. + +``` +gitops/ +├── clusters/ +│ ├── staging/ # cluster entry point: Flux Kustomizations, or Argo CD Applications/ApplicationSets +│ │ ├── infrastructure.yaml +│ │ └── apps.yaml +│ └── production/ +│ ├── infrastructure.yaml +│ └── apps.yaml +├── infrastructure/ # controllers and shared config, reconciled FIRST +│ ├── base/ +│ ├── staging/ +│ └── production/ +├── apps/ +│ └── web/ +│ ├── base/ # kustomization.yaml + plain manifests (or a HelmRelease) +│ └── overlays/ +│ ├── staging/ # version, replicas, hostnames of staging +│ └── prod/ # version/digest pinned by pull request +├── charts/ # charts you own (published to an OCI registry by CI) +│ └── web/ +└── README.md # who owns what, how to promote, how to roll back +``` + +Rules: + +- Infrastructure is reconciled before apps (`dependsOn` in Flux, sync waves or separate Applications in Argo CD). +- Environments are folders, not long-lived branches. +- `clusters/` is the only place that knows which cluster gets which overlay. +- Secrets are encrypted files or references only (devops-secrets). + +Promotion flow: + +``` +CI (devops-cicd) builds image app:1.4.2 + digest, publishes chart web 1.4.2 to OCI + | + v automatic (Flux image automation or CI commit) +apps/web/overlays/staging -> controller applies -> smoke test passes + | + v pull request: only the version/digest line changes, CI attaches the rendered diff +apps/web/overlays/prod -> reviewer approves -> merge -> controller applies -> verify + | + v rollback = git revert of that pull request's commit (data is not rolled back) +``` diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/kustomize/base/deployment.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/kustomize/base/deployment.yaml new file mode 100644 index 0000000..4632d16 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/kustomize/base/deployment.yaml @@ -0,0 +1,33 @@ +# Minimal base. Pod hardening, probes and PDB: see devops-kubernetes templates. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + labels: + app.kubernetes.io/name: web +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: web + template: + metadata: + labels: + app.kubernetes.io/name: web + spec: + automountServiceAccountToken: false + containers: + - name: web + image: registry.example.com/org/web:0.0.0 # replaced by overlays through images: + ports: + - name: http + containerPort: 8080 + envFrom: + - configMapRef: + name: web-config + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 256Mi diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/kustomize/base/kustomization.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/kustomize/base/kustomization.yaml new file mode 100644 index 0000000..3bab2c8 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/kustomize/base/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - deployment.yaml +configMapGenerator: + - name: web-config + literals: + - LOG_LEVEL=info diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/kustomize/components/ingress-tls/kustomization.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/kustomize/components/ingress-tls/kustomization.yaml new file mode 100644 index 0000000..85304ea --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/kustomize/components/ingress-tls/kustomization.yaml @@ -0,0 +1,14 @@ +# Optional feature packaged as a Component (referenced from overlays under `components:`). +# It patches the Deployment with an annotation as a minimal example; replace with the real feature +# (for example an HTTPRoute resource, see devops-kubernetes templates/gateway-httproute.yaml). +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +patches: + - target: + kind: Deployment + name: web + patch: |- + - op: add + path: /metadata/annotations + value: + example.com/tls: "terminated-at-gateway" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/kustomize/overlays/prod/kustomization.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/kustomize/overlays/prod/kustomization.yaml new file mode 100644 index 0000000..d752830 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-gitops-packaging/references/templates/kustomize/overlays/prod/kustomization.yaml @@ -0,0 +1,27 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: shop +resources: + - ../../base +components: + - ../../components/ingress-tls +images: + - name: registry.example.com/org/web + newTag: 1.4.2 + # digest: sha256:<64 hex> # pin by digest for production once known +replicas: + - name: web + count: 3 +patches: + - target: + kind: Deployment + name: web + patch: |- + - op: replace + path: /spec/template/spec/containers/0/resources/limits/memory + value: 512Mi +configMapGenerator: + - name: web-config + behavior: merge + literals: + - LOG_LEVEL=warn diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/SKILL.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/SKILL.md new file mode 100644 index 0000000..e876fc4 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/SKILL.md @@ -0,0 +1,169 @@ +--- +name: devops-iac +description: Use when writing, reviewing or operating infrastructure as code with OpenTofu or Terraform - choosing between them, project layout, provider pinning and the lock file, remote state (S3-compatible/MinIO, HTTP, PostgreSQL, Kubernetes), locking and state encryption, state surgery (import/moved/removed blocks, state mv/rm, force-unlock), modules, variables and validation, environments (workspaces vs directories vs Terragrunt), the saved-plan review/apply workflow, drift detection, tofu/terraform test, tflint/checkov/trivy, and the providers for Proxmox VE, OpenStack, Kubernetes/Helm, Docker and libvirt. Not for Ansible (devops-ansible), Packer and cloud-init images (devops-virtualization), Proxmox or OpenStack platform internals (devops-proxmox, devops-openstack), secret stores (devops-secrets), CI pipeline authoring (devops-cicd) or public-cloud providers. +versions: + opentofu: "1.13.9 (stable line); 1.12.12 also supported" + terraform: "1.17.1; 1.16.7" + terragrunt: "1.2.1" + bpg-proxmox-provider: "0.121.0" + openstack-provider: "3.4.0" + kubernetes-provider: "3.5.0" + helm-provider: "3.3.0" + docker-provider: "4.6.0" + libvirt-provider: "0.9.9" + tflint: "0.66.0" + trivy: "0.76.2" +user-invocable: true +references: + - references/tofu-vs-terraform.md + - references/layout-providers-lock.md + - references/state-backends-encryption.md + - references/state-surgery.md + - references/modules-variables.md + - references/environments.md + - references/plan-apply-protocol.md + - references/testing-quality.md + - references/provider-proxmox.md + - references/provider-openstack.md + - references/provider-k8s-docker-libvirt.md + - references/templates/root-module.tf + - references/templates/encryption.tf + - references/templates/module-skeleton.tf + - references/templates/proxmox-vm-cloudinit.tf + - references/templates/openstack-instance-network.tf + - references/templates/import-moved-removed.tf + - references/templates/main.tftest.hcl +related-skills: + - devops-proxmox + - devops-openstack + - devops-kubernetes + - devops-gitops-packaging + - devops-docker + - devops-virtualization + - devops-ansible + - devops-cicd + - devops-secrets + - devops-databases + - devops-sre-ops +--- + + +Covers infrastructure as code with OpenTofu and Terraform for a self-hosted +audience (Proxmox, OpenStack, Kubernetes, Docker, libvirt): the tool choice, +layout and version pinning, remote state with locking and encryption, safe state +surgery, modules, environments, a plan-then-apply protocol that never applies an +unreviewed plan, drift detection, tests and static scanning. The main danger is +destruction through state; every destructive command is shown with its exact +effect. Does not cover Ansible, Packer/cloud-init, platform internals, secret +stores, pipeline authoring or public clouds. + + +# Infrastructure as Code (OpenTofu / Terraform) + +## Agent Workflow (MANDATORY) + +Before ANY change to `.tf`, `.tofu`, `.tfvars`, `.tftest.hcl`, `terragrunt.hcl` or state, spawn in parallel, one `Task` call each: + +1. **fuse-ai-pilot:explore-codebase** - find the root modules, backends, providers, lock file, workspaces, wrappers (Makefile, Terragrunt, CI) and which binary (`tofu` or `terraform`) the repo uses +2. **fuse-ai-pilot:research-expert** - re-read the registry page and docs of every provider/resource you will write (versions move; the Proxmox and OpenStack providers are community-maintained) +3. **mcp__context7__query-docs** (`/websites/opentofu`, `/websites/developer_hashicorp_terraform`, `/bpg/terraform-provider-proxmox`) - confirm each argument before writing it + +Read-only commands first ([plan-apply-protocol.md](references/plan-apply-protocol.md)). After file changes, run **fuse-ai-pilot:sniper**. + +--- + +## Overview + +| Area | What matters on 2026-11-10 | +|------|----------------------------| +| **Tool** | OpenTofu (Linux Foundation fork) vs Terraform (Business Source License 1.1, licensor IBM, 1.6.0 and later); same HCL core, diverging features | +| **Only OpenTofu** | State and plan encryption (1.7), `-exclude` (1.9), provider `for_each` (1.9), `enabled` meta-argument (1.11), dynamic `prevent_destroy` and `destroy = false` (1.12) | +| **Both** | Ephemeral resources and write-only arguments (Terraform 1.11+, OpenTofu 1.11+), `import`/`moved`/`removed` blocks, native S3 locking `use_lockfile` | +| **Only Terraform** | Actions and `terraform query` list resources (1.14), Terraform Policy GA (1.17), `-minimal-refresh` (1.17) | +| **Providers** | `bpg/proxmox` is the maintained Proxmox provider; `Telmate/proxmox` publishes only release candidates on its 3.x line (last stable 2.9.14); OpenStack provider last released 2025-11-11 | + +--- + +## Critical Rules + +1. **Plan, read, show, then apply that exact plan** - `plan -out=FILE`, read the summary line and every `replace` (`-/+`) and `destroy`, show it to the user, `apply FILE` only after the user's go. Never apply from a dirty tree, a stale plan or an unreviewed plan. +2. **Destruction through state is proposed, never run on your own** - `destroy`, applying a plan that replaces or destroys, `state rm`, `state push`, `force-unlock`, `-target`, `-replace`, deleting a backend bucket/table or a lock: ACTION / EFFECT / PROOF / WAIT-FOR-GO ([plan-apply-protocol.md](references/plan-apply-protocol.md)). +3. **Back up state before any surgery** - `state pull > backup` plus bucket versioning; with encryption on, the key is part of the backup. +4. **Never invent an argument, block or provider** - every attribute here comes from the provider docs of the pinned version; the rest is under "Not verified". +5. **Pin everything and commit the lock file** - `required_version`, provider `version`, exact module versions, `.terraform.lock.hcl`. +6. **Secrets never in `.tf`, `.tfvars` committed or state when avoidable** - ephemeral values and write-only arguments first, then encrypted state; the secret store is `devops-secrets`. +7. **Say which binary** - OpenTofu documents compatibility with state files created by Terraform up to 1.5.x; never switch binaries on a live state without the migration guide and a state backup. + +--- + +## Architecture + +``` +infra/ +├── live/// # root modules: backend, providers, module calls (one state each) +│ ├── versions.tf backend.tf main.tf variables.tf outputs.tf +│ ├── encryption.tf # OpenTofu only +│ ├── .terraform.lock.hcl # committed +│ └── terraform.tfvars # non-secret values only +├── modules// # reusable, versioned, tested (tests/*.tftest.hcl) +└── .tflint.hcl .trivyignore # static checks +``` + +--- + +## Reference Guide + +### Concepts + +| Topic | Reference | When to Consult | +|-------|-----------|-----------------| +| **Tool choice** | [tofu-vs-terraform.md](references/tofu-vs-terraform.md) | License, governance, feature divergence, migration | +| **Layout & providers** | [layout-providers-lock.md](references/layout-providers-lock.md) | Project tree, version pinning, lock file, registries | +| **State** | [state-backends-encryption.md](references/state-backends-encryption.md) | Backend choice, S3/MinIO, http, pg, kubernetes, locking, encryption | +| **State surgery** | [state-surgery.md](references/state-surgery.md) | import, moved, removed, state mv/rm/push, force-unlock | +| **Modules** | [modules-variables.md](references/modules-variables.md) | Module design, versioning, variables, validation, outputs, locals | +| **Environments** | [environments.md](references/environments.md) | Workspaces vs directories vs Terragrunt | +| **Plan/apply** | [plan-apply-protocol.md](references/plan-apply-protocol.md) | Saved plans, destructive catalogue, drift, CI pointer | +| **Testing** | [testing-quality.md](references/testing-quality.md) | fmt, validate, test, tflint, checkov, trivy | +| **Proxmox** | [provider-proxmox.md](references/provider-proxmox.md) | Which provider, auth, VM from template | +| **OpenStack** | [provider-openstack.md](references/provider-openstack.md) | clouds.yaml auth, network + instance | +| **K8s/Helm/Docker/libvirt** | [provider-k8s-docker-libvirt.md](references/provider-k8s-docker-libvirt.md) | Provider choice, v3 schema changes, pitfalls | + +### Templates + +| Template | When to Use | +|----------|-------------| +| [root-module.tf](references/templates/root-module.tf) | Root module with pinned providers and an S3-compatible backend | +| [encryption.tf](references/templates/encryption.tf) | OpenTofu state and plan encryption with migration fallback | +| [module-skeleton.tf](references/templates/module-skeleton.tf) | Reusable module with validated inputs and outputs | +| [proxmox-vm-cloudinit.tf](references/templates/proxmox-vm-cloudinit.tf) | VM cloned from a cloud-init template (bpg provider) | +| [openstack-instance-network.tf](references/templates/openstack-instance-network.tf) | Network, router, security group and instance | +| [import-moved-removed.tf](references/templates/import-moved-removed.tf) | Adopt, rename and forget resources without destroying them | +| [main.tftest.hcl](references/templates/main.tftest.hcl) | Offline plan-only tests for the module skeleton, including negative validation tests | + +--- + +## Quick Reference + +```bash +tofu version && tofu providers # binary, versions, provider tree +tofu fmt -check -recursive && tofu init -backend=false && tofu validate # offline checks +tofu init && tofu plan -out=tfplan -detailed-exitcode # exit 0 no diff, 1 error, 2 changes +tofu show tfplan # read it; show the user; wait for the go +tofu apply tfplan # only the reviewed file +``` + +--- + +## Best Practices + +### DO +- One state per stack and environment; remote backend with locking; bucket versioning on +- Provider and module versions pinned; lock file reviewed in code review +- `moved` blocks for renames, `removed` with `lifecycle { destroy = false }` to forget +- `prevent_destroy` on state-like resources (databases, volumes) and VM templates + +### DON'T +- Run `apply` without a saved plan outside a throwaway lab; use `-target` as a routine tool +- Run `state push -force` or `force-unlock` without proof of the cause +- Mix `tofu` and `terraform` on one state; commit `.tfstate`, plan files or `.tfvars` with secrets diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/environments.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/environments.md new file mode 100644 index 0000000..53eaa70 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/environments.md @@ -0,0 +1,86 @@ +--- +name: environments +description: Separating dev/staging/prod - workspaces vs directories vs Terragrunt, with blast-radius rules +when-to-use: Designing environment separation, adding a new environment, or deciding whether to adopt Terragrunt +keywords: workspace, terraform.workspace, directory per environment, terragrunt, run --all, filter, dependency, DRY, blast radius +priority: high +related: state-backends-encryption.md, modules-variables.md, plan-apply-protocol.md +--- + +# Environments + +**Load when:** a second environment appears, or one `apply` could touch production by accident. + +## Decision + +| Approach | Use when | Avoid when | +|----------|----------|-----------| +| **Directories** (`live///`, each with its own backend key and tfvars) | Default. Environments differ in size, endpoints, guards; prod needs its own credentials and review | Dozens of near-identical stacks (copy drift) | +| **Workspaces** (`tofu workspace`) | Short-lived copies of one stack in the same account (feature branches, tests) | Prod vs non-prod: same code, same backend credentials, and the current workspace is invisible state - a wrong `workspace select` targets production | +| **Terragrunt** (v1.2.1) | Many stacks per environment, dependencies between them, shared backend/provider boilerplate | A handful of stacks; the extra tool is a second thing to learn and pin | + +Rule: **prod credentials are separate from non-prod credentials** whatever the layout, so a mistaken command cannot reach prod. + +## Directories + +``` +live/ +├── dev/ network/ compute/ k8s-addons/ +├── prod/ network/ compute/ k8s-addons/ +modules/ (versioned, shared) +``` + +- Each leaf is a root module with its own `backend` key (`live/prod/network/terraform.tfstate`), its own lock file and `terraform.tfvars`. +- Cross-stack data: read outputs with `terraform_remote_state` (needs the consumer to read the other state; with OpenTofu encryption configure `remote_state_data_sources`) or, better, look objects up by name/tag with data sources so stacks stay decoupled. +- Promote by changing the module `ref`/version in `dev`, then `prod`, in separate reviewed PRs. + +## Workspaces + +```bash +tofu workspace list; tofu workspace show # always print before any plan/apply +tofu workspace new feature-x # state lives under the same backend (S3 default prefix "env:") +tofu workspace select feature-x +tofu workspace delete -force feature-x # dangling resources if it still tracks any +``` + +`terraform.workspace` can drive names and sizes; never drive security guards (`prevent_destroy`) from it without a variable that is set per environment in code. `workspace delete` without `-force` refuses when resources are tracked. + +## Terragrunt (when chosen) + +Verified at https://docs.terragrunt.com : an empty `terragrunt.hcl` next to a `.tf` root is enough to call `terragrunt` instead of `tofu`/`terraform`, with automatic `init`. + +```bash +terragrunt run --all plan # every unit in the current stack +terragrunt run --all --filter 'prod/**' -- plan # filter by path +terragrunt run --all --filter-affected -- plan # units changed between the default branch and HEAD (plan/apply only, not -destroy) +terragrunt run -- plan -no-color # `--` separates Terragrunt flags from tofu/terraform flags +``` + +- `terragrunt run --all apply` applies many states in one command: it needs the same saved-plan discipline - plan all, review all summaries, apply after the go; never auto-approve against prod from a laptop. +- Pick the binary explicitly with `--tf-path` (`TG_TF_PATH`) when both are installed. +- The `terraform { source = ... }` block accepts local, Git, `tfr://` registry and `oci://` sources; pin versions exactly (an omitted `?version=` fetches the latest stable). +- Block syntax for `include`, `dependency`, `remote_state`, `generate`, `unit`/`stack`: read https://docs.terragrunt.com/reference/hcl/blocks before writing; this skill does not reproduce it. + +## Pitfalls + +- Same backend key for two environments: the second `init` shares state. +- Workspaces plus a single shared `terraform.tfvars`: prod gets dev values. +- `terragrunt run --all destroy` / `tofu destroy` in a directory named `prod`: propose-only (`plan-apply-protocol.md`). + +## Verification + +```bash +tofu workspace show && grep -n "key" backend.tf # the environment you think you are in +terragrunt run --all --filter-affected -- plan # only intended units appear +``` + +## Sources + +- https://docs.terragrunt.com/getting-started/quick-start/ , https://docs.terragrunt.com/reference/cli/commands/run/ , https://docs.terragrunt.com/reference/hcl/blocks/ +- https://github.com/gruntwork-io/terragrunt/releases (v1.2.1) +- https://opentofu.org/docs/cli/commands/workspace/delete/ , https://opentofu.org/docs/language/settings/backends/s3/ + +## Not verified + +- Terragrunt `include`/`dependency`/`remote_state`/`stack` syntax and its auto-approve/non-interactive flags (not read in detail). +- Terragrunt-specific exit codes and behaviour with OpenTofu state encryption. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/layout-providers-lock.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/layout-providers-lock.md new file mode 100644 index 0000000..3d9ef87 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/layout-providers-lock.md @@ -0,0 +1,86 @@ +--- +name: layout-providers-lock +description: Project layout, required_version, provider version pinning and the dependency lock file for OpenTofu and Terraform +when-to-use: Creating a root module, bumping a provider, reviewing a lock-file diff or fixing a checksum error +keywords: required_providers, version constraint, .terraform.lock.hcl, tofu init -upgrade, providers lock, registry, checksum +priority: high +related: tofu-vs-terraform.md, modules-variables.md, testing-quality.md +--- + +# Layout, providers and the lock file + +**Load when:** a root module is created, a provider version changes, or `init` reports a checksum mismatch. + +## Layout rules + +- One root module = one state = one blast radius. Split by lifecycle and ownership (network, compute, k8s add-ons), not by file size. +- Per root module: `versions.tf` (terraform + required_providers), `backend.tf`, `providers.tf` (provider blocks), `main.tf`, `variables.tf`, `outputs.tf`. Sibling files are merged; the split is for readers. +- Reusable code goes in `modules/` or a versioned registry/Git source (`modules-variables.md`); root modules hold only module calls, provider config and backend. +- Both binaries read `.tf`; OpenTofu also reads `.tofu` files (same-name `.tofu` wins over `.tf`). Use `.tofu` only for OpenTofu-only syntax that Terraform must not see (Terraform's handling of `.tofu` is not verified, so keep a separate unique file name such as `encryption.tf` and do not run Terraform on that root). + +## Pinning + +```hcl +terraform { + required_version = ">= 1.11.0, < 2.0.0" + required_providers { + proxmox = { source = "bpg/proxmox", version = "~> 0.121.0" } # 0.x: minors may break + openstack = { source = "terraform-provider-openstack/openstack", version = "~> 3.4" } + } +} +``` + +| Constraint | Meaning | Use for | +|------------|---------|---------| +| `= 0.121.0` | exactly | providers below 1.0 in production | +| `~> 0.121.0` | `>= 0.121.0, < 0.122.0` | 0.x providers (bpg README: 0.x is not guaranteed backward-compatible across minors) | +| `~> 3.4` | `>= 3.4, < 4.0` | 1.0+ providers following semver | + +Terraform 1.17 adds variables and locals in provider requirements; OpenTofu allows variables in module sources, backend and encryption config only when values are assigned to root variables - keep the pin literal anyway. + +## The lock file (`.terraform.lock.hcl`) + +- Created and updated by `init`; lives next to the root module; **commit it** and review its diff like code (OpenTofu docs: dependency-lock). +- It tracks providers only. Remote module versions are not locked: pin them exactly (`version = "1.4.2"` or a Git ref). +- Existing selection is reused even when newer versions exist; `tofu init -upgrade` re-selects the newest matching the constraint. +- Trust on first use: every installed package must match a recorded checksum. A mismatch is an alert, not a nuisance: confirm the provider version and source before touching the file. +- OpenTofu 1.12 records the full `zh:` and `h1:` checksum set on `init`; `tofu providers lock` is needed only when `init` uses a mirror or alternative installation source. Set `OPENTOFU_ENFORCE_GPG_VALIDATION=true` to require signed checksums. + +## Bumping a provider (procedure) + +```bash +git switch -c chore/provider-bump +# edit the constraint in versions.tf +tofu init -upgrade # rewrites the lock file +tofu validate && tofu plan -out=tfplan -detailed-exitcode +tofu show tfplan # a bump must plan zero or explainable changes +``` + +Exit code 0 means no changes; 2 means a diff to read (a provider upgrade that wants to replace resources is a stop: read the provider changelog). Commit the lock diff in the same PR. + +## Pitfalls + +- `required_providers` without `source` for non-HashiCorp providers resolves to the wrong namespace. +- Two roots sharing one `.terraform` directory or one plugin cache across versions: use per-root directories; OpenTofu 1.10 added a global provider cache lock for concurrent CI jobs. +- Provider block credentials in HCL: use environment variables or ephemeral values (`state-backends-encryption.md`). +- Switching Proxmox providers (`Telmate` to `bpg`) changes resource types: it is a migration with `state rm` + `import`, never a rename (`provider-proxmox.md`). + +## Verification + +```bash +tofu providers # tree of required providers per module +tofu version # shows the installed provider versions too +git diff --stat -- .terraform.lock.hcl +``` + +## Sources + +- https://opentofu.org/docs/language/files/dependency-lock/ +- https://opentofu.org/blog/opentofu-1-12-0/ , https://opentofu.org/blog/opentofu-1-10-0/ +- https://registry.terraform.io/v1/providers/bpg/proxmox (0.121.0, 2026-11-09) +- https://raw.githubusercontent.com/bpg/terraform-provider-proxmox/main/README.md + +## Not verified + +- Exact flags of `tofu providers lock` (read the command page before use). +- Terraform's handling of `.tofu` files. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/modules-variables.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/modules-variables.md new file mode 100644 index 0000000..95f4b0a --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/modules-variables.md @@ -0,0 +1,90 @@ +--- +name: modules-variables +description: Module design and versioning, variables with validation, outputs, locals and meta-arguments (for_each, count, enabled, lifecycle) +when-to-use: Writing or reviewing a reusable module, adding inputs, or deciding how to version and consume modules +keywords: module, variable validation, sensitive, ephemeral, outputs, locals, for_each, count, enabled, prevent_destroy, precondition +priority: high +related: layout-providers-lock.md, testing-quality.md, state-surgery.md +--- + +# Modules, variables, outputs + +**Load when:** code is repeated in two stacks, a module interface changes, or an input needs guarding. + +## Design rules + +- A module wraps one coherent unit (a VM with its disks and firewall, a network with its subnets), exposes few inputs, and hides provider details. Extract at the third repetition, not before. +- No `provider` blocks inside reusable modules; declare `required_providers` (source + minimum version) and let the root configure providers. +- Inputs: typed (`object({...})` over loose maps), described, validated; outputs: only what callers need, each described. Sensitive outputs are marked `sensitive = true` - this hides them from the UI but they remain in state. +- `for_each` over a map/set with stable keys, not `count` over a list (index shifts replace resources). Use `moved` when migrating (`state-surgery.md`). +- Zero-or-one instances: `count = var.x ? 1 : 0` everywhere; OpenTofu 1.11 adds `lifecycle { enabled = var.x }` for resources and module calls. +- Destruction guards: `lifecycle { prevent_destroy = true }` on data-bearing resources and VM templates; from OpenTofu 1.12 it can reference module variables, so one module serves dev (false) and prod (true). + +## Variables and validation + +```hcl +variable "cpu_cores" { + type = number + description = "vCPU count per VM." + default = 2 + validation { + condition = var.cpu_cores >= 1 && var.cpu_cores <= 32 + error_message = "cpu_cores must be between 1 and 32." + } +} +variable "admin_token" { + type = string + sensitive = true # hidden in output, still in state unless the provider offers a write-only argument + ephemeral = true # not persisted in state or plan (OpenTofu 1.11+; Terraform version: see Not verified) +} +``` + +Complete example with object types, `optional()`, cross-field checks and outputs: [templates/module-skeleton.tf](templates/module-skeleton.tf). Validation errors are testable without any provider (`tofu test` with `expect_failures`, see `testing-quality.md`). + +- Where values come from, from weakest to strongest: defaults, `terraform.tfvars`/`*.auto.tfvars`, `-var-file`, `-var`, environment `TF_VAR_name`. Secrets only through `TF_VAR_*` from a secret store (`devops-secrets`), never committed `.tfvars`. +- Ephemeral variables are accepted only in ephemeral contexts (write-only arguments, provider config, other ephemeral values); using them elsewhere is an error at plan time. +- Preconditions/postconditions (`lifecycle { precondition {...} }`) check assumptions about data sources (an image exists, a template VM is a template) before changes are made. + +## Locals + +Use `locals` to name a computed value used twice or more (tags merged with defaults, naming scheme). No logic that would be clearer as a variable default; no locals that read secrets (they end up in state via resources anyway, but keep them out of outputs). + +## Versioning and consuming + +| Source | Pin | Notes | +|--------|-----|-------| +| Registry (`source = "ns/name/provider"`) | `version = "1.4.2"` exact for production | Remote module versions are not recorded in the lock file (OpenTofu docs), so a range changes silently | +| Git (`git::https://...//subdir?ref=v1.4.2`) | tag or commit SHA, never a branch | Terragrunt downloads the whole repo before the `//` | +| OCI registry | `oci://host/repo?tag=...` for Terragrunt; OpenTofu 1.10 supports OCI for providers and modules | Useful in air-gapped networks | +| Local path | none | Fine inside one repo; promote to Git tags when two repos need it | + +- Semver the module: removing/renaming an input or output is a major bump; changing a resource address needs a `moved` block inside the module so callers' states follow. +- OpenTofu 1.10 supports marking variables and outputs as deprecated; use it for a release cycle before removing. +- Document each module with a README of inputs/outputs and keep `tests/*.tftest.hcl` next to it. + +## Pitfalls + +- `count` and `for_each` values must be known at plan time; deriving them from resource attributes created in the same run errors. +- `depends_on` on modules forces deferral of data sources and produces noisy plans; prefer real references. +- Output of a whole resource object leaks every attribute (including secrets) into state of consumers and `terraform_remote_state`: output only the fields needed. +- Overriding `prevent_destroy` by deleting the line is a normal code change - require review on those lines. + +## Verification + +```bash +tofu fmt -check -recursive && tofu init -backend=false && tofu validate +tofu test # validation and plan assertions (testing-quality.md) +``` + +## Sources + +- https://opentofu.org/docs/language/files/dependency-lock/ (modules not locked) +- https://opentofu.org/blog/opentofu-1-10-0/ , .../1-11-0/ , .../1-12-0/ +- https://opentofu.org/docs/language/ephemerality/ +- https://opentofu.org/docs/cli/commands/test/ (`expect_failures`) +- https://docs.terragrunt.com/reference/hcl/blocks/ (`tfr://`, `oci://` sources) + +## Not verified + +- Exact syntax of the OpenTofu deprecation attribute for variables/outputs (read the 1.10 what's-new page). +- Terraform `ephemeral = true` variable version (the ephemeral block page was read; the first supported version number was not). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/plan-apply-protocol.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/plan-apply-protocol.md new file mode 100644 index 0000000..3a47297 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/plan-apply-protocol.md @@ -0,0 +1,88 @@ +--- +name: plan-apply-protocol +description: The plan, review, apply protocol with saved plans, the exact effect of every destructive command, drift detection and the CI pointer +when-to-use: Before ANY plan/apply/destroy, whenever -target, -replace, state rm, state push, force-unlock or a backend deletion is considered +keywords: plan -out, apply tfplan, detailed-exitcode, destroy, replace, -/+, target, exclude, refresh-only, drift, prevent_destroy +priority: critical +related: state-surgery.md, state-backends-encryption.md, testing-quality.md +--- + +# Plan, review, apply + +**Load when:** infrastructure will change. Read-only commands (`fmt`, `validate`, `plan`, `show`, `state list/show/pull`, `providers`, `output`) need no go; everything that writes state or infrastructure follows this file. + +## The protocol + +1. **Target check** - which binary, which directory, which workspace/backend key, which provider endpoint/cloud. Print them: `tofu version; tofu workspace show; grep -n 'key\|endpoint' backend.tf`. +2. **Clean inputs** - `git status` clean (or the dirty files are named and intended); lock file unchanged; variables from files/env, not typed ad hoc. +3. **Plan to a file** - `tofu plan -out=tfplan -detailed-exitcode` (exit 0 = no changes, 1 = error, 2 = changes). +4. **Read it** - `tofu show tfplan`. Read the summary line `Plan: X to add, Y to change, Z to destroy.` and **every** resource marked destroy (`-`) or replace (`-/+`, "must be replaced"). Read forced-replacement reasons (`# forces replacement`). +5. **Show the user** - summary line, the list of destroys/replaces, the reason for each, the data at risk (disks, databases, volumes, floating IPs), and the exact apply command. +6. **Wait for the go** naming the plan. +7. **Apply that file** - `tofu apply tfplan`. A saved plan is applied without a prompt and accepts no extra planning options (the file carries the decisions). +8. **Verify** - `tofu plan -detailed-exitcode` again must return 0; check the real object (VM running, network reachable). + +Never: apply from a plan produced before the code or the state changed (re-plan), apply a plan someone else produced without reading it, run `apply -auto-approve` outside a disposable lab, or `apply` without a file against shared infrastructure. A speculative `plan` without `-out` can differ from the final effect because other changes happen meanwhile - re-check the final plan. + +Plan files contain resolved values, secrets included: keep them out of Git and artifacts shared widely; OpenTofu plan encryption (`state-backends-encryption.md`) protects stored plans. + +## Destructive catalogue (each: ACTION / EFFECT / PROOF / WAIT-FOR-GO) + +``` +ACTION tofu apply tfplan (plan contains: proxmox_virtual_environment_vm.db -/+ must be replaced) +EFFECT the VM and its local disks are destroyed, then created empty from the template; guest data is lost +PROOF `tofu show tfplan` excerpt; last verified backup of the VM (devops-storage-backup); `# forces replacement` reason +WAITING for your go +``` + +| Command | Exact effect | Required proof before proposing | +|---------|--------------|---------------------------------| +| `tofu destroy` (= `apply -destroy`) | Destroys every remote object in the state; `lifecycle.destroy = false` objects are forgotten and the command exits non-zero | `tofu plan -destroy -out=...` shown in full; backups; the environment name in the user's words | +| `apply` of a plan with destroy/replace | See the plan; replace = destroy then create (or create then destroy with `create_before_destroy`) | The reviewed plan file and backups | +| `-replace=ADDR` | Plans a replace of that instance even if config is unchanged; cannot combine with `-destroy` | Why the object is degraded; same checks as above | +| `-target=ADDR` / `-exclude=ADDR` / `-exclude-file` / `-target-file` | Limits the plan to (or away from) addresses and their dependencies/dependents; the rest of the config is NOT reconciled and state/config diverge. The docs reserve `-exclude` for exceptional cases (recovering from mistakes, tool limits) | State the reason; run a full plan afterwards | +| `-refresh=false` | Skips reading real objects; plan ignores external changes and can be wrong | Only for speed in a reviewed pipeline; never before a destroy | +| `apply -refresh-only` | Writes the real attributes into state (accepts drift) without changing infrastructure | Read the drift first; it may hide a manual change someone must revert | +| `state rm`, `state push`, `force-unlock`, `workspace delete -force` | See `state-surgery.md` | Backup file + proof | +| Deleting the state bucket/object, a DB schema, or a lock object | The state is gone; every resource becomes unmanaged; a lock file object (`use_lockfile`) removed by hand has the same effect as `force-unlock` | Versioned bucket + backup + proof nothing runs; never in a script | +| Removing a `prevent_destroy` line, or a resource block without a `removed` block | The next plan includes the destroy | Review the diff as a destroy request | + +## Drift detection + +```bash +tofu plan -refresh-only -detailed-exitcode # exit 2 = real objects changed outside the tool; no changes proposed to infrastructure +tofu plan -detailed-exitcode # exit 2 = config differs from reality (drift or pending code) +``` + +Run it on a schedule per root module from a read-only identity; alert on exit 2 or 1. Resolve drift by one of: revert the manual change (apply the config), codify it (edit config), or accept it (`apply -refresh-only` after the user's go). Never "fix" drift by applying blindly. + +## CI pointer (pipeline authoring is `devops-cicd`) + +- Pull request: `fmt -check`, `validate`, `tflint`, config scan, `plan -out`; post the summary line and destroy/replace list as a comment. +- Merge to main: a protected environment requires a human approval, then applies **the same plan artifact** (store it encrypted, expire it). +- Credentials in CI: short-lived OIDC/JWT roles rather than static keys (the S3 backend docs recommend this); lock per state; never `-lock=false`. +- `-json-into=FILE` (OpenTofu 1.12) writes machine-readable output next to the normal log. + +## Pitfalls + +- Reading only the final count: a "1 to destroy" can be the production database. +- Trusting `# (known after apply)` for a name that forces a downstream replace. +- Provider upgrade and code change in one plan: bump providers in their own change (`layout-providers-lock.md`). + +## Verification + +```bash +tofu plan -out=tfplan -detailed-exitcode; echo "exit=$?" +tofu show -json tfplan | jq -r '.resource_changes[] | select(.change.actions != ["no-op"] and .change.actions != ["read"]) | "\(.change.actions|join(",")) \(.address)"' +``` + +## Sources + +- https://opentofu.org/docs/cli/commands/plan/ , .../apply/ , .../destroy/ , .../state/ , .../force-unlock/ +- https://opentofu.org/docs/language/settings/backends/s3/ (OIDC in CI) , https://opentofu.org/blog/opentofu-1-12-0/ (`-json-into`) + +## Not verified + +- The JSON plan layout (`resource_changes[].change.actions`) used in the `jq` line: confirm against `tofu show -json` output of the installed version. +- Exact wording of the "saved plan is stale" error and its conditions. +- Terraform equivalents of `-exclude`, `-target-file`, `-json-into`. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/provider-k8s-docker-libvirt.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/provider-k8s-docker-libvirt.md new file mode 100644 index 0000000..95e06c6 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/provider-k8s-docker-libvirt.md @@ -0,0 +1,112 @@ +--- +name: provider-k8s-docker-libvirt +description: Kubernetes, Helm, Docker and libvirt providers - versions, authentication, schema changes (Helm v3, libvirt 0.9 rewrite) and when not to use IaC for workloads +when-to-use: Declaring cluster add-ons, Helm releases, Docker hosts or KVM guests in HCL, or upgrading these providers +keywords: hashicorp/kubernetes, hashicorp/helm, helm_release, kreuzwerker/docker, dmacvicar/libvirt, libvirt_domain, kubeconfig, bootstrap +priority: medium +related: layout-providers-lock.md, state-backends-encryption.md, provider-proxmox.md +--- + +# Kubernetes, Helm, Docker, libvirt providers + +**Load when:** one of these providers is added, bumped or debugged. Workload manifests belong to `devops-kubernetes` and GitOps/Helm authoring to `devops-gitops-packaging`; Docker/Compose to `devops-docker`; libvirt/KVM internals and Packer to `devops-virtualization`. + +## Versions (registry, 2026-11-10) + +| Provider | Source | Version | Tier / date | +|----------|--------|---------|-------------| +| Kubernetes | `hashicorp/kubernetes` | 3.5.0 | official, 2026-11-05 | +| Helm | `hashicorp/helm` | 3.3.0 | official, 2026-09-02 | +| Docker | `kreuzwerker/docker` | 4.6.0 | community, 2026-09-01 | +| libvirt | `dmacvicar/libvirt` | 0.9.9 | community, 2026-08-30 | + +## When to use IaC for Kubernetes + +Use the providers for **bootstrap and platform add-ons**: the cluster's namespaces, the GitOps controller install (Flux/Argo CD), CRDs, a CNI or ingress Helm release. Application workloads belong in Git, reconciled by the GitOps controller (`devops-gitops-packaging`); two controllers owning one object is a fight. + +## Kubernetes provider + +```hcl +provider "kubernetes" { + config_path = "~/.kube/config" # the provider does not read KUBECONFIG by default + config_context = "prod-eu" # always name the context explicitly +} +``` + +- Requires Terraform 1.0 or later (docs). Resources come in plain and `_v1` forms: read the docs page for the one you use. +- **Never create the cluster and the Kubernetes/Helm resources in the same module/apply**: provider blocks are evaluated before resources, producing intermittent errors (provider docs warning). Two roots: cluster, then add-ons; pass data via data sources. +- Credentials with exec plugins or tokens come from the environment; they land in the plan if passed as variables (encrypt plans). + +## Helm provider (v3 schema) + +The v3 provider moved to the Plugin Framework: blocks became attributes. Pin `~> 3.3` and rewrite v2 configs. + +```hcl +provider "helm" { + kubernetes = { config_path = "~/.kube/config" } # was: kubernetes { ... } +} +resource "helm_release" "addon" { + name = "addon" + repository = "" + chart = "" + version = "" # always pin the chart + set = [{ name = "replicaCount", value = "2" }] # was: set { ... } blocks +} +``` + +`registry` blocks became `registries = [...]`; `set_list`/`set_sensitive` are lists of objects; v3 speaks plugin protocol 6 (Terraform 1.0+), 2.x stays for older Terraform. A release `destroy` uninstalls the chart and may delete PVC-backed data depending on the chart: treat `-/+` on `helm_release` as destructive. Chart repository URLs and versions: verify at the chart's own docs (not reproduced here). + +## Docker provider + +```hcl +provider "docker" { host = "unix:///var/run/docker.sock" } # or "ssh://user@remote-host:22" with ssh_opts +resource "docker_image" "app" { name = "registry.example.internal/app:1.4.2" } +resource "docker_container" "app" { name = "app" image = docker_image.app.image_id } +``` + +- Pin images by tag or digest, never `latest`. Remote hosts apply daemon defaults (for example log options) that show as diffs: use `lifecycle { ignore_changes = [...] }` for those attributes (provider docs). +- Registry credentials via `registry_auth`; the Docker config file is read from the machine running the tool, not the Docker host. +- IaC-managed containers on a long-lived host belong to a Compose/Quadlet workflow in many teams (`devops-docker`); use this provider when the Docker host itself is part of a larger Terraform graph (for example the VM it runs on). +- The 4.x provider documents "actions" (container exec, image import/save, system prune) for Terraform; availability in OpenTofu is **not verified**. + +## libvirt provider (0.9 rewrite) + +v0.9.0+ is a complete rewrite on the Plugin Framework that models the libvirt XML schema directly; v0.8.x and earlier live on a legacy branch. Do not assume a 0.8 configuration works on 0.9: resource arguments are different (XML-shaped nested objects). + +```hcl +provider "libvirt" { uri = "qemu:///system" } # also qemu+ssh://user@host/system, qemu+sshcmd://, qemu+tls:// +resource "libvirt_domain" "example" { + name = "example-vm" type = "kvm" memory = 512 memory_unit = "MiB" vcpu = 1 + os = { type = "hvm" type_arch = "x86_64" type_machine = "q35" } +} +``` + +Resources: `libvirt_domain`, `libvirt_network`, `libvirt_pool`, `libvirt_volume` (README); `libvirt_cloudinit_disk` and `libvirt_combustion` appear in the registry docs list. Pools support `create`/`destroy` lifecycle-control objects marked experimental (they steer post-define and pre-undefine operations such as build, start, autostart and delete). Read https://registry.terraform.io/providers/dmacvicar/libvirt/latest/docs and `docs/schema-mapping.md` before writing disks, networks and cloud-init. + +## Pitfalls + +- All four: `terraform { required_providers }` pins and the lock file (`layout-providers-lock.md`); a major provider bump is its own change with a plan that must show no resource changes. +- libvirt URIs over SSH use the Go SSH library or the native `ssh` command (`sshcmd` honours `~/.ssh/config`). +- Secrets in Helm `set_sensitive` or container env vars are in state: encrypt state. + +## Verification + +```bash +tofu providers && tofu validate +tofu plan -out=tfplan -detailed-exitcode # after a provider bump: exit 0 +``` + +## Sources + +- https://registry.terraform.io/v1/providers/hashicorp/kubernetes , .../hashicorp/helm , .../kreuzwerker/docker , .../dmacvicar/libvirt +- https://raw.githubusercontent.com/hashicorp/terraform-provider-kubernetes/main/docs/index.md +- https://raw.githubusercontent.com/hashicorp/terraform-provider-helm/main/docs/guides/v3-upgrade-guide.md +- https://raw.githubusercontent.com/kreuzwerker/terraform-provider-docker/master/docs/index.md +- https://raw.githubusercontent.com/dmacvicar/terraform-provider-libvirt/main/README.md + +## Not verified + +- Chart repository URLs, chart names and versions (placeholders in the example). +- libvirt resource arguments beyond the README example (disks, networks, cloud-init). +- Kubernetes provider resource coverage for the cluster's Kubernetes version. +- OpenTofu support for Terraform "actions" exposed by the Docker provider. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/provider-openstack.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/provider-openstack.md new file mode 100644 index 0000000..605e431 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/provider-openstack.md @@ -0,0 +1,82 @@ +--- +name: provider-openstack +description: The OpenStack provider - maintainer and release status, clouds.yaml and application-credential auth, network, router, security group and instance resources +when-to-use: Declaring Neutron/Nova/Cinder objects in HCL against a private OpenStack cloud +keywords: openstack provider, clouds.yaml, application credential, openstack_compute_instance_v2, networking_network_v2, router_interface, boot from volume +priority: high +related: layout-providers-lock.md, state-surgery.md, modules-variables.md +--- + +# OpenStack provider + +**Load when:** HCL manages OpenStack projects' networks, instances or volumes. Cloud operation (Kolla-Ansible, Neutron/OVN, quotas, flavors, `openstack` CLI) is `devops-openstack`. + +## Status (registry and GitHub, 2026-11-10) + +| Item | Value | +|------|-------| +| Source | `terraform-provider-openstack/openstack` (GitHub org `terraform-provider-openstack`), community tier | +| Latest release | 3.4.0, published 2025-11-11; no newer version in the registry list read today (about one year ago) | +| History | 2.0.0 and 3.0.0 are major bumps; the docs index still shows an old `~> 1.53.0` example - ignore it | +| Maintainer detail | Registry owner field shows an individual account; maintainer team and support policy **not verified** | + +Verdict: it is the provider to use, but check repository activity and open issues before relying on a new OpenStack release; test every provider bump against a lab project. Pin `~> 3.4`. + +## Authentication + +```hcl +provider "openstack" { + cloud = var.os_cloud # entry in clouds.yaml (OS_CLOUD) + # region = "RegionOne" # or OS_REGION_NAME +} +``` + +- `cloud` selects an entry of `clouds.yaml` (openstacksdk format); alternatively set `auth_url` plus credentials, or `OS_*` environment variables. +- Prefer an **application credential** (`application_credential_id` + `application_credential_secret`, Identity v3) limited to one project: revocable, no user password in automation. Create it per `devops-openstack`; keep the secret in `devops-secrets`. +- `clouds.yaml` and `secure.yaml` are per-user files outside the repo (`~/.config/openstack/`); the CI job receives them as secret files. +- Use a TLS CA bundle for private clouds through the clouds.yaml `cacert`, not `insecure`. + +## Resources (names from the provider docs) + +| Need | Resource | +|------|----------| +| Network, subnet | `openstack_networking_network_v2`, `openstack_networking_subnet_v2` (`network_id`, `cidr`, `ip_version`, `enable_dhcp`, `gateway_ip`, `allocation_pool`) | +| Router and attachment | `openstack_networking_router_v2` (`external_network_id`, `enable_snat` needs the ext-gw-mode extension), `openstack_networking_router_interface_v2` (`router_id`, `subnet_id`) | +| Security group | `openstack_networking_secgroup_v2`, `openstack_networking_secgroup_rule_v2` | +| Port | `openstack_networking_port_v2` (`fixed_ip`, `security_group_ids`) | +| Floating IP | `openstack_networking_floatingip_v2` (`pool`), `openstack_networking_floatingip_associate_v2` | +| Volume | `openstack_blockstorage_volume_v3`, `openstack_compute_volume_attach_v2` | +| Instance | `openstack_compute_instance_v2` (`image_id`, `flavor_id`, `key_pair`, `security_groups`, `network {}`, `block_device {}`, `metadata`) | + +Template: [templates/openstack-instance-network.tf](templates/openstack-instance-network.tf). + +## Patterns and pitfalls + +- Boot from volume uses a `block_device` block with `source_type = "image"`, `destination_type = "volume"`, `boot_index = 0`, `delete_on_termination` - decide `delete_on_termination` deliberately: `true` destroys the data disk with the instance. +- The docs' boot-from-volume examples omit `image_id` on the instance (the image is referenced in `block_device.uuid`); copy that shape. +- Attach extra volumes with `openstack_compute_volume_attach_v2`, not by editing `block_device` on a running instance (replacement). +- Resource arguments marked "Changing this creates a new ..." (for example router and port ownership, `tenant_id`) mean `-/+` in the plan: read it. +- All instance arguments, the admin password included, are stored in plain text in state: encrypt state (`state-backends-encryption.md`) and prefer key pairs. +- Image and flavor IDs differ per cloud; pass them as variables or look them up with data sources (names in the provider docs); never hard-code UUIDs from another cloud. +- `openstack_networking_router_interface_v2` imports by **port id** (`openstack port list --router `); the import formats of the other resources are in each resource page. + +## Verification + +```bash +openstack --os-cloud network list ; openstack --os-cloud server list # platform side (devops-openstack) +tofu plan -out=tfplan -detailed-exitcode +openstack --os-cloud server show -c status -c addresses +``` + +## Sources + +- https://registry.terraform.io/v1/providers/terraform-provider-openstack/openstack (3.4.0) +- https://github.com/terraform-provider-openstack/terraform-provider-openstack/releases +- https://raw.githubusercontent.com/terraform-provider-openstack/terraform-provider-openstack/main/docs/index.md , .../docs/resources/compute_instance_v2.md , .../networking_network_v2.md , .../networking_subnet_v2.md , .../networking_router_v2.md , .../networking_router_interface_v2.md + +## Not verified + +- Who maintains the provider and whether releases resumed after 2025-11-11 (the registry showed none). +- Compatibility with the current OpenStack release series (check `devops-openstack` for the release in use). +- Names of image/flavor data sources and the import ID of instances, networks and volumes. +- `cacert` handling by the provider (clouds.yaml field per the openstacksdk docs, not tested with this provider). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/provider-proxmox.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/provider-proxmox.md new file mode 100644 index 0000000..982b88d --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/provider-proxmox.md @@ -0,0 +1,87 @@ +--- +name: provider-proxmox +description: Which Proxmox VE provider to use (bpg/proxmox vs Telmate), authentication and least privilege, and a VM cloned from a cloud-init template +when-to-use: Managing Proxmox VMs, containers, storage or SDN with OpenTofu/Terraform, or migrating between Proxmox providers +keywords: bpg/proxmox, Telmate, proxmox_virtual_environment_vm, proxmox_cloned_vm, api_token, ssh, sudoers, clone, initialization, cloud-init +priority: high +related: state-surgery.md, layout-providers-lock.md, modules-variables.md +--- + +# Proxmox VE provider + +**Load when:** a Proxmox guest, template clone, file or network object is declared in HCL. Platform behaviour (clusters, storage, HA, backups, cloud-init template creation) is `devops-proxmox`; the API user/token itself is `devops-proxmox/references/api-access.md`. + +## Which provider (registry facts, 2026-11-10) + +| Provider | Latest at registry | Maintainer / status | Verdict | +|----------|--------------------|---------------------|---------| +| `bpg/proxmox` | 0.121.0, published 2026-11-09 | BPG Labs (individual-run, not affiliated with Proxmox Server Solutions); targets PVE 9.x, PVE 8.x supported with limits, 7.x unsupported; Terraform and OpenTofu both documented | **Use this.** Still 0.x: minors may break | +| `Telmate/proxmox` | 3.0.2-rc10 (2026-08-30); last non-RC listed 2.9.14 | Community (registry owner ggongaware); all 3.x entries are release candidates; README lists known limitations (disk size mismatch, failed-task noise on update, LXC needs `rootfs`) | Existing stacks only; plan a migration | +| `danitso/proxmox` | registry entry exists | bpg's README says upstream maintenance stopped in 2021 and bpg forked it | Do not use | + +Other forks exist on the registry (e.g. `Terraform-for-Proxmox`, `TheGameProfi`, `lucian-tx`): not evaluated; do not adopt without reading their repository activity. Maintainer status beyond what the registry and READMEs state is **not verified**. + +Docs: https://bpg.sh/docs (mirrored in the repository `docs/` folder). The Terraform and OpenTofu registries serve the same releases. + +## Authentication and privileges + +```hcl +provider "proxmox" { + endpoint = var.pve_endpoint # https://pve.example.internal:8006/ + api_token = var.pve_api_token # "user@realm!tokenid=" from the environment, never the repo + # ssh { username = "terraform" agent = true } # only if a resource needs SSH +} +``` + +- Order of precedence: API token (recommended), auth ticket, username/password. Environment variables such as `PROXMOX_VE_USERNAME`/`PROXMOX_VE_PASSWORD` exist; for tokens read the Argument Reference for the exact name. +- Requires TLS 1.3 on the API endpoint (`min_tls` allows older); `insecure = true` only for lab self-signed certificates. +- SSH is optional: needed for snippets and some file uploads. With a token, set the SSH username; the sudoers rule shown in the docs is `terraform ALL=(root) NOPASSWD: /usr/sbin/pvesm apiinfo`. **Never grant sudo on all of `qm` or `pvesm`**: the provider docs state both are root-equivalent (`qm set --args`, `--hookscript`, `pvesm export`). +- The docs' sample `Terraform` role lists many privileges and says it is likely too permissive for PVE 9. Start from the minimal role in `devops-proxmox` (`pveum role add ... --privs`, pool-scoped ACL, privilege-separated token with expiry) and add privileges until plans pass; keep `SDN.Audit` or `SDN.Use` to avoid "interface not found" on network reads. +- The token secret is shown once; keep it in `devops-secrets`. Rotation (`pveum user token modify ... --regenerate 1`) cuts every consumer at once. + +## Resources you will use + +| Need | Resource | Notes | +|------|----------|-------| +| VM from a cloud-init template | `proxmox_virtual_environment_vm` with `clone { vm_id = ... }` | Legacy (SDKv2) resource, still maintained. Template: [templates/proxmox-vm-cloudinit.tf](templates/proxmox-vm-cloudinit.tf) | +| VM clone managed as a delta | `proxmox_cloned_vm` | Newer short-prefix resource: `clone = { source_vm_id, full, target_datastore, source_node_name }`; `network`/`disk` as maps; migrating from the legacy resource recreates the VM (migration guide) | +| Image download to a node | `proxmox_virtual_environment_download_file` (`content_type = "import"`) | File must be renamed `*.qcow2` for import disks | +| Cloud-init snippets | `proxmox_virtual_environment_file` | Needs SSH | +| LXC | `proxmox_virtual_environment_container` | Same SSH/privilege notes | + +Both resource families ship in one binary; the Plugin Framework migration of the VM resource is a prerequisite for provider 1.0 (README), so expect the VM schema to change before 1.0. + +## Pitfalls + +- `agent.enabled` defaults to false; when true and the guest agent is absent, refresh waits up to the agent `timeout` (default 15m). Use `wait_for_ip { disabled = true }` only when you accept empty IP attributes. +- Cloning: if you change any attribute of an inherited disk, restate every non-default attribute (`size`, `discard`, `cache`, `aio`) or the schema defaults override the source values. +- Cross-node clone with non-shared datastores clones to the source node first, then migrates; set `clone.datastore_id` when the datastore name does not exist on the target node. +- `started = true` needs a bootable disk; set `stop_on_destroy = true` or destroying a running VM without an agent fails. +- A cloud-init template is a Proxmox object created outside HCL (`devops-proxmox`, `devops-virtualization` for Packer); never let the provider destroy the template: `prevent_destroy` on its resource if managed, data source lookup otherwise. +- Replacing a VM (`-/+`) discards guest disks: read plan reasons (`plan-apply-protocol.md`). + +## Migration Telmate to bpg + +Resource types and schemas differ; a rename is impossible. Per VM: `state rm` the old address, write the bpg resource, `import` it (`state-surgery.md`), plan to zero changes. Do it one VM at a time with a state backup; propose, then go. + +## Verification + +```bash +tofu providers | grep -i proxmox +tofu plan -out=tfplan -detailed-exitcode # after adopting a VM: exit 0 +pvesh get /nodes//qemu//config # platform side (devops-proxmox) +``` + +## Sources + +- https://registry.terraform.io/v1/providers/bpg/proxmox , .../Telmate/proxmox , https://registry.terraform.io/v2/providers?filter%5Bname%5D=proxmox +- https://raw.githubusercontent.com/bpg/terraform-provider-proxmox/main/README.md , .../docs/index.md , .../docs/resources/virtual_environment_vm.md +- https://raw.githubusercontent.com/Telmate/terraform-provider-proxmox/master/README.md +- Context7 `/bpg/terraform-provider-proxmox` (clone, cloned_vm migration guide, sudoers, role) + +## Not verified + +- The `import` ID format for `proxmox_virtual_environment_vm` (read the resource page's Import section; commonly `node/vmid` but unconfirmed). +- Environment variable name for the API token; exact minimal privilege set per resource. +- Whether `danitso/proxmox` and the other forks are abandoned today (only bpg's README statement is quoted). +- Context7 indexes provider v0.118.1; the registry shows 0.121.0. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/state-backends-encryption.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/state-backends-encryption.md new file mode 100644 index 0000000..4b960b7 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/state-backends-encryption.md @@ -0,0 +1,99 @@ +--- +name: state-backends-encryption +description: Remote state backends (S3-compatible/MinIO, HTTP, PostgreSQL, Kubernetes), locking, credentials handling and OpenTofu state encryption +when-to-use: Choosing or changing a backend, configuring MinIO state, enabling encryption, or diagnosing a lock +keywords: backend s3, minio, use_lockfile, pg, http, kubernetes secret, force-unlock, state encryption, pbkdf2, aws_kms, fallback +priority: critical +related: state-surgery.md, plan-apply-protocol.md, environments.md +--- + +# State backends, locking and encryption + +**Load when:** state needs a home, a backend changes, or state must be encrypted. The state holds every attribute of every resource, secrets included (the OpenStack instance resource states that all arguments, admin password too, are stored in plain text in state). + +## Backend choice + +| Backend | Locking | Pick when | Limits | +|---------|---------|-----------|--------| +| `s3` (AWS or S3-compatible such as MinIO) | `use_lockfile = true` (conditional writes, `If-None-Match`) or DynamoDB | You run an object store; most common self-hosted choice | Enable bucket versioning; lock objects add versions, add a lifecycle rule | +| `pg` | Postgres advisory locks; `force-unlock` not supported (locks drop with the session) | You already run PostgreSQL 10+ | Several tool versions under 1.10 on one database are unsafe; keep `public` schema access | +| `http` | `LOCK`/`UNLOCK` methods; 423 or 409 means held | A service (GitLab, custom) exposes a state API | Locking disabled unless `lock_address` set | +| `kubernetes` | Lease resource | State of in-cluster tooling | Secret size limit 1 MB; secret named `tfstate-{workspace}-{secret_suffix}` | +| `local` | none across machines | Throwaway lab only | Never for shared infrastructure | + +## S3-compatible (MinIO) configuration + +```hcl +terraform { + backend "s3" { + bucket = "tfstate" + key = "live/prod/network/terraform.tfstate" + region = "us-east-1" # dummy value for non-AWS stores + endpoints = { s3 = "https://minio.example.internal:9000" } + use_path_style = true + use_lockfile = true + skip_credentials_validation = true + skip_requesting_account_id = true + skip_metadata_api_check = true + skip_region_validation = true + skip_s3_checksum = true # only if the store rejects checksums + } +} +``` + +Credentials come from the environment, never the file: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` (store the values in `devops-secrets`). Pass secrets with a partial configuration or environment variables: values given inline or via `-backend-config` are written into `.terraform/` and into plan files (OpenTofu backend docs warning). Full file: [templates/root-module.tf](templates/root-module.tf). + +Terraform's S3 backend documents `use_lockfile` and marks DynamoDB locking as deprecated; OpenTofu documents both as supported. + +## Locks + +- A lock is released by the process that took it. `force-unlock LOCK_ID` removes it without touching infrastructure; run it only after proving the holder is dead (ACTION / EFFECT / PROOF in `plan-apply-protocol.md`). `pg` has no `force-unlock`: end the holding session instead (`pg_locks`). +- `-lock=false` on any state command is dangerous when others can run concurrently; never in automation. + +## OpenTofu state and plan encryption + +Template: [templates/encryption.tf](templates/encryption.tf). Rules from the OpenTofu docs: + +1. Encrypts state and plan **at rest** (local and any backend). It does not protect against data loss, replay of an older state/plan, or the person running `tofu`. +2. Without the key the state is unrecoverable: back up the unencrypted state and the keys first, and exercise the recovery before enabling. +3. Existing project: add a `method "unencrypted"` and a `fallback` on the state method, apply once, then remove the fallback and set `enforced = true`. +4. Never rename key providers or methods after encrypting (metadata stores their names); roll over with a `fallback` block or set `encrypted_metadata_alias`. +5. Key providers verified in the docs: `pbkdf2` (passphrase, at least 16 characters), `aws_kms`, `gcp_kms`, `external` (a command); the page also lists Azure Key Vault and OpenBao providers (read their exact block names there). Use one KMS key per state, not one for many. +6. `TF_ENCRYPTION` environment variable supplies the same config and merges over code; use it with `enforced = true` in code so unencrypted writes are impossible. +7. Reading an encrypted state through `terraform_remote_state` is configured in the `remote_state_data_sources` block. + +## Procedure: enable encryption (propose, then go) + +```bash +tofu state pull > backup-$(date +%F).tfstate # plain backup, stored encrypted elsewhere +# add encryption.tf with the unencrypted fallback, then: +tofu plan -out=tfplan && tofu apply tfplan # first write is encrypted +aws --endpoint-url s3 cp s3://tfstate/ - | head -c 300 # proof: the stored object is not readable JSON resources (state pull itself may print decrypted state: not verified) +# remove the fallback, set enforced = true, plan again: expect no changes +``` + +## Pitfalls + +- Bucket versioning off: a bad `state push` or delete is unrecoverable. +- One bucket and key shared by two roots: the second `init` adopts the first root's state. +- Putting the passphrase default in `variable` (as the docs sample does) - supply it through the environment. +- MinIO and `use_lockfile`: confirm the server honours conditional writes before trusting the lock (not verified). + +## Verification + +```bash +tofu init -reconfigure && tofu state list | head # backend reachable, state readable +aws --endpoint-url https://minio.example.internal:9000 s3api get-bucket-versioning --bucket tfstate # Status: Enabled +``` + +## Sources + +- https://opentofu.org/docs/language/settings/backends/s3/ , .../pg/ , .../http/ , .../kubernetes/ +- https://opentofu.org/docs/language/state/encryption/ , https://opentofu.org/docs/cli/commands/force-unlock/ +- https://developer.hashicorp.com/terraform/language/backend/s3 + +## Not verified + +- MinIO behaviour for conditional writes, checksum support and `use_path_style` requirements; test on the actual server. +- `init -migrate-state` / `-reconfigure` semantics (read the init command page before moving state between backends). +- Terraform-side encryption options other than backend features. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/state-surgery.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/state-surgery.md new file mode 100644 index 0000000..5c3ddc0 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/state-surgery.md @@ -0,0 +1,80 @@ +--- +name: state-surgery +description: Changing what state tracks without destroying infrastructure - import, moved and removed blocks, state mv/rm/push, force-unlock, with exact effects and backups +when-to-use: Renaming or moving a resource, adopting existing infrastructure, forgetting an object, recovering from a lock or a broken state +keywords: import block, moved block, removed block, lifecycle destroy false, state mv, state rm, state push, force-unlock, workspace delete +priority: critical +related: state-backends-encryption.md, plan-apply-protocol.md, modules-variables.md +--- + +# State surgery + +**Load when:** the config and the state disagree about names, ownership or existence. Order of preference: **declarative blocks reviewed in a plan** > CLI state commands > `state push`. + +## Rule 0 - backup before anything + +```bash +tofu state pull > "backup-$(date +%F-%H%M).tfstate" # keep it outside the repo +tofu state list > state-list-before.txt +``` + +Bucket versioning is the second net. With encryption on, the key is part of the backup (`state-backends-encryption.md`). + +## Declarative blocks (preferred) + +| Need | Block | Effect | +|------|-------|--------| +| Rename a resource/module, add `count`/`for_each` | `moved { from = a to = b }` | Before planning, the object at `from` is re-bound to `to`; no destroy. New environments without `from` ignore it. Not usable with `ephemeral` blocks | +| Adopt an existing object | `import { to = ADDR id = "..." }` (or `identity = {...}` where the resource type has an identity schema) | Plan shows the import; apply writes it to state. The `resource` block must exist (or `-generate-config-out`, experimental) | +| Forget an object, keep it running | `removed { from = ADDR lifecycle { destroy = false } }` | State entry removed at apply; remote object untouched. `from` cannot carry an instance key. OpenTofu warns if `lifecycle` is missing | +| Keep a resource in config but never destroy it | `lifecycle { destroy = false }` (OpenTofu 1.12) | `destroy` and replace plans "forget" it instead; persisted in state; `tofu destroy` exits non-zero when something was forgotten (`-suppress-forget-errors` silences) | +| Block destruction | `lifecycle { prevent_destroy = true }` (dynamic in 1.12) | Any plan that destroys it errors | + +Template: [templates/import-moved-removed.tf](templates/import-moved-removed.tf). Procedure: add the block, `plan -out`, confirm the plan says "will be imported/moved/forgotten" and **zero** unrelated create/destroy, show it, apply the saved plan after the go. Keep `moved` blocks for as long as any consumer's state may still hold the old address. + +## CLI state commands (each is a proposal) + +``` +ACTION tofu state mv 'proxmox_virtual_environment_vm.web' 'module.web.proxmox_virtual_environment_vm.this' +EFFECT rebinds the existing VM to the new address in state only; if the config is not changed to match, + the next plan destroys the VM at one address and creates it at the other +PROOF backup file exists; `tofu state mv -dry-run` output; plan after the move shows no changes +WAITING for your go +``` + +| Command | Exact effect | Guard | +|---------|--------------|-------| +| `tofu state mv [-dry-run] SRC DST` | Re-associates the remote object with a new address of the same kind and type | Tell collaborators: a plan between your code change and the move destroys and recreates | +| `tofu state rm [-dry-run] ADDR...` | State forgets the object; it keeps running; the next plan proposes to create a new one, which can fail on name conflicts | `-dry-run` first; prefer a `removed` block (reviewable) | +| `tofu import ADDR ID` / `import` block | Binds an existing object to a resource address | Run the plan afterwards; a non-empty diff means config drift to resolve before apply | +| `tofu state pull` / `state push [-force] FILE` | Read / overwrite the whole remote state. `push` refuses a different lineage or a lower serial; `-force` disables both checks and **overwrites the destination** | Only to restore a backup; never `-force` without the backup of the current remote state | +| `tofu force-unlock [-force] LOCK_ID` | Removes the backend lock without touching infrastructure; if the holder is alive, two writers can corrupt state | Prove the holder died (CI job gone, host destroyed); `pg` backend does not support it | +| `tofu workspace delete -force NAME` | Deletes a workspace that still tracks resources: they become dangling, running but unmanaged | Without `-force` it refuses; run `state list` in it first | +| `tofu state replace-provider` | Rewrites the provider address in state | Required when moving between provider namespaces; read the command page first | +| `-lock=false` (any state command) | Skips the lock | Never when others can run concurrently | + +## Pitfalls + +- A resource deleted from config with no `removed` block is destroyed at the next apply. +- Re-adding a forgotten resource at the same address makes the tool try to create it (conflict) - import it instead. +- `moved` across modules needs both module addresses; test with `plan` before sharing. +- `import` with the legacy `id` string differs per resource type: read the provider's Import section, do not guess the format (for Proxmox VMs and OpenStack resources see the provider references). + +## Verification + +```bash +tofu plan -out=tfplan -detailed-exitcode # after surgery: exit 0 (no changes) is the goal +tofu state list | diff - state-list-before.txt # only the intended addresses changed +``` + +## Sources + +- https://opentofu.org/docs/language/modules/develop/refactoring/ , https://opentofu.org/docs/language/import/ +- https://opentofu.org/docs/language/resources/syntax/ (removed block) , https://opentofu.org/docs/language/resources/behavior/ +- https://opentofu.org/docs/cli/commands/state/rm/ , .../state/mv/ , .../state/push/ , .../force-unlock/ , .../destroy/ , .../workspace/delete/ +- https://opentofu.org/docs/cli/state/recover/ + +## Not verified + +- Terraform's `removed` block and `import` `identity` support versions; the Terraform equivalents of `lifecycle { destroy = false }`. +- Exact `tofu state replace-provider` and `tofu import` flags (not fetched). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/encryption.tf b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/encryption.tf new file mode 100644 index 0000000..c432dc0 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/encryption.tf @@ -0,0 +1,43 @@ +# OpenTofu-only state and plan encryption. Terraform rejects this block: never run Terraform on this root. +# Source: https://opentofu.org/docs/language/state/encryption/ +# +# Before enabling on an existing state: tofu state pull > backup.tfstate (kept elsewhere), back up the passphrase. +# Supply the passphrase from the environment: export TF_VAR_state_passphrase=... (secret store), at least 16 characters. +# Step 1 (migration): apply once with the fallback below. Step 2: delete the fallback blocks, set enforced = true, apply again. +# Never rename the key provider or the methods after the first encrypted write. + +variable "state_passphrase" { + type = string + sensitive = true +} + +terraform { + encryption { + key_provider "pbkdf2" "main" { + passphrase = var.state_passphrase + } + + method "aes_gcm" "main" { + keys = key_provider.pbkdf2.main + } + + # Migration only: lets OpenTofu read the existing unencrypted state/plan. + method "unencrypted" "migrate" {} + + state { + method = method.aes_gcm.main + fallback { + method = method.unencrypted.migrate + } + # enforced = true # enable after the migration, with the fallback removed + } + + plan { + method = method.aes_gcm.main + fallback { + method = method.unencrypted.migrate + } + # enforced = true + } + } +} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/import-moved-removed.tf b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/import-moved-removed.tf new file mode 100644 index 0000000..c4cb879 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/import-moved-removed.tf @@ -0,0 +1,50 @@ +# Declarative state changes, reviewed in a plan and applied from a saved plan file. +# Procedure: tofu state pull > backup; add the blocks; tofu plan -out=tfplan; the plan must show only +# "will be imported", "has moved" or "will be removed from state" for these addresses and nothing else; +# show it, wait for the go, tofu apply tfplan. Docs: https://opentofu.org/docs/language/import/ and +# https://opentofu.org/docs/language/modules/develop/refactoring/ +# Addresses and IDs below are placeholders. Import ID formats are per resource type: read its Import section. + +# 1. Rename a resource without recreating it (old address -> new address). +moved { + from = openstack_networking_network_v2.app_net + to = openstack_networking_network_v2.app +} + +# 2. Turn a single resource into a for_each instance, keeping the existing object. +moved { + from = proxmox_virtual_environment_vm.web + to = proxmox_virtual_environment_vm.this["web"] +} + +# 3. Adopt an existing object. The resource block must exist (or use -generate-config-out, experimental). +import { + to = openstack_networking_secgroup_v2.legacy + id = "00000000-0000-0000-0000-000000000000" # UUID from: openstack security group list +} + +resource "openstack_networking_secgroup_v2" "legacy" { + name = "legacy" + description = "adopted from the console" +} + +# 4. Forget an object but leave it running (OpenTofu 1.7+, lifecycle block recommended). +# `from` cannot carry an instance key. The block can stay as a record or be removed after apply. +removed { + from = openstack_networking_floatingip_v2.old + + lifecycle { + destroy = false + } +} + +# 5. Keep a resource in the configuration but never destroy it (OpenTofu 1.12): destroy and replace plans +# forget it instead; tofu destroy then exits non-zero. Persisted in state: to destroy it later, remove the +# argument (or add a removed block with destroy = true) and apply again. +# resource "openstack_blockstorage_volume_v3" "data" { +# name = "data" +# size = 100 +# lifecycle { +# destroy = false +# } +# } diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/main.tftest.hcl b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/main.tftest.hcl new file mode 100644 index 0000000..e860553 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/main.tftest.hcl @@ -0,0 +1,64 @@ +# Plan-only tests for module-skeleton.tf (save both in one module directory as main.tf and main.tftest.hcl). +# `command = plan` never creates infrastructure; the default (`apply`) would. Run: tofu test +# Docs: https://opentofu.org/docs/cli/commands/test/ + +variables { + name = "web" + environment = "dev" + nics = [{ bridge = "vmbr0" }] +} + +run "defaults_produce_expected_name" { + command = plan + + assert { + condition = output.full_name == "dev-web" + error_message = "full_name must be -." + } + + assert { + condition = output.tags["managed-by"] == "opentofu" + error_message = "the managed-by tag is missing." + } +} + +run "prod_enables_protection" { + command = plan + + variables { + environment = "prod" + } + + assert { + condition = output.protect == true + error_message = "prod must request destroy protection." + } +} + +run "rejects_unknown_environment" { + command = plan + + variables { + environment = "test" + } + + expect_failures = [ + var.environment, + ] +} + +run "rejects_zero_cpu" { + command = plan + + variables { + cpu_cores = 0 + } + + expect_failures = [ + var.cpu_cores, + ] +} + +# For modules that create resources, add a mocked provider so plans need no platform access, for example: +# mock_provider "proxmox" {} +# then assert on planned values only (computed attributes such as ids need mock_resource defaults). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/module-skeleton.tf b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/module-skeleton.tf new file mode 100644 index 0000000..155ea62 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/module-skeleton.tf @@ -0,0 +1,97 @@ +# Reusable module skeleton: typed, described and validated inputs, locals, documented outputs. +# It declares no provider block and no resource, so `tofu test` (main.tftest.hcl) runs fully offline. +# Add resources of one provider below, declaring it in required_providers with a minimum version only. + +terraform { + required_version = ">= 1.11.0" +} + +variable "name" { + type = string + description = "Short guest name, lowercase letters, digits and dashes." + + validation { + condition = can(regex("^[a-z][a-z0-9-]{1,30}$", var.name)) + error_message = "name must start with a letter and contain only lowercase letters, digits and dashes (2 to 31 characters)." + } +} + +variable "environment" { + type = string + description = "Deployment environment." + + validation { + condition = contains(["dev", "staging", "prod"], var.environment) + error_message = "environment must be one of: dev, staging, prod." + } +} + +variable "cpu_cores" { + type = number + description = "vCPU count." + default = 2 + + validation { + condition = var.cpu_cores >= 1 && var.cpu_cores <= 32 && floor(var.cpu_cores) == var.cpu_cores + error_message = "cpu_cores must be a whole number between 1 and 32." + } +} + +variable "memory_mb" { + type = number + description = "Memory in MiB." + default = 2048 + + validation { + condition = var.memory_mb >= 512 && var.memory_mb % 256 == 0 + error_message = "memory_mb must be at least 512 and a multiple of 256." + } +} + +variable "nics" { + type = list(object({ + bridge = string + vlan_id = optional(number) + })) + description = "Network interfaces; the first entry is the primary NIC." + + validation { + condition = length(var.nics) >= 1 + error_message = "at least one NIC is required." + } + + validation { + condition = alltrue([for n in var.nics : n.vlan_id == null ? true : (n.vlan_id >= 1 && n.vlan_id <= 4094)]) + error_message = "vlan_id must be between 1 and 4094 when set." + } +} + +variable "extra_tags" { + type = map(string) + description = "Additional tags merged over the defaults." + default = {} +} + +locals { + full_name = "${var.environment}-${var.name}" + tags = merge( + { environment = var.environment, managed-by = "opentofu" }, + var.extra_tags, + ) + protect = var.environment == "prod" # feed lifecycle.prevent_destroy of module resources (dynamic since OpenTofu 1.12) +} + +output "full_name" { + description = "Environment-prefixed guest name." + value = local.full_name +} + +output "tags" { + description = "Effective tags." + value = local.tags +} + +output "protect" { + description = "True when destroy protection should be enforced by the resources of this module." + value = local.protect +} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/openstack-instance-network.tf b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/openstack-instance-network.tf new file mode 100644 index 0000000..9d88b32 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/openstack-instance-network.tf @@ -0,0 +1,115 @@ +# Project network, router, security group and one instance with the OpenStack provider (3.4.0). +# Arguments follow the provider docs: networking_network_v2, subnet_v2, router_v2, router_interface_v2, +# secgroup_v2, secgroup_rule_v2 and compute_instance_v2 (https://github.com/terraform-provider-openstack/terraform-provider-openstack/tree/main/docs/resources). +# Auth: clouds.yaml entry via var.cloud (OS_CLOUD). Prefer an application credential scoped to one project. +# Image and flavor are passed as IDs because they differ per cloud. + +terraform { + required_providers { + openstack = { + source = "terraform-provider-openstack/openstack" + version = "~> 3.4" + } + } +} + +variable "cloud" { + type = string + description = "Entry name in clouds.yaml." +} + +variable "external_network_id" { + type = string + description = "UUID of the external (provider) network used as router gateway." +} + +variable "image_id" { + type = string + description = "Glance image UUID." +} + +variable "flavor_id" { + type = string + description = "Nova flavor ID." +} + +variable "key_pair" { + type = string + description = "Name of an existing Nova key pair." +} + +variable "ssh_source_cidr" { + type = string + description = "CIDR allowed to reach SSH (never 0.0.0.0/0 in production)." + + validation { + condition = can(cidrhost(var.ssh_source_cidr, 0)) && var.ssh_source_cidr != "0.0.0.0/0" + error_message = "ssh_source_cidr must be a valid CIDR and not 0.0.0.0/0." + } +} + +provider "openstack" { + cloud = var.cloud +} + +resource "openstack_networking_network_v2" "app" { + name = "app-net" + admin_state_up = true +} + +resource "openstack_networking_subnet_v2" "app" { + name = "app-subnet" + network_id = openstack_networking_network_v2.app.id + cidr = "192.168.50.0/24" + ip_version = 4 +} + +resource "openstack_networking_router_v2" "app" { + name = "app-router" + admin_state_up = true + external_network_id = var.external_network_id +} + +resource "openstack_networking_router_interface_v2" "app" { + router_id = openstack_networking_router_v2.app.id + subnet_id = openstack_networking_subnet_v2.app.id +} + +resource "openstack_networking_secgroup_v2" "ssh" { + name = "app-ssh" + description = "SSH from the admin network" +} + +resource "openstack_networking_secgroup_rule_v2" "ssh" { + direction = "ingress" + ethertype = "IPv4" + protocol = "tcp" + port_range_min = 22 + port_range_max = 22 + remote_ip_prefix = var.ssh_source_cidr + security_group_id = openstack_networking_secgroup_v2.ssh.id +} + +resource "openstack_compute_instance_v2" "app" { + name = "app-01" + image_id = var.image_id + flavor_id = var.flavor_id + key_pair = var.key_pair + security_groups = [openstack_networking_secgroup_v2.ssh.name] + + network { + name = openstack_networking_network_v2.app.name + } + + # The subnet and router attachment must exist before the instance boots. + depends_on = [openstack_networking_router_interface_v2.app] + + metadata = { + managed-by = "opentofu" + } +} + +output "instance_id" { + description = "Nova server UUID; read its addresses with: openstack server show -c addresses." + value = openstack_compute_instance_v2.app.id +} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/proxmox-vm-cloudinit.tf b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/proxmox-vm-cloudinit.tf new file mode 100644 index 0000000..62d5762 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/proxmox-vm-cloudinit.tf @@ -0,0 +1,128 @@ +# VMs cloned from an existing cloud-init template with the bpg/proxmox provider (0.121.0). +# The template VM is created outside this stack (devops-proxmox cloud-init template, or Packer). +# Arguments follow https://github.com/bpg/terraform-provider-proxmox/blob/main/docs/resources/virtual_environment_vm.md +# Needs a provider "proxmox" block (see root-module.tf). `prevent_destroy = var.protect` needs OpenTofu 1.12+; +# on Terraform replace it by a literal true/false. + +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "~> 0.121.0" + } + } +} + +variable "node_name" { + type = string + description = "Proxmox node that hosts the VMs." +} + +variable "template_vm_id" { + type = number + description = "VMID of the cloud-init template to clone (must be a template with qemu-guest-agent installed when agent_enabled is true)." +} + +variable "datastore_id" { + type = string + description = "Datastore for the cloned disks and the cloud-init drive." + default = "local-lvm" +} + +variable "agent_enabled" { + type = bool + description = "Wait for the QEMU guest agent. When true and the agent is missing, refresh waits up to the agent timeout (15 minutes by default)." + default = true +} + +variable "ssh_public_key" { + type = string + description = "Public key injected through cloud-init (no password is set)." +} + +variable "protect" { + type = bool + description = "Block destruction of the VMs (true in prod)." + default = true +} + +variable "vms" { + type = map(object({ + vm_id = number + cores = optional(number, 2) + memory = optional(number, 2048) + bridge = optional(string, "vmbr0") + vlan_id = optional(number) + address = string # CIDR, for example 10.20.0.11/24 + gateway = string + })) + description = "VMs to create, keyed by name (stable keys: renaming a key replaces the VM)." +} + +variable "dns_servers" { + type = list(string) + default = ["10.20.0.2"] +} + +resource "proxmox_virtual_environment_vm" "this" { + for_each = var.vms + + name = each.key + node_name = var.node_name + vm_id = each.value.vm_id + tags = ["opentofu"] + + clone { + vm_id = var.template_vm_id + full = true + datastore_id = var.datastore_id + } + + agent { + enabled = var.agent_enabled + } + stop_on_destroy = !var.agent_enabled # a VM without agent cannot be shut down gracefully + + cpu { + cores = each.value.cores + type = "x86-64-v2-AES" + } + + memory { + dedicated = each.value.memory + } + + network_device { + bridge = each.value.bridge + vlan_id = each.value.vlan_id + } + + initialization { + datastore_id = var.datastore_id + + dns { + servers = var.dns_servers + } + + ip_config { + ipv4 { + address = each.value.address + gateway = each.value.gateway + } + } + + user_account { + username = "ubuntu" # match the template's default user + keys = [trimspace(var.ssh_public_key)] + } + } + + lifecycle { + prevent_destroy = var.protect + } +} + +output "vm_ids" { + description = "Proxmox VMIDs by VM name." + value = { for name, vm in proxmox_virtual_environment_vm.this : name => vm.vm_id } +} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/root-module.tf b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/root-module.tf new file mode 100644 index 0000000..685fda2 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/templates/root-module.tf @@ -0,0 +1,55 @@ +# Root module skeleton: pinned tool and providers, S3-compatible (MinIO) backend with native locking. +# Split into versions.tf / backend.tf / providers.tf / variables.tf in a real project. +# Credentials: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY from the environment (secret store), never here. +# Pair with encryption.tf (OpenTofu only). Review: enable bucket versioning before the first apply. + +terraform { + required_version = ">= 1.11.0" + + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "~> 0.121.0" # 0.x provider: minor versions may break; bump in its own change + } + } + + backend "s3" { + bucket = "tfstate" + key = "live/prod/compute/terraform.tfstate" # unique per root module + region = "us-east-1" # dummy value for non-AWS stores + + endpoints = { + s3 = "https://minio.example.internal:9000" + } + + use_path_style = true + use_lockfile = true # needs conditional-write support on the server: test it + + skip_credentials_validation = true + skip_requesting_account_id = true + skip_metadata_api_check = true + skip_region_validation = true + skip_s3_checksum = true # only if the store rejects checksum headers + } +} + +variable "pve_endpoint" { + type = string + description = "Proxmox VE API URL, for example https://pve.example.internal:8006/." +} + +variable "pve_api_token" { + type = string + description = "API token in the form user@realm!tokenid=secret, supplied via TF_VAR_pve_api_token." + sensitive = true +} + +provider "proxmox" { + endpoint = var.pve_endpoint + api_token = var.pve_api_token +} + +output "state_key" { + description = "Backend key of this root module, to eyeball the environment before an apply." + value = "live/prod/compute/terraform.tfstate" +} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/testing-quality.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/testing-quality.md new file mode 100644 index 0000000..81ea52c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/testing-quality.md @@ -0,0 +1,76 @@ +--- +name: testing-quality +description: Offline checks and tests for IaC - fmt, validate, tofu/terraform test with mock providers, tflint, checkov and trivy config scanning +when-to-use: Before proposing any plan, when adding a module, or when wiring checks into CI +keywords: tofu test, tftest.hcl, mock_provider, expect_failures, fmt, validate, tflint, checkov, trivy config, plan scan +priority: high +related: modules-variables.md, plan-apply-protocol.md, layout-providers-lock.md +--- + +# Testing and static quality + +**Load when:** HCL is written or reviewed. The ladder, cheapest first: `fmt` > `validate` > `tflint` > config scan > `test` (plan only, mocked) > `test` (apply, real infra, lab only) > reviewed `plan`. + +## Offline checks (no credentials, no network to the platform) + +```bash +tofu fmt -check -recursive # exit 0 = formatted; non-zero lists files (use -diff to see changes) +tofu init -backend=false # installs providers/modules, does not touch any backend +tofu validate # needs the initialized dir; checks syntax, types, references +``` + +## `tofu test` / `terraform test` + +- Runs every `*.tftest.hcl` / `*.tofutest.hcl` (and `.json`) in the module dir or `tests/` (`-test-directory=path`; `-filter=file` repeatable; `-verbose` prints each plan/state). +- A `run` block runs `plan` or `apply` (**default `apply`**, which creates real infrastructure and destroys it at the end) and evaluates `assert` blocks. Always set `command = plan` unless the test targets a throwaway lab. +- `expect_failures = [var.x]` asserts that a validation or check fails (negative tests); `variables {}` sets inputs; `override_resource`, `override_data`, `override_module`, `mock_provider` replace remote calls so tests run with no platform at all. +- OpenTofu: a `.tofutest.hcl` file wins over a same-named `.tftest.hcl`, letting a module keep one test per binary. +- Variables used in backend/module-source/encryption config must be given values when running `tofu test`. + +Template: [templates/main.tftest.hcl](templates/main.tftest.hcl) (plan-only, mocked provider, one negative test). + +Test pyramid for this audience: validation rules and naming logic with `command = plan` + `mock_provider` (fast, in CI); real Proxmox/OpenStack applies only in a lab project with its own credentials and quota, never against shared infrastructure. + +## Static analysis + +| Tool | Version read | Use | Command | +|------|--------------|-----|---------| +| `tflint` | 0.66.0 (supports Terraform 1.17) | Language rules, unused declarations, deprecated syntax; provider rulesets cover AWS/Azure/GCP - none read for Proxmox/OpenStack, so expect language rules only | `tflint --init && tflint --recursive` (flags: read `tflint --help`) | +| `trivy config` | 0.76.2 | Misconfiguration scan of Terraform, Dockerfile, Helm, Kubernetes manifests | `trivy config ./infra` | +| `checkov` | current on PyPI (not read) | Policy scan of `.tf` and of the JSON plan; plan scanning sees resolved values and `__change_actions__` (delete detection) | `checkov -d .` ; `tofu show -json tfplan > tfplan.json && checkov -f tfplan.json` | + +- Plan JSON can contain secrets: scan it inside the pipeline, never upload it as a public artifact. +- Scanner output is advice: a finding on a self-hosted target (for example "bucket not encrypted with KMS") may not apply; record a justified ignore (`.trivyignore` or inline skip) with the reason, never a blanket skip. +- Pin scanner versions in CI; they ship new rules continuously and break builds silently otherwise. + +## CI order (pointer: `devops-cicd`) + +`fmt -check` > `init -backend=false` > `validate` > `tflint` > `trivy config` / `checkov` > `test` > (with credentials) `plan -out` > review > `apply`. + +## Pitfalls + +- `tofu test` with the default `apply` against production credentials. +- Mock providers return generated values: assertions on computed attributes (`id`, IPs) need `mock_resource` defaults. +- `validate` passes while a provider rejects a value at plan: only plan/apply talk to the platform. +- Installing scanners with `curl | sh`: download release assets and verify checksums (tflint documents `gh attestation verify`). + +## Verification + +```bash +tofu test -verbose 2>&1 | tail -20 # run summary: N passed, 0 failed +tflint --version && trivy --version # pinned tool versions in use +``` + +## Sources + +- https://opentofu.org/docs/cli/commands/test/ , .../validate/ , .../fmt/ +- https://github.com/terraform-linters/tflint (README, release v0.66.0) +- https://trivy.dev/docs/latest/scanner/misconfiguration/ , https://github.com/aquasecurity/trivy/releases (v0.76.2) +- https://www.checkov.io/7.Scan%20Examples/Terraform%20Plan%20Scanning.html + +## Not verified + +- tflint `--recursive` and `--init` flags beyond what the README shows (`tflint --init` is documented there; `--recursive` was not read). +- Whether any maintained tflint or checkov rule pack exists for Proxmox or OpenStack resources. +- `terraform test` differences from `tofu test` (the Terraform command page was not read). +- Checkov's current version number. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/tofu-vs-terraform.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/tofu-vs-terraform.md new file mode 100644 index 0000000..ad2acbb --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-iac/references/tofu-vs-terraform.md @@ -0,0 +1,79 @@ +--- +name: tofu-vs-terraform +description: Choosing OpenTofu or Terraform - license, governance, current versions, feature divergence and migration +when-to-use: Starting a project, reviewing which binary a repo uses, or writing HCL that must run on both +keywords: opentofu, terraform, BSL, license, state encryption, ephemeral, write-only, exclude, migration +priority: high +related: state-backends-encryption.md, layout-providers-lock.md, testing-quality.md +--- + +# OpenTofu or Terraform + +**Load when:** a new stack is started, a repo mixes both binaries, or a feature must be chosen by version. + +## Decision rules + +| Question | Answer | +|----------|--------| +| Need state/plan encryption in the tool itself? | OpenTofu (1.7+). Terraform: no equivalent found in the docs read (not verified as absent) | +| Need Terraform-only features (actions, `terraform query`, Terraform Policy)? | Terraform 1.14+ / 1.17 | +| Need a vendor-neutral project outside the BSL? | OpenTofu (created in response to the BSL switch, given to the Linux Foundation, per its FAQ) | +| Internal use of Terraform? | Allowed by the BSL additional use grant; offering Terraform to third parties as a competing paid hosted/embedded product is not | +| Existing Terraform state at 1.5.x or earlier? | Migrate with the OpenTofu guide; later Terraform states need the guide's per-version notes | +| Team has no preference | OpenTofu for self-hosted Proxmox/OpenStack work: same providers, encryption included | + +Default recommendation for this plugin's audience: **OpenTofu**, pinned to the 1.13 line, unless the repo already runs Terraform. Do not switch a live state on your own: migration is a proposal with a state backup (`state-surgery.md`). + +## Versions read on 2026-11-10 + +| Tool | Version | Source | +|------|---------|--------| +| OpenTofu | 1.13.9 (tag, 2026-11); 1.12.12 also tagged 2026-11-05 | https://github.com/opentofu/opentofu/tags | +| Terraform | 1.17.1 (2026-10-28); 1.16.7 (2026-10-22); 1.15 EOL 2026-10-21 (research doc, not re-read) | https://github.com/hashicorp/terraform/releases | +| Terraform license | BSL 1.1, licensor IBM, Terraform 1.6.0 or later, change license MPL 2.0 four years after publication | https://raw.githubusercontent.com/hashicorp/terraform/main/LICENSE | + +## Feature divergence (verified at the source) + +| Feature | OpenTofu | Terraform | +|---------|----------|-----------| +| State and plan encryption | 1.7 (`terraform { encryption {} }`, key providers include pbkdf2, aws_kms, gcp_kms, external) | not found | +| Provider-defined functions | 1.7 (plus dynamic ones, OpenTofu-only) | exists (not re-verified here) | +| `removed` block, loopable `import` | 1.7 | `removed`/`import` exist (not re-verified) | +| Provider `for_each`, `-exclude` | 1.9 | not found | +| OCI registry for providers/modules, native S3 locking, `-exclude-file` | 1.10 | S3 `use_lockfile` documented; DynamoDB locking deprecated | +| Ephemeral resources, write-only attributes, `enabled` meta-argument | 1.11 | write-only needs 1.11+; `enabled` not found | +| Dynamic `prevent_destroy`, `lifecycle { destroy = false }`, full `h1:`+`zh:` lock hashes on `init` | 1.12 | not found | +| Actions (`action_trigger`), list resources (`*.tfquery.hcl`), `terraform query` | not found | 1.14 | +| Terraform Policy GA, variables in provider requirements, `-minimal-refresh` | not found | 1.17 | +| `.tofu` / `.tofutest.hcl` file extensions take precedence over same-named `.tf` / `.tftest.hcl` | yes | no | + +## Writing HCL that runs on both + +- Keep OpenTofu-only blocks (`encryption`, `enabled`, `destroy = false`) in separate files and say so in the PR. +- `required_version = ">= 1.11.0"` lets both binaries use ephemeral and write-only features; the Proxmox provider doc states write-only needs Terraform 1.11+ or OpenTofu 1.10+, while the OpenTofu 1.11 announcement introduces them in 1.11 - use 1.11. +- Registries differ: OpenTofu installs from `registry.opentofu.org`; `required_providers` sources are the same strings. Lock files are per binary - commit the one you run. + +## Pitfalls + +- Treating the two as drop-in forever: after Terraform 1.5.x the projects diverge; read the migration guide for the Terraform version in use. +- `terragrunt` picks a binary via `--tf-path`/`TG_TF_PATH` when both are installed; set it explicitly. + +## Verification + +```bash +tofu version; terraform version # which one is installed; never assume +grep -rn "encryption\|lifecycle" --include=*.tf . | head # OpenTofu-only syntax present? +``` + +## Sources + +- https://opentofu.org/faq/ , https://opentofu.org/docs/intro/migration/ +- https://opentofu.org/blog/opentofu-1-7-0/ , .../1-9-0/ , .../1-10-0/ , .../1-11-0/ , .../1-12-0/ +- https://github.com/hashicorp/terraform/releases , https://raw.githubusercontent.com/hashicorp/terraform/v1.14/CHANGELOG.md +- https://developer.hashicorp.com/terraform/language/block/ephemeral , https://developer.hashicorp.com/terraform/language/backend/s3 + +## Not verified + +- Whether Terraform has any state-encryption feature other than backend-side encryption. +- Terraform `removed`/`import`/provider-defined-function version numbers and the exact Terraform EOL policy. +- Which license text OpenTofu itself ships (not read); governance is only stated as Linux Foundation per the OpenTofu FAQ. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/SKILL.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/SKILL.md new file mode 100644 index 0000000..cabbf20 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/SKILL.md @@ -0,0 +1,161 @@ +--- +name: devops-k8s-cluster +description: Use when building, upgrading, repairing or choosing the Kubernetes cluster itself - kubeadm, k3s or Talos (RKE2 and Cluster API as pointers), HA control plane and API endpoint, etcd health/backup/restore, version skew and upgrade order, node join/drain/replace/remove, certificate expiry, Cilium as CNI (kube-proxy replacement, Gateway API support), bare-metal LoadBalancer IPs (Cilium LB IPAM and L2 announcements, MetalLB, kube-vip) and cluster-level troubleshooting (NotReady nodes, etcd quorum loss, API server down, CNI broken). Not for workloads, Services, Gateway resources, RBAC or day-to-day kubectl (devops-kubernetes), Helm/Kustomize/Argo CD/Flux (devops-gitops-packaging), CSI backend internals (devops-storage-backup) or VM provisioning (devops-proxmox, devops-virtualization). +versions: + kubernetes: "1.37.2 (supported minors 1.37, 1.36, 1.35)" + k3s: "v1.37.2+k3s1" + talos: "v1.14.5" + cilium: "1.20.4" + metallb: "v0.16.2" + etcd: "v3.7.3 (GitHub tag, single source)" + kubeadm-config-api: "kubeadm.k8s.io/v1beta4" +user-invocable: true +references: + - references/distribution-choice.md + - references/kubeadm-ha-bootstrap.md + - references/k3s-cluster.md + - references/talos-cluster.md + - references/etcd-backup-restore.md + - references/version-skew-upgrades.md + - references/node-lifecycle.md + - references/cni-cilium.md + - references/baremetal-loadbalancing.md + - references/cluster-troubleshooting.md + - references/templates/kubeadm-ha-config.yaml + - references/templates/k3s-server-config.yaml + - references/templates/talos-patches.yaml + - references/templates/cilium-values.yaml + - references/templates/cilium-lb-ipam-l2.yaml + - references/templates/metallb-l2.yaml + - references/templates/etcd-snapshot-restore.md + - references/templates/cluster-upgrade-checklist.md +related-skills: + - devops-kubernetes + - devops-gitops-packaging + - devops-storage-backup + - devops-networking + - devops-linux + - devops-proxmox + - devops-virtualization + - devops-openstack + - devops-iac + - devops-ansible + - devops-observability + - devops-secrets + - devops-sre-ops +--- + + +Covers the cluster itself: choosing a distribution, bootstrapping an HA control +plane, etcd backup/restore/health, version skew and upgrades, node lifecycle, +certificate expiry, Cilium as the default CNI, bare-metal LoadBalancer IPs and +cluster-level troubleshooting. Does not cover workloads, Services, Gateway +resources, RBAC or kubectl usage (devops-kubernetes), packaging and GitOps +(devops-gitops-packaging), CSI backends (devops-storage-backup) or VM +provisioning (devops-proxmox, devops-virtualization). + + +# DevOps Kubernetes Cluster + +## Agent Workflow (MANDATORY) + +Before ANY cluster change, spawn in parallel, one `Task` call each: + +1. **fuse-ai-pilot:explore-codebase** - find existing cluster config (kubeadm YAML, k3s `config.yaml`, Talos patches, inventories, Terraform/Ansible that built the nodes) +2. **fuse-ai-pilot:research-expert** - verify flags and versions at kubernetes.io, docs.k3s.io, docs.siderolabs.com, docs.cilium.io, etcd.io +3. **mcp__context7__query-docs** - `/websites/etcd_io`, `/websites/cilium_io_en_stable`, `/websites/kubernetes_io` + +Then inspect read-only first (`kubectl get nodes -o wide`, `kubectl version`, etcd health, certificate expiry). After changes to files, run **fuse-ai-pilot:sniper**. + +## Versions (checked 2026-11-10) + +Kubernetes v1.37.2 / v1.36.6 / v1.35.10 are the latest patches; supported minors are 1.37, 1.36, 1.35 (https://kubernetes.io/releases/version-skew-policy/). k3s v1.37.2+k3s1, Talos v1.14.5 (docs.siderolabs.com/talos/v1.14), Cilium 1.20.4, MetalLB v0.16.2. etcd v3.7.3 comes from a GitHub tag only; the Kubernetes etcd page names 3.4.29+ and 3.5.11+ as the minimum for production. Cilium documents Gateway API CRDs v1.6.3 although Gateway API v1.7.0 exists: install the version Cilium documents. Kubernetes 1.37 kubelet defaults to cgroup v2 only (`FailCgroupV1: true`). + +## Overview + +| Area | Decision in one line | +|------|----------------------| +| **Distribution** | kubeadm = upstream, you own OS and etcd; k3s = single binary, embedded etcd HA; Talos = immutable, API-only, no SSH; RKE2 and Cluster API are pointers ([distribution-choice.md](references/distribution-choice.md)) | +| **Control plane** | 3 stacked nodes minimum behind a TCP endpoint (VIP or load balancer); kubeadm `controlPlaneEndpoint` must equal that address | +| **etcd** | Odd member count; 3 tolerate 1 loss, 5 tolerate 2; snapshot on a schedule, off the node, restore-tested | +| **Upgrades** | One minor at a time, API server first, kubelet may lag up to three minors; drain before a minor kubelet upgrade | +| **CNI** | Cilium by default; install it before joining more nodes; kube-proxy replacement needs `k8sServiceHost` and `k8sServicePort` | +| **Bare-metal LB** | Cilium LB IPAM + L2 announcements if Cilium is the CNI; MetalLB otherwise; kube-vip for the control-plane VIP | +| **Troubleshooting** | kubelet and container runtime first, then etcd, then API server, then CNI ([cluster-troubleshooting.md](references/cluster-troubleshooting.md)) | + +## Critical Rules + +1. **Version first** - read server, kubelet and etcd versions before any upgrade; never skip a minor ([version-skew-upgrades.md](references/version-skew-upgrades.md)) +2. **Backup before touching etcd** - a fresh snapshot, copied off the node, with `etcdutl snapshot status` output attached ([etcd-backup-restore.md](references/etcd-backup-restore.md)) +3. **Destructive verbs are proposed, not run** - `kubeadm reset`, `etcdutl snapshot restore`, `etcdctl member remove`, `talosctl reset`, `talosctl bootstrap --recover-from`, `k3s server --cluster-reset`, `kubectl delete node`, `k3s-uninstall.sh`: show ACTION / EFFECT / QUORUM / PROOF and wait for the user's go +4. **Quorum arithmetic before any member change** - state members, quorum and how many can still fail after the action +5. **One control-plane node at a time** - verify `Ready` and etcd health before the next +6. **Explicit target** - name the node and the kubeconfig/context in every command; Talos commands carry `--nodes` +7. **Never invent a flag or config key** - cite the official page; unknown = "not verified" +8. **Secrets stay secret** - kubeadm certificate key, join tokens, etcd snapshots (they hold the CA keys and all Secrets) and Talos `secrets.yaml` are never printed or committed + +## Architecture + +``` +cluster-repo/ +├── kubeadm/ClusterConfiguration + InitConfiguration # or k3s/config.yaml, or talos/patches/ +├── cni/cilium-values.yaml # pinned chart version +├── lb/pool + announcement policy # Cilium LB IPAM/L2 or MetalLB +├── backup/etcd-snapshot job + restore runbook +└── upgrades/checklist per minor +``` + +## Reference Guide + +### Concepts + +| Topic | Reference | When to Consult | +|-------|-----------|-----------------| +| **Distribution choice** | [distribution-choice.md](references/distribution-choice.md) | New cluster, kubeadm vs k3s vs Talos, RKE2, Cluster API | +| **kubeadm HA** | [kubeadm-ha-bootstrap.md](references/kubeadm-ha-bootstrap.md) | init/join, stacked vs external etcd, endpoint, tokens | +| **k3s** | [k3s-cluster.md](references/k3s-cluster.md) | Embedded-etcd HA, config file, snapshots, uninstall | +| **Talos** | [talos-cluster.md](references/talos-cluster.md) | Machine config, bootstrap, upgrades, disaster recovery | +| **etcd** | [etcd-backup-restore.md](references/etcd-backup-restore.md) | Health, snapshot, restore, member replacement, defrag | +| **Upgrades and certificates** | [version-skew-upgrades.md](references/version-skew-upgrades.md) | Skew limits, upgrade order, certificate renewal | +| **Node lifecycle** | [node-lifecycle.md](references/node-lifecycle.md) | Join, cordon/drain, replace, remove, reset | +| **CNI** | [cni-cilium.md](references/cni-cilium.md) | Cilium install, kube-proxy replacement, Gateway API support | +| **Bare-metal LB** | [baremetal-loadbalancing.md](references/baremetal-loadbalancing.md) | LoadBalancer IPs, ARP/BGP, control-plane VIP | +| **Troubleshooting** | [cluster-troubleshooting.md](references/cluster-troubleshooting.md) | NotReady node, quorum loss, API server, CNI | + +### Templates + +| Template | When to Use | +|----------|-------------| +| [kubeadm-ha-config.yaml](references/templates/kubeadm-ha-config.yaml) | `kubeadm init --config` for an HA control plane (v1beta4) | +| [k3s-server-config.yaml](references/templates/k3s-server-config.yaml) | `/etc/rancher/k3s/config.yaml` for an HA server | +| [talos-patches.yaml](references/templates/talos-patches.yaml) | Talos patch: no default CNI, no kube-proxy, install disk | +| [cilium-values.yaml](references/templates/cilium-values.yaml) | Helm values: kube-proxy replacement, L2 announcements, Gateway API | +| [cilium-lb-ipam-l2.yaml](references/templates/cilium-lb-ipam-l2.yaml) | LoadBalancer IP pool and L2 announcement policy | +| [metallb-l2.yaml](references/templates/metallb-l2.yaml) | MetalLB address pool and L2 advertisement | +| [etcd-snapshot-restore.md](references/templates/etcd-snapshot-restore.md) | Snapshot, verify, restore sequence with proposal blocks | +| [cluster-upgrade-checklist.md](references/templates/cluster-upgrade-checklist.md) | Minor-version upgrade checklist | + +## Quick Reference + +```bash +kubectl get nodes -o wide # Ready? versions? InternalIP correct? +kubectl get --raw='/readyz?verbose' # API server readiness checks +kubeadm certs check-expiration # kubeadm clusters, on a control-plane node +sudo kubeadm upgrade plan # read-only: what can be upgraded +ETCDCTL_API=3 etcdctl --endpoints=$EP --cacert=$CA --cert=$CRT --key=$KEY endpoint status -w table +talosctl -n etcd status # Talos clusters +``` + +## Best Practices + +### DO +- Put the control-plane endpoint on a DNS name that resolves to the VIP or load balancer, set once at `kubeadm init` +- Keep etcd on SSD-backed, dedicated or isolated disks; heartbeat timeouts under disk or network starvation cause leader loss +- Pin Cilium, MetalLB and Talos versions and read each release's upgrade notes before bumping +- Rebalance CoreDNS after the control-plane nodes have joined (`kubectl -n kube-system rollout restart deployment coredns`) + +### DON'T +- Run an even number of etcd members, or auto-scale etcd +- Remove kube-proxy on a live cluster without a plan: existing Service connections break +- Run `kubeadm reset` on a node whose etcd member you have not accounted for in the quorum arithmetic +- Restore an etcd snapshot while any API server is running diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/baremetal-loadbalancing.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/baremetal-loadbalancing.md new file mode 100644 index 0000000..39c4007 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/baremetal-loadbalancing.md @@ -0,0 +1,90 @@ +--- +name: baremetal-loadbalancing +description: Load balancing on bare metal - control-plane VIP (kube-vip, keepalived + HAProxy) and LoadBalancer Service IPs (Cilium LB IPAM + L2 announcements, MetalLB) +when-to-use: A cluster outside a cloud needs a stable API endpoint or Services of type LoadBalancer get no external IP +keywords: kube-vip, metallb, cilium lb ipam, l2 announcements, CiliumLoadBalancerIPPool, CiliumL2AnnouncementPolicy, IPAddressPool, L2Advertisement, strictARP, keepalived, haproxy +priority: high +related: cni-cilium.md, kubeadm-ha-bootstrap.md, cluster-troubleshooting.md +--- + +# Bare-Metal Load Balancing + +**Load when:** `EXTERNAL-IP` stays ``, or the API endpoint has no HA. Two separate problems, two separate components. Templates: [cilium-lb-ipam-l2.yaml](templates/cilium-lb-ipam-l2.yaml), [metallb-l2.yaml](templates/metallb-l2.yaml). The Service/Gateway objects that consume these IPs are in devops-kubernetes. + +## Decision + +| Problem | Pick | Notes | +|---------|------|-------| +| API server endpoint (port 6443, before any CNI exists) | **kube-vip static pod** (ARP or BGP) or **keepalived + HAProxy** (same subnet), or a hardware/virtual appliance | Must exist before `kubeadm init`; the address equals `controlPlaneEndpoint` | +| `type: LoadBalancer` Services, Cilium is the CNI with kube-proxy replacement | **Cilium LB IPAM + L2 announcements** (or BGP Control Plane) | One component less; L2 announcements are **beta** | +| `type: LoadBalancer` Services, other CNI or you want a mature standalone | **MetalLB** (L2 or BGP) | v0.16.2; FRR-K8s mode recommended for BGP, native mode for L2-only/simple BGP | +| Both control-plane VIP and Service IPs from one tool | **kube-vip** with `--controlplane --services` | Watches `LoadBalancer` Services and advertises via ARP/BGP | + +Never run two controllers that both assign or announce the same Service IPs (for example Cilium L2 announcements and MetalLB over one range); pick one per address range. + +## Control-plane VIP + +kube-vip as a static pod (https://kube-vip.io/docs/installation/static/): generate the manifest with the kube-vip image, write it to `/etc/kubernetes/manifests/kube-vip.yaml` on the first node **before** `kubeadm init --control-plane-endpoint `, then copy it to each additional control-plane node's manifest directory. + +```bash +export VIP=192.168.0.40 INTERFACE=ens160 KVVERSION= +kube-vip manifest pod --interface $INTERFACE --address $VIP --controlplane --services --arp --leaderElection | sudo tee /etc/kubernetes/manifests/kube-vip.yaml +``` + +(`kube-vip` is an alias for running the image with host networking, defined on the page for containerd or Docker.) keepalived (VRRP, health script curling `https://localhost:/healthz`) + HAProxy (TCP mode, TLS stays at the apiserver) is the long-established alternative; all hosts must share a subnet. + +## Cilium LB IPAM + L2 announcements + +```bash +helm upgrade cilium cilium/cilium --version 1.20.4 -n kube-system --reuse-values \ + --set l2announcements.enabled=true --set k8sClientRateLimit.qps= --set k8sClientRateLimit.burst= +kubectl apply -f cilium-lb-ipam-l2.yaml +kubectl get ippools ; kubectl -n kube-system get lease | grep cilium-l2announce +``` + +- LB IPAM is always present but dormant until the first `CiliumLoadBalancerIPPool` exists; it only **allocates** IPs. `CiliumL2AnnouncementPolicy` (`cilium.io/v2alpha1`) makes nodes answer ARP/NDP for them (`loadBalancerIPs: true`; both IP types default to false, so a policy without one does nothing). +- Prerequisites: kube-proxy replacement on; the announcing interfaces must be among Cilium's `devices`. Size `k8sClientRateLimit` - announcements raise API traffic (see the sizing section of the page). +- One node per Service answers ARP (a Kubernetes Lease per Service, first come first served): that node carries all north-south traffic for the IP and nodes can be unevenly loaded. Failover moves the Lease. +- **Incompatible with `externalTrafficPolicy: Local`** (IP may be announced on nodes without pods). +- Pools must not overlap (the later one is marked `Conflicting`); disabling a pool stops new allocations only; **editing a pool can reassign Service IPs** (GitHub issue 40358). +- Services need `loadBalancerClass` unset or `io.cilium/l2-announcer` to be announced by a policy. By default every IP in a CIDR is used; `allowFirstLastIPs: "No"` reserves the network/broadcast addresses. + +## MetalLB + +```bash +# only if kube-proxy runs in IPVS mode: set ipvs.strictARP: true in the kube-proxy ConfigMap (diff first) +kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.16.2/config/manifests/metallb-native.yaml +kubectl apply -f metallb-l2.yaml +``` + +Installs into `metallb-system` (controller Deployment + speaker DaemonSet); it stays idle until `IPAddressPool` and an `L2Advertisement` (or BGP resources) exist, all in that same namespace. An `L2Advertisement` without a pool selector applies to every pool. In L2 mode one node per IP answers ARP; limit candidates with `nodeSelectors`, prefer nodes with `preferredNodeSelectors`, or restrict `interfaces`. The speaker needs elevated privileges: on namespaces enforcing Pod Security Admission, label `metallb-system` accordingly. Check the manifest URL: it deploys from the repo at the named tag, and MetalLB advises a stable release for production. + +## Network preconditions + +L2 mode needs the client network and the nodes in the same broadcast domain (switch port security, ARP inspection, and some virtual switches suppress gratuitous ARP). BGP needs a peer router configured by devops-networking. On Proxmox/KVM bridges check that MAC/IP spoofing filters do not drop the announced MAC. + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| Pool range overlaps DHCP or a real host | IP conflict, flapping ARP | +| MetalLB with kube-proxy IPVS and no `strictARP` | Wrong ARP replies | +| `externalTrafficPolicy: Local` + Cilium L2 | Drops on nodes without pods | +| VIP address differs from `controlPlaneEndpoint` | Joins and kubeconfigs fail | +| Cilium L2 policy with both IP types left false | Nothing announced | + +## Verification + +`kubectl get svc -A | grep LoadBalancer` shows an IP; from a host on the same L2, `arping -I ` or `ip neigh` returns one MAC; `curl` to the IP reaches the backend; power off the announcing node in a rehearsal and measure how long the IP takes to move. + +## Sources (read 2026-11-10) + +- https://docs.cilium.io/en/stable/network/lb-ipam/ , https://docs.cilium.io/en/stable/network/l2-announcements/ +- https://metallb.io/installation/ , https://metallb.io/configuration/ , https://metallb.io/configuration/_advanced_l2_configuration/ +- https://kube-vip.io/docs/installation/static/ , https://github.com/kubernetes/kubeadm/blob/main/docs/ha-considerations.md + +## Not verified + +- kube-vip's current release (use the GitHub release list), its BGP mode, and the 6443 ARP behaviour with Kubernetes versions that changed kubeadm kubeconfigs. +- Cilium BGP Control Plane and MetalLB BGP/FRR-K8s configuration; MetalLB Helm values; MetalLB PSA label names. +- `arping` verification and switch-level ARP behaviour are operational practice. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/cluster-troubleshooting.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/cluster-troubleshooting.md new file mode 100644 index 0000000..8573dc1 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/cluster-troubleshooting.md @@ -0,0 +1,107 @@ +--- +name: cluster-troubleshooting +description: Cluster-level troubleshooting in fixed order - NotReady nodes, kubelet, API server unreachable, etcd quorum loss, CNI failures, expired certificates +when-to-use: Nodes are NotReady, kubectl cannot reach the cluster, the control plane is flapping, or pods hang in ContainerCreating on several nodes +keywords: NotReady, kubelet, journalctl, readyz, etcd no leader, quorum lost, cni, NetworkUnavailable, certificate expired, static pod +priority: critical +related: etcd-backup-restore.md, cni-cilium.md, node-lifecycle.md, version-skew-upgrades.md, baremetal-loadbalancing.md +--- + +# Cluster Troubleshooting + +**Load when:** the problem is the cluster, not one application (application-level debugging: devops-kubernetes `troubleshooting-playbook.md`). The order is fixed: **scope -> node agent -> control plane -> etcd -> network**. Every step is read-only until a fix is proposed; fixes that touch etcd, certificates or nodes follow the ACTION / EFFECT / QUORUM / PROOF / WAIT-FOR-GO blocks of the sibling files. + +## Step 0 - scope the blast radius + +```bash +kubectl get nodes -o wide # which nodes, which versions, which InternalIP +kubectl get --raw='/readyz?verbose' # needs a reachable API; [+] ok / [-] failed checks (etcd, informers) +kubectl -n kube-system get pods -o wide +kubectl get events -A --sort-by=.lastTimestamp | tail -n 30 +``` + +Cannot reach the API at all? Distinguish: endpoint/VIP/LB down (`nc -zv 6443`; test each control-plane IP directly), kubeconfig certificate expired, apiserver down, etcd down. Test `curl -k https://:6443/livez?verbose` on a node: HTTP 200 = alive; the verbose output names failing checks. + +## Step 1 - NotReady node + +```bash +kubectl describe node # Conditions: Ready, MemoryPressure, DiskPressure, PIDPressure, NetworkUnavailable; Taints; Events +kubectl get node -o yaml +``` + +A node that stops posting status gets `Unknown` conditions and the `node.kubernetes.io/unreachable` taints; its pods are evicted after about five minutes. Then on the node (SSH; Talos: `talosctl -n service`, `dmesg`, `logs`): + +```bash +systemctl status kubelet ; journalctl -xeu kubelet --no-pager | tail -n 100 +``` + +Typical causes: kubelet stopped or crash-looping (swap on, cgroup v1 on Kubernetes 1.37, wrong container runtime socket), runtime not running, disk or memory pressure, clock skew, expired kubelet client certificate (see the kubeadm troubleshooting page, "Kubelet client certificate rotation fails"), lost network to the endpoint. + +## Step 2 - control plane (kubeadm) + +Control-plane components are static pods whose manifests are in `/etc/kubernetes/manifests/`; the kubelet runs them, so a broken kubelet means a broken control plane. Check logs through the container runtime or `journalctl`; on non-systemd setups the logs are `/var/log/kube-apiserver.log`, `kube-scheduler.log`, `kube-controller-manager.log`. Failure modes (official page): apiserver down -> no changes to pods/services, existing pods keep running; apiserver backing store lost -> it will not become healthy; scheduler/controller-manager down -> same consequences as the apiserver. + +`kubeadm certs check-expiration` on every control-plane node: expired certificates make the apiserver or etcd refuse clients ([version-skew-upgrades.md](version-skew-upgrades.md)). + +## Step 3 - etcd + +```bash +etcdctl ... endpoint health --cluster ; etcdctl ... endpoint status --cluster -w table ; etcdctl ... member list -w table # certs as in etcd-backup-restore.md +talosctl -n ,, etcd status ; talosctl -n service etcd # Talos +k3s etcd-snapshot ls ; journalctl -u k3s # k3s +``` + +| Symptom | Meaning | Next | +|---------|---------|------| +| One member down, 3-member cluster | Still has quorum (2/3); cluster works | Replace the member; snapshot now ([etcd-backup-restore.md](etcd-backup-restore.md)) | +| `no leader`, flapping leader, slow requests | Disk or network starvation causing heartbeat timeouts; resource contention on the node or the VM host | Fix the cause (dedicated fast disk, noisy neighbours); do not restart members in a loop | +| Majority of members permanently lost | Quorum lost: no state change possible | Restore from snapshot after the user's go | +| `NOSPACE` alarm | Quota (2 GiB default) hit; writes stop | Defragment, raise quota, disarm alarm ([talos-cluster.md](talos-cluster.md) shows the Talos commands) | +| Member joined but mismatched cluster ID | Stale data directory from another cluster | Treat as a new member: remove, wipe data dir, add | + +If the majority is lost and a restore is the only way: stop, state members/quorum, find the latest snapshot and its hash, propose the restore block. Do not run `--force-new-cluster` as a first move. + +## Step 4 - network (CNI) + +Signs: nodes `NotReady` with the network plugin not ready, `NetworkUnavailable` condition, many pods `ContainerCreating` with sandbox/network errors, CoreDNS unreachable, Services unreachable but pods reachable. + +```bash +kubectl -n kube-system get pods -l k8s-app=cilium -o wide ; kubectl -n kube-system logs ds/cilium --tail=100 +kubectl -n kube-system exec ds/cilium -- cilium-dbg status +ls /etc/cni/net.d # one CNI config only; stale files from a previous CNI conflict +kubectl get nodes -o wide # InternalIP on the right interface +``` + +Common causes: no CNI installed after `kubeadm init` (expected `NotReady` until installed), kube-proxy removed before Cilium replaced it, wrong `k8sServiceHost/Port`, kernel too old, MTU or overlay port blocked between nodes (firewall), pod CIDR mismatch with `podSubnet`, two CNIs. For external traffic problems on bare metal see [baremetal-loadbalancing.md](baremetal-loadbalancing.md). + +## Fix discipline + +One change at a time; each change has a before/after read-only check; if a change makes it worse, roll back before stacking another (never the same fix twice). Restarting kubelet or a CNI agent on one node is low risk; touching etcd members, certificates or running `kubeadm reset` is not: propose with the blocks used in the sibling files. + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| Rebooting all control-plane nodes at once | Quorum lost until a majority is back | +| Deleting the Node object of a flapping node | Hides the symptom, loses events; fix kubelet first | +| Restoring etcd to "fix" a networking problem | Data loss for no reason | +| Treating `NotReady` after a fresh init as an error | It is expected until the CNI runs | +| Debugging pods before checking nodes and control plane | Wasted time | + +## Verification + +All nodes `Ready`, `/readyz?verbose` all `[+]`, etcd one leader and equal raft index, a new test Pod gets an IP and resolves a Service name, and the original symptom no longer occurs after a full reconcile interval. + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/tasks/debug/debug-cluster/ +- https://kubernetes.io/docs/reference/using-api/health-checks/ +- https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/ (`systemctl status kubelet`, `journalctl -xeu kubelet`) +- https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/ , https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/ + +## Not verified + +- `crictl` usage for inspecting control-plane containers and the kubeadm troubleshooting page (linked from the certificates page, not read). +- Causes listed for kubelet failures (swap, cgroup v1, runtime socket) combine documented requirements with operational experience; confirm against the kubelet log. +- `curl -k https://:6443/livez?verbose` works without credentials only when anonymous access to health endpoints is enabled (default RBAC); otherwise use credentials. +- "Mismatched cluster ID" handling is operational practice. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/cni-cilium.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/cni-cilium.md new file mode 100644 index 0000000..dc8da57 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/cni-cilium.md @@ -0,0 +1,102 @@ +--- +name: cni-cilium +description: Cilium as the default CNI - requirements, install order per distribution, kube-proxy replacement, Gateway API support at the cluster level, validation and upgrade rules +when-to-use: Choosing or installing the CNI, replacing kube-proxy, enabling Cilium's Gateway API controller, or debugging a node stuck NotReady because of the CNI +keywords: cilium, cni, kube-proxy replacement, kubeProxyReplacement, k8sServiceHost, gatewayAPI.enabled, helm, hubble, l7Proxy, flannel +priority: high +related: baremetal-loadbalancing.md, kubeadm-ha-bootstrap.md, k3s-cluster.md, talos-cluster.md, cluster-troubleshooting.md +--- + +# CNI: Cilium + +**Load when:** the cluster needs a CNI or the CNI is suspect. Values template: [cilium-values.yaml](templates/cilium-values.yaml). Gateway/HTTPRoute resources and NetworkPolicy objects belong to devops-kubernetes; here only the cluster-level switches. + +## Why Cilium by default + +One component gives pod networking, eBPF Service load balancing (kube-proxy replacement), network policy enforcement, LoadBalancer IP management with L2/BGP announcement, and a Gateway API controller. Flannel (k3s default) gives pod networking only. + +## Requirements (https://docs.cilium.io/en/stable/operations/system_requirements/) + +- AMD64 or AArch64; Linux kernel >= 5.10 (or 4.18 on RHEL 8.10) with the BPF options enabled (standard distribution kernels have them). Debian >= 10, Ubuntu >= 20.04, RHEL >= 8.6, Talos >= 1.5.0 are listed. +- Kernel-level features beyond the base need newer kernels: read the "advanced features" table before enabling a feature. +- `kubelet --node-ip` must be right on multi-homed nodes (`kubectl get nodes -o wide` shows `InternalIP`); kube-proxy replacement may not work otherwise. +- Pod CIDR consistency: the Cilium IPAM setting must match the kubeadm `podSubnet` when you use one. + +## Install order + +| Distribution | Before the CNI | +|--------------|----------------| +| kubeadm | `kubeadm init` with `proxy.disabled: true` in `ClusterConfiguration` (v1beta4) or `--skip-phases=addon/kube-proxy`; nodes stay `NotReady` until Cilium runs | +| k3s | `flannel-backend: none`, `disable-network-policy: true`, `disable-kube-proxy: true` on **all** servers | +| Talos | patch deletes `KubeFlannelCNIConfig` (v1.14+) and sets `KubeProxyConfig enabled: false`; apply before bootstrap; install Cilium within about ten minutes | + +```bash +helm repo add cilium https://helm.cilium.io/ +helm install cilium cilium/cilium --version 1.20.4 --namespace kube-system -f cilium-values.yaml +kubectl -n kube-system get pods -l k8s-app=cilium +``` + +With `kubeProxyReplacement: true` and no kube-proxy, the agent cannot use the in-cluster `kubernetes` Service, so `k8sServiceHost`/`k8sServicePort` must name the API server: a control-plane node IP and `6443` in Cilium's example. For an HA control plane use the `controlPlaneEndpoint` (VIP/DNS) so the agent survives the loss of one node. On Talos use `localhost:7445` (KubePrism) and the Talos-specific values (IPAM `kubernetes`, cgroup auto-mount off, drop `SYS_MODULE`) in the template. + +## Switching a LIVE cluster off kube-proxy (destructive) + +``` +ACTION kubectl -n kube-system delete ds kube-proxy ; kubectl -n kube-system delete cm kube-proxy ; (each node, root) iptables-save | grep -v KUBE | iptables-restore +EFFECT removing kube-proxy breaks existing Service connections and stops Service traffic until Cilium's replacement is serving; + the two mechanisms do not share NAT state, so toggling either on a running cluster drops live connections. Deleting the ConfigMap stops kubeadm re-adding it on upgrade +QUORUM none, but the API Service VIP is affected on nodes without a working replacement +PROOF Cilium installed with kubeProxyReplacement=true and `cilium-dbg status` reporting it True on every node first; maintenance window announced +WAIT-FOR-GO +``` + +On a new cluster with no traffic this is not an issue: skip kube-proxy from the start. + +## Validate + +```bash +kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement # True [...] +kubectl -n kube-system exec ds/cilium -- cilium-dbg service list # Service frontends/backends present +kubectl get nodes -o wide ; kubectl -n kube-system get pods -o wide # nothing stuck +``` + +`kubeProxyReplacement: true` makes the agent refuse to start when the kernel lacks support (good: fails loudly). The Helm default is `false` (ClusterIP in-cluster balancing only). + +## Gateway API controller (cluster-level) + +Prerequisites per the Cilium page: kube-proxy replacement on; `l7Proxy=true` (default); nodes provide iptables and the netfilter modules for the default TPROXY implementation (`bpf.tproxy=true` is beta); an environment that provides `LoadBalancer` Services (see [baremetal-loadbalancing.md](baremetal-loadbalancing.md)) or host-network mode (since Cilium 1.16). Install the Gateway API CRDs **at the version Cilium documents (v1.6.3 for 1.20.4)**, not the newest upstream release (v1.7.0 exists), with `kubectl apply --server-side` and the standard channel unless you need experimental fields. Then: + +```bash +helm upgrade cilium cilium/cilium --version 1.20.4 -n kube-system --reuse-values --set kubeProxyReplacement=true --set gatewayAPI.enabled=true +kubectl -n kube-system rollout restart deployment/cilium-operator ; kubectl -n kube-system rollout restart ds/cilium +kubectl get gatewayclass +``` + +Supported resources include GatewayClass, Gateway, HTTPRoute, GRPCRoute, TLSRoute, BackendTLSPolicy, ReferenceGrant (and TCP/UDP routes, ListenerSet). The Gateway/HTTPRoute manifests live in devops-kubernetes `services-gateway-api.md`. + +## Upgrading Cilium + +Pin the chart version, read the upgrade guide (https://docs.cilium.io/en/stable/operations/upgrade/) first - it carries breaking changes, e.g. Gateway API CRD and `TLSRoute` notes for 1.20 - upgrade after the control plane, and after `kubeadm upgrade apply` follow the "CNI provider" step. + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| Installing the newest Gateway API CRDs instead of the documented version | Controller/CRD mismatch | +| `helm upgrade` without `--reuse-values` or the full values file | Settings (KPR, L2 announcements) silently revert | +| Wrong `InternalIP` on a multi-homed node | KPR misroutes | +| Two CNIs (Flannel left on) | Config conflict in `/etc/cni/net.d` | +| Enabling L2 announcements without KPR | Documented prerequisite missing | + +## Sources (read 2026-11-10) + +- https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/ +- https://docs.cilium.io/en/stable/network/servicemesh/gateway-api/gateway-api/ (+ Context7 `/websites/cilium_io_en_stable`) +- https://docs.cilium.io/en/stable/operations/system_requirements/ +- https://docs.siderolabs.com/kubernetes-guides/cni/deploying-cilium.md +- https://docs.k3s.io/networking/basic-network-options + +## Not verified + +- The Cilium upgrade guide contents, Hubble setup, encryption (WireGuard/IPsec) and BGP Control Plane. +- Using the VIP/DNS endpoint as `k8sServiceHost` on kubeadm is inferred from the requirement (the page's example uses a control-plane IP). +- Context7 snippets show Gateway API CRD v1.6.1 and Helm 1.20.2 (older indexed copies); the fetched stable page says v1.6.3 and 1.20.4. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/distribution-choice.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/distribution-choice.md new file mode 100644 index 0000000..e341a89 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/distribution-choice.md @@ -0,0 +1,66 @@ +--- +name: distribution-choice +description: Choosing how to build the cluster - kubeadm, k3s, Talos, with RKE2 and Cluster API as pointers +when-to-use: Before creating a new cluster or when asked "which Kubernetes distribution" +keywords: kubeadm, k3s, talos, rke2, cluster api, distribution, immutable os, embedded etcd, ha +priority: high +related: kubeadm-ha-bootstrap.md, k3s-cluster.md, talos-cluster.md, version-skew-upgrades.md +--- + +# Distribution Choice + +**Load when:** a cluster is about to be created, rebuilt or migrated. The choice decides who owns the OS, etcd, upgrades and recovery. + +## Decision table + +| Need | Pick | Why (source) | +|------|------|--------------| +| Upstream behaviour, you manage Linux hosts, exact control of every component | **kubeadm** | Bootstraps control plane as static pods; stacked etcd by default; HA needs an external endpoint you provide | +| Few nodes, edge/homelab, single binary, fastest path to HA | **k3s** | HA = 3+ servers with embedded etcd via `--cluster-init`; config in `/etc/rancher/k3s/config.yaml`; Flannel by default | +| No SSH, immutable OS, API-managed nodes, reproducible machine config | **Talos** | Everything through the Talos API (`talosctl`); A-B image upgrades with rollback | +| Hardened, FIPS/government-style distribution from Rancher | **RKE2** (pointer) | https://docs.rke2.io/ - "Rancher's enterprise-ready next-generation Kubernetes distribution"; not covered here, read its docs before use | +| Many clusters, declarative lifecycle, GitOps-driven | **Cluster API** (pointer) | https://cluster-api.sigs.k8s.io/user/quick-start - needs a management cluster (a "real" cluster with backup/DR for production, at least v1.20.0) plus an infrastructure provider; `clusterctl` or the Cluster API Operator | +| Managed cloud Kubernetes | out of scope | EKS/GKE/AKS | + +## Rules of thumb + +1. **Pick the distribution before the CNI and the LB** - Talos and k3s ship defaults (Flannel for k3s; Talos installs Flannel unless you delete it) that you must disable to run Cilium ([cni-cilium.md](cni-cilium.md)). +2. **Ownership of the OS** - kubeadm: you patch Linux, container runtime, kubelet (devops-linux). Talos: the OS is part of the cluster and is upgraded with `talosctl upgrade`. k3s: one binary, but the host OS is still yours. +3. **etcd** - kubeadm stacked (default) needs a minimum of three control-plane nodes; external etcd needs three control plane hosts plus three etcd hosts (https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/). k3s embeds etcd with an odd number of servers. Talos runs etcd on the control-plane nodes. +4. **Kubernetes version lag** - k3s and Talos each document a supported Kubernetes range; check Talos' support matrix (https://docs.siderolabs.com/talos/v1.14/getting-started/support-matrix.md) and the k3s release for the minor you want before committing. +5. **Single control-plane node is a lab** - a lost disk is a lost cluster unless snapshots exist ([etcd-backup-restore.md](etcd-backup-restore.md)). Talos' own guide says snapshots matter even more with one control-plane node. +6. **Where the VMs come from** - Proxmox/KVM provisioning is devops-proxmox / devops-virtualization; node configuration at scale is devops-ansible / devops-iac. + +## What each choice costs you + +| | kubeadm | k3s | Talos | +|-|---------|-----|-------| +| Upgrade unit | per node: packages, `kubeadm upgrade`, kubelet restart | per node: install script or binary swap | `talosctl upgrade` per node (OS) and `talosctl upgrade-k8s` (Kubernetes) | +| API endpoint HA | you build it (VIP/LB) | optional fixed registration address, `--tls-san` | LB or multi-A DNS record, plus KubePrism for in-node access | +| Backup tool | `etcdctl snapshot save` | `k3s etcd-snapshot` (local or S3) | `talosctl etcd snapshot` | +| Node removal | drain, `kubectl delete node`, `kubeadm reset` | drain, delete node, uninstall script | drain, delete node, `talosctl reset` | + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| Choosing a distribution by blog post without reading its version-support page | Kubernetes minor you need is not offered yet, or is nearly end of life (1.34 ended 2026-10-27) | +| Mixing distributions in one cluster | No supported upgrade or recovery path | +| Starting with one control-plane node "to be expanded later" on kubeadm | Needs `controlPlaneEndpoint` set at `kubeadm init` to expand cleanly - set it from day one ([kubeadm-ha-bootstrap.md](kubeadm-ha-bootstrap.md)) | + +## Verification + +State the choice with its reasons, the Kubernetes minor, the number of control-plane nodes (odd), and the answer to "where is the etcd snapshot stored and who tested the restore?". + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/ +- https://docs.k3s.io/datastore/ha-embedded , https://docs.k3s.io/networking/basic-network-options +- https://docs.siderolabs.com/talos/v1.14/overview/what-is-talos.md , https://docs.siderolabs.com/talos/v1.14/getting-started/prodnotes.md +- https://docs.rke2.io/ , https://cluster-api.sigs.k8s.io/user/quick-start + +## Not verified + +- RKE2 install, HA and upgrade procedures (only the one-line description was read). +- Cluster API provider list, current Cluster API version (v1.14.4 from a GitHub tag only) and the Proxmox/OpenStack provider status. +- Talos support-matrix contents (page named, not read). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/etcd-backup-restore.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/etcd-backup-restore.md new file mode 100644 index 0000000..b1a8cc0 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/etcd-backup-restore.md @@ -0,0 +1,120 @@ +--- +name: etcd-backup-restore +description: etcd for Kubernetes - quorum arithmetic, health checks, snapshots, restore, replacing a failed member, defragmentation +when-to-use: Before any control-plane change, on a schedule for backups, and during quorum loss or etcd member failure +keywords: etcd, etcdctl, etcdutl, snapshot save, snapshot restore, member remove, member add, quorum, defrag, bump-revision, mark-compacted +priority: critical +related: kubeadm-ha-bootstrap.md, node-lifecycle.md, cluster-troubleshooting.md, k3s-cluster.md, talos-cluster.md +--- + +# etcd Backup and Restore + +**Load when:** touching etcd, planning backups, or etcd is unhealthy. k3s and Talos have their own wrappers ([k3s-cluster.md](k3s-cluster.md), [talos-cluster.md](talos-cluster.md)); the raw `etcdctl`/`etcdutl` flow below is for kubeadm (stacked or external). Runbook template: [etcd-snapshot-restore.md](templates/etcd-snapshot-restore.md). + +## Quorum arithmetic (state it before every member change) + +Quorum = floor(n/2)+1. Tolerated failures = n - quorum. + +| Members | Quorum | Can lose | +|---------|--------|----------| +| 1 | 1 | 0 | +| 3 | 2 | 1 | +| 5 | 3 | 2 | +| 4 | 3 | 1 (no better than 3) | + +Kubernetes recommends an odd count, a static five-member cluster for production at any supported scale, never auto-scaling etcd; 3 to 5 is the only reasonable scaling step. Below quorum the cluster cannot change its state: scheduled pods may keep running but no new pod can be scheduled. Minimum etcd versions for production: 3.4.29+ and 3.5.11+. + +## Tools + +`etcdctl` talks to a running cluster (snapshot **save**, member list, health). `etcdutl` works on data files (snapshot **status**, snapshot **restore**, defrag offline). `etcdctl snapshot status/restore` are deprecated since 3.5 and removed from 3.6: use `etcdutl`. + +## Health (read-only) + +kubeadm stacked etcd serves TLS on the control-plane node; certificates live in `/etc/kubernetes/pki/etcd/`. + +```bash +export ETCDCTL_API=3 +EP=https://127.0.0.1:2379 +CA=/etc/kubernetes/pki/etcd/ca.crt ; CRT=/etc/kubernetes/pki/etcd/server.crt ; KEY=/etc/kubernetes/pki/etcd/server.key +sudo etcdctl --endpoints=$EP --cacert=$CA --cert=$CRT --key=$KEY member list -w table +sudo etcdctl --endpoints=$EP --cacert=$CA --cert=$CRT --key=$KEY endpoint health --cluster +sudo etcdctl --endpoints=$EP --cacert=$CA --cert=$CRT --key=$KEY endpoint status --cluster -w table +``` + +Expect: all members `started`, one leader, equal raft index, DB size well under the quota (default 2 GiB, 8 GiB suggested maximum). Missing `etcdctl` on a kubeadm node: run it from the etcd image or install a release matching the server version. + +## Backup + +```bash +sudo etcdctl --endpoints=$EP --cacert=$CA --cert=$CRT --key=$KEY snapshot save /var/backups/etcd/snap-$(date +%F-%H%M).db +sudo etcdutl --write-out=table snapshot status /var/backups/etcd/snap-.db # HASH, REVISION, TOTAL KEYS, TOTAL SIZE +``` + +- Taking a snapshot from a live member does not hurt its performance. A snapshot copied from `member/snap/db` of a stopped member has no integrity hash and may miss data still in the WAL. +- The file holds every Secret and the cluster state: encrypt it, copy it **off the node and off the failure domain** (devops-storage-backup), keep several generations, schedule it (a CronJob on a control-plane node or a systemd timer). +- A backup counts only after a restore rehearsal on spare VMs; log the evidence. + +## Restore (destructive - propose, then the user's go) + +Official order: stop **all** API servers, restore state on all etcd members, restart API servers (and restart scheduler, controller-manager and kubelets so they drop stale state). Restoring under a running API server is unsupported. + +``` +ACTION etcdutl snapshot restore snap.db --data-dir /var/lib/etcd-restore [--bump-revision 1000000000 --mark-compacted] + (kubeadm stacked: run on EVERY control-plane node with a fresh --data-dir and the member-specific --name/--initial-* flags, + then point the etcd static pod's `etcd-data` hostPath at the new directory and update etcd.local.dataDir in the kubeadm-config ConfigMap) +EFFECT creates a NEW logical cluster from the snapshot: member and cluster IDs are overwritten so old members cannot join; + all writes after the snapshot are lost; clients may see revisions go backwards +QUORUM cluster is unavailable from the moment the API servers stop until a majority of restored members run +PROOF snapshot hash/revision from `etcdutl snapshot status` recorded; current state snapshotted first if etcd still answers; list of API servers to stop +WAIT-FOR-GO +``` + +- **Revision bump**: for Kubernetes use `--bump-revision --mark-compacted` so informer caches are invalidated (N must exceed the revisions written since the snapshot; the etcd docs suggest 1,000,000,000 for a week-old snapshot at under 1,500 writes/s). Documented in the etcd v3.6/v3.8 recovery pages; check the page for your etcd version. +- `--force-new-cluster` on `etcd` rewrites membership but is "strongly discouraged"; it panics if old members are alive. +- The exact stacked-kubeadm multi-node sequence (moving static pod manifests out of `/etc/kubernetes/manifests`, per-member `--initial-cluster` flags) is assembled from the two official pages; rehearse it first. +- After restore: `kubectl get nodes`, `kubectl get pods -A`, and compare object counts with what you expect; expect controllers to re-sync for a while. + +## Replace a failed member + +Replace failed members immediately and one by one. Sequence from the Kubernetes etcd page: get the member ID with `member list` from healthy endpoints; stop API servers pointing only at the broken member (or remove it from `--etcd-servers`); stop etcd on the broken node; `etcdctl member remove `; `etcdctl member add --peer-urls=https://:2380`; start the new member with `ETCD_INITIAL_CLUSTER_STATE=existing`. + +``` +ACTION etcdctl member remove +EFFECT the member is dropped from the cluster permanently; quorum shrinks immediately (3 -> 2 members: zero failures tolerated) +QUORUM state members and quorum before and after; never remove a healthy member of a 3-member cluster, never add a member before removing a dead one when quorum is already at risk +PROOF member list output with the target ID matching the dead node; fresh snapshot stored off-node +WAIT-FOR-GO +``` + +On a kubeadm cluster prefer joining a replacement control-plane node (`kubeadm join --control-plane`) and removing the dead node ([node-lifecycle.md](node-lifecycle.md)); `kubeadm reset` removes the local stacked member of that node by itself. + +## Maintenance + +Defragmentation is expensive and blocks the member while it runs; do it rarely, one member at a time (the Kubernetes docs recommend a tool such as `etcd-defrag` and show a CronJob). `NOSPACE` alarms mean the quota was hit: defrag, then raise `quota-backend-bytes` only to match need. Before an etcd upgrade take a snapshot. + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| 4 or 6 members | More cost, same tolerance | +| Backup only on the same disk/node | Lost together with the node | +| Restoring without stopping API servers | Unsupported; stale data stays cached | +| Skipping `--bump-revision` with live controllers | Controllers may see a time-warped history | +| etcd on shared slow disks or noisy VMs | Heartbeat timeouts, leader flapping, "no leader" and a frozen control plane | + +## Verification + +`endpoint health --cluster` all healthy; `snapshot status` of the newest backup prints a hash; restore rehearsal log exists; nodes `Ready` and workloads reconciled after a real restore. + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/ +- https://etcd.io/docs/v3.6/op-guide/recovery/ , https://etcd.io/docs/v3.6/faq/ +- Context7 `/websites/etcd_io` (v3.8 recovery page and cluster-status page for `--bump-revision`, `endpoint health/status`) +- https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-reset/ + +## Not verified + +- The exact certificate files to use with `etcdctl` on every kubeadm version (the Kubernetes page example uses `server.crt/server.key`; `etcd-healthcheck-client` certificates also exist). +- Whether the v3.7.3 line changed restore flags (docs read were v3.6 and v3.8). +- The recommended `--bump-revision` value for your write rate: computed by you, not by a source. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/k3s-cluster.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/k3s-cluster.md new file mode 100644 index 0000000..7c9a8b1 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/k3s-cluster.md @@ -0,0 +1,109 @@ +--- +name: k3s-cluster +description: k3s clusters - HA with embedded etcd, config file, agent join, upgrades, etcd snapshots and restore, uninstall +when-to-use: Building or operating a k3s cluster (servers, agents, snapshots, restore, removal) +keywords: k3s, cluster-init, embedded etcd, config.yaml, etcd-snapshot, cluster-reset, flannel-backend, tls-san, k3s-uninstall.sh +priority: high +related: etcd-backup-restore.md, version-skew-upgrades.md, node-lifecycle.md, cni-cilium.md +--- + +# k3s Cluster + +**Load when:** the cluster runs k3s (v1.37.2+k3s1 today). Template: [k3s-server-config.yaml](templates/k3s-server-config.yaml). + +## HA with embedded etcd + +- Three or more **server** nodes, an **odd** number. Quorum is n/2+1; adding a 4th server raises quorum without adding fault tolerance. +- Optional fixed registration address (VIP, load balancer or DNS) for agents; add it to every server's `tls-san`. + +```bash +# first server +curl -sfL https://get.k3s.io | K3S_TOKEN= sh -s - server --cluster-init --tls-san= +# servers 2 and 3 +curl -sfL https://get.k3s.io | K3S_TOKEN= sh -s - server --server https://:6443 --tls-san= +# agents +curl -sfL https://get.k3s.io | K3S_TOKEN= sh -s - agent --server https://:6443 +``` + +Prefer the config file `/etc/rancher/k3s/config.yaml` (drop-ins in `config.yaml.d/*.yaml`): the install script persists env vars and arguments into the service, and **re-running the script without them loses the original values**; the config file is not touched by the script. CLI arguments override the file. + +**Flags that must be identical on every server**: `--cluster-dns`, `--cluster-domain`, `--cluster-cidr`, `--service-cidr`, `--disable-helm-controller`, `--disable-kube-proxy`, `--disable-network-policy`, anything passed to `--disable`, `--secrets-encryption`. A mismatch fails with `critical configuration value mismatch`. Flannel options are server-only and must match too. + +A single-node SQLite cluster converts to etcd by restarting the server with `--cluster-init`. Embedded etcd on slow disks (SD cards) performs badly. + +## Networking + +Default CNI is Flannel (`vxlan`; `wireguard-native` encrypts). For Cilium: `flannel-backend: none` and `disable-network-policy: true`, install Cilium before agents are expected `Ready`, and add `disable-kube-proxy: true` for kube-proxy replacement ([cni-cilium.md](cni-cilium.md)). Dual-stack must be chosen at creation. + +## Upgrades + +Servers first, one at a time, then agents. The Kubernetes skew policy applies and the k3s docs state that the system-upgrade-controller does not protect against unsupported jumps, so never skip a minor ([version-skew-upgrades.md](version-skew-upgrades.md)). + +```bash +kubectl drain --ignore-daemonsets # optional but advised: the script does not drain +curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.37.2+k3s1 sh -s - +kubectl uncordon +``` + +Channels: `stable` (default), `latest`, or one per minor (`INSTALL_K3S_CHANNEL`). `INSTALL_K3S_SKIP_START=true` downloads without restarting. Pods keep running while k3s restarts. + +## Snapshots + +Scheduled snapshots exist alongside on-demand ones; default retention is 5, applied per server node for local files and cluster-wide for S3. Confirm the schedule is active with `k3s etcd-snapshot ls`. + +```bash +k3s etcd-snapshot save # on-demand, name prefix on-demand-- +k3s etcd-snapshot ls +k3s etcd-snapshot prune --snapshot-retention 5 +kubectl get etcdsnapshotfile # cluster-wide view of snapshots on all members +``` + +S3 flags (`--etcd-s3`, `--etcd-s3-endpoint`, `--etcd-s3-bucket`, `--etcd-s3-folder`, `--etcd-s3-retention`, ...) or a Secret of type `etcd.k3s.cattle.io/s3-config-secret` in `kube-system` (not usable during a restore: pass S3 flags on the CLI then). A snapshot contains the CA keys and all Secrets; whoever holds it plus the server token can extract encrypted data. Store it in a bucket with access control and server-side encryption. + +## Restore (destructive - propose, then the user's go) + +``` +ACTION systemctl stop k3s ; k3s server --cluster-reset --cluster-reset-restore-path= (on ONE server) +EFFECT current etcd files move to /server/db/etcd-old-/; the snapshot is extracted; every other etcd member is removed + from the cluster; membership is reset to this single node; a reset-flag file blocks a second reset +QUORUM cluster is a one-member etcd until the other servers rejoin; they must start with their old etcd database files removed +PROOF fresh snapshot of the current state first; snapshot name, size and k3s version recorded; server token saved +WAIT-FOR-GO +``` + +After "Managed etcd cluster membership has been reset, restart without --cluster-reset flag now", run `systemctl start k3s`. A snapshot from an older k3s that you could not upgrade directly from must not be restored. Restoring to **new hosts** needs the original server token (`/var/lib/rancher/k3s/server/token`) passed with `--token`, and old Node objects must be deleted by hand afterwards. + +## Removal + +``` +ACTION /usr/local/bin/k3s-uninstall.sh (server) or /usr/local/bin/k3s-agent-uninstall.sh (agent) +EFFECT stops K3s and all pods; deletes the local datastore, local-storage PV data, node config and the k3s binaries/scripts +QUORUM a server uninstall removes one etcd member: state members before/after; never take a 3-server cluster below 2 +PROOF node drained and deleted from the cluster first (removes the node password secret needed for a clean rejoin); snapshot copied off-node +WAIT-FOR-GO +``` + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| Re-running the install script with different/no arguments | Service args change silently | +| Server flags differ between servers | Join fails or components diverge | +| Snapshots only on the node's disk | Lost with the node; configure S3 or copy off-node | +| Rejoining a node with the same name without deleting the Node | Node password secret mismatch | + +## Verification + +`kubectl get nodes` shows the servers with roles `control-plane,etcd,master`; `k3s etcd-snapshot ls` shows fresh snapshots; a restore rehearsal on spare VMs succeeded. + +## Sources (read 2026-11-10) + +- https://docs.k3s.io/datastore/ha-embedded , https://docs.k3s.io/installation/configuration +- https://docs.k3s.io/cli/etcd-snapshot , https://docs.k3s.io/upgrades/manual +- https://docs.k3s.io/installation/uninstall , https://docs.k3s.io/networking/basic-network-options + +## Not verified + +- The "Multiple Servers" restore tab (page tabs did not render): the per-node steps for the other servers are inferred from the numbered restore description ("restart to rejoin after removing old database files"). Read the tab before executing a multi-server restore. +- The snapshot schedule flag names (the "Creating Snapshots" section was empty in the fetch). +- Certificate rotation for k3s. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/kubeadm-ha-bootstrap.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/kubeadm-ha-bootstrap.md new file mode 100644 index 0000000..e57935b --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/kubeadm-ha-bootstrap.md @@ -0,0 +1,106 @@ +--- +name: kubeadm-ha-bootstrap +description: Bootstrapping a highly available kubeadm control plane - API endpoint, init config, joining control-plane and worker nodes, tokens and certificate keys +when-to-use: Creating a kubeadm cluster, adding a control-plane node, or re-issuing join credentials +keywords: kubeadm init, kubeadm join, controlPlaneEndpoint, upload-certs, certificate-key, stacked etcd, external etcd, token create, v1beta4 +priority: critical +related: etcd-backup-restore.md, node-lifecycle.md, cni-cilium.md, baremetal-loadbalancing.md +--- + +# kubeadm HA Bootstrap + +**Load when:** a kubeadm control plane is created or extended. Template: [kubeadm-ha-config.yaml](templates/kubeadm-ha-config.yaml). + +## Decisions + +| Question | Rule | +|----------|------| +| Stacked or external etcd? | Stacked is the kubeadm default and needs at least three control-plane nodes; external needs three control-plane + three etcd hosts and decouples failures (https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/). Default to stacked unless etcd must be isolated | +| Endpoint | A DNS name resolving to a TCP load balancer or VIP in front of all control-plane nodes on `:6443`; the health check is a TCP check. Set it at init as `controlPlaneEndpoint`; the LB address must always match it | +| LB options | keepalived + HAProxy (same subnet), kube-vip static pod, or an external appliance ([baremetal-loadbalancing.md](baremetal-loadbalancing.md)) | +| Config style | A YAML file (`kubeadm.k8s.io/v1beta4`) with `--config`, not a pile of flags; the file can be kept in Git | +| CNI | Decide first; Cilium with kube-proxy replacement means skipping kube-proxy at init ([cni-cilium.md](cni-cilium.md)) | + +## Preconditions on every node + +- Swap disabled; container runtime installed and its socket known (`nodeRegistration.criSocket`). +- Hosts reach `registry.k8s.io` or the images are preloaded. +- Kubernetes packages from `pkgs.k8s.io` (the `apt.kubernetes.io` / `yum.kubernetes.io` repositories are frozen since 2023-09-13). Hold `kubeadm kubelet kubectl` so they upgrade only deliberately. +- kubelet on Kubernetes 1.37 supports cgroup v2 only by default (`FailCgroupV1: true`): check `stat -fc %T /sys/fs/cgroup` returns `cgroup2fs`. +- If a node has several interfaces, set the node IP explicitly (`nodeRegistration.kubeletExtraArgs` name `node-ip`); Cilium's kube-proxy replacement misbehaves with a wrong `InternalIP`. + +## First control-plane node + +```bash +nc -zv -w 2 6443 # refused is expected before init; a timeout means the LB cannot reach the node +sudo kubeadm init --config kubeadm-ha-config.yaml --upload-certs +``` + +- `--config` and `--certificate-key` cannot be mixed: put `certificateKey` in `InitConfiguration` if you want a fixed key (generate with `kubeadm certs certificate-key`). +- `--upload-certs` encrypts the shared control-plane certificates into the `kubeadm-certs` Secret; the Secret and the key expire after two hours, and the key "gives access to cluster sensitive data" - never paste it in a ticket. +- Save the printed `kubeadm join` commands. Re-issue later: + +```bash +sudo kubeadm init phase upload-certs --upload-certs # new certificate key, on an existing control-plane node +kubeadm token create --print-join-command # new worker join command (default TTL 24h) +``` + +Install the CNI now. Nothing else works until a CNI runs ([cni-cilium.md](cni-cilium.md)). Watch `kubectl get pod -n kube-system -w`. + +## Remaining control-plane nodes + +One at a time, waiting for `Ready` and etcd health between nodes: + +```bash +sudo kubeadm join :6443 --token --discovery-token-ca-cert-hash sha256: \ + --control-plane --certificate-key +``` + +`--control-plane` creates a new control-plane instance and, with stacked etcd, a local etcd member. Run `kubectl -n kube-system rollout restart deployment coredns` after at least one extra node joined, because all CoreDNS pods usually start on the first node. + +## Workers + +```bash +sudo kubeadm join :6443 --token --discovery-token-ca-cert-hash sha256: +``` + +## External etcd variant + +Build the etcd cluster first, copy `ca.crt`, `apiserver-etcd-client.crt` and `.key` to the first control-plane node, and set `etcd.external.endpoints/caFile/certFile/keyFile` in `ClusterConfiguration` (shown in the official HA page). Everything else is as above. + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| `controlPlaneEndpoint` omitted or set to the first node's IP | The cluster cannot become HA without regenerating certificates | +| LB is an HTTP proxy | The API server terminates TLS and uses mutual TLS: use a TCP load balancer | +| Joining two control-plane nodes at once | Racing etcd member adds; join sequentially | +| Certificate key expired (2 h) | `kubeadm join --control-plane` fails to download certs: re-run `upload-certs` | +| Even number of control-plane nodes | Same fault tolerance as one fewer, more ways to lose quorum | +| `kubeadm init` run again on an initialised node | Use `kubeadm reset` only after the effect is understood ([node-lifecycle.md](node-lifecycle.md)) | + +## Verification + +```bash +kubectl get nodes -o wide # all Ready, InternalIP as intended +kubectl -n kube-system get pods -o wide # 3x apiserver, controller-manager, scheduler, etcd +kubectl get --raw='/readyz?verbose' | tail -n 3 # "readyz check passed" +sudo kubeadm certs check-expiration # each control-plane node +``` + +Then run the etcd health check in [etcd-backup-restore.md](etcd-backup-restore.md) and take the first snapshot. + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/ +- https://kubernetes.io/docs/reference/config-api/kubeadm-config.v1beta4/ +- https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-token/ +- https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/ (package repositories, cgroup v2, swap) +- https://github.com/kubernetes/kubeadm/blob/main/docs/ha-considerations.md (keepalived + HAProxy) +- https://kubernetes.io/docs/reference/using-api/health-checks/ + +## Not verified + +- Container runtime installation steps and the exact `kubeadm join` flags for `JoinConfiguration` control-plane mode (only the CLI join form was read). +- The "joining sequentially" rule is stated by the official page as a recommendation for external etcd; applying it to stacked etcd is operational practice. +- `stat -fc %T /sys/fs/cgroup` is a common check, not read in an official page. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/node-lifecycle.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/node-lifecycle.md new file mode 100644 index 0000000..039ea50 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/node-lifecycle.md @@ -0,0 +1,93 @@ +--- +name: node-lifecycle +description: Node lifecycle - joining, maintenance (cordon/drain/uncordon), replacing and removing workers and control-plane nodes, kubeadm reset and its leftovers +when-to-use: Adding, patching, replacing or decommissioning a node, or recovering a node that was reset +keywords: kubeadm join, cordon, drain, uncordon, kubectl delete node, kubeadm reset, node replacement, etcd member, decommission +priority: high +related: etcd-backup-restore.md, kubeadm-ha-bootstrap.md, version-skew-upgrades.md, cluster-troubleshooting.md +--- + +# Node Lifecycle + +**Load when:** a node enters, leaves or is taken out temporarily. Workload-side effects of `drain` (PDBs, emptyDir, eviction) are detailed in devops-kubernetes `kubectl-safe-ops.md`; this file adds the cluster-infrastructure side. + +## Lifecycle at a glance + +| Step | kubeadm | k3s | Talos | +|------|---------|-----|-------| +| Join worker | `kubeadm join --token ... --discovery-token-ca-cert-hash ...` (new token: `kubeadm token create --print-join-command`) | `curl -sfL https://get.k3s.io \| K3S_TOKEN=... sh -s - agent --server https://:6443` | `talosctl apply-config --insecure --nodes --file worker.yaml` | +| Join control plane | `kubeadm join ... --control-plane --certificate-key ` | `... sh -s - server --server https://:6443` | `apply-config` with `controlplane.yaml` (bootstrap only on the first node, ever) | +| Maintenance | `kubectl drain --ignore-daemonsets` ... `kubectl uncordon ` | same | same, plus `talosctl upgrade`/`reboot` | +| Remove | drain -> `kubectl delete node` -> `kubeadm reset` | drain -> `kubectl delete node` -> `k3s-agent-uninstall.sh` / `k3s-uninstall.sh` | drain -> `kubectl delete node` -> `talosctl reset` | + +## Maintenance (reboot, kernel patch, runtime update) + +```bash +kubectl cordon # no new pods; reversible +kubectl get pdb -A # ALLOWED DISRUPTIONS must be > 0 for what runs there +kubectl drain --ignore-daemonsets # evicts; add --delete-emptydir-data only if the data is disposable +# ... patch / reboot ... +kubectl uncordon +``` + +Drain is **required** before a minor kubelet upgrade. Do worker nodes one at a time or in batches that leave capacity for the workloads. When a node is unreachable it becomes `NotReady`, taints are added, and pods are evicted after about five minutes ([cluster-troubleshooting.md](cluster-troubleshooting.md)). + +## Replace a control-plane node (kubeadm, stacked etcd) + +Order: **add first, remove second** so the member count never drops below quorum. + +1. Snapshot etcd and record `member list` ([etcd-backup-restore.md](etcd-backup-restore.md)). +2. Provision the new node, install the same package versions, `kubeadm join --control-plane` (re-issue the certificate key if older than two hours). +3. Verify the new member is `started` and the cluster is healthy. +4. Drain the old node, then remove it (below). If the old node is dead and its etcd member remains, remove that member explicitly with the quorum arithmetic stated. +5. Update the load balancer/VIP target list and monitoring. + +## Remove a node (propose, then the user's go) + +``` +ACTION kubectl --context delete node +EFFECT deletes the Node object; pods bound to it are garbage-collected; it does NOT stop kubelet or remove the host's state; + if kubelet is still running it re-registers itself +QUORUM none for workers; for a control-plane node the etcd member is NOT removed by this command +PROOF node drained, `kubectl get pods -A -o wide --field-selector spec.nodeName=` shows only DaemonSet pods; backups ok +WAIT-FOR-GO + +ACTION sudo kubeadm reset (on the node; add --dry-run first to see the plan; -f skips the confirmation prompt) +EFFECT phases preflight, remove-etcd-member, cleanup-node: reverts files created by init/join, and on a control-plane node removes + that node's local stacked etcd member from the etcd cluster +QUORUM with the replacement already joined: 4 -> 3 members (1 failure tolerated); without a replacement: 3 -> 2 (zero failures tolerated) - state which case applies +PROOF node drained and deleted from the API; `etcdctl member list` taken before; replacement control-plane node already joined +WAIT-FOR-GO +``` + +`kubeadm reset` leaves behind: **external etcd data** (clean it with `etcdctl del "" --prefix` only if you mean to wipe that etcd), **`/etc/cni/net.d`** (a stale CNI config conflicts with a different CNI on reuse; back up then remove), **iptables/nftables/IPVS rules** from kube-proxy (clean with `kube-proxy --cleanup` in the kube-proxy image, or reboot into a fresh image), and **`$HOME/.kube/config`**, whose admin credentials stay valid while the cluster exists - delete it. A control-plane node being reset should have the apiserver shut down gracefully first if `--shutdown-delay-duration` is configured. + +k3s: `k3s-uninstall.sh` deletes the local datastore and Local Storage PV data; delete the Node first so the node password secret goes away before a rejoin. Talos: `talosctl reset` semantics are in [talos-cluster.md](talos-cluster.md) (not verified here); wipe-by-label flags are shown in the disaster-recovery procedure. + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| `kubectl delete node` instead of drain | Pods lost without graceful termination handling | +| `kubeadm reset` on a control-plane node with no replacement | Quorum shrinks; 2 of 3 left is one failure from outage | +| Re-using a reset host with a different CNI without cleaning `/etc/cni/net.d` | Config conflict | +| Rejoining with a stale token or expired certificate key | Join fails: create a new token / re-upload certs | +| Draining the node that holds the last healthy etcd member's workloads without PDB check | Self-inflicted outage | + +## Verification + +`kubectl get nodes -o wide` (expected set, versions, InternalIP), `kubectl -n kube-system get pods -o wide` (control-plane pods only on the expected nodes), etcd `member list`/`endpoint health` green, and the load balancer's target list matches. + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-reset/ +- https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-token/ +- https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/ (drain before minor kubelet upgrade) +- https://kubernetes.io/docs/tasks/debug/debug-cluster/ (NotReady eviction) +- https://docs.k3s.io/installation/uninstall , https://docs.k3s.io/datastore/ha-embedded + +## Not verified + +- That `kubectl delete node` does not stop the kubelet, that a running kubelet re-registers, and that pods bound to the node are garbage-collected: standard behaviour, not read in a kubectl page today. +- "Add first, remove second" for control-plane replacement is operational practice consistent with the etcd quorum rules, not a quoted procedure. +- `talosctl reset` defaults and Talos node removal from etcd. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/talos-cluster.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/talos-cluster.md new file mode 100644 index 0000000..bf6e5e1 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/talos-cluster.md @@ -0,0 +1,106 @@ +--- +name: talos-cluster +description: Talos Linux clusters - production bootstrap, machine config patches, OS and Kubernetes upgrades, etcd operations and disaster recovery with talosctl +when-to-use: Building or operating a cluster on Talos Linux (no SSH, API-managed nodes) +keywords: talos, talosctl, gen config, apply-config, bootstrap, upgrade-k8s, etcd snapshot, recover-from, machine config, kubeprism +priority: high +related: etcd-backup-restore.md, cni-cilium.md, version-skew-upgrades.md, node-lifecycle.md +--- + +# Talos Cluster + +**Load when:** nodes run Talos (v1.14.5 docs: https://docs.siderolabs.com/talos/v1.14/). There is no SSH: `talosctl` is the only interface, so every command carries `--nodes ` (and `--endpoints` from your `talosconfig`). Patch template: [talos-patches.yaml](templates/talos-patches.yaml). + +## Production bootstrap (order matters) + +1. Boot three control-plane ISOs (Image Factory https://factory.talos.dev/) and the workers. A node without config is in **maintenance mode**, whose API is unauthenticated by design: anyone who can reach it can apply a config. Keep that network closed or apply config at boot. +2. Kubernetes endpoint = a TCP load balancer on `:6443` or several A/AAAA records, one per control-plane node. It cannot be an HTTP load balancer. If nodes are not directly reachable, TCP port 50000 (Talos API, gRPC + mutual TLS) must be forwarded by a TCP load balancer. +3. Secrets and configs: + +```bash +talosctl gen secrets -o secrets.yaml # keep out of Git; it holds every key +talosctl gen config --with-secrets secrets.yaml https://:6443 +# -> controlplane.yaml, worker.yaml, talosconfig +talosctl machineconfig patch controlplane.yaml --patch controlplane-patch-1.yaml --output controlplane.yaml +``` + + `talosctl` version should match the Talos version on the nodes; otherwise pass `--talos-version v1.14` and check the support matrix for the Kubernetes version. +4. `talosctl get disks --insecure --nodes ` and `talosctl get links --insecure --nodes ` to choose the install disk (prefer `disk.serial`/`disk.wwid` to `/dev/sdX`) and the interface. +5. Apply: `talosctl apply-config --insecure --nodes --file controlplane.yaml` for each control-plane node, `worker.yaml` for workers. +6. `talosctl config merge ./talosconfig` ; `talosctl config endpoint `. +7. **Bootstrap once, on one node**: `talosctl bootstrap --nodes `. The official guide's warning: run it ONCE on a SINGLE control-plane node. +8. `talosctl kubeconfig --nodes ` then `kubectl get nodes`. + +With Cilium: set CNI to none (and disable kube-proxy) in the patch **before** bootstrap; the node then waits at "node not ready" until a CNI runs, and you have about ten minutes to install it ([cni-cilium.md](cni-cilium.md)). KubePrism (port 7445) is the in-node endpoint Cilium uses (`k8sServiceHost: localhost`, `k8sServicePort: 7445`). + +## Configuration discipline + +Talos 1.14 splits the monolithic `v1alpha1` config into small documents (`KubeAPIServerConfig`, `KubeProxyConfig`, `KubeFlannelCNIConfig`, `DHCPv4Config`, `UnattendedInstallConfig`, ...); old fields still work but a deprecated field and its replacement document cannot both be set. Patches are strategic-merge YAML; from 1.13 do not use RFC 6902 `op/path/value` patches. After `talosctl upgrade-k8s` the live config differs from a stored `controlplane.yaml`: re-applying a stale full file can downgrade components. Store **patches**, regenerate the config, and back up the live config with `talosctl -n get mc v1alpha1 -o yaml`. + +## Upgrades + +```bash +talosctl -n upgrade --image factory.talos.dev/metal-installer/:v1.14.5 # one node at a time +talosctl -n upgrade-k8s --to --dry-run # then without --dry-run +``` + +- Since Talos 1.14 `ghcr.io/siderolabs/installer` is no longer published: use the Image Factory installer with the **schematic ID of the image the node was installed from** (it carries the system extensions); the empty-schematic ID is for nodes without customizations. A wrong schematic drops extensions. +- A-B images: a failed boot rolls back; manual rollback is `talosctl rollback`. Migration is tested between adjacent minors: go through the latest patch of each intermediate minor. +- OS upgrade does not upgrade Kubernetes; `upgrade-k8s` pre-pulls images, patches control-plane component images, updates kube-proxy and kubelets, re-applies bootstrap manifests with server-side apply and (1.13+) prunes unneeded resources (`--manifests-no-prune` to keep them). It can be restarted after a failure. +- Read the "Before upgrade to v1.14.5" section (a multipath extension config must be migrated first). + +## etcd operations + +```bash +talosctl -n ,, etcd members ; talosctl -n ,, etcd status +talosctl -n etcd alarm list # NOSPACE at the 2 GiB default quota stops etcd writes +talosctl -n etcd defrag # one node at a time: blocks reads/writes on that member +talosctl -n etcd snapshot db.snapshot # schedule it; copy off-node +``` + +Raise the quota only to match real usage (recommended maximum 8 GiB): `cluster.etcd.extraArgs.quota-backend-bytes`, then reboot and `talosctl etcd alarm disarm`. + +## Disaster recovery (quorum lost) - destructive + +Pre-check: compare `etcd members` on every healthy node and `talosctl service etcd`; if quorum can be restored, do that instead. No node may be of the deprecated `init` machine type. + +``` +ACTION talosctl -n reset --graceful=false --reboot --system-labels-to-wipe=EPHEMERAL (only on a control-plane node whose etcd is down) +EFFECT wipes the EPHEMERAL partition, which holds the etcd data directory; the node reboots with etcd in Preparing state +QUORUM that member's data is gone; recovery then rebuilds the cluster from the snapshot +PROOF a snapshot (regular or copied with `talosctl -n cp /var/lib/etcd/member/snap/db .`) is already stored off-node and its hash noted +WAIT-FOR-GO + +ACTION talosctl -n bootstrap --recover-from=./db.snapshot (one control-plane node; add --recover-skip-hash-check for a copied db file) +EFFECT etcd on that node is rebuilt from the snapshot; the other control-plane nodes join once the endpoint is up; state after the snapshot time is lost +PROOF all etcd services show Preparing; snapshot hash/revision printed matches the file +WAIT-FOR-GO +``` + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| `talosctl bootstrap` on several nodes or twice | Violates the official "ONCE on a SINGLE node" warning; consequence not documented, assume etcd inconsistency | +| Wrong schematic in `upgrade --image` | Extensions (drivers, iSCSI, etc.) vanish after reboot | +| Storing full `controlplane.yaml` in Git across Kubernetes upgrades | Re-apply downgrades components | +| Exposing maintenance mode on an open network | Anyone can apply a config | + +## Verification + +`talosctl etcd status` shows one leader and equal raft indexes; `kubectl get nodes` all `Ready`; a restore rehearsal from the latest snapshot succeeded on a throwaway cluster. + +## Sources (read 2026-11-10) + +- https://docs.siderolabs.com/talos/v1.14/getting-started/getting-started.md , .../prodnotes.md +- https://docs.siderolabs.com/talos/v1.14/configure-your-talos-cluster/lifecycle-management/upgrading-talos.md +- https://docs.siderolabs.com/kubernetes-guides/advanced-guides/upgrading-kubernetes.md +- https://docs.siderolabs.com/talos/v1.14/build-and-extend-talos/cluster-operations-and-maintenance/disaster-recovery.md +- https://docs.siderolabs.com/talos/v1.12/build-and-extend-talos/cluster-operations-and-maintenance/etcd-maintenance.md (v1.12 page; v1.14 equivalent not read) +- https://docs.siderolabs.com/kubernetes-guides/cni/deploying-cilium.md + +## Not verified + +- `talosctl reset` default flags (graceful behaviour, which partitions are wiped without `--system-labels-to-wipe`) and node removal from etcd (`talosctl etcd remove-member`): not read; read the CLI reference before proposing them. +- The CEL expression syntax of `diskSelector.match` beyond the documented `disk.dev_path == "..."` form (the `disk.serial` / `disk.wwid` property names are named by the guide; the template uses `disk.serial == "..."`). +- Talos certificate/secret rotation, KubePrism configuration details, `talosctl health`, and the support-matrix contents. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/cilium-lb-ipam-l2.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/cilium-lb-ipam-l2.yaml new file mode 100644 index 0000000..42fded1 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/cilium-lb-ipam-l2.yaml @@ -0,0 +1,39 @@ +# Cilium LB IPAM pool + L2 announcement policy (Cilium 1.20.x). +# Requires Helm values: kubeProxyReplacement: true and l2announcements.enabled: true (see cilium-values.yaml). +# The pool range must be unused on the LAN (outside DHCP, not a host address) and in the nodes' broadcast domain. +# Check: kubectl get ippools | kubectl -n kube-system get lease | grep cilium-l2announce +# Sources: https://docs.cilium.io/en/stable/network/lb-ipam/ https://docs.cilium.io/en/stable/network/l2-announcements/ +apiVersion: "cilium.io/v2" +kind: CiliumLoadBalancerIPPool +metadata: + name: lan-pool +spec: + blocks: + # start/stop form ONE block; keep them in the same list item + - start: "" + stop: "" + # Only Services labelled pool=lan may take addresses from this pool. + serviceSelector: + matchLabels: + pool: lan +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumL2AnnouncementPolicy +metadata: + name: lan-l2 +spec: + serviceSelector: + matchLabels: + pool: lan + # Announce from workers only; one node per Service answers ARP/NDP (Lease-based election). + nodeSelector: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist + # Regexes of interfaces that may announce; they must also be in Cilium's "devices". + interfaces: + - ^eth[0-9]+ + # Both default to false: a policy needs at least one set to true. + loadBalancerIPs: true + externalIPs: false +# Do not combine with Services using externalTrafficPolicy: Local (incompatible with L2 announcements). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/cilium-values.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/cilium-values.yaml new file mode 100644 index 0000000..6c47a93 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/cilium-values.yaml @@ -0,0 +1,39 @@ +# Cilium 1.20.4 Helm values: kube-proxy replacement, optional L2 announcements and Gateway API. +# Install: helm install cilium cilium/cilium --version 1.20.4 -n kube-system -f cilium-values.yaml +# Upgrade: helm upgrade cilium cilium/cilium --version -n kube-system -f cilium-values.yaml (always the full file, or --reuse-values) +# Sources: https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/ +# https://docs.cilium.io/en/stable/network/l2-announcements/ +# https://docs.cilium.io/en/stable/network/servicemesh/gateway-api/gateway-api/ +# https://docs.siderolabs.com/kubernetes-guides/cni/deploying-cilium.md (Talos block) + +# Kube-proxy replacement. The cluster must have been created WITHOUT kube-proxy (kubeadm: proxy.disabled / --skip-phases=addon/kube-proxy). +kubeProxyReplacement: true +# The agent cannot use the in-cluster "kubernetes" Service without kube-proxy: name the API server directly. +# kubeadm / k3s: the controlPlaneEndpoint host and port. Talos: host "localhost", port 7445 (KubePrism). +k8sServiceHost: "" +k8sServicePort: 6443 + +# L2 announcements (beta): answers ARP/NDP for LoadBalancer IPs. Requires kubeProxyReplacement: true. +# Raises API traffic: size the client rate limit using the sizing section of the Cilium page, then uncomment. +l2announcements: + enabled: false +# k8sClientRateLimit: +# qps: +# burst: + +# Gateway API controller. Install the Gateway API CRDs at the version Cilium documents (v1.6.3 for Cilium 1.20.4) FIRST, +# then restart cilium-operator and the cilium DaemonSet after enabling. +gatewayAPI: + enabled: false + +# --- Talos only: uncomment the whole block --- +# ipam: +# mode: kubernetes +# securityContext: +# capabilities: +# ciliumAgent: [CHOWN, KILL, NET_ADMIN, NET_RAW, IPC_LOCK, SYS_ADMIN, SYS_RESOURCE, DAC_OVERRIDE, FOWNER, SETGID, SETUID] +# cleanCiliumState: [NET_ADMIN, SYS_ADMIN, SYS_RESOURCE] +# cgroup: +# autoMount: +# enabled: false +# hostRoot: /sys/fs/cgroup diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/cluster-upgrade-checklist.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/cluster-upgrade-checklist.md new file mode 100644 index 0000000..8a32642 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/cluster-upgrade-checklist.md @@ -0,0 +1,54 @@ +# Kubernetes Minor Upgrade Checklist + +Copy per upgrade. Tick with evidence (command output pasted or file path). One minor at a time. Sources: https://kubernetes.io/releases/version-skew-policy/ , https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/ . + +``` +Cluster: ____ Distribution: kubeadm | k3s | Talos Date: ____ Window: ____ +Current -> target: v1.__.__ -> v1.__.__ (latest patch of target read at https://kubernetes.io/releases/) +``` + +## 1. Plan + +- [ ] Target is exactly one minor above the oldest control-plane component (no skip) +- [ ] Current minor is on its latest patch (upgrade within the minor first if not) +- [ ] Release notes and deprecation/removal list of the target read; manifests using removed APIs found (`talosctl upgrade-k8s --dry-run` lists them on Talos) +- [ ] Skew table checked: kubelets <= apiserver and within three minors; kubectl within one minor; kube-proxy within three +- [ ] Admission webhooks and CRDs/operators support the new minor (vendor pages read) +- [ ] CNI (Cilium), LB (MetalLB/kube-vip), CSI and ingress/Gateway controller versions support the target; their upgrade order noted +- [ ] OS prerequisites: cgroup v2, swap off, kernel supported by Cilium (>= 5.10) +- [ ] Maintenance window announced; rollback = restore from snapshot (data loss window stated) + +## 2. Backup + +- [ ] etcd snapshot taken, `etcdutl snapshot status` output attached, copy stored off-node (templates/etcd-snapshot-restore.md) +- [ ] `/etc/kubernetes` (kubeadm) or machine configs / `secrets.yaml` (Talos) or k3s config + token backed up +- [ ] Application backups verified (devops-storage-backup) + +## 3. Health gate (before touching anything) + +- [ ] `kubectl get nodes` all Ready; `kubectl -n kube-system get pods` all Running +- [ ] `kubectl get --raw='/readyz?verbose'` all `[+]` +- [ ] etcd `endpoint health --cluster` green, one leader, DB size below quota +- [ ] `kubeadm certs check-expiration` (kubeadm) shows > 30 days +- [ ] PodDisruptionBudgets leave room to drain (`kubectl get pdb -A`) + +## 4. Execute (one node at a time; stop on the first red check) + +Control plane: +- [ ] Node 1: upgrade kubeadm package -> `kubeadm upgrade plan` -> `kubeadm upgrade apply v` (k3s: install script with `INSTALL_K3S_VERSION`; Talos: `talosctl upgrade` per node then `upgrade-k8s`) +- [ ] CNI provider upgrade step done if it is not a DaemonSet-only chart +- [ ] Node 1: drain -> kubelet/kubectl packages -> restart kubelet -> uncordon -> Ready, etcd green +- [ ] Node 2 and 3: `kubeadm upgrade node`, then the same drain/kubelet/uncordon sequence, one at a time + +Workers (one or a few at a time while capacity remains): +- [ ] drain -> upgrade -> restart kubelet -> uncordon -> Ready + +## 5. Verify + +- [ ] `kubectl get nodes` shows the target version on every node +- [ ] `kubectl -n kube-system get pods` healthy; CoreDNS rebalanced (`rollout restart deployment coredns` if all on one node) +- [ ] Cilium: `cilium-dbg status` OK, Services reachable; LoadBalancer IPs still announced +- [ ] Certificates renewed (`kubeadm certs check-expiration`) +- [ ] Workload smoke tests pass (devops-kubernetes); alerts quiet for one scrape/evaluation window +- [ ] `/etc/kubernetes/tmp/kubeadm-backup-*` reviewed and cleaned; packages re-held (`apt-mark hold`) +- [ ] Result recorded: versions, duration, anything unexpected (devops-sre-ops change log) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/etcd-snapshot-restore.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/etcd-snapshot-restore.md new file mode 100644 index 0000000..df54232 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/etcd-snapshot-restore.md @@ -0,0 +1,63 @@ +# etcd Snapshot and Restore Runbook (kubeadm, stacked etcd) + +Fill the variables, run Part A freely (read-only plus a snapshot), run Part B only after the user's go. Sources: https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/ and https://etcd.io/docs/v3.6/op-guide/recovery/ . k3s and Talos have their own tools (k3s-cluster.md, talos-cluster.md). + +```bash +export ETCDCTL_API=3 +EP=https://127.0.0.1:2379 +CA=/etc/kubernetes/pki/etcd/ca.crt ; CRT=/etc/kubernetes/pki/etcd/server.crt ; KEY=/etc/kubernetes/pki/etcd/server.key +E="sudo etcdctl --endpoints=$EP --cacert=$CA --cert=$CRT --key=$KEY" +SNAP=/var/backups/etcd/snap-$(date +%F-%H%M).db +``` + +## Part A - snapshot and verify (no go needed) + +```bash +$E member list -w table +$E endpoint health --cluster +$E endpoint status --cluster -w table +sudo mkdir -p /var/backups/etcd && $E snapshot save "$SNAP" +sudo etcdutl --write-out=table snapshot status "$SNAP" # record HASH / REVISION / TOTAL KEYS +sha256sum "$SNAP" # record; copy off-node and encrypt the copy +``` + +Success = `snapshot status` prints a hash, the file is stored off the control-plane node, a second person can find it. + +## Part B - restore (destructive) + +Proposal block to present first: + +``` +ACTION restore etcd on control-plane nodes from +EFFECT API servers stop; each member's data directory is rebuilt as a NEW logical cluster; every change after is lost; + controllers re-sync; revisions may go backwards unless --bump-revision/--mark-compacted is used +QUORUM 3 members, quorum 2: the cluster is unavailable from API-server stop until 2 restored members run +PROOF snapshot hash matches; current state snapshotted too (if etcd still answers); node list and etcd member names recorded +WAIT-FOR-GO +``` + +Steps after the go (rehearse on spare VMs first): + +1. On every control-plane node, stop the API server: move `/etc/kubernetes/manifests/kube-apiserver.yaml` out of the directory (kubelet removes the static pod) and confirm `crictl ps` / process list shows it gone. Stop etcd the same way by moving `etcd.yaml`. +2. Copy the same snapshot to every node. On each node `` is the etcd member name from the old manifest: + +```bash +sudo etcdutl snapshot restore "$SNAP" \ + --name \ + --data-dir /var/lib/etcd-restore \ + --initial-cluster =https://:2380,=https://:2380,=https://:2380 \ + --initial-cluster-token \ + --initial-advertise-peer-urls https://:2380 \ + --bump-revision 1000000000 --mark-compacted +``` + +3. Edit the moved `etcd.yaml`: set the `etcd-data` hostPath to `/var/lib/etcd-restore`, and check that the manifest's cluster flags (`--initial-cluster-token`, `--initial-cluster`, `--name`) agree with the ones used at restore (exact manifest edits: see "Not verified"). On kubeadm set `etcd.local.dataDir` in the `kubeadm-config` ConfigMap to the new directory, or the next `kubeadm upgrade` writes the old path. +4. Move `etcd.yaml` back on all three nodes; wait for `member list` to show 3 started members; then move `kube-apiserver.yaml` back. Restart scheduler, controller-manager and kubelets if they hold stale state. +5. Verify: `$E endpoint health --cluster`, `kubectl get nodes`, `kubectl get pods -A`, application smoke tests (devops-kubernetes). + +If etcd's client URLs changed, restart API servers with the new `--etcd-servers`. + +## Not verified + +- Steps 1, 3 and 4 combine two official pages and the static-pod mechanism (kubelet watches the manifest directory); no single official page lists the full stacked-kubeadm multi-node procedure. `crictl` usage and the exact flags etcd needs in the manifest after restore must be checked against your etcd version before a real restore. +- `--bump-revision` is documented in the etcd v3.6/v3.8 recovery pages; v3.7 not read. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/k3s-server-config.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/k3s-server-config.yaml new file mode 100644 index 0000000..3461ecf --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/k3s-server-config.yaml @@ -0,0 +1,31 @@ +# /etc/rancher/k3s/config.yaml for an HA k3s SERVER (embedded etcd), Cilium as CNI without kube-proxy. +# Keys map to CLI flags (repeatable flags are lists). CLI arguments override this file. +# Flags marked IDENTICAL must be the same on every server or the join fails +# ("critical configuration value mismatch"). Source: https://docs.k3s.io/installation/configuration +# and https://docs.k3s.io/datastore/ha-embedded +# +# First server: cluster-init: true (and no "server:" key) +# Servers 2 and 3: remove cluster-init, set server: "https://:6443" +# The shared secret: prefer the file/env over committing it to Git. +cluster-init: true +# server: "https://:6443" +token: "" +tls-san: + - "" +write-kubeconfig-mode: "0600" +node-label: + - "role=control-plane" + +# IDENTICAL on all servers -------------------------------------------------- +cluster-cidr: "10.42.0.0/16" +service-cidr: "10.43.0.0/16" +flannel-backend: "none" # Cilium provides pod networking +disable-network-policy: true # avoid two policy engines +disable-kube-proxy: true # Cilium kube-proxy replacement; remove if you keep kube-proxy +# disable: +# - servicelb # if MetalLB / Cilium LB IPAM provides LoadBalancer IPs (verify the component name in the k3s docs for your version) + +# Snapshots ------------------------------------------------------------------ +etcd-snapshot-retention: 5 # local retention is per server node +# S3 target (preferred): etcd-s3: true / etcd-s3-endpoint / etcd-s3-bucket / etcd-s3-folder / etcd-s3-retention +# Snapshot schedule key: see https://docs.k3s.io/cli/etcd-snapshot (the schedule section was not readable when this template was written) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/kubeadm-ha-config.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/kubeadm-ha-config.yaml new file mode 100644 index 0000000..d2c47db --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/kubeadm-ha-config.yaml @@ -0,0 +1,39 @@ +# kubeadm HA control plane, stacked etcd, API objects kubeadm.k8s.io/v1beta4. +# Use: sudo kubeadm init --config kubeadm-ha-config.yaml --upload-certs +# Do NOT also pass --certificate-key on the command line (it cannot be mixed with --config). +# Replace every . Verify field names with: kubeadm config print init-defaults +# Source: https://kubernetes.io/docs/reference/config-api/kubeadm-config.v1beta4/ +apiVersion: kubeadm.k8s.io/v1beta4 +kind: InitConfiguration +nodeRegistration: + # criSocket: "unix:///var/run/containerd/containerd.sock" # set only if auto-detection is ambiguous + kubeletExtraArgs: + - name: node-ip + value: "" # needed on multi-homed nodes (Cilium kube-proxy replacement relies on a correct InternalIP) +localAPIEndpoint: + advertiseAddress: "" + bindPort: 6443 +--- +apiVersion: kubeadm.k8s.io/v1beta4 +kind: ClusterConfiguration +kubernetesVersion: "v" # e.g. the latest 1.37 patch; kubeadm, kubelet and kubectl should match +clusterName: "" +# DNS name of the TCP load balancer / VIP in front of ALL control-plane nodes. Must match the LB address forever. +controlPlaneEndpoint: ":6443" +networking: + serviceSubnet: "10.96.0.0/16" + podSubnet: "10.244.0.0/16" # must agree with the CNI's IPAM settings + dnsDomain: "cluster.local" +apiServer: + certSANs: + - "" + - "" +etcd: + local: + dataDir: "/var/lib/etcd" +# Keep kube-proxy off when Cilium replaces it (kubeProxyReplacement=true). Remove this block to keep kube-proxy. +proxy: + disabled: true +# Optional: ECDSA keys and explicit validity (defaults: RSA-2048, 8760h leaf, 87600h CA) +# encryptionAlgorithm: ECDSA-P256 +# certificateValidityPeriod: 8760h diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/metallb-l2.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/metallb-l2.yaml new file mode 100644 index 0000000..183220c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/metallb-l2.yaml @@ -0,0 +1,29 @@ +# MetalLB v0.16.2, Layer 2 mode. All resources live in the namespace MetalLB runs in (metallb-system). +# Install first: kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.16.2/config/manifests/metallb-native.yaml +# kube-proxy in IPVS mode needs ipvs.strictARP: true in the kube-proxy ConfigMap (kubectl diff first). +# Do not give the same range to Cilium LB IPAM. +# Sources: https://metallb.io/installation/ https://metallb.io/configuration/ https://metallb.io/configuration/_advanced_l2_configuration/ +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: lan-pool + namespace: metallb-system +spec: + addresses: + - - # or a CIDR; must be unused on the LAN +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: lan-l2 + namespace: metallb-system +spec: + # Without ipAddressPools, the advertisement applies to ALL pools. + ipAddressPools: + - lan-pool + # Optional hard filter on candidate announcing nodes, or interfaces: + # nodeSelectors: + # - matchLabels: + # kubernetes.io/hostname: + # interfaces: + # - diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/talos-patches.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/talos-patches.yaml new file mode 100644 index 0000000..2e9c3b7 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/templates/talos-patches.yaml @@ -0,0 +1,27 @@ +# Talos v1.14 control-plane patch: no default Flannel CNI, no kube-proxy (Cilium replaces both), +# DHCP on the node interface, install disk chosen by a stable property. +# Use (before bootstrap): +# talosctl machineconfig patch controlplane.yaml --patch controlplane-patch-1.yaml --output controlplane.yaml +# or at generation time: +# talosctl gen config --with-secrets secrets.yaml https://:6443 --config-patch-control-plane @controlplane-patch-1.yaml +# Find values with: talosctl --nodes get links --insecure | talosctl get disks --insecure --nodes +# Sources: https://docs.siderolabs.com/kubernetes-guides/cni/deploying-cilium.md +# https://docs.siderolabs.com/talos/v1.14/getting-started/prodnotes.md +# On Talos < 1.14 use the legacy form instead: cluster: { network: { cni: { name: none } }, proxy: { disabled: true } } +apiVersion: v1alpha1 +kind: KubeFlannelCNIConfig +$patch: delete +--- +apiVersion: v1alpha1 +kind: KubeProxyConfig +enabled: false +--- +apiVersion: v1alpha1 +kind: DHCPv4Config +name: # interface whose OPER state is up +--- +apiVersion: v1alpha1 +kind: UnattendedInstallConfig +provisioning: + diskSelector: + match: disk.serial == "" # prefer serial/wwid over /dev/sdX; dev_path form: disk.dev_path == "/dev/" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/version-skew-upgrades.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/version-skew-upgrades.md new file mode 100644 index 0000000..c3a59b6 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-k8s-cluster/references/version-skew-upgrades.md @@ -0,0 +1,94 @@ +--- +name: version-skew-upgrades +description: Kubernetes version skew policy, upgrade order, kubeadm upgrade procedure and certificate expiry/renewal +when-to-use: Planning or running a minor or patch upgrade, checking whether a component version combination is supported, or certificates are about to expire +keywords: version skew, kubeadm upgrade plan, kubeadm upgrade apply, upgrade node, kubelet, certs check-expiration, certs renew, certificateValidityPeriod +priority: critical +related: kubeadm-ha-bootstrap.md, node-lifecycle.md, etcd-backup-restore.md, k3s-cluster.md, talos-cluster.md +--- + +# Version Skew, Upgrades, Certificates + +**Load when:** any version change or certificate question. Checklist template: [cluster-upgrade-checklist.md](templates/cluster-upgrade-checklist.md). + +## Skew limits (newest kube-apiserver = 1.37 in the examples) + +| Component | Rule | +|-----------|------| +| kube-apiserver (HA) | Newest and oldest instance within **one** minor | +| kubelet | Not newer than the apiserver; up to **three** minors older (1.37 -> 1.37..1.34) | +| kube-proxy | Not newer than the apiserver; up to three minors older; within three minors of the kubelet beside it | +| controller-manager, scheduler, cloud-controller-manager | Not newer than the apiserver they talk to; same minor or one older | +| kubectl | Within one minor of the apiserver, older or newer | + +During a mixed HA upgrade the narrower limits apply: with apiservers at 1.37 and 1.36, kubelets must be 1.36..1.34 and controller-manager/scheduler 1.36. A kubelet that stays three minors behind blocks the next control-plane upgrade. Maintained minors today: 1.37, 1.36, 1.35 (1.34 ended 2026-10-27). + +## Upgrade order (never skip a minor, even with one control-plane node) + +1. Control-plane apiserver(s) -> 2. controller-manager, scheduler, cloud-controller-manager -> 3. kubelet and kube-proxy on every node. Prefer to be on the latest patch of the current minor first, then upgrade to the latest patch of the target. Admission webhooks must handle the new API versions (use `matchPolicy: Equivalent`). + +## kubeadm procedure (https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/) + +Before: read the release notes; fresh etcd snapshot ([etcd-backup-restore.md](etcd-backup-restore.md)); swap off; backups of app state. Control-plane nodes **one at a time**. + +```bash +# first control-plane node +sudo apt-mark unhold kubeadm && sudo apt-get update && sudo apt-get install -y kubeadm='1.37.x-*' && sudo apt-mark hold kubeadm +kubeadm version ; sudo kubeadm upgrade plan +sudo kubeadm upgrade apply v1.37.x # other control-plane nodes: sudo kubeadm upgrade node +# every node, after kubeadm: drain (needed for a MINOR kubelet upgrade), then kubelet/kubectl +kubectl drain --ignore-daemonsets +sudo apt-mark unhold kubelet kubectl && sudo apt-get install -y kubelet='1.37.x-*' kubectl='1.37.x-*' && sudo apt-mark hold kubelet kubectl +sudo systemctl daemon-reload && sudo systemctl restart kubelet +kubectl uncordon ; kubectl get nodes +``` + +Facts to respect: `kubeadm upgrade` renews the certificates it manages (opt out with `--certificate-renewal=false`); all containers restart because the spec hash changes; the CNI provider is upgraded manually (first node, unless it is a DaemonSet-only chart); since 1.28 addons (CoreDNS, kube-proxy) upgrade only after the last control-plane instance; a stalled `kubeadm upgrade` is idempotent - re-run it, or `kubeadm upgrade apply --force` on the same version; backups are written to `/etc/kubernetes/tmp/kubeadm-backup-etcd-*` and `kubeadm-backup-manifests-*` and must be cleaned up by hand. `kubeadm upgrade` cannot reconfigure a cluster. During an etcd upgrade, in-flight apiserver requests stall; the docs suggest `killall -s SIGTERM kube-apiserver; sleep 20` before `kubeadm upgrade`. For RHEL-family hosts use the `dnf` forms with the `kubernetes` repo excludes disabled as shown on the page. + +``` +ACTION kubeadm upgrade apply v1.37.x (control-plane node 1 of 3) +EFFECT rewrites static pod manifests, restarts apiserver/controller-manager/scheduler and possibly etcd on this node; renews its certificates +QUORUM etcd keeps 2 of 3 members on the other nodes; do the next node only when this one is Ready and etcd health is green +PROOF snapshot taken and listed; `kubeadm upgrade plan` output reviewed; skew table checked +WAIT-FOR-GO +``` + +k3s: servers first one at a time, then agents; the install script does not drain ([k3s-cluster.md](k3s-cluster.md)). Talos: OS and Kubernetes upgrades are separate commands ([talos-cluster.md](talos-cluster.md)). + +## Certificates (kubeadm) + +Client certificates generated by kubeadm last **1 year** (CAs 10 years). `kubelet.conf` is not listed because the kubelet rotates its own certificates under `/var/lib/kubelet/pki`. + +```bash +sudo kubeadm certs check-expiration # every control-plane node; "EXTERNALLY MANAGED" rows are yours to renew +sudo kubeadm certs renew all # or one name (apiserver, ...) - run on EVERY control-plane node +``` + +Upgrading at least yearly renews everything automatically. After `renew`, restart the control-plane static pods (the new certs are read at start) and refresh the admin kubeconfig you copied to `~/.kube/config`. Only non-CA certificates are renewable by `kubeadm certs renew`; validity can be set with `certificateValidityPeriod` / `caCertificateValidityPeriod` in `ClusterConfiguration`. kubeadm cannot manage certificates signed by an external CA ("External CA" mode when `ca.key` is absent). Alert on expiry in devops-observability (30 days). + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| Jumping 1.35 -> 1.37 | Unsupported; API deprecation and storage-version assumptions break | +| Upgrading kubelet before the apiserver | Kubelet newer than apiserver is unsupported | +| Leaving `apt` unheld | `apt upgrade` moves kubelet independently of the plan | +| Certificates expire after a year without upgrades | apiserver/etcd clients fail; control plane unreachable until renewed | +| Forgetting the CNI upgrade step | Old agent against new API/kernel features | + +## Verification + +`kubectl get nodes` (all `Ready`, versions as planned), `kubectl version`, `kubeadm certs check-expiration` (fresh dates), `kubectl -n kube-system get pods`, then smoke-test the workloads (devops-kubernetes). + +## Sources (read 2026-11-10) + +- https://kubernetes.io/releases/version-skew-policy/ +- https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/ +- https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-certs/ +- https://kubernetes.io/docs/reference/config-api/kubeadm-config.v1beta4/ +- https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-certs/ (`kubeadm certs renew all`: unconditional renewal, `--cert-dir`, `--config`) + +## Not verified + +- The restart of control-plane static pods after `certs renew` (the page's text after "After running th..." was cut off; the restart is the usual step, confirm on the page). +- Deprecated-API scanning commands for kubeadm clusters; Talos/k3s certificate rotation. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/SKILL.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/SKILL.md new file mode 100644 index 0000000..c25fcff --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/SKILL.md @@ -0,0 +1,151 @@ +--- +name: devops-kubernetes +description: Use when writing, reviewing, applying or debugging workloads on an existing Kubernetes cluster - Deployment, StatefulSet, DaemonSet, Job/CronJob, ConfigMap/Secret usage, Services, Gateway API (HTTPRoute) or a migration away from Ingress/Ingress NGINX, PV/PVC/StorageClass, RBAC and ServiceAccounts, requests/limits, probes, HPA, NetworkPolicy, safe kubectl (diff, server dry-run, delete/drain/scale/rollback) and pod troubleshooting (Pending, CrashLoopBackOff, DNS, no endpoints). Not for installing or upgrading clusters, etcd or CNI choice (devops-k8s-cluster), Helm/Kustomize/Argo CD/Flux (devops-gitops-packaging), secret stores (devops-secrets) or a cluster security audit (fuse-security:security-expert). +versions: + kubernetes: "1.37.2 (supported minors 1.37, 1.36, 1.35)" + gateway-api: "1.7.0" +user-invocable: true +references: + - references/workloads-scheduling.md + - references/probes-lifecycle.md + - references/services-gateway-api.md + - references/network-policy.md + - references/storage-csi.md + - references/config-secrets.md + - references/rbac-serviceaccounts.md + - references/resources-qos-hpa.md + - references/kubectl-safe-ops.md + - references/troubleshooting-playbook.md + - references/templates/deployment-production.yaml + - references/templates/statefulset-pvc.yaml + - references/templates/cronjob.yaml + - references/templates/gateway-httproute.yaml + - references/templates/networkpolicy-default-deny.yaml + - references/templates/rbac-least-privilege.yaml + - references/templates/hpa.yaml +related-skills: + - devops-k8s-cluster + - devops-gitops-packaging + - devops-secrets + - devops-observability + - devops-storage-backup + - devops-networking + - devops-docker + - devops-sre-ops +--- + + +Covers using a running Kubernetes cluster: workload controllers, configuration, +Services and traffic entry through Gateway API, storage claims, RBAC, +resources/QoS/autoscaling, probes, NetworkPolicy, safe kubectl operation and a +fixed troubleshooting order. Does not cover building or upgrading the cluster, +etcd, CNI selection (devops-k8s-cluster), packaging and GitOps controllers +(devops-gitops-packaging), external secret stores (devops-secrets) or a +security audit (fuse-security:security-expert). + + +# DevOps Kubernetes + +## Agent Workflow (MANDATORY) + +Before ANY manifest or cluster change, spawn in parallel, one `Task` call each: + +1. **fuse-ai-pilot:explore-codebase** - find existing manifests, overlays, charts, the target namespace layout +2. **fuse-ai-pilot:research-expert** - verify the API fields for the cluster's minor at kubernetes.io / gateway-api.sigs.k8s.io +3. **mcp__context7__query-docs** - `/websites/kubernetes_io`, `/websites/gateway-api_sigs_k8s_io` + +Then read the cluster read-only (`kubectl version`, `kubectl api-resources`, `kubectl get crd | grep gateway`). +After changes, run **fuse-ai-pilot:sniper**. + +## Versions (checked 2026-11-10) + +Kubernetes v1.37.2 / v1.36.6 / v1.35.10 are the latest patches (GitHub release tags, 2026-10-28; https://kubernetes.io/releases/ still listed 1.37.1); 1.34 reached end of life 2026-10-27. Gateway API v1.7.0 (2026-10-29). Each minor deprecates or graduates fields: read the release blog of the target minor (https://kubernetes.io/blog/) before writing manifests for it. + +## Overview + +| Area | Decision in one line | +|------|----------------------| +| **Workloads** | Stateless = Deployment; stable identity/storage = StatefulSet; one per node = DaemonSet; run-to-completion = Job/CronJob | +| **Traffic entry** | Gateway API (`gateway.networking.k8s.io/v1`); Ingress API is frozen; Ingress NGINX is retired | +| **Storage** | PVC + StorageClass (`WaitForFirstConsumer`, `allowVolumeExpansion`), `Retain` for data that must survive | +| **Config** | ConfigMap for config, Secret for credentials; Secrets are unencrypted in etcd by default | +| **Access** | One ServiceAccount per workload, namespaced Role, `automountServiceAccountToken: false` unless needed | +| **Capacity** | Requests on every container, memory limit always, HPA `autoscaling/v2` with `behavior` | +| **Isolation** | Default-deny NetworkPolicy per namespace, then explicit allows (needs a CNI that enforces it) | + +## Critical Rules + +1. **Version first** - read the server minor (`kubectl version`) and the CRDs present before writing a field; a field that is stable in 1.37 may be beta or absent in 1.35 +2. **Explicit target on every command** - `--context -n `; never rely on the current context +3. **Diff and server dry-run before apply** - `kubectl diff` then `kubectl apply --dry-run=server`; never `kubectl apply --prune` (alpha) +4. **Destructive verbs are proposed, not run** - `delete`, `drain`, `scale --replicas=0`, `rollout undo`, `--force`: show the exact effect and blast radius, wait for the user's go ([kubectl-safe-ops.md](references/kubectl-safe-ops.md)) +5. **Troubleshoot in order** - describe → events → logs → exec/debug → node; no fix before the evidence ([troubleshooting-playbook.md](references/troubleshooting-playbook.md)) +6. **Never print a Secret value** - names and keys only; values come from the secret store (devops-secrets) +7. **Never invent an apiVersion or field** - cite the kubernetes.io or gateway-api page; unknown = "not verified" + +## Architecture + +``` +k8s/ +├── namespace.yaml # PSA labels: pod-security.kubernetes.io/enforce: restricted +├── rbac.yaml # ServiceAccount + Role + RoleBinding (namespaced) +├── networkpolicy.yaml # default-deny + explicit allows +├── config.yaml # ConfigMap (Secrets come from devops-secrets tooling) +├── deployment.yaml # Deployment + Service + PodDisruptionBudget +├── hpa.yaml # autoscaling/v2 +└── route.yaml # HTTPRoute attached to the shared Gateway +``` + +## Reference Guide + +### Concepts + +| Topic | Reference | When to Consult | +|-------|-----------|-----------------| +| **Workloads** | [workloads-scheduling.md](references/workloads-scheduling.md) | Choosing a controller, rollout strategy, PDB, spreading | +| **Probes and lifecycle** | [probes-lifecycle.md](references/probes-lifecycle.md) | startup/readiness/liveness, sidecars, graceful shutdown | +| **Services and Gateway API** | [services-gateway-api.md](references/services-gateway-api.md) | Exposing an app, leaving Ingress NGINX | +| **NetworkPolicy** | [network-policy.md](references/network-policy.md) | Namespace isolation, egress control | +| **Storage** | [storage-csi.md](references/storage-csi.md) | PVC sizing, reclaim policy, expansion, access modes | +| **Config and Secrets** | [config-secrets.md](references/config-secrets.md) | Mounting config, rollout on change, Secret handling | +| **RBAC** | [rbac-serviceaccounts.md](references/rbac-serviceaccounts.md) | Least privilege, `auth can-i`, Pod Security Admission | +| **Resources and HPA** | [resources-qos-hpa.md](references/resources-qos-hpa.md) | Requests/limits, QoS, in-place resize, autoscaling | +| **Safe kubectl** | [kubectl-safe-ops.md](references/kubectl-safe-ops.md) | Any mutating command | +| **Troubleshooting** | [troubleshooting-playbook.md](references/troubleshooting-playbook.md) | Pending, CrashLoopBackOff, DNS, no endpoints | + +### Templates + +| Template | When to Use | +|----------|-------------| +| [deployment-production.yaml](references/templates/deployment-production.yaml) | Stateless app: probes, limits, restricted securityContext, Service, PDB | +| [statefulset-pvc.yaml](references/templates/statefulset-pvc.yaml) | Stateful app with headless Service and per-replica PVC | +| [cronjob.yaml](references/templates/cronjob.yaml) | Scheduled job with time zone, concurrency and history limits | +| [gateway-httproute.yaml](references/templates/gateway-httproute.yaml) | Shared Gateway, HTTPS redirect, app HTTPRoute, ReferenceGrant | +| [networkpolicy-default-deny.yaml](references/templates/networkpolicy-default-deny.yaml) | Default deny + DNS + gateway ingress allows | +| [rbac-least-privilege.yaml](references/templates/rbac-least-privilege.yaml) | ServiceAccount + Role + RoleBinding | +| [hpa.yaml](references/templates/hpa.yaml) | CPU-based HPA with scale-down stabilization | + +## Quick Reference + +```bash +CTX=prod-eu; NS=shop # always explicit +kubectl --context "$CTX" version # server minor +kubectl --context "$CTX" -n "$NS" diff -f k8s/ # exit 0 none, 1 diff, >1 error +kubectl --context "$CTX" -n "$NS" apply --dry-run=server -f k8s/ +kubectl --context "$CTX" -n "$NS" apply -f k8s/ # after the user's go +kubectl --context "$CTX" -n "$NS" rollout status deploy/web --timeout=5m +kubectl --context "$CTX" -n "$NS" auth can-i list secrets --as=system:serviceaccount:"$NS":web +``` + +## Best Practices + +### DO +- Pin the Gateway API CRD release you install (`standard-install.yaml` of a named tag) and match the controller's conformance +- Give every Deployment with more than one replica a PodDisruptionBudget +- Use a startupProbe for slow starters instead of a long liveness `initialDelaySeconds` +- Keep manifests in Git, pin images by tag and digest (devops-docker), apply through the pipeline (devops-gitops-packaging) + +### DON'T +- Start new work on Ingress NGINX (retired) or new Ingress features (API frozen) +- Use `kubectl edit` or `kubectl patch` on objects owned by GitOps - the controller reverts it +- Put CPU limits by reflex - they throttle; decide per workload ([resources-qos-hpa.md](references/resources-qos-hpa.md)) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/config-secrets.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/config-secrets.md new file mode 100644 index 0000000..5023fb2 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/config-secrets.md @@ -0,0 +1,97 @@ +--- +name: config-secrets +description: Consuming ConfigMaps and Secrets in workloads, update semantics, rollout on change, Secret handling rules +when-to-use: Wiring configuration or credentials into pods, config changes not picked up, reviewing Secret exposure +keywords: configmap, secret, envFrom, volume mount, subPath, immutable, encryption at rest, checksum annotation +priority: high +related: rbac-serviceaccounts.md, kubectl-safe-ops.md +--- + +# Config and Secrets (usage) + +**Load when:** passing configuration or credentials to a workload, or a changed value does not reach the pods. +Where secret values come from (SOPS, Vault/OpenBao, External Secrets) is devops-secrets; this file covers how pods consume them. + +## Facts that drive the decisions (verified) + +| Fact | Consequence | +|------|-------------| +| Secrets are stored **unencrypted** in etcd by default; anyone with etcd access can read them | Encryption at rest is a cluster setting (devops-k8s-cluster); base64 in YAML is encoding, not protection | +| Anyone allowed to create a Pod in a namespace can mount any Secret of that namespace | RBAC on `pods` create is effectively Secret read - namespace = trust boundary | +| ConfigMap data is limited to 1 MiB | Large files go to a volume or an image, not a ConfigMap | +| Mounted ConfigMap/Secret volumes are refreshed by the kubelet on its periodic sync | Files change eventually, the app must re-read them | +| Values consumed as **env vars are not updated**; a pod restart is required | Env-based config needs a rollout on change | +| A **`subPath`** mount never receives updates | Avoid `subPath` for config that changes | +| `immutable: true` (stable since v1.21) on ConfigMaps and Secrets | Protects against accidental edits and cuts kubelet watch load; change = new object | + +## Decision rules + +1. Non-sensitive settings → ConfigMap; credentials, tokens, keys, certificates → Secret. Never put a credential in a ConfigMap, an env literal in the manifest, or an image. +2. Prefer **volume mounts** for Secrets over env vars: env vars leak into crash dumps, `/proc//environ`, child processes and debug output. +3. Choose one update model and state it: + - **Immutable + versioned name** (`app-config-v7`): a change creates a new object and the Deployment references it → automatic rollout, trivial rollback. Best default. + - **Mutable + checksum annotation** on the pod template (`checksum/config: `), computed by the packaging tool (devops-gitops-packaging) → rollout when content changes. + - **Mutable, live reload**: only if the app watches its files; no `subPath`. +4. Mark optional references explicitly (`optional: true`) - otherwise a missing key keeps the pod in `CreateContainerConfigError`. + +## Consumption patterns + +```yaml +spec: + containers: + - name: app + envFrom: + - configMapRef: { name: app-config-v7 } # non-sensitive only + env: + - name: DB_USER + valueFrom: + secretKeyRef: { name: db-credentials, key: username } + volumeMounts: + - name: db-credentials + mountPath: /var/run/secrets/db + readOnly: true + volumes: + - name: db-credentials + secret: + secretName: db-credentials + defaultMode: 0400 +``` + +## Handling rules for the agent + +- Never `kubectl get secret -o yaml`, `-o json`, `describe` with values, or `base64 -d` a value into the conversation or a log. To check presence and keys only: + +```bash +kubectl --context "$CTX" -n "$NS" get secret db-credentials -o jsonpath='{.data}' | jq 'keys' +``` + +- Never write a real value in a manifest committed to Git; generate Secrets from the secret store (devops-secrets). +- Creating a Secret by hand is a last resort; if the user insists, read the value from a file with restricted permissions (`kubectl create secret generic ... --from-file=`) rather than a command-line literal that lands in shell history, and use `--dry-run=client -o yaml` only into an encrypted pipeline, never to stdout in the chat. +- Redact values before quoting any manifest, event or log. + +## Pitfalls + +| Symptom | Cause | Fix | +|---------|-------|-----| +| `CreateContainerConfigError` | Missing ConfigMap/Secret or key | `kubectl describe pod` → event names the missing ref | +| New value not seen | Env var or `subPath` | Roll out, or switch to a plain volume mount | +| Rollback restores old code but new config | Mutable shared ConfigMap | Versioned immutable names | +| Secret readable by a team that should not | They can create pods in the namespace | Split namespaces; tighten RBAC | + +## Verification + +```bash +kubectl --context "$CTX" -n "$NS" get pod "$POD" -o jsonpath='{.spec.containers[*].envFrom}{"\n"}{.spec.volumes}' +kubectl --context "$CTX" -n "$NS" exec "$POD" -c app -- ls -l /var/run/secrets/db # names and modes, never cat +``` + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/concepts/configuration/configmap/ +- https://kubernetes.io/docs/concepts/configuration/secret/ + +## Not verified + +- The exact kubelet refresh delay for mounted ConfigMaps/Secrets (depends on sync period and cache TTL; not quantified here). +- `jq` availability on the operator machine (any JSON tool that prints only keys works). +- The env-var leak paths listed in rule 2 are general Linux behaviour, not taken from the Kubernetes page. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/kubectl-safe-ops.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/kubectl-safe-ops.md new file mode 100644 index 0000000..d3f63c9 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/kubectl-safe-ops.md @@ -0,0 +1,93 @@ +--- +name: kubectl-safe-ops +description: Safe kubectl operation - explicit context and namespace, diff and server dry-run, and the exact effect of destructive verbs +when-to-use: Before ANY mutating kubectl command, and whenever delete, drain, scale to zero, rollout undo or --force is considered +keywords: kubectl, context, namespace, diff, dry-run=server, server-side apply, delete, drain, cordon, scale, rollout undo, force +priority: critical +related: troubleshooting-playbook.md, workloads-scheduling.md, storage-csi.md +--- + +# Safe kubectl Operation + +**Load when:** about to change anything in a cluster. Read-only commands (`get`, `describe`, `logs`, `events`, `top`, `auth can-i`, `diff`) need no go; everything else follows this file. + +## Rule 1 - explicit target, every time + +```bash +kubectl config get-contexts # list; read which is current +CTX=prod-eu; NS=shop +kubectl --context "$CTX" -n "$NS" get deploy # every command carries both +``` + +Never rely on the current context or the namespace stored in kubeconfig: one `kubectl config use-context` in another terminal turns a staging command into a production one. Name the context and namespace in every proposal to the user. + +## Rule 2 - diff, then server dry-run, then apply + +```bash +kubectl --context "$CTX" -n "$NS" diff -f k8s/ # exit 0 = no diff, 1 = differences, >1 = error +kubectl --context "$CTX" -n "$NS" apply --dry-run=server -f k8s/ # admission, validation, PSA, quotas, without persisting +kubectl --context "$CTX" -n "$NS" apply -f k8s/ # only after showing the diff and getting the go +kubectl --context "$CTX" -n "$NS" rollout status deploy/web --timeout=5m +``` + +- `--dry-run=server` submits the request to the API server **without persisting** it; `--dry-run=client` only prints what would be sent and misses admission - use server. +- Server-side apply (`apply --server-side`) tracks field ownership; `--force-conflicts` takes fields owned by another manager (often a GitOps controller) - treat it as destructive. +- `kubectl apply --prune` is still **alpha** (kubectl apply page): do not use it; deletions go through an explicit, reviewed `delete` or the GitOps tool. +- On a GitOps-managed object, change Git, not the cluster (devops-gitops-packaging). + +## Rule 3 - destructive verbs: propose, then the user's go + +The agent writes the exact command with context, namespace and object, states the effect and the blast radius, and runs it only after the user says go. Read-only evidence first (what exists, what depends on it, backups). + +| Command | Exact effect (kubectl reference) | Before proposing | +|---------|----------------------------------|------------------| +| `kubectl delete ` | Graceful deletion; dependents removed by the default `--cascade=background`; `--wait=true` waits for finalizers | List dependents; for PVC/PV/namespace see storage-csi.md | +| `kubectl delete ... --force --grace-period=0` | Removes the object from the API immediately, bypassing graceful deletion; does not wait for the pod's processes to stop - they may keep running and, with shared storage, two instances can write at once; immediate deletion may cause inconsistency or data loss | Only for a pod on a node confirmed dead; never on StatefulSet pods without that proof | +| `kubectl delete pods --all` / `-l` | Deletes every match in the namespace | Run the same selector with `get` first and show the list; prefer `--interactive` | +| `kubectl delete namespace` | Deletes everything inside, PVCs included (data gone if reclaim policy is `Delete`) | Inventory with `kubectl api-resources --verbs=list --namespaced -o name` + backups | +| `kubectl drain ` | Cordons the node, then evicts (or deletes if eviction is unsupported) every pod except mirror pods; refuses DaemonSet pods without `--ignore-daemonsets`; refuses pods without a controller unless `--force`; `--delete-emptydir-data` destroys emptyDir data; `--disable-eviction` bypasses PodDisruptionBudgets | `kubectl get pdb -A` (ALLOWED DISRUPTIONS > 0), pods on the node, spare capacity; `kubectl uncordon` after | +| `kubectl scale --replicas=0` | Stops every pod of the workload: outage for a Deployment; ordered stop for a StatefulSet | Who calls it; add `--current-replicas=N` so the command fails if the state changed | +| `kubectl rollout undo deploy/` | Rolls the template back to the previous revision (or `--to-revision=N`); config in shared mutable ConfigMaps is not rolled back | `kubectl rollout history` and `--dry-run=server` first; GitOps will re-apply Git unless Git is reverted too | +| `kubectl replace --force` | Deletes and recreates the object - pods restart, generated fields reset | Prefer apply | +| `kubectl debug --copy-to=X --replace` | Creates a copy of the pod and deletes the original | Use without `--replace` | + +Proposal format (one block per action): + +``` +ACTION kubectl --context prod-eu -n shop scale deploy/web --current-replicas=4 --replicas=0 +EFFECT all 4 pods of web stop; Gateway route shop.example.com returns 503 +EVIDENCE rollout history attached; no PDB involved; restore = same command with --replicas=4 +WAITING for your go +``` + +## Rule 4 - never in a loop without a canary + +A shell loop over contexts, namespaces or nodes runs once on one target, gets verified, then continues - and only after the user's go for the whole loop. + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| `kubectl apply -f dir/` with a stray `Namespace` file | Applies to the wrong namespace or creates one | +| `kubectl delete -f dir/` on a shared directory | Deletes objects other apps depend on (CRDs, namespaces) | +| `kubectl edit` on a live object | Drift from Git, reverted by GitOps or the next apply | +| Stuck `Terminating` namespace "fixed" by removing finalizers | Orphaned external resources (volumes, LBs, DNS) - find the blocking controller first | + +## Verification + +After any change: `rollout status`, `get` of the touched objects, events since the change, and the user-facing check (HTTP call through the Gateway). A command that returned 0 is not proof the service works. + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/reference/kubectl/generated/kubectl_diff/ +- https://kubernetes.io/docs/reference/kubectl/generated/kubectl_apply/ +- https://kubernetes.io/docs/reference/kubectl/generated/kubectl_delete/ +- https://kubernetes.io/docs/reference/kubectl/generated/kubectl_drain/ +- https://kubernetes.io/docs/reference/kubectl/generated/kubectl_scale/ +- https://kubernetes.io/docs/reference/kubectl/generated/kubectl_rollout/kubectl_rollout_undo/ +- https://kubernetes.io/docs/reference/kubectl/generated/kubectl_debug/ + +## Not verified + +- `kubectl replace --force` semantics (page not read today; long-standing behaviour). +- Finalizer-removal consequences are operational experience, not from the pages above. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/network-policy.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/network-policy.md new file mode 100644 index 0000000..7911ab3 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/network-policy.md @@ -0,0 +1,95 @@ +--- +name: network-policy +description: Namespace isolation with NetworkPolicy - default deny, explicit allows, egress control +when-to-use: Isolating a namespace, restricting egress, debugging traffic blocked by a policy +keywords: networkpolicy, default deny, podSelector, namespaceSelector, ipBlock, egress, dns, policyTypes +priority: high +related: services-gateway-api.md, troubleshooting-playbook.md +--- + +# NetworkPolicy + +**Load when:** a namespace needs isolation, egress must be restricted, or traffic is suddenly blocked after a policy change. + +## Ground rules (verified at the NetworkPolicy page) + +- Policies are enforced by the **network plugin**. Creating one in a cluster whose CNI does not implement it has **no effect** - check the CNI first (devops-k8s-cluster). +- Policies are additive allow-lists. A pod selected by any policy with `Ingress` in `policyTypes` is isolated for ingress; only what some policy allows gets in. Same for `Egress`. +- If `policyTypes` is omitted, `Ingress` is always set and `Egress` only if the policy has egress rules - write `policyTypes` explicitly. +- Every namespace carries the label `kubernetes.io/metadata.name: ` - select namespaces by it instead of custom labels that someone may forget. +- `apiVersion: networking.k8s.io/v1`. + +## Rollout order (never lock out production at once) + +1. Inventory flows: who calls the app, what the app calls (DB, DNS, external APIs, metrics scrape). +2. Write the allow policies first: DNS egress, ingress from the Gateway namespace, egress to the DB, ingress from monitoring. +3. Apply the allows, then the default deny, in one change after `kubectl diff` and server dry-run. +4. Test each flow right after; keep the delete command of the deny policy ready as rollback (run only after the user's go). + +## Selector semantics - the classic trap + +```yaml +# ONE peer: pods labelled app=gw IN namespaces named infra (AND) +- from: + - namespaceSelector: + matchLabels: { kubernetes.io/metadata.name: infra } + podSelector: + matchLabels: { app: gw } +# TWO peers: any pod in infra, OR pods app=gw in the policy's own namespace (OR) +- from: + - namespaceSelector: + matchLabels: { kubernetes.io/metadata.name: infra } + - podSelector: + matchLabels: { app: gw } +``` + +One dash too many turns an AND into an OR and opens the namespace. Review every `-` in `from`/`to`. + +## Egress essentials + +- Default-deny egress also blocks DNS: allow UDP and TCP 53 to the cluster DNS pods (label is distribution-specific - `k8s-app: kube-dns` is common; read it with `kubectl -n kube-system get pods --show-labels`). +- `ipBlock` with `except` for external ranges; pod IPs are ephemeral, never put them in `ipBlock`. +- Traffic to the API server from pods (operators, controllers) needs an explicit egress allow; its address comes from `kubectl get endpointslices -n default -l kubernetes.io/service-name=kubernetes`. +- `endPort` allows a port range on one rule. + +## What NetworkPolicy does not do + +- No L7 (paths, methods), no FQDN rules, no explicit deny rules, no logging - those are CNI-specific extensions (e.g. Cilium policies, devops-k8s-cluster). +- Node-level and host-network pods are generally not covered the same way; behaviour varies by plugin. + +→ Template: [networkpolicy-default-deny.yaml](templates/networkpolicy-default-deny.yaml) + +## Debugging a blocked flow + +```bash +kubectl --context "$CTX" -n "$NS" get networkpolicy +kubectl --context "$CTX" -n "$NS" describe networkpolicy default-deny-all +# which policies select the pod? compare its labels with every podSelector +kubectl --context "$CTX" -n "$NS" get pod "$POD" --show-labels +# test from an ephemeral debug container in the source pod (see troubleshooting-playbook.md) +kubectl --context "$CTX" -n "$NS" debug -it "$POD" --image=busybox:1.37 --target=app -- \ + wget -qO- -T 3 http://db.data.svc.cluster.local:5432 || echo blocked-or-closed +``` + +## Pitfalls + +| Symptom | Cause | +|---------|-------| +| Everything still reachable after default deny | CNI does not enforce NetworkPolicy | +| Name resolution fails after egress deny | DNS egress not allowed | +| Gateway gets 503 after deny | Missing ingress allow from the Gateway's namespace/pods | +| Prometheus targets down | Missing ingress allow from the monitoring namespace | + +## Verification + +A flow is verified only by a real connection test from the source pod, before and after the change - not by reading the YAML. + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/concepts/services-networking/network-policies/ +- https://hub.docker.com/v2/repositories/library/busybox/tags/1.37 (tag exists, updated 2026-10-28) + +## Not verified + +- The "What NetworkPolicy does not do" list (no L7, FQDN, deny rules, logging; host-network behaviour) is general CNI knowledge, not quoted from the page. +- The exact DNS pod label on a given distribution (`k8s-app: kube-dns` is a convention, read it on the cluster). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/probes-lifecycle.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/probes-lifecycle.md new file mode 100644 index 0000000..f8b562b --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/probes-lifecycle.md @@ -0,0 +1,109 @@ +--- +name: probes-lifecycle +description: Startup, readiness and liveness probes, native sidecars and graceful shutdown +when-to-use: Adding health checks, fixing restart loops caused by probes, adding a sidecar, tuning shutdown +keywords: startupProbe, readinessProbe, livenessProbe, grpc probe, sidecar, restartPolicy Always, terminationGracePeriodSeconds +priority: high +related: workloads-scheduling.md, troubleshooting-playbook.md +--- + +# Probes and Pod Lifecycle + +**Load when:** writing health checks, a pod restarts without an app error, a rollout never becomes Ready, or adding a sidecar. + +## What each probe does + +| Probe | On failure | Use for | +|-------|------------|---------| +| `startupProbe` | Container killed after `failureThreshold * periodSeconds`; liveness/readiness wait until it succeeds once | Slow first start (migrations, JIT, cache warm-up) | +| `readinessProbe` | Pod removed from Service endpoints, not restarted; runs for the whole life of the container | "Can I take traffic now?" - dependencies, warm-up, overload | +| `livenessProbe` | Container restarted | Deadlock detection only | + +Readiness and liveness do not depend on each other. If readiness must wait, use `initialDelaySeconds` or a `startupProbe`. + +## Decision rules + +1. **Always readiness** on anything behind a Service. +2. **Liveness only when the process can hang while alive** - and it must check the process itself, never a database or a downstream API. A liveness probe that checks a shared dependency restarts every replica at once when that dependency blips. +3. **startupProbe instead of a big `initialDelaySeconds`** on liveness: the official example gives `failureThreshold: 30` x `periodSeconds: 10` = 300 s to start, then liveness with `failureThreshold: 1` reacts fast. +4. Separate endpoints: `/readyz` (dependencies allowed) and `/livez` (process only). +5. `timeoutSeconds` defaults to 1 s - raise it for endpoints that do real work. + +## Handlers + +| Handler | Notes | +|---------|-------| +| `httpGet` | 2xx/3xx = success; use a named port | +| `tcpSocket` | Port open only - weak signal | +| `grpc` | Built-in gRPC health checking protocol; the app must implement it | +| `exec` | Runs in the container: costs a process per probe, avoid on high-frequency probes | + +```yaml +ports: + - name: http + containerPort: 8080 +startupProbe: + httpGet: { path: /livez, port: http } + failureThreshold: 30 + periodSeconds: 10 +livenessProbe: + httpGet: { path: /livez, port: http } + periodSeconds: 10 + failureThreshold: 3 +readinessProbe: + httpGet: { path: /readyz, port: http } + periodSeconds: 5 + timeoutSeconds: 2 +``` + +## Native sidecars (stable since v1.33) + +An entry in `initContainers` with `restartPolicy: Always` is a sidecar: it starts before the app containers, keeps running alongside them, supports probes, and does not block Job completion. + +```yaml +initContainers: + - name: log-shipper + image: registry.example.com/shipper:1.4.2 + restartPolicy: Always +containers: + - name: app + image: registry.example.com/app:2.3.0 +``` + +Use it instead of a plain second container whenever start order or Job completion matters (proxies, log shippers, credential refreshers). + +## Graceful shutdown + +On delete/eviction the kubelet sends SIGTERM, waits `terminationGracePeriodSeconds` (pod field), then SIGKILL. In parallel the pod leaves the Service endpoints - but proxies and Gateways may still route to it for a short moment. + +- The app must handle SIGTERM: stop accepting, drain in-flight requests, exit. +- If the image cannot do that, a `preStop` hook (`exec` sleep) gives load balancers time to drop the endpoint before SIGTERM. +- Set `terminationGracePeriodSeconds` longer than the longest request or job step. + +## Pitfalls + +| Symptom | Cause | Fix | +|---------|-------|-----| +| Restarts every few minutes, app log clean | Liveness timeout too short under load | Raise `timeoutSeconds`/`failureThreshold`, or drop liveness | +| All replicas restart together | Liveness checks a shared dependency | Liveness = process only | +| Rollout stuck at 0 available | Readiness path wrong or needs auth | `kubectl describe pod` → probe failure event | +| 502 during deploys | No drain time before SIGTERM | `preStop` delay + app SIGTERM handling | + +## Verification + +```bash +kubectl --context "$CTX" -n "$NS" describe pod "$POD" | sed -n '/Liveness/,/Environment/p' +kubectl --context "$CTX" -n "$NS" get events --field-selector involvedObject.name="$POD" +kubectl --context "$CTX" -n "$NS" get pod "$POD" -o jsonpath='{.status.containerStatuses[*].restartCount}' +``` + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +- https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/ + +## Not verified + +- Default values of `periodSeconds`, `failureThreshold`, `successThreshold` (not stated on the task page read; only `timeoutSeconds` = 1 s is). +- The probe page mentions newer options (`protocol: HTTP2` on HTTP probes, TLS mode for gRPC probes); their feature state was not read - do not use them without checking. +- `preStop` `sleep` action (instead of `exec` sleep): feature state not read today. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/rbac-serviceaccounts.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/rbac-serviceaccounts.md new file mode 100644 index 0000000..6955a01 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/rbac-serviceaccounts.md @@ -0,0 +1,99 @@ +--- +name: rbac-serviceaccounts +description: Least-privilege RBAC, ServiceAccounts, token mounting, Pod Security Admission and the restricted securityContext +when-to-use: Granting a workload or a person API access, checking who can do what, hardening pod settings +keywords: rbac, role, rolebinding, clusterrole, serviceaccount, automountServiceAccountToken, auth can-i, pod security admission, securityContext +priority: high +related: config-secrets.md, network-policy.md +--- + +# RBAC, ServiceAccounts and Pod Security + +**Load when:** a workload needs to call the API, a user needs access, a `Forbidden` error appears, or pod security settings must be written. +A full cluster security audit belongs to `fuse-security:security-expert`; this file is the operator's day-to-day. + +## Decision rules + +1. **One ServiceAccount per workload**, never `default`. Most apps never call the API: set `automountServiceAccountToken: false` (on the ServiceAccount or the pod spec) so no token is mounted. +2. **Namespaced first**: `Role` + `RoleBinding`. A `ClusterRole` bound with a **RoleBinding** stays namespaced - use that to reuse a rule set across namespaces. `ClusterRoleBinding` only for truly cluster-wide needs. +3. **Name resources and verbs exactly** - `resourceNames` when the workload touches one object; no `*` on verbs, resources or apiGroups. +4. **Dangerous grants need an explicit reason** (`rbac.authorization.k8s.io/v1` page): `escalate` and `bind` verbs on roles, `impersonate`, `create` on pods or workload controllers (it implies reading every Secret of the namespace), `get/list/watch` on secrets, `nodes/proxy`. +5. Humans get access through groups from the identity provider, bound to roles - not per-user bindings scattered across namespaces. + +## Checking access (read-only) + +```bash +kubectl --context "$CTX" auth whoami +kubectl --context "$CTX" -n "$NS" auth can-i --list --as=system:serviceaccount:"$NS":web +kubectl --context "$CTX" -n "$NS" auth can-i create pods --as=system:serviceaccount:"$NS":web +kubectl --context "$CTX" get rolebindings,clusterrolebindings -A -o wide | grep -w web +``` + +`--as` uses impersonation: the operator needs `impersonate` rights for it to work. A `no` from `can-i` is the proof that a grant is absent. + +→ Template: [rbac-least-privilege.yaml](templates/rbac-least-privilege.yaml) + +## Pod Security Admission (namespace labels) + +Labels `pod-security.kubernetes.io/: ` and `pod-security.kubernetes.io/-version: ` on the Namespace. +Modes: `enforce` (reject), `audit` (annotate audit log), `warn` (client warning). Levels: `privileged`, `baseline`, `restricted`. + +```yaml +apiVersion: v1 +kind: Namespace +metadata: + name: shop + labels: + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: v1.37 + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/audit: restricted +``` + +Pinning `enforce-version` stops a control-plane upgrade from tightening the policy under running apps; bump it deliberately. +Introduce `restricted` with `warn` + `audit` first, read the warnings, then add `enforce` - and test with a server dry-run: + +```bash +kubectl --context "$CTX" label --dry-run=server --overwrite ns shop pod-security.kubernetes.io/enforce=restricted +``` + +## The `restricted` securityContext (verified at the Pod Security Standards page) + +| Field | Required value | +|-------|----------------| +| `allowPrivilegeEscalation` | `false` | +| `runAsNonRoot` | `true` (pod or container level) | +| `seccompProfile.type` | `RuntimeDefault` or `Localhost` | +| `capabilities.drop` | list that includes `ALL`; only `NET_BIND_SERVICE` may be added | +| volumes | no `hostPath`; only the allowed volume types | + +Add on top (not required by the level, but cheap): `readOnlyRootFilesystem: true` with an `emptyDir` for `/tmp`, an explicit numeric `runAsUser`/`runAsGroup` so `runAsNonRoot` can be checked against images without a numeric `USER`. + +## Pitfalls + +| Symptom | Cause | Fix | +|---------|-------|-----| +| `forbidden: User "system:serviceaccount:ns:web" cannot list ...` | Missing Role rule or binding in that namespace | Add the narrowest rule; prove with `auth can-i` | +| Pod rejected `violates PodSecurity "restricted:..."` | Missing securityContext field | Message names each field; fix the manifest, not the label | +| `container has runAsNonRoot and image will run as root` | Image `USER` is root or non-numeric | Set `runAsUser` to the image's UID, or fix the image (devops-docker) | +| Token found in a pod that never calls the API | automount left on | `automountServiceAccountToken: false` | + +## Verification + +```bash +kubectl --context "$CTX" -n "$NS" get pod "$POD" -o jsonpath='{.spec.serviceAccountName}{" automount="}{.spec.automountServiceAccountToken}{"\n"}' +kubectl --context "$CTX" get ns "$NS" --show-labels | tr ',' '\n' | grep pod-security +``` + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/reference/access-authn-authz/rbac/ +- https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +- https://kubernetes.io/docs/concepts/security/pod-security-standards/ +- https://kubernetes.io/docs/concepts/security/pod-security-admission/ +- https://kubernetes.io/docs/tasks/configure-pod-container/enforce-standards-namespace-labels/ (`kubectl label --dry-run=server` to evaluate a level) + +## Not verified + +- The full list of volume types allowed by `restricted` (see the Pod Security Standards page before relying on a specific type). +- That `impersonate`, `nodes/proxy` belong on the dangerous list is general RBAC guidance; only `escalate`/`bind` were matched on the RBAC page today. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/resources-qos-hpa.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/resources-qos-hpa.md new file mode 100644 index 0000000..b3ce9ee --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/resources-qos-hpa.md @@ -0,0 +1,98 @@ +--- +name: resources-qos-hpa +description: Requests and limits, QoS classes, in-place resize, pod-level resources and HorizontalPodAutoscaler v2 +when-to-use: Sizing containers, OOMKilled or throttled pods, eviction under pressure, setting up autoscaling +keywords: requests, limits, qos, guaranteed, burstable, besteffort, oomkilled, in-place resize, hpa, autoscaling/v2, behavior, tolerance +priority: high +related: workloads-scheduling.md, troubleshooting-playbook.md +--- + +# Resources, QoS and Autoscaling + +**Load when:** choosing requests/limits, a pod is `OOMKilled` or slow, nodes evict pods, or an HPA is needed. + +## What the scheduler and kubelet do with the numbers + +- **Requests** are what the scheduler reserves; a pod that does not fit any node's free requests stays `Pending`. +- **Memory limit**: exceeding it lets the kernel OOM-kill the container (`Reason: OOMKilled`, exit code 137). +- **CPU limit**: enforced by throttling, never by a kill. +- Limit set without a request → the request is copied from the limit (unless an admission default applies). + +## QoS classes (verified at the Pod QoS page) + +| Class | Rule | Eviction order under node pressure | +|-------|------|------------------------------------| +| `Guaranteed` | Every container has CPU and memory requests = limits, all > 0 | Last | +| `Burstable` | At least one request or limit, not Guaranteed | Middle | +| `BestEffort` | No requests or limits at all | First | + +Under resource-pressure eviction, only pods exceeding their requests are candidates - accurate requests protect a pod. + +## Decision rules + +1. **Every container gets CPU and memory requests**, set from observed usage (p95 over a representative week, devops-observability), not guesses. +2. **Memory limit always**; equal to the request for latency-critical or stateful apps (predictable, Guaranteed if CPU also matches). +3. **CPU limit is a choice, not a reflex**: it caps bursts through throttling. Set it for noisy batch neighbours or when Guaranteed QoS is required; leave it out for latency-sensitive services whose requests are accurate. +4. Namespace guard rails: a `LimitRange` for defaults and a `ResourceQuota` for totals, so a manifest without resources does not land as BestEffort. +5. JVM, Node.js, Go: make the runtime respect the container limit (heap/GOMEMLIMIT) or the OOM kill comes before GC. + +## In-place resize (stable since v1.35) + +CPU and memory of a running container can change without recreating the pod, through the pod's `resize` subresource: + +```bash +kubectl --context "$CTX" -n "$NS" edit pod "$POD" --subresource resize +``` + +`resizePolicy` per resource: `NotRequired` (default, apply live) or `RestartContainer` (restart that container). Pod-level resource resize and memory-backed `emptyDir` resize are alpha in v1.37 (feature gates off). +For managed workloads change the Deployment/StatefulSet template in Git - a hand resize of one pod is lost at the next rollout. + +## Pod-level resources (beta since v1.34, on by default) + +`spec.resources` at pod level sets a shared budget for all containers; useful with sidecars. Check the cluster minor and feature gate before relying on it. + +## HorizontalPodAutoscaler (`autoscaling/v2`) + +- Needs the **Metrics Server** (installed separately) for CPU/memory, or an adapter for custom/external metrics. +- Utilization targets are a percentage **of the request** - no request, no CPU-based HPA. +- `behavior.scaleDown.stabilizationWindowSeconds: 300` (the documented default for scale-down) avoids flapping; `policies` cap the rate (`Pods` or `Percent` per `periodSeconds`), `selectPolicy` picks `Max`/`Min`/`Disabled`. +- `behavior.scaleUp.tolerance` / `scaleDown.tolerance` (stable since v1.37) override the global 10% tolerance per direction; the global default is a kube-controller-manager flag, not settable through the API. +- `minReplicas: 0` is beta since v1.37 and only for object/external metrics. +- Do not let the HPA and a hard-coded `replicas:` fight: omit `replicas` from the Deployment in Git once an HPA owns it (or the GitOps tool resets it). +- HPA and a VPA in auto mode must not act on the same resource metric. + +```bash +kubectl --context "$CTX" -n "$NS" autoscale deploy web --min=2 --max=10 --cpu=70% --dry-run=server -o yaml +``` + +→ Template: [hpa.yaml](templates/hpa.yaml) + +## Pitfalls + +| Symptom | Cause | Fix | +|---------|-------|-----| +| `OOMKilled`, exit 137 | Memory limit below the real peak, or runtime ignores the limit | Raise limit from metrics; configure the runtime | +| Latency spikes, CPU below request on average | CPU throttling by a tight limit | Raise or remove the CPU limit; check `container_cpu_cfs_throttled_periods_total` | +| HPA `/70%` | Metrics Server missing or no CPU request | Install metrics-server (cluster work), set requests | +| Pods evicted on a busy node | BestEffort/Burstable over requests under pressure | Accurate requests, memory limit = request | + +## Verification + +```bash +kubectl --context "$CTX" -n "$NS" top pods --containers +kubectl --context "$CTX" -n "$NS" get pod "$POD" -o jsonpath='{.status.qosClass}' +kubectl --context "$CTX" -n "$NS" describe hpa web # conditions AbleToScale, ScalingActive +``` + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ +- https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/ +- https://kubernetes.io/docs/tasks/configure-pod-container/resize-container-resources/ +- https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/ + +## Not verified + +- Rule 3 (CPU limits optional for latency-sensitive services) and rule 5 (runtime memory settings) are operational practice, not from the pages above. +- `--cpu=70%` is the form shown on the HPA page today; older kubectl clients used `--cpu-percent` - check `kubectl autoscale --help` on the client in use. +- The metric name `container_cpu_cfs_throttled_periods_total` (cAdvisor) was not re-read today. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/services-gateway-api.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/services-gateway-api.md new file mode 100644 index 0000000..4ba5ad9 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/services-gateway-api.md @@ -0,0 +1,101 @@ +--- +name: services-gateway-api +description: Services, traffic entry with Gateway API, and leaving Ingress / Ingress NGINX +when-to-use: Exposing an app, writing a Gateway or HTTPRoute, migrating off Ingress NGINX +keywords: service, clusterip, loadbalancer, headless, gateway api, httproute, referencegrant, ingress, ingress-nginx, ingress2gateway +priority: high +related: network-policy.md, troubleshooting-playbook.md +--- + +# Services and Gateway API + +**Load when:** exposing a workload inside or outside the cluster, or an Ingress / Ingress NGINX setup must be replaced. + +## Status on 2026-11-10 (verified) + +| Item | Status | Source | +|------|--------|--------| +| Ingress API | Stable since v1.19 and **frozen**: still GA, no new features; the project recommends Gateway | https://kubernetes.io/docs/concepts/services-networking/ingress/ | +| Ingress NGINX | **Retired**: best-effort maintenance until March 2026, then no releases, no bugfixes, no security fixes; existing installs keep running | https://kubernetes.io/blog/2025/11/11/ingress-nginx-retirement/ | +| `kubernetes/ingress-nginx` repo | Archived (`"archived": true`, last push 2026-03-23) | https://api.github.com/repos/kubernetes/ingress-nginx | +| Gateway API | v1.7.0 released 2026-10-29 | https://github.com/kubernetes-sigs/gateway-api/releases/tag/v1.7.0 | +| ingress2gateway | v1.2.0 (2026-07-07) | https://github.com/kubernetes-sigs/ingress2gateway/releases | + +Decision: new exposure = Gateway API. A cluster still on Ingress NGINX carries unpatched CVE risk: plan the migration, do not extend it. + +## Services + +| Type | Use | +|------|-----| +| `ClusterIP` (default) | In-cluster callers; the backend of every HTTPRoute | +| headless (`clusterIP: None`) | StatefulSet per-pod DNS, client-side discovery | +| `NodePort` | Rarely directly; building block for external LBs | +| `LoadBalancer` | L4 exposure; on bare metal needs an LB implementation (devops-k8s-cluster) | + +`spec.trafficDistribution: PreferSameZone` or `PreferSameNode` keeps traffic local when endpoints exist there; `PreferClose` is the deprecated older alias of `PreferSameZone` (Service page). +A Service with no endpoints = selector does not match pod labels, or no pod is Ready. + +## Gateway API model + +| Resource (`gateway.networking.k8s.io/v1`) | Owner | Role | +|-------------------------------------------|-------|------| +| `GatewayClass` | Controller install | Which implementation | +| `Gateway` | Platform team | Listeners (port, protocol, hostname, TLS), `allowedRoutes` | +| `ListenerSet` | App team | Extra listeners merged onto a shared Gateway (Standard since v1.5) | +| `HTTPRoute` | App team | Host/path/header matching, filters, weighted `backendRefs` | +| `TLSRoute` | App team | SNI routing (Standard since v1.5) | +| `TCPRoute`, `UDPRoute` | App team | L4 routing (Standard and `v1` since v1.6; `v1alpha2` deprecated) | +| `ReferenceGrant` | Owner of the target namespace | Allows a cross-namespace reference (e.g. Gateway → Secret); `v1` since v1.5 | + +Standard additions: HTTPRoute CORS filter and client certificate validation (v1.5); HTTPRoute `retry` (`codes`, `attempts`, `backoff`) in Standard since v1.7 - `backoff` behaviour is implementation-specific. +Since v1.6, new experimental resources live in `gateway.networking.x-k8s.io` with an `X` prefix (`XBackend`, `XMesh`); anything in that group is experimental. + +## Install the CRDs (pinned tag) + +```bash +kubectl --context "$CTX" apply --server-side --dry-run=server \ + -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.7.0/standard-install.yaml +kubectl --context "$CTX" apply --server-side \ + -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.7.0/standard-install.yaml +``` + +Use `experimental-install.yaml` only on purpose. Many controllers bundle their own CRDs - check before installing a second copy, and check the controller's conformance report for the release you install. + +## Routing patterns + +- One shared Gateway per environment in an infra namespace; app namespaces attach via `allowedRoutes.namespaces.from: Selector`. +- HTTP listener carries only a `RequestRedirect` route (`scheme: https`, `statusCode: 301`); the HTTPS listener (`tls.mode: Terminate`, `certificateRefs`) carries the app routes. +- `parentRefs[].sectionName` pins a route to one listener. +- Canary: two `backendRefs` with `weight`. + +→ Template: [gateway-httproute.yaml](templates/gateway-httproute.yaml) + +## Migrating from Ingress NGINX + +1. Inventory: `kubectl --context "$CTX" get ingress -A -o yaml > ingress-inventory.yaml`; list every `nginx.ingress.kubernetes.io/*` annotation - each one needs an explicit equivalent or is dropped. +2. Convert with ingress2gateway as a draft, then review by hand (annotations do not all map). +3. Deploy the Gateway controller next to Ingress NGINX; point a test hostname at the new Gateway. +4. Move hostnames one by one (DNS or LB switch), watch errors, keep the old path ready. +5. Remove Ingress objects and the controller only after the user's go ([kubectl-safe-ops.md](kubectl-safe-ops.md)). + +## Verification + +```bash +kubectl --context "$CTX" get gatewayclass +kubectl --context "$CTX" -n infra get gateway shared -o jsonpath='{.status.conditions}' +kubectl --context "$CTX" -n "$NS" get httproute web -o yaml | grep -A5 'parents:' # Accepted / ResolvedRefs +kubectl --context "$CTX" -n "$NS" get endpointslices -l kubernetes.io/service-name=web +``` + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/concepts/services-networking/ingress/ , https://kubernetes.io/docs/concepts/services-networking/service/ +- https://kubernetes.io/blog/2026/04/21/gateway-api-v1-5/ , https://kubernetes.io/blog/2026/08/03/gateway-api-v1-6-release/ +- https://gateway-api.sigs.k8s.io/guides/getting-started/ (install commands), Context7 `/websites/gateway-api_sigs_k8s_io` (redirect, multiple-ns guides) + +## Not verified + +- Minimum Kubernetes version for Gateway API v1.7 (the v1.5 post says 1.30 or later; not re-read for v1.7). +- GRPCRoute and BackendTLSPolicy channel/version (not read today). +- Which controllers are conformant with v1.7.0; ingress2gateway's annotation coverage. +- Feature state of `trafficDistribution` values `PreferSameZone`/`PreferSameNode`. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/storage-csi.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/storage-csi.md new file mode 100644 index 0000000..e0870cf --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/storage-csi.md @@ -0,0 +1,104 @@ +--- +name: storage-csi +description: PersistentVolumes, claims, StorageClasses, CSI expansion, snapshots and data-safety rules +when-to-use: Giving a workload persistent storage, resizing a volume, choosing access modes or reclaim policy +keywords: pv, pvc, storageclass, csi, reclaimPolicy, WaitForFirstConsumer, ReadWriteOncePod, volumesnapshot, volumeattributesclass +priority: high +related: workloads-scheduling.md, kubectl-safe-ops.md +--- + +# Storage: PV, PVC, StorageClass, CSI + +**Load when:** a workload needs data that outlives its pods, a PVC is Pending, or a volume must grow. + +## Model + +- The app asks with a **PVC** (size, access mode, `storageClassName`). +- A **StorageClass** names a CSI provisioner and its parameters; dynamic provisioning creates the **PV**. +- StatefulSets create one PVC per ordinal from `volumeClaimTemplates`. +- Backend choice (Ceph RBD, NFS, local, ZFS, cloud disks) and backups are in devops-storage-backup; installing CSI drivers is cluster work (devops-k8s-cluster). + +## Decision rules + +| Question | Rule | +|----------|------| +| Reclaim policy | `Delete` (usual default) removes the backing volume with the PVC. Data that must survive a mistaken PVC delete → a StorageClass with `reclaimPolicy: Retain`. `Recycle` is deprecated | +| Binding mode | `volumeBindingMode: WaitForFirstConsumer` for topology-bound storage (zonal or local disks): the volume is created where the pod is scheduled | +| Access mode | `ReadWriteOnce` = one node; `ReadWriteOncePod` = one pod cluster-wide (stable since v1.29, CSI volumes only); `ReadWriteMany` only if the backend supports it (NFS, CephFS) | +| Growth | StorageClass `allowVolumeExpansion: true`; then raise `spec.resources.requests.storage` on the PVC. Shrinking is not supported | +| Default class | Exactly one class annotated `storageclass.kubernetes.io/is-default-class: "true"`; PVCs without a class get it, retroactively too (stable since v1.28) | +| Changing IOPS/throughput live | VolumeAttributesClass (`storage.k8s.io/v1`) if the CSI driver supports modification | + +## Expansion (CSI, stable since v1.24) + +```bash +kubectl --context "$CTX" -n "$NS" get sc "$(kubectl --context "$CTX" -n "$NS" get pvc data-db-0 -o jsonpath='{.spec.storageClassName}')" \ + -o jsonpath='{.allowVolumeExpansion}' +kubectl --context "$CTX" -n "$NS" get pvc data-db-0 -o yaml > pvc-before.yaml # evidence +# edit spec.resources.requests.storage in Git, then diff + server dry-run + apply +kubectl --context "$CTX" -n "$NS" get pvc data-db-0 -w # watch conditions until capacity changes +``` + +For a StatefulSet, resize each existing PVC directly; changing `volumeClaimTemplates` does not resize PVCs that already exist (and the API may reject the template change - see Not verified). + +## Snapshots (stable since v1.20) + +`VolumeSnapshot` / `VolumeSnapshotClass` in `snapshot.storage.k8s.io/v1` need the snapshot CRDs, the snapshot controller and a CSI driver that supports snapshots. +A snapshot on the same storage system is not a backup - copy off-system (devops-storage-backup). + +```yaml +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshot +metadata: + name: data-db-0-pre-upgrade + namespace: shop +spec: + volumeSnapshotClassName: csi-snapclass + source: + persistentVolumeClaimName: data-db-0 +``` + +Take one before any risky change to stateful data (schema migration, major upgrade, resize). + +## Data-safety rules + +1. Never delete a PVC, PV or namespace holding PVCs without: the list of affected PVCs, their reclaim policy, a verified backup, and the user's go ([kubectl-safe-ops.md](kubectl-safe-ops.md)). +2. Before deleting a StatefulSet, read `persistentVolumeClaimRetentionPolicy` - `whenDeleted: Delete` removes the data. +3. To keep a `Delete`-policy volume when its PVC must go: switch the PV's `persistentVolumeReclaimPolicy` to `Retain` first (a mutation - propose it). +4. `emptyDir` is scratch: it dies with the pod and is wiped by `kubectl drain --delete-emptydir-data`. + +## PVC Pending - diagnosis + +```bash +kubectl --context "$CTX" -n "$NS" describe pvc data-db-0 # Events: provisioning errors +kubectl --context "$CTX" get sc # class exists? default? +kubectl --context "$CTX" get csidrivers # driver registered? +``` + +| Event / state | Cause | +|---------------|-------| +| `waiting for first consumer to be created before binding` | Normal with `WaitForFirstConsumer` until a pod uses it | +| `storageclass ... not found` | Typo or class not installed | +| provisioning failed / quota | Backend full, quota, credentials of the CSI driver | +| Pod `Pending` with volume node affinity conflict | Zonal volume in a zone where the pod cannot run | + +## Verification + +```bash +kubectl --context "$CTX" -n "$NS" get pvc -o wide +kubectl --context "$CTX" get pv -o custom-columns=NAME:.metadata.name,CLAIM:.spec.claimRef.name,POLICY:.spec.persistentVolumeReclaimPolicy,STATUS:.status.phase +``` + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/concepts/storage/persistent-volumes/ +- https://kubernetes.io/docs/concepts/storage/storage-classes/ +- https://kubernetes.io/docs/concepts/storage/volume-snapshots/ +- https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + +## Not verified + +- VolumeAttributesClass GA version: the concept page banner says "Stable since v1.36", while the blog index lists "Kubernetes v1.34: VolumeAttributesClass for Volume Modification GA". The two disagree; check `kubectl api-resources | grep volumeattributesclasses` on the target cluster. +- Whether `volumeClaimTemplates` updates are allowed on 1.35-1.37 (the StatefulSet page read today says nothing about it; historically immutable). +- Exact PVC condition names shown during an online expansion (driver-dependent). +- The `csi-snapclass` name is a placeholder - read `kubectl get volumesnapshotclass`. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/cronjob.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/cronjob.yaml new file mode 100644 index 0000000..30640ac --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/cronjob.yaml @@ -0,0 +1,65 @@ +# Scheduled job (example: nightly export). Replace: shop, name, schedule, image, command. +# timeZone takes an IANA name (stable since v1.27); CRON_TZ/TZ inside schedule is rejected. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: nightly-export + namespace: shop +spec: + schedule: "15 2 * * *" + timeZone: "Europe/Zurich" + concurrencyPolicy: Forbid # never two exports at once + startingDeadlineSeconds: 600 # a run missed by more than 10 min is skipped + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + suspend: false + jobTemplate: + spec: + backoffLimit: 2 + activeDeadlineSeconds: 3600 + ttlSecondsAfterFinished: 86400 + podReplacementPolicy: Failed # no overlap; with podFailurePolicy set, Failed is the default and the only allowed value + podFailurePolicy: + rules: + - action: FailJob # exit code 2 = bad input, retrying will not help + onExitCodes: + containerName: export + operator: In + values: [2] + - action: Ignore # evictions/drains do not count against backoffLimit + onPodConditions: + - type: DisruptionTarget + template: + metadata: + labels: + app.kubernetes.io/name: nightly-export + spec: + restartPolicy: Never # required with podFailurePolicy + serviceAccountName: nightly-export + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: export + image: registry.example.com/shop/export:1.8.0 + args: ["--target", "s3://exports/shop"] + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/deployment-production.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/deployment-production.yaml new file mode 100644 index 0000000..fa737aa --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/deployment-production.yaml @@ -0,0 +1,137 @@ +# Production-grade stateless app: Deployment + Service + PodDisruptionBudget. +# Replace: shop (namespace), web (name), image, ports, probe paths, resources. +# Assumes: namespace labelled pod-security.kubernetes.io/enforce=restricted, +# ServiceAccount "web" from rbac-least-privilege.yaml, an HPA owning replicas (hpa.yaml). +# Apply: kubectl --context CTX -n shop diff -f . ; kubectl --context CTX -n shop apply --dry-run=server -f . +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + namespace: shop + labels: + app.kubernetes.io/name: web + app.kubernetes.io/part-of: shop +spec: + # replicas omitted on purpose: the HPA owns it. Set it only if no HPA exists. + revisionHistoryLimit: 5 + progressDeadlineSeconds: 600 + minReadySeconds: 10 + selector: + matchLabels: + app.kubernetes.io/name: web + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + labels: + app.kubernetes.io/name: web + app.kubernetes.io/part-of: shop + spec: + serviceAccountName: web + automountServiceAccountToken: false + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: web + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: web + containers: + - name: app + image: registry.example.com/shop/web:2.3.0 # pin a tag, preferably @sha256 digest + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 + protocol: TCP + envFrom: + - configMapRef: + name: web-config-v1 # immutable, versioned name + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + memory: 256Mi # memory limit always; CPU limit deliberately omitted + startupProbe: + httpGet: + path: /livez + port: http + periodSeconds: 10 + failureThreshold: 30 + livenessProbe: + httpGet: + path: /livez # process health only, never a dependency + port: http + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: http + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 3 + lifecycle: + preStop: + exec: + command: ["sh", "-c", "sleep 5"] # drop from endpoints before SIGTERM; needs a shell in the image + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: + sizeLimit: 64Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: web + namespace: shop + labels: + app.kubernetes.io/name: web +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: web + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: web + namespace: shop +spec: + maxUnavailable: 1 + unhealthyPodEvictionPolicy: AlwaysAllow + selector: + matchLabels: + app.kubernetes.io/name: web diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/gateway-httproute.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/gateway-httproute.yaml new file mode 100644 index 0000000..4c1e333 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/gateway-httproute.yaml @@ -0,0 +1,99 @@ +# Gateway API (standard channel, gateway.networking.k8s.io/v1) - shared Gateway + HTTPS redirect + app route. +# Prerequisites: CRDs from a pinned release (v1.7.0 standard-install.yaml, server-side apply) and a +# controller providing the GatewayClass "shared-gateway-class" - read `kubectl get gatewayclass`. +# ReferenceGrant is v1 since Gateway API v1.5; on older CRDs use the version they serve. +# Layout: Gateway in "infra", certificate Secret in "certs", app route + Service in "shop". +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: shared + namespace: infra +spec: + gatewayClassName: shared-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + hostname: "*.example.com" + allowedRoutes: + namespaces: + from: Same # only the redirect route in infra attaches here + - name: https + protocol: HTTPS + port: 443 + hostname: "*.example.com" + tls: + mode: Terminate + certificateRefs: + - kind: Secret + group: "" + name: wildcard-example-com + namespace: certs # cross-namespace: needs the ReferenceGrant below + allowedRoutes: + namespaces: + from: Selector + selector: + matchLabels: + shared-gateway-access: "true" # label app namespaces deliberately +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: ReferenceGrant +metadata: + name: allow-infra-gateways + namespace: certs # lives in the namespace being referenced +spec: + from: + - group: gateway.networking.k8s.io + kind: Gateway + namespace: infra + to: + - group: "" + kind: Secret + name: wildcard-example-com # always set name: omitting it grants every Secret in "certs" +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: https-redirect + namespace: infra +spec: + parentRefs: + - name: shared + sectionName: http + hostnames: + - "*.example.com" + rules: + - filters: + - type: RequestRedirect + requestRedirect: + scheme: https + statusCode: 301 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: web + namespace: shop # namespace must carry shared-gateway-access: "true" +spec: + parentRefs: + - name: shared + namespace: infra + sectionName: https + hostnames: + - shop.example.com + rules: + - matches: + - path: + type: PathPrefix + value: /api + backendRefs: + - name: api + port: 80 + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: web # canary: add a second backendRef and split with weight + port: 80 + weight: 100 diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/hpa.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/hpa.yaml new file mode 100644 index 0000000..923c14a --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/hpa.yaml @@ -0,0 +1,42 @@ +# CPU-based autoscaling for Deployment "web" (autoscaling/v2). +# Requires: Metrics Server in the cluster and CPU requests on every container +# (utilization is a percentage of the request). Remove "replicas" from the Deployment in Git. +# The behavior below restates the documented defaults with a slower scale-down rate. +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: web + namespace: shop +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: web + minReplicas: 2 # keep >= 2 so the PodDisruptionBudget can allow one eviction + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 # documented default + policies: + - type: Percent + value: 50 # remove at most half of the pods per minute + periodSeconds: 60 + selectPolicy: Min + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 15 + - type: Pods + value: 4 + periodSeconds: 15 + selectPolicy: Max + # tolerance: 0.05 # per-direction tolerance, stable since v1.37 - only on 1.37+ control planes diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/networkpolicy-default-deny.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/networkpolicy-default-deny.yaml new file mode 100644 index 0000000..dfbcaae --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/networkpolicy-default-deny.yaml @@ -0,0 +1,140 @@ +# Namespace isolation for "shop": default deny (ingress + egress) plus the minimum allows. +# Only effective if the cluster's CNI enforces NetworkPolicy. +# Apply the allows and the deny in ONE change after diff + server dry-run, then test every flow. +# Check the DNS pod labels on your distribution: kubectl -n kube-system get pods --show-labels +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-all + namespace: shop +spec: + podSelector: {} # every pod in the namespace + policyTypes: + - Ingress + - Egress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-dns-egress + namespace: shop +spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: # same list item as namespaceSelector -> AND + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-gateway-to-web + namespace: shop +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: web + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: infra # namespace running the Gateway data plane + ports: + - protocol: TCP + port: 8080 # container port, not the Service port +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-web-to-db + namespace: shop +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: web + policyTypes: + - Egress + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: db + ports: + - protocol: TCP + port: 5432 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-db-from-web + namespace: shop +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: db + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: web + ports: + - protocol: TCP + port: 5432 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-db-peers # replication between StatefulSet replicas; without it a clustered DB splits + namespace: shop +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: db + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: db + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: db +--- +# Any other outbound flow (CronJob export to object storage, external APIs) needs its own +# egress allow, e.g. an ipBlock for the storage endpoint - default deny blocks it otherwise. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-monitoring-scrape + namespace: shop +spec: + podSelector: {} + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + ports: + - protocol: TCP + port: 9090 # metrics port of your pods diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/rbac-least-privilege.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/rbac-least-privilege.yaml new file mode 100644 index 0000000..e8d0a57 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/rbac-least-privilege.yaml @@ -0,0 +1,53 @@ +# Least-privilege identity for a workload that reads ONE ConfigMap and watches pods of its own app. +# Most apps need no API access at all: then keep only the ServiceAccount (automount false) and drop Role/RoleBinding. +# Prove the result: kubectl --context CTX -n shop auth can-i --list --as=system:serviceaccount:shop:web +apiVersion: v1 +kind: ServiceAccount +metadata: + name: web + namespace: shop +automountServiceAccountToken: false # pods that need the token set automountServiceAccountToken: true in their spec +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: web + namespace: shop +rules: + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["web-feature-flags"] # one named object + verbs: ["get", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] # read-only; never "create" (implies reading every Secret) +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: web + namespace: shop +subjects: + - kind: ServiceAccount + name: web + namespace: shop +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: web +--- +# Read-only access for a human team, reusing the built-in "view" ClusterRole but scoped +# to this namespace through a RoleBinding (a RoleBinding never grants cluster-wide). +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: team-shop-view + namespace: shop +subjects: + - kind: Group + apiGroup: rbac.authorization.k8s.io + name: team-shop # group name as asserted by your identity provider +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: view diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/statefulset-pvc.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/statefulset-pvc.yaml new file mode 100644 index 0000000..ac65994 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/templates/statefulset-pvc.yaml @@ -0,0 +1,118 @@ +# StatefulSet with headless Service and one PVC per replica. +# Replace: shop, db, image, port, storage class, size, probes. +# Data safety: PVCs are retained on StatefulSet delete and scale-down (stable since v1.32). +# The StorageClass "fast-retain" should set reclaimPolicy: Retain and +# volumeBindingMode: WaitForFirstConsumer - read `kubectl get sc` for the real name. +apiVersion: v1 +kind: Service +metadata: + name: db-headless + namespace: shop + labels: + app.kubernetes.io/name: db +spec: + clusterIP: None # headless: per-pod DNS db-0.db-headless.shop.svc + publishNotReadyAddresses: false + selector: + app.kubernetes.io/name: db + ports: + - name: tcp-db + port: 5432 + targetPort: tcp-db +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: db + namespace: shop + labels: + app.kubernetes.io/name: db +spec: + serviceName: db-headless + replicas: 3 + podManagementPolicy: OrderedReady + minReadySeconds: 10 + revisionHistoryLimit: 5 + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + updateStrategy: + type: RollingUpdate + rollingUpdate: + partition: 0 # raise to stage a canary on the highest ordinals + selector: + matchLabels: + app.kubernetes.io/name: db + template: + metadata: + labels: + app.kubernetes.io/name: db + spec: + serviceAccountName: db + automountServiceAccountToken: false + terminationGracePeriodSeconds: 120 # time to flush and shut down cleanly + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: db + containers: + - name: db + image: registry.example.com/shop/db:17.6 # placeholder: pin the real tag/digest + ports: + - name: tcp-db + containerPort: 5432 + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "1" + memory: 2Gi # requests = limits -> Guaranteed QoS + startupProbe: + tcpSocket: + port: tcp-db + periodSeconds: 10 + failureThreshold: 60 + readinessProbe: + tcpSocket: + port: tcp-db # replace with an exec/app-level check where the image provides one + periodSeconds: 10 + timeoutSeconds: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: data + mountPath: /var/lib/db + volumeClaimTemplates: + - metadata: + name: data # PVCs are named data-db-0, data-db-1, ... + spec: + accessModes: ["ReadWriteOncePod"] # CSI volumes only; use ReadWriteOnce otherwise + storageClassName: fast-retain + resources: + requests: + storage: 50Gi +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: db + namespace: shop +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: db diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/troubleshooting-playbook.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/troubleshooting-playbook.md new file mode 100644 index 0000000..5ab109c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/troubleshooting-playbook.md @@ -0,0 +1,110 @@ +--- +name: troubleshooting-playbook +description: Fixed-order troubleshooting of pods, Services, DNS and nodes - describe, events, logs, exec/debug, node +when-to-use: A pod is Pending, CrashLoopBackOff, not Ready, a Service has no endpoints, DNS fails, or a node misbehaves +keywords: troubleshooting, pending, crashloopbackoff, imagepullbackoff, oomkilled, events, logs --previous, kubectl debug, ephemeral container, dns, endpoints +priority: critical +related: kubectl-safe-ops.md, probes-lifecycle.md, resources-qos-hpa.md, network-policy.md +--- + +# Troubleshooting Playbook + +**Load when:** anything in a workload does not behave. The order is fixed; skip a step only when the previous one already names the cause with evidence. Every step below is read-only except `debug`, which adds an ephemeral container (propose it on production). + +```bash +CTX=prod-eu; NS=shop; POD=web-7d9c8b6f5-abcde +``` + +## Step 1 - describe + +```bash +kubectl --context "$CTX" -n "$NS" get pod "$POD" -o wide +kubectl --context "$CTX" -n "$NS" describe pod "$POD" +``` + +Read: `Status`/`Reason`, each container's `State` and `Last State` (exit code, `OOMKilled`), `Restart Count`, probe definitions, `Conditions` (`PodScheduled`, `Initialized`, `ContainersReady`, `Ready`), node, QoS class. + +## Step 2 - events + +```bash +kubectl --context "$CTX" -n "$NS" events --for pod/"$POD" +kubectl --context "$CTX" -n "$NS" get events --sort-by=.lastTimestamp | tail -30 +``` + +Events expire (about an hour by default on most clusters) - capture them early. Scheduler, kubelet, probe and volume errors appear here, not in app logs. + +## Step 3 - logs + +```bash +kubectl --context "$CTX" -n "$NS" logs "$POD" -c app --tail=200 +kubectl --context "$CTX" -n "$NS" logs "$POD" -c app --previous --tail=200 # the crashed instance +kubectl --context "$CTX" -n "$NS" logs deploy/web --all-containers --since=15m +``` + +For CrashLoopBackOff, `--previous` is the one that shows the crash. + +## Step 4 - exec / debug + +```bash +kubectl --context "$CTX" -n "$NS" exec -it "$POD" -c app -- sh # if the image has a shell +kubectl --context "$CTX" -n "$NS" debug -it "$POD" --image=busybox:1.37 --target=app # ephemeral container sharing app's process namespace +kubectl --context "$CTX" -n "$NS" debug "$POD" -it --copy-to=web-debug --container=app -- sh # copy with a changed command (crashing image) +``` + +- `--target` attaches to the app's processes for distroless images. +- `--profile` (`general`, `baseline`, `restricted`, `netadmin`, `sysadmin`) sets the debug container's privileges; under PSA `restricted` use `--profile=restricted`. +- A `--copy-to` pod is a new pod: delete it afterwards (propose), never use `--replace` on production. + +## Step 5 - node + +```bash +kubectl --context "$CTX" describe node "$NODE" # Conditions: MemoryPressure, DiskPressure, PIDPressure, Ready; Allocated resources +kubectl --context "$CTX" top node "$NODE" +kubectl --context "$CTX" debug node/"$NODE" -it --image=busybox:1.37 # host namespaces, host filesystem at /host +``` + +Node-level repair (kubelet, container runtime, disk) moves to devops-k8s-cluster and devops-linux. + +## Symptom map + +| Symptom | First evidence | Usual causes | +|---------|----------------|--------------| +| `Pending` | Events: `FailedScheduling` with the reason | Requests exceed free capacity, taints without tolerations, node affinity, PVC unbound (storage-csi.md), quota | +| `ImagePullBackOff` / `ErrImagePull` | Events: pull error text | Wrong tag/digest, private registry without `imagePullSecrets`, rate limit | +| `CreateContainerConfigError` | Events: missing ConfigMap/Secret/key | config-secrets.md | +| `CrashLoopBackOff` | `logs --previous`, exit code | App error, missing env, permissions under `readOnlyRootFilesystem`/non-root, liveness killing it | +| `OOMKilled`, exit 137 | `Last State` | Memory limit below peak (resources-qos-hpa.md) | +| Running, not Ready | Events: `Readiness probe failed` | Wrong path/port, dependency down, too short timeout (probes-lifecycle.md) | +| `Terminating` forever | `metadata.finalizers`, node status | Node unreachable, finalizer controller gone | +| Service has no endpoints | `get endpointslices -l kubernetes.io/service-name=` | Selector/label mismatch, no Ready pod, wrong `targetPort` | +| Route returns 404/503 | HTTPRoute `status.parents` conditions | Route not `Accepted` (allowedRoutes), `ResolvedRefs` false (Service/port), NetworkPolicy | + +## DNS checks (official dnsutils procedure) + +```bash +kubectl --context "$CTX" apply -f https://k8s.io/examples/admin/dns/dnsutils.yaml # creates pod dnsutils in default ns (propose) +kubectl --context "$CTX" exec -i -t dnsutils -- nslookup kubernetes.default +kubectl --context "$CTX" exec -i -t dnsutils -- nslookup web.shop +kubectl --context "$CTX" exec -i -t dnsutils -- cat /etc/resolv.conf +kubectl --context "$CTX" -n kube-system logs -l k8s-app=kube-dns --tail=50 +``` + +If `kubernetes.default` fails: cluster DNS or NetworkPolicy egress (network-policy.md). If only the app name fails: Service name/namespace, or a missing headless Service. + +## Reporting a diagnosis + +State the step that produced the evidence, quote the event or log line (secrets redacted), name the cause, then propose the fix with its command per kubectl-safe-ops.md. "Restarted it and it works" is not a diagnosis. + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/ +- https://kubernetes.io/docs/reference/kubectl/generated/kubectl_debug/ +- https://kubernetes.io/docs/reference/kubectl/generated/kubectl_events/ (`kubectl events --for pod/`) +- https://kubernetes.io/docs/tasks/administer-cluster/dns-debugging-resolution/ +- https://kubernetes.io/docs/tasks/debug/debug-application/debug-service/ + +## Not verified + +- Event retention ("about an hour") is the common kube-apiserver default (`--event-ttl`), not re-read today; it is cluster-configurable. +- The `k8s-app=kube-dns` label is the one used on the official DNS debugging page; a given distribution may label its DNS pods differently. +- The meaning of the HTTPRoute `Accepted` / `ResolvedRefs` conditions is taken from the Gateway API status model, not re-read today. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/workloads-scheduling.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/workloads-scheduling.md new file mode 100644 index 0000000..a117e7c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-kubernetes/references/workloads-scheduling.md @@ -0,0 +1,111 @@ +--- +name: workloads-scheduling +description: Choosing and configuring workload controllers, rollouts, disruption budgets and pod placement +when-to-use: Writing a Deployment, StatefulSet, DaemonSet, Job or CronJob, or deciding where pods land +keywords: deployment, statefulset, daemonset, job, cronjob, pdb, rollout, topologySpreadConstraints +priority: high +related: probes-lifecycle.md, storage-csi.md, resources-qos-hpa.md +--- + +# Workloads and Scheduling + +**Load when:** choosing a controller, setting a rollout strategy, protecting availability during node maintenance, or spreading replicas. + +## Decision rules + +| Need | Controller | Why | +|------|------------|-----| +| Stateless replicas, any pod interchangeable | `apps/v1` Deployment | Rolling updates and `rollout undo` via ReplicaSet history | +| Stable name, ordinal, per-replica volume | `apps/v1` StatefulSet | Pods `web-0..N-1`, one PVC per ordinal from `volumeClaimTemplates` | +| One pod per (selected) node: agents, log shippers | `apps/v1` DaemonSet | Follows nodes; ignores cordon for its own pods | +| Run to completion | `batch/v1` Job | Retries with `backoffLimit`, `podFailurePolicy` | +| Run on a schedule | `batch/v1` CronJob | Creates Jobs; `timeZone`, `concurrencyPolicy` | + +A bare Pod is never the answer for a service: nothing recreates it. + +## Deployment rollout + +- `strategy.rollingUpdate.maxUnavailable: 0` + `maxSurge: 1` keeps capacity during rollout (needs room for one extra pod). +- `minReadySeconds` delays "available" until a pod stayed Ready that long - catches crash-after-start. +- `revisionHistoryLimit` bounds the ReplicaSets kept for `rollout undo`; `0` disables rollback. +- `progressDeadlineSeconds` marks a stuck rollout as failed so `rollout status` exits non-zero. + +```bash +kubectl --context "$CTX" -n "$NS" rollout status deploy/web --timeout=5m +kubectl --context "$CTX" -n "$NS" rollout history deploy/web +``` + +## StatefulSet specifics (verified at the StatefulSet page) + +| Field | State | Use | +|-------|-------|-----| +| `serviceName` + headless Service | stable | DNS per pod `web-0...svc` | +| `minReadySeconds` | stable since v1.25 | Same meaning as Deployment | +| `ordinals.start` | stable since v1.31 | Start numbering at N (migrations) | +| `apps.kubernetes.io/pod-index` label | stable since v1.32 | Select one replica | +| `persistentVolumeClaimRetentionPolicy` (`whenDeleted`, `whenScaled`) | stable since v1.32 | `Retain` keeps PVCs; `Delete` removes them after the pods | +| `updateStrategy.rollingUpdate.maxUnavailable` | beta since v1.37, on by default | Faster rollouts for replicas that tolerate it | +| `updateStrategy.type: Recreate` | alpha since v1.37, off by default | Do not use without the feature gate | + +Default to `whenDeleted: Retain` and `whenScaled: Retain` for databases: a mistaken delete or scale-down then keeps the data. +`podManagementPolicy: Parallel` only for apps that do not need ordered start. + +## Jobs and CronJobs + +- `ttlSecondsAfterFinished` cleans finished Jobs; `activeDeadlineSeconds` caps runtime. +- `podFailurePolicy` lets non-retriable exit codes fail fast instead of burning `backoffLimit`. +- `backoffLimitPerIndex` for Indexed Jobs (stable since v1.33); `successPolicy` to finish early. +- `podReplacementPolicy: Failed` (delayed replacement pods, stable since v1.34) creates a replacement only once the old pod is fully terminated - no overlap. +- CronJob `timeZone` (stable since v1.27) takes an IANA name; `CRON_TZ`/`TZ` inside `schedule` is rejected by the API. +- `concurrencyPolicy`: `Allow` (default), `Forbid` (skip if previous still runs), `Replace` (kill previous). Backups and migrations: `Forbid`. +- `startingDeadlineSeconds` bounds how late a missed run may still start. + +→ Template: [cronjob.yaml](templates/cronjob.yaml) + +## Availability during disruption + +A PodDisruptionBudget (`policy/v1`) limits voluntary evictions (drain, cluster autoscaler), never node crashes. +- Use `maxUnavailable: 1` for most apps; `minAvailable` with a percentage when replicas vary with HPA. +- A PDB that allows zero disruptions blocks `kubectl drain` forever - check `ALLOWED DISRUPTIONS` first. +- `unhealthyPodEvictionPolicy: AlwaysAllow` lets drain evict pods that are already unhealthy instead of blocking on them. + +```bash +kubectl --context "$CTX" -n "$NS" get pdb +``` + +## Placement + +| Tool | When | +|------|------| +| `topologySpreadConstraints` on `topology.kubernetes.io/zone` / `kubernetes.io/hostname` | Spread replicas; `whenUnsatisfiable: ScheduleAnyway` unless a hard rule is required | +| `affinity.podAntiAffinity` | Hard "never two on one node" for small replica counts | +| `nodeSelector` / node affinity | Hardware or pool constraints | +| taints + tolerations | Reserve nodes (GPU, ingress) for workloads that tolerate them | +| `priorityClassName` | Let critical pods preempt batch work | + +## Pitfalls + +- Changing a Deployment `selector` is rejected - create a new Deployment. +- Deleting a StatefulSet does not wait for ordered pod termination; scale to 0 first only when the app needs ordered shutdown, and only after the user's go. +- A DaemonSet without tolerations skips tainted control-plane nodes - usually intended, confirm it. + +## Verification + +```bash +kubectl --context "$CTX" -n "$NS" get deploy,sts,ds,job,cronjob -o wide +kubectl --context "$CTX" -n "$NS" get pods -o wide # spread across nodes/zones? +``` + +## Sources (read 2026-11-10) + +- https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/ +- https://kubernetes.io/docs/concepts/workloads/controllers/job/ +- https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/ +- https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ +- https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + +## Not verified + +- Exact feature-state versions for `podFailurePolicy` and `successPolicy` (fields listed on the Job page; per-field version not extracted). +- `maxUnavailable: 0` + `maxSurge: 1`, `progressDeadlineSeconds`, placement table: long-stable Deployment/scheduling fields stated from the API, not re-read today. +- The feature-state version of `unhealthyPodEvictionPolicy` (field confirmed on the PDB page, version not extracted). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/SKILL.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/SKILL.md new file mode 100644 index 0000000..df6c179 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/SKILL.md @@ -0,0 +1,149 @@ +--- +name: devops-linux +description: Use when administering a Debian/Ubuntu or RHEL-family server - systemd units, timers, journald, sandboxing and resource limits, apt/dnf and unattended upgrades, users/groups/sudo, sshd and ssh client config, partitions/LVM/filesystems/fstab, sysctl and limits, SELinux or AppArmor operation, time sync, CPU/memory/disk/network troubleshooting with the USE method, or robust bash scripts. Not for firewall/DNS/TLS/VPN (devops-networking), ZFS/Ceph/backups (devops-storage-backup), KVM/LXC (devops-virtualization), Ansible (devops-ansible), or security audits and hardening benchmarks (fuse-security:security-expert). +versions: + debian: "13.7 (trixie)" + ubuntu: "26.04.1 LTS (resolute)" + rhel: "10.2 / 9.8" + systemd-man-pages: "262 (freedesktop latest; shipped versions differ)" + bash-manual: "5.3" +user-invocable: true +references: + - references/systemd-units-journald.md + - references/systemd-hardening-resources.md + - references/packages-updates.md + - references/users-sudo-ssh.md + - references/storage-lvm-fs.md + - references/kernel-sysctl-limits.md + - references/selinux-apparmor.md + - references/time-sync.md + - references/perf-troubleshooting.md + - references/bash-scripting.md + - references/templates/hardened.service + - references/templates/job.service + - references/templates/job.timer + - references/templates/journald-retention.conf + - references/templates/sshd-10-hardening.conf + - references/templates/sudoers-ops + - references/templates/52unattended-upgrades-local + - references/templates/dnf-automatic.conf + - references/templates/script-skeleton.sh +related-skills: + - devops-networking + - devops-storage-backup + - devops-virtualization + - devops-ansible + - devops-observability + - devops-sre-ops + - devops-secrets + - devops-docker +--- + + +Covers day-2 administration of Debian/Ubuntu and RHEL-family servers: systemd +(units, drop-ins, timers, journald, sandboxing, resource control), apt/dnf and +automatic updates, users/groups/sudo, sshd and ssh client configuration, +partitions/LVM/filesystems/fstab, sysctl and limits, SELinux and AppArmor +operation, time sync, performance troubleshooting (USE method) and robust bash. +Does not cover firewall/DNS/TLS/VPN, ZFS/Ceph/backups, hypervisors, configuration +management, or security audits/benchmarks - see related skills. + + +# DevOps Linux + +## Agent Workflow (MANDATORY) + +Before ANY change, spawn in parallel, one `Task` call each: + +1. **fuse-ai-pilot:explore-codebase** - Find existing units, scripts, configs in the repo +2. **fuse-ai-pilot:research-expert** - Verify every directive/flag at the official man page for the target release +3. **mcp__fuse-browser__browser_fetch** - Fast-path read of freedesktop.org / man7.org / distro docs + +On a live host: discovery (read-only) -> proposal with rollback -> owner's go -> change -> verify. +After file changes, run **fuse-ai-pilot:sniper**. + +## Overview + +| Area | Debian / Ubuntu | RHEL family | +|------|-----------------|-------------| +| Packages | `apt`, `apt-mark hold`, `unattended-upgrades` | `dnf`, `dnf versionlock`, `dnf-automatic` (dnf5: `dnf5-automatic.timer`) | +| MAC | AppArmor (`aa-status`) | SELinux (`getenforce`, `semanage`) | +| Admin group (convention - read `/etc/sudoers`) | `sudo` | `wheel` | +| Services, logs, timers | systemd + journald (both) | systemd + journald (both) | + +Releases verified 2026-11-10: Debian 13.7 "trixie" (https://www.debian.org/releases/), +Ubuntu 26.04.1 LTS (https://releases.ubuntu.com/), RHEL 10.2 and 9.8 +(https://access.redhat.com/articles/3078). systemd man pages are read at +https://www.freedesktop.org/software/systemd/man/latest/ (systemd 262); every +directive there carries "Added in version N" - compare with `systemctl --version`. + +## Critical Rules + +1. **Discover before you touch** - `systemctl status`, `journalctl -u`, `cat /etc/os-release`, `systemctl --version` first; never assume the distro or systemd version +2. **Never edit vendor files** - units via `systemctl edit` drop-ins, sshd via `sshd_config.d/`, sudo via `visudo -f /etc/sudoers.d/`, sysctl via `/etc/sysctl.d/` +3. **Validate before reload** - `systemd-analyze verify`, `sshd -t`, `visudo -c`, `findmnt --verify`, `sysctl -p ` +4. **Lock-out risk = second session open** - touching `sshd`, `sudoers`, PAM, network or firewall: keep a working root session open until the new login is proven +5. **Destructive = propose, then the owner's go** - `rm -rf`, filesystem creation, `dd`, `wipefs`, `lvremove`, `userdel -r`, `systemctl mask`, partition tools: state the exact effect and target, wait for the go +6. **Never disable SELinux/AppArmor to "fix" a denial** - read the denial, fix the label/boolean/profile +7. **Never invent a directive** - verify at the man page of the shipped version; unverified -> "I don't know" + +## Reference Guide + +### Concepts + +| Topic | Reference | When to Consult | +|-------|-----------|-----------------| +| **Units, timers, journald** | [systemd-units-journald.md](references/systemd-units-journald.md) | Writing a service, cron -> timer, log retention | +| **Sandboxing, resources** | [systemd-hardening-resources.md](references/systemd-hardening-resources.md) | `ProtectSystem=`, `MemoryMax=`, `systemd-analyze security` | +| **Packages, updates** | [packages-updates.md](references/packages-updates.md) | apt/dnf, pinning, holds, unattended upgrades | +| **Users, sudo, SSH** | [users-sudo-ssh.md](references/users-sudo-ssh.md) | Accounts, sudoers drop-ins, sshd/ssh config | +| **Storage** | [storage-lvm-fs.md](references/storage-lvm-fs.md) | Disks, LVM grow, ext4/xfs, fstab, swap | +| **Kernel, limits** | [kernel-sysctl-limits.md](references/kernel-sysctl-limits.md) | sysctl drop-ins, `LimitNOFILE=`, limits.conf | +| **SELinux, AppArmor** | [selinux-apparmor.md](references/selinux-apparmor.md) | Denials, labels, booleans, profile modes | +| **Time sync** | [time-sync.md](references/time-sync.md) | chrony / timesyncd, clock drift | +| **Troubleshooting** | [perf-troubleshooting.md](references/perf-troubleshooting.md) | Slow host, OOM, full disk, USE method | +| **Bash** | [bash-scripting.md](references/bash-scripting.md) | Strict mode, traps, locking, shellcheck | + +### Templates + +| Template | When to Use | +|----------|-------------| +| [hardened.service](references/templates/hardened.service) | Long-running daemon with sandboxing | +| [job.service](references/templates/job.service) + [job.timer](references/templates/job.timer) | Replace a cron job | +| [journald-retention.conf](references/templates/journald-retention.conf) | Persistent journal with a size cap | +| [sshd-10-hardening.conf](references/templates/sshd-10-hardening.conf) | Key-only sshd drop-in | +| [sudoers-ops](references/templates/sudoers-ops) | Group-scoped sudo rules | +| [52unattended-upgrades-local](references/templates/52unattended-upgrades-local) | Debian/Ubuntu automatic security updates | +| [dnf-automatic.conf](references/templates/dnf-automatic.conf) | RHEL-family automatic security updates | +| [script-skeleton.sh](references/templates/script-skeleton.sh) | Any new operational script | + +## Quick Reference + +```bash +# Discovery (read-only) +cat /etc/os-release; systemctl --version | head -1; uptime +systemctl --failed; journalctl -p err -b --no-pager | tail -50 +df -hT; df -i; lsblk -f; free -h + +# Unit change with validation +systemctl edit myapp.service # drop-in, then: +systemd-analyze verify /etc/systemd/system/myapp.service +systemctl restart myapp.service && systemctl status myapp.service + +# sshd change with lock-out guard (keep current session open) +sshd -t && systemctl reload ssh # Debian/Ubuntu unit name; RHEL: sshd +``` + +## Best Practices + +### DO +- Ship every change as a drop-in file with a comment naming its owner and source URL +- Prefer timers over cron: logs in the journal, `Persistent=`, dependency ordering +- Score services with `systemd-analyze security ` and tighten iteratively +- Grow storage online (`lvextend -r`), verify with `df -h` and `findmnt` + +### DON'T +- Run `systemctl daemon-reload` and walk away - check `systemctl status` and the journal +- Use `setenforce 0` / `aa-complain` as a permanent state +- Put a `.` in a `/etc/sudoers.d/` file name - sudo skips it silently +- Plan on shrinking XFS - only shrinking inside the last allocation group is implemented; treat it as grow-only diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/bash-scripting.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/bash-scripting.md new file mode 100644 index 0000000..1037444 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/bash-scripting.md @@ -0,0 +1,106 @@ +--- +name: bash-scripting +description: Robust operational bash - strict mode and its blind spots, traps for cleanup, locking, idempotence, dry-run, shellcheck +when-to-use: Writing or reviewing any shell script that runs on a server, in a timer, or over SSH +keywords: set -euo pipefail, errexit, inherit_errexit, trap EXIT, trap ERR, mktemp, flock, shellcheck, SC2155, SC2086, idempotent +priority: high +related: systemd-units-journald.md +--- + +# Bash Scripting for Operations + +**Load when:** a new script, a cron/timer job, or a review of an existing one. +Anything beyond ~200 lines of logic or with data structures: write it in Python/Go instead. + +Sources (read 2026-11-10): GNU Bash Reference Manual 5.3 - +https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html, +https://www.gnu.org/software/bash/manual/html_node/The-Shopt-Builtin.html, +https://www.gnu.org/software/bash/manual/html_node/Bourne-Shell-Builtins.html (`trap`), +https://www.gnu.org/software/bash/manual/html_node/Bash-Variables.html; https://man7.org/linux/man-pages/man1/mktemp.1.html, +https://man7.org/linux/man-pages/man1/flock.1.html; https://www.shellcheck.net/wiki/SC2155, `SC2086`, `SC2164`. + +--- + +## Strict Mode - What It Does and Does Not Do + +```bash +set -Eeuo pipefail +shopt -s inherit_errexit # bash >= 4.4 +``` + +| Option | Effect (Bash manual) | Blind spot | +|--------|----------------------|------------| +| `-e` (errexit) | exit on a failing command | NOT triggered in `if`/`while`/`until` tests, any command of a `&&`/`\|\|` list except the last, or a pipeline element other than the last (without `pipefail`) | +| `-u` (nounset) | unset variable = error | use `${VAR:-}` for optional variables | +| `-o pipefail` | pipeline status = last non-zero status | `grep -q` closing early can make the producer fail with SIGPIPE | +| `-E` (errtrace) | ERR trap inherited by functions, command substitutions, subshells | without it an ERR trap misses errors inside functions | +| `inherit_errexit` | command substitution inherits `-e` | otherwise `x=$(false; echo ok)` succeeds | + +`local x=$(cmd)` / `export x=$(cmd)` hide `cmd`'s status (SC2155): declare first, assign on the next line. + +--- + +## Traps + +- `trap cleanup EXIT` - runs when the shell exits, for any reason handled by bash: the one place for cleanup. +- `trap 'on_err $LINENO' ERR` - report the failing line; `BASH_COMMAND` holds the command being executed. +- Add `INT TERM` handlers that `exit` with 130/143 so EXIT cleanup still runs. + +--- + +## Patterns + +| Need | Pattern | +|------|---------| +| Temp files | `tmpdir=$(mktemp -d)` + `rm -rf -- "$tmpdir"` in the EXIT trap (guard: only if it is set and a directory) | +| Single instance | `exec 9>/run/lock/job.lock; flock -n 9 \|\| { echo "already running"; exit 0; }` (`-n` = fail instead of waiting) | +| Dry run | `DRY_RUN=1` default for destructive scripts; a `run()` wrapper that prints instead of executing | +| Idempotence | check state before acting (`id user`, `grep -q line file`, `systemctl is-enabled`) - safe to rerun | +| Quoting | quote every expansion `"$var"` (SC2086); arrays for argument lists `"${args[@]}"` | +| `cd` | `cd -- "$dir" \|\| exit 1` (SC2164) - or rely on `-e`, but be explicit in functions | +| Logging | to stderr with a timestamp; under systemd the journal captures it | +| Exit codes | 0 success, non-zero failure; document any special code (like `dnf check-update` uses 100) | + +Template: [script-skeleton.sh](templates/script-skeleton.sh) - strict mode, traps, lock, dry-run, argument parsing. + +--- + +## Remote Execution + +- Read-only discovery first: `ssh host 'bash -s' < discover.sh`. +- Fleet: one canary host, check, then the rest - or hand over to devops-ansible for anything repeated. +- Never pipe an untrusted URL into a shell (`curl ... | bash`) on a server; download, read, checksum, then run. + +--- + +## Destructive Operations Inside Scripts + +| Construct | Risk | Rule | +|-----------|------|------| +| `rm -rf -- "$dir"/` | empty/unset `$dir` targets `/` | `-u` + `[[ -n $dir && -d $dir ]]` guard + `--`; default to dry-run | +| loops over `find ... -delete` | deletes more than intended | print the list first (`-print`), then propose | +| `> file` truncation | loses data on a wrong path | write to a temp file, `mv` atomically | + +Scripts that delete or overwrite are shown to the owner with the dry-run output first. + +--- + +## Verification + +```bash +bash -n script.sh # syntax only +shellcheck script.sh # static analysis - zero warnings or each one justified inline +DRY_RUN=1 ./script.sh --target x # behaviour preview +``` + +## Pitfalls + +- Trusting `-e` inside `if myfunc; then` - errexit is disabled for the whole function body there +- `cmd | while read` modifying variables - the loop runs in a subshell; use `while ...; done < <(cmd)` +- Parsing `ls` output - use globs or `find -print0` with `read -d ''` + +## Not verified + +- macOS ships bash 3.2 (no `inherit_errexit`); the template guards it - server distros ship bash 5.x (not checked per release) +- The `while ... done < <(cmd)` and `find -print0` notes are standard bash behaviour, not re-read this session +- The `grep -q` + `pipefail` SIGPIPE interaction is general knowledge, not sourced this session diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/kernel-sysctl-limits.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/kernel-sysctl-limits.md new file mode 100644 index 0000000..37220d4 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/kernel-sysctl-limits.md @@ -0,0 +1,115 @@ +--- +name: kernel-sysctl-limits +description: Persistent kernel tunables via sysctl.d, process limits via systemd and pam_limits, kernel log reading +when-to-use: Tuning memory/VM writeback, raising inotify or file-descriptor limits, "too many open files" +keywords: sysctl, sysctl.d, systemd-sysctl, vm.swappiness, vm.dirty_ratio, inotify, LimitNOFILE, limits.conf, pam_limits +priority: medium +related: systemd-hardening-resources.md, perf-troubleshooting.md +--- + +# Kernel Parameters and Limits + +**Load when:** changing any `/proc/sys` value, or a process hits a resource limit. +Network-stack sysctls (forwarding, rp_filter, conntrack) belong to **devops-networking**. + +Sources (read 2026-11-10): https://man7.org/linux/man-pages/man8/sysctl.8.html, +https://www.freedesktop.org/software/systemd/man/latest/sysctl.d.html, +https://docs.kernel.org/admin-guide/sysctl/vm.html, https://docs.kernel.org/admin-guide/sysctl/kernel.html, +https://man7.org/linux/man-pages/man7/inotify.7.html, https://man7.org/linux/man-pages/man5/limits.conf.5.html, +https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html (`LimitNOFILE=`). + +## Decision Rules + +| Need | Where | +|------|-------| +| Persistent kernel tunable | `/etc/sysctl.d/-.conf`, applied at boot by `systemd-sysctl.service` | +| Test a value now (lost at reboot) | `sysctl -w key=value` | +| Limit for a **service** | `LimitNOFILE=` / `LimitNPROC=` in the unit (drop-in) - limits.conf does not apply to services | +| Limit for **interactive logins** | `/etc/security/limits.d/*.conf` (pam_limits: login sessions only) | +| Memory/CPU cap for a service | cgroup settings, see [systemd-hardening-resources.md](systemd-hardening-resources.md) | + +Never tune by folklore: record the metric that motivated the change (see [perf-troubleshooting.md](perf-troubleshooting.md)) +and re-measure after. + +## sysctl.d Precedence + +- Directories: `/etc/sysctl.d/` > `/run/sysctl.d/` > `/usr/local/lib/sysctl.d/` > `/usr/lib/sysctl.d/`; a file in `/etc` + replaces a vendor file **with the same name**. +- All files are then sorted by name across directories; for the same key, **the lexicographically latest file wins** + (the opposite of sshd, where the first value wins). Prefix files with a number: `90-local-vm.conf`. +- Only `*.conf` files are read. + +```text +# /etc/sysctl.d/90-local-vm.conf - owner: ops, reason: , source: docs.kernel.org/admin-guide/sysctl/vm.html +vm.swappiness = 10 +fs.inotify.max_user_watches = 524288 +``` + +## Commonly Touched Keys + +| Key | Meaning (kernel docs) | Note | +|-----|-----------------------|------| +| `vm.swappiness` | relative IO cost of swap vs page cache, 0-200, default 60 | lower = prefer dropping page cache; workload-dependent | +| `vm.dirty_ratio` | % of available memory at which a writing process must flush itself | lower to cut write stalls on big-RAM hosts | +| `vm.dirty_background_ratio` | % at which background flusher threads start | keep below `dirty_ratio` | +| `fs.inotify.max_user_watches` | max inotify watches per real UID | file watchers, IDE servers, log shippers | +| `kernel.panic` | 0 = loop forever on panic, >0 = reboot after N seconds | servers usually want auto-reboot | + +```bash +sysctl vm.swappiness # read +sysctl -w vm.swappiness=10 # runtime only +sysctl -p /etc/sysctl.d/90-local-vm.conf # load one file, prints what it set +sysctl --system # reload all directories in precedence order +``` + +## File-Descriptor Limits + +systemd.exec advises against raising `LimitNOFILE=` blindly: raising the soft limit above 1024 breaks +programs using `select(2)` (fds above 1023); the hard limit already defaults to 524288 per the current +man page, and applications that do not use `select()` should raise their own soft limit to the hard one. +Set it in the unit only when the application cannot. + +```ini +# systemctl edit myapp.service +[Service] +LimitNOFILE=65536 +``` + +```bash +cat /proc/$(systemctl show -p MainPID --value myapp)/limits # effective limits of the running process +ulimit -n # current shell only +``` + +## Kernel Messages + +```bash +journalctl -k -b --no-pager | tail -100 # kernel ring buffer via journald +dmesg -T | tail -50 # human timestamps (can drift after suspend) +``` + +## Destructive / Risky Operations + +| Change | Effect | Rule | +|--------|--------|------| +| `kernel.panic`, `vm.overcommit_*`, `kernel.sysrq` | alters crash/memory semantics host-wide | propose with the measured reason, wait for the go | +| `vm.drop_caches` | drops page cache - performance cliff, never a fix | do not use as a routine | +| Wrong value in `/etc/sysctl.d/` | applied at every boot | test with `sysctl -w` first, then persist | + +## Verification + +```bash +sysctl --system 2>&1 | grep -i -E 'error|invalid' # parse errors +systemctl status systemd-sysctl.service --no-pager +sysctl vm.swappiness fs.inotify.max_user_watches # value in effect +``` + +## Pitfalls + +- Editing `/usr/lib/sysctl.d/` - overwritten by the next package update +- Two local files setting the same key - the later file name wins silently +- Raising `nofile` in limits.conf for a daemon - ignored for systemd services + +## Not verified + +- Whether each distro still ships `/etc/sysctl.conf` (prefer `sysctl.d` regardless) +- `vm.overcommit_*` and `kernel.sysrq` semantics were not re-read this session - read vm.html / kernel.html before changing diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/packages-updates.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/packages-updates.md new file mode 100644 index 0000000..8c3e7bd --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/packages-updates.md @@ -0,0 +1,113 @@ +--- +name: packages-updates +description: apt and dnf package management, pinning/holds, history rollback, automatic security updates, reboot detection +when-to-use: Installing/upgrading packages, freezing a version, enabling unattended upgrades, deciding on a reboot +keywords: apt, apt-mark, apt_preferences, unattended-upgrades, dnf, versionlock, dnf-automatic, needs-restarting, reboot-required +priority: high +related: systemd-units-journald.md +--- + +# Packages and Updates + +**Load when:** any package install/upgrade, version freeze, automatic update setup, or "does it need a reboot?". + +Sources (read 2026-11-10): https://manpages.debian.org/trixie/apt/apt-get.8.en.html, +https://manpages.debian.org/trixie/apt/apt-mark.8.en.html, https://manpages.debian.org/trixie/apt/apt_preferences.5.en.html, +https://manpages.debian.org/trixie/unattended-upgrades/unattended-upgrade.8.en.html, +https://documentation.ubuntu.com/server/how-to/software/automatic-updates/, +https://raw.githubusercontent.com/mvo5/unattended-upgrades/master/data/50unattended-upgrades.Debian, +https://dnf.readthedocs.io/en/latest/command_ref.html, https://dnf.readthedocs.io/en/latest/automatic.html, +https://dnf5.readthedocs.io/en/latest/dnf5_plugins/automatic.8.html, +https://dnf-plugins-core.readthedocs.io/en/latest/versionlock.html, https://dnf-plugins-core.readthedocs.io/en/latest/needs_restarting.html. + +## Side by Side + +| Task | Debian / Ubuntu | RHEL family | +|------|-----------------|-------------| +| Refresh metadata | `apt-get update` | `dnf makecache` (or implicit) | +| List pending | `apt list --upgradeable` | `dnf check-update` (exit 100 = updates available) | +| Simulate | `apt-get -s upgrade` | `dnf upgrade --assumeno` | +| Security only | unattended-upgrades origins (below) | `dnf upgrade --security` | +| Install | `apt-get install -y pkg` | `dnf install -y pkg` | +| Who owns a file | `dpkg -S /path` | `rpm -qf /path` | +| Files of a package | `dpkg -L pkg` | `rpm -ql pkg` | +| Freeze version | `apt-mark hold pkg` / `apt-mark showhold` | `dnf versionlock add pkg` (plugin) | +| Undo a transaction | no native undo - reinstall the previous version explicitly | `dnf history` then `dnf history undo ` | +| Reboot needed? | `test -f /var/run/reboot-required` | `needs-restarting -r` (exit 1 = reboot needed) | + +Check the dnf generation first: `dnf --version`. dnf5 renames parts of the tooling +(`dnf5-automatic.timer`, vendor config `/usr/share/dnf5/dnf5-plugins/automatic.conf`). + +## Decision Rules + +- Production upgrade window: list -> simulate -> snapshot/backup (devops-storage-backup) -> upgrade -> verify services -> reboot if flagged. +- Pin with `apt_preferences` (`Pin-Priority:`) when you need "prefer this origin"; use `apt-mark hold` when you need "never move this package". +- Never mix distro releases in `sources.list` to get one newer package - use backports or a vendor repo with a pin. +- Automatic updates: security only, automatic reboot off by default; turn reboot on only with a maintenance window and monitoring. + +## Pinning (Debian/Ubuntu) + +```text +# /etc/apt/preferences.d/pin-perl +Package: perl +Pin: version 5.32* +Pin-Priority: 1001 +``` + +Priority > 1000 allows downgrades to the pinned version. Inspect with `apt-cache policy perl`. + +## Automatic Security Updates + +**Debian/Ubuntu** - `unattended-upgrades` driven by `apt-daily.timer` and `apt-daily-upgrade.timer`. +Enable via `/etc/apt/apt.conf.d/20auto-upgrades`: + +```text +APT::Periodic::Update-Package-Lists "1"; +APT::Periodic::Unattended-Upgrade "1"; +``` + +Local policy goes in a separate file so package upgrades do not conflict with your edits: +[52unattended-upgrades-local](templates/52unattended-upgrades-local) (keys: `Origins-Pattern`, +`Package-Blacklist`, `Automatic-Reboot`, `Automatic-Reboot-Time`, `Mail`, `Remove-Unused-Dependencies`). + +```bash +unattended-upgrade --dry-run --debug # simulate with the current policy +less /var/log/unattended-upgrades/unattended-upgrades.log +``` + +**RHEL family** - `dnf-automatic` reads `/etc/dnf/automatic.conf` ([dnf-automatic.conf](templates/dnf-automatic.conf)): +`upgrade_type = security`, `apply_updates = yes`, `reboot = never|when-changed|when-needed`. + +```bash +systemctl enable --now dnf-automatic.timer # behaves as automatic.conf says +systemctl enable --now dnf-automatic-notifyonly.timer # report only +``` + +## Destructive Operations + +| Command | Exact effect | Rule | +|---------|--------------|------| +| `apt-get purge pkg` / `dnf remove pkg` | removes the package and may remove dependants (apt purge also deletes its config) | simulate first (`-s` / `--assumeno`), show the removal list, wait for the go | +| `apt-get autoremove` | removes packages marked as automatically installed and no longer needed | show the list; old kernels are a fallback | +| `dnf history undo ` | reverses a whole transaction | show `dnf history info ` first | + +## Verification + +```bash +apt-get -s upgrade | tail -5 ; apt-mark showhold +dnf check-update; echo "exit=$?"; dnf versionlock list +systemctl list-timers 'apt-daily*' 'dnf*automatic*' +systemctl --failed # services broken by the upgrade +``` + +## Pitfalls + +- `Automatic-Reboot "true"` on a single-node service without a window - unplanned downtime +- `dnf check-update` exit code 100 treated as failure in scripts +- Editing `50unattended-upgrades` directly - conflicts on package upgrade; use a `52...-local` file + +## Not verified + +- `dpkg -S`/`dpkg -L`/`rpm -qf`/`rpm -ql` were not re-read at a man page this session (standard queries, check `--help` on the host) +- Which dnf generation RHEL 10 ships (docs.redhat.com returned Access Denied) - run `dnf --version` +- Whether `needs-restarting` is installed by default on RHEL 9/10 (it comes from dnf-plugins-core / dnf-utils) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/perf-troubleshooting.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/perf-troubleshooting.md new file mode 100644 index 0000000..857ccaf --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/perf-troubleshooting.md @@ -0,0 +1,118 @@ +--- +name: perf-troubleshooting +description: USE-method triage of CPU, memory, disk and network on a Linux host, plus full-disk, OOM and failed-service playbooks +when-to-use: "The server is slow", load spikes, OOM kills, disk full, a service keeps failing +keywords: USE method, utilization, saturation, errors, vmstat, mpstat, pidstat, iostat, sar, PSI, free, df, lsof, ss, OOM +priority: high +related: kernel-sysctl-limits.md, storage-lvm-fs.md, systemd-hardening-resources.md +--- + +# Performance and Troubleshooting + +**Load when:** any "slow / down / full / killed" report on a host. Everything here is read-only; +fixes are proposed with the measurement that justifies them. Fleet dashboards and alerting are +**devops-observability**; incident process is **devops-sre-ops**; deep network captures are **devops-networking**. + +Sources (read 2026-11-10): https://www.brendangregg.com/usemethod.html, +https://www.brendangregg.com/USEmethod/use-linux.html, https://docs.kernel.org/accounting/psi.html, +man7.org pages for `vmstat(8)`, `mpstat(1)`, `pidstat(1)`, `iostat(1)`, `sar(1)`, `free(1)`, `df(1)`, +`ss(8)`, `lsof(8)`, `dmesg(1)`. sysstat (`mpstat`, `pidstat`, `iostat`, `sar`) may need installing. + +--- + +## The USE Method + +"For every resource, check utilization, saturation, and errors." - Utilization: time busy (%). +Saturation: queued work it cannot service. Errors: count of error events. Walk every resource +before blaming one; a 100%-utilized resource with no saturation is not necessarily the problem. + +| Resource | Utilization | Saturation | Errors | +|----------|-------------|------------|--------| +| CPU | `mpstat -P ALL 1` (all but idle/iowait); `pidstat 1` per process | `vmstat 1` column `r` > CPU count; `/proc/pressure/cpu` | `journalctl -k` MCE lines | +| Memory | `free -m` (use `available`, not `free`) | `vmstat 1` `si`/`so` > 0 (swapping); `/proc/pressure/memory`; OOM kills | `journalctl -k` for hardware errors | +| Disk | `iostat -xz 1` `%util` (misleading on SSD/RAID that serve in parallel) | `iostat -xz 1` `aqu-sz`, high await; `/proc/pressure/io` | `journalctl -k` I/O errors; SMART (devops-storage-backup) | +| Network | `sar -n DEV 1` throughput vs link speed | `sar -n EDEV 1` drops; `ss -s`; TCP retransmits (`sar -n ETCP 1`) | `ip -s link` errors | +| Software limits | fds (`/proc//limits`), `TasksMax=`, cgroup memory | throttling in `systemd-cgtop` | `EMFILE`/`ENOMEM` in the journal | + +PSI: "some" = share of time at least some tasks stalled on the resource; "full" = all non-idle tasks stalled. + +--- + +## First 60 Seconds (read-only) + +```bash +uptime # load averages trend +journalctl -k -b --no-pager | tail -30 # OOM, I/O errors, segfaults +vmstat 1 5 # r, b, si/so, wa, st (steal = noisy neighbour on a VM) +mpstat -P ALL 1 3 # one hot CPU = single-threaded bottleneck +pidstat 1 3 # who is using CPU +iostat -xz 1 3 # per-device await, aqu-sz, %util +free -m; cat /proc/pressure/{cpu,memory,io} +sar -n DEV 1 3; ss -s +systemctl --failed +``` + +--- + +## Playbooks + +**Disk full** +```bash +df -hT; df -i # space vs inodes +du -xh --max-depth=1 /var | sort -h | tail # -x stays on one filesystem +lsof +L1 # deleted-but-open files still holding space +journalctl --disk-usage +``` +Fix: rotate/vacuum logs, restart the process holding deleted files, grow the volume +([storage-lvm-fs.md](storage-lvm-fs.md)). Never delete files under `/var/lib/` to free space. + +**OOM kills** +```bash +journalctl -k -b | grep -iE 'out of memory|oom-kill|killed process' +systemctl show -p MemoryMax -p MemoryHigh +``` +Decide: leak (memory grows until kill) -> app fix; legitimate peak -> `MemoryHigh=`/`MemoryMax=` per unit, +more RAM, or swap ([systemd-hardening-resources.md](systemd-hardening-resources.md)). + +**Service keeps failing** +```bash +systemctl status --no-pager -l +journalctl -u -b -n 200 --no-pager +systemctl show -p NRestarts -p ExecMainStatus -p Result +``` +Then check MAC denials ([selinux-apparmor.md](selinux-apparmor.md)) and sandbox directives. + +**What listens where** +```bash +ss -tlnp # TCP listeners with owning process +ss -tunp state established # live connections +``` + +--- + +## Risky "Fixes" to Refuse + +| Action | Why not | +|--------|---------| +| `kill -9` on a database or the heaviest process | data loss, hides the cause - capture state first (`pidstat`, journal) | +| `echo 3 > /proc/sys/vm/drop_caches` | kernel docs: not a cache control and can cause performance problems | +| Deleting logs with `rm` while the writer holds them | space not freed; use `journalctl --vacuum-*` or logrotate | +| Rebooting before collecting evidence | the cause disappears with it - save the journal and the commands above first | + +--- + +## Verification + +Re-run the same measurement that showed the problem (same command, same interval) and report before/after numbers. + +## Pitfalls + +- Reading `free` "free" column as available memory - Linux uses spare RAM for cache +- `%util` 100% on an NVMe/RAID device - may still have headroom; look at await and queue +- Load average counts tasks in uninterruptible I/O wait - high load with idle CPU = I/O or NFS stall + +## Not verified + +- The `du -x`, `ss state established`, `systemctl show -p NRestarts` details were not re-read at a man page this session +- Brendan Gregg's 60-second checklist article itself was not fetched; the command list follows the USE Linux checklist page +- Load average including uninterruptible tasks: general Linux knowledge, not sourced this session diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/selinux-apparmor.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/selinux-apparmor.md new file mode 100644 index 0000000..9eec690 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/selinux-apparmor.md @@ -0,0 +1,115 @@ +--- +name: selinux-apparmor +description: Operating SELinux (RHEL family) and AppArmor (Debian/Ubuntu) - reading denials, fixing labels, ports, booleans and profile modes +when-to-use: A service fails with permission denied although Unix permissions look right, or a non-default path/port is introduced +keywords: SELinux, getenforce, semanage fcontext, restorecon, setsebool, semanage port, ausearch, audit2allow, AppArmor, aa-status, aa-complain, apparmor_parser +priority: high +related: systemd-hardening-resources.md +--- + +# SELinux and AppArmor (operation) + +**Load when:** "permission denied" with correct owner/mode, a service moved to a non-default path or port, +or a profile blocks an application. Policy audits and benchmarks belong to `fuse-security:security-expert`. + +Sources (read 2026-11-10): SELinux userspace man pages at https://github.com/SELinuxProject/selinux +(`python/semanage/semanage-fcontext.8`, `semanage-port.8`, `semanage-boolean.8`, `semanage-permissive.8`, +`policycoreutils/setfiles/restorecon.8`, `policycoreutils/setsebool/setsebool.8`, +`libselinux/man/man8/getenforce.8`, `python/audit2allow/audit2allow.1`), +https://github.com/linux-audit/audit-userspace/blob/master/docs/ausearch.8, +https://documentation.ubuntu.com/server/how-to/security/apparmor/. + +## Decision Rules + +1. Denial suspected -> **read the log first**, do not change modes. +2. Wrong label on a custom path -> `semanage fcontext` + `restorecon` (never `chcon` as the fix - a relabel undoes it). +3. Service on a non-default port -> `semanage port -a`. +4. Documented behaviour switch exists -> boolean (`setsebool -P`). +5. Only then, and only for a vendor gap -> local module from `audit2allow`, reviewed line by line. +6. Diagnosing one domain -> `semanage permissive -a ` (that domain only), never `setenforce 0` host-wide. + +Detect which MAC is active: `getenforce` (SELinux) / `aa-status` (AppArmor). Ubuntu installs and loads +AppArmor by default (Ubuntu docs); Debian uses AppArmor and RHEL family SELinux - confirm on the host. + +## SELinux - Read the Denial + +```bash +getenforce # Enforcing | Permissive | Disabled +ausearch -m avc,user_avc,selinux_err,user_selinux_err -i -ts recent # last 10 minutes +ls -Z /srv/www; ps -eZ | grep httpd # file and process contexts +``` + +## SELinux - Fix + +```bash +# Custom content path for a web server (example from semanage-fcontext(8)) +semanage fcontext -a -t httpd_sys_content_t "/web(/.*)?" +restorecon -R -v /web + +# Label a tree like an existing one +semanage fcontext -a -e /home /home1 && restorecon -R -v /home1 + +# Non-default port +semanage port -a -t http_port_t -p tcp 81 +semanage port -l | grep http_port_t + +# Boolean - persistent across reboot only with -P +setsebool -P httpd_can_sendmail on +semanage boolean -l -C # locally changed booleans + +# Preview a relabel without changing anything +restorecon -R -n -v /web +``` + +Local module (last resort, review the generated `.te` before loading): + +```bash +ausearch -m avc -ts recent | audit2allow -M local_myapp # writes local_myapp.te / .pp +# review local_myapp.te, then: semodule -i local_myapp.pp +``` + +## AppArmor (Debian/Ubuntu) + +```bash +aa-status # loaded profiles and their modes +journalctl -k -b | grep -i apparmor # DENIED lines name the profile and path +aa-complain /path/to/bin # log instead of block (diagnosis only) +aa-enforce /path/to/bin # back to enforcing +apparmor_parser -r /etc/apparmor.d/profile.name # reload after editing +``` + +Local additions go in the profile's `local/` include (if the profile ships one) instead of the vendor file. +Disable one profile (documented method): link it into `/etc/apparmor.d/disable/` and unload with +`apparmor_parser -R /etc/apparmor.d/profile.name`. + +## Destructive / Risky Operations + +| Command | Exact effect | Rule | +|---------|--------------|------| +| `setenforce 0` | whole host permissive until reboot or `setenforce 1` | diagnosis only, minutes, with the owner's go; prefer per-domain permissive | +| SELINUX=disabled in `/etc/selinux/config` | removes MAC at next boot; files created meanwhile get no label | never as a fix | +| `semodule -i` of an `audit2allow` module | permanently widens policy | review the `.te`, propose, wait for the go | +| `restorecon -R -F /` | resets every label including customised user/role/range | scope to the path at issue | +| AppArmor profile disabled | the confined program runs unconfined | name the profile and reason, wait for the go | + +## Verification + +```bash +ls -Z /web | head; restorecon -R -n -v /web # no output = labels already correct +ausearch -m avc -ts recent # no new denials after retest +aa-status | grep -A2 'profiles are in enforce mode' +``` + +## Pitfalls + +- `chcon` used as a fix - lost on the next relabel or `restorecon` +- `setsebool` without `-P` - reverts at reboot +- `audit2allow` run on every denial - grants the attack along with the bug + +## Not verified + +- `semodule -i`, `chcon`, `ls -Z`/`ps -eZ` were not re-read at a man page this session +- The AppArmor `local/` include convention varies by profile - check the profile's `include` lines +- RHEL docs (docs.redhat.com) returned Access Denied; RHEL-specific guidance was not cross-checked there +- Default MAC per distro (Debian AppArmor, RHEL SELinux enforcing) and the relabel needed after re-enabling SELinux +- `restorecon -F` resets user/role/range as well as type (from restorecon(8)); `-R -F /` scope advice is practice diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/storage-lvm-fs.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/storage-lvm-fs.md new file mode 100644 index 0000000..7800f8a --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/storage-lvm-fs.md @@ -0,0 +1,115 @@ +--- +name: storage-lvm-fs +description: Disks, partitions, LVM, ext4/XFS grow, fstab entries and swap on a single host +when-to-use: Adding a disk, growing a volume, writing fstab, adding swap, full-disk triage +keywords: lsblk, blkid, findmnt, pvcreate, vgextend, lvextend, resizefs, xfs_growfs, resize2fs, fstab, nofail, swap +priority: high +related: perf-troubleshooting.md +--- + +# Storage: Partitions, LVM, Filesystems, fstab + +**Load when:** a disk is added, a filesystem is full, or an fstab line is written. +ZFS, Ceph, NFS servers and backups belong to **devops-storage-backup**. + +Sources (read 2026-11-10): https://man7.org/linux/man-pages/man8/lvextend.8.html, `lvcreate.8`, `lvresize.8`, +`pvcreate.8`, `vgcreate.8`, `vgextend.8`, `lvremove.8`, `xfs_growfs.8`, `resize2fs.8`, `lsblk.8`, `findmnt.8`, +`mount.8`, `swapon.8`, https://man7.org/linux/man-pages/man5/fstab.5.html, +https://www.freedesktop.org/software/systemd/man/latest/systemd.mount.html. + +## Discovery (read-only, always first) + +```bash +lsblk -f # tree: disks, partitions, LVs, FS type, UUID, mountpoint +findmnt -D # mounted filesystems with usage +pvs; vgs; lvs # LVM state (VFree = room to grow) +df -hT; df -i # space and inodes +cat /etc/fstab +``` + +Identify the target by **WWN/by-id + size** (`lsblk -o NAME,SIZE,MODEL,WWN`, `ls -l /dev/disk/by-id/`; +`lsblk --list-columns` shows what else is available), never by `/dev/sdX` alone - names can change +between boots. + +## Decision Rules + +| Situation | Action | +|-----------|--------| +| LV full, VG has free extents | `lvextend -r -l +100%FREE vg/lv` (grows LV and FS together) | +| LV full, VG full | add a disk -> `pvcreate` -> `vgextend` -> `lvextend -r` | +| VM disk enlarged by the hypervisor | grow the partition (growpart / parted, see Not verified) -> `pvresize` -> `lvextend -r` | +| Need to shrink | ext4: offline only (`resize2fs` shrinks unmounted FS); XFS: treat as grow-only (only shrinking inside the last AG is implemented) | +| Non-critical mount (data disk, NFS) | `nofail` in fstab so boot does not hang | +| Network FS used rarely | `x-systemd.automount` + `x-systemd.idle-timeout=` | + +## Grow Online + +```bash +lvextend -r -L +20G vg0/data # -r|--resizefs calls the FS resizer +# Manual equivalent if the LV was already extended: +xfs_growfs /srv/data # XFS: takes the MOUNT POINT, FS must be mounted +resize2fs /dev/vg0/data # ext4: grows a mounted FS online +``` + +## New Volume End-to-End + +```bash +pvcreate /dev/disk/by-id/ +vgcreate vg_data /dev/disk/by-id/ +lvcreate -n srv -l 100%FREE vg_data +# create the filesystem with the mkfs. tool for xfs or ext4 - DESTRUCTIVE, see below +blkid /dev/vg_data/srv # copy the UUID +``` + +fstab line (use `UUID=`, never `/dev/sdX`): + +```text +UUID= /srv xfs defaults,nofail,x-systemd.device-timeout=30s 0 0 +``` + +Fields: device, mount point, type, options, dump, fsck pass (`1` root, `2` other checked filesystems, +`0` or absent = never checked at boot). + +## Swap + +```bash +swapon --show # current swap +# File-based swap: allocate, chmod 600, mkswap, swapon; persist with an fstab line " none swap defaults 0 0" +``` + +Tuning `vm.swappiness` lives in [kernel-sysctl-limits.md](kernel-sysctl-limits.md). + +## Destructive Operations + +| Command | Exact effect | Rule | +|---------|--------------|------| +| `mkfs.*` / `mkswap` on a device | writes a new filesystem/swap signature - existing data is gone | confirm device by serial + `lsblk -f` showing no FS, propose, wait for the go | +| `wipefs -a` | erases all filesystem/RAID/partition-table signatures | same; `wipefs` without `-a` only lists | +| `dd of=/dev/...` | overwrites the raw device | never on a guessed device name | +| partition tools (`fdisk`, `sfdisk`, `parted`, `sgdisk`) | rewrite the partition table | dump the table first (`sfdisk -d /dev/X > table.bak`) | +| `lvremove vg/lv` | deletes the LV and its data | `lvs` + `findmnt` showing it unmounted, backup proven, propose, wait for the go | +| `lvreduce` / `lvresize -L -` | truncates the LV; FS corruption if the FS was not shrunk first | ext4 only, offline, `-r`, backup first | +| `vgremove` / `pvremove` | destroys LVM metadata | same | + +## Verification + +```bash +findmnt --verify # parses and checks /etc/fstab +mount -a && findmnt /srv # mounts all non-noauto entries now +systemctl daemon-reload # regenerate mount units after editing fstab +df -hT /srv; lvs vg_data +``` + +## Pitfalls + +- Editing fstab and rebooting without `findmnt --verify` + `mount -a` - emergency shell at boot +- Forgetting `nofail` on a secondary disk - boot waits then drops to emergency mode +- `lvextend` without `-r` - LV grows, filesystem does not (`df` unchanged) +- `xfs_growfs` on an unmounted filesystem - it must be mounted to be grown +- `systemctl daemon-reload` skipped after an fstab edit - generators keep the old mount units + +## Not verified + +- `growpart` (cloud-utils) and `parted resizepart` syntax - read their man pages on the host before use +- `sfdisk -d` dump syntax was not re-read this session (`sfdisk --help`) +- `pvresize`, `vgremove`, `pvremove`, `wipefs` pages were not fetched this session diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/systemd-hardening-resources.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/systemd-hardening-resources.md new file mode 100644 index 0000000..ec3bda4 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/systemd-hardening-resources.md @@ -0,0 +1,108 @@ +--- +name: systemd-hardening-resources +description: systemd sandboxing directives and cgroup resource control for services +when-to-use: Confining a daemon, capping memory/CPU/tasks, reading systemd-analyze security output +keywords: ProtectSystem, NoNewPrivileges, DynamicUser, CapabilityBoundingSet, SystemCallFilter, MemoryMax, CPUQuota, TasksMax +priority: high +related: systemd-units-journald.md, kernel-sysctl-limits.md +--- + +# systemd Sandboxing and Resource Control + +**Load when:** writing a new daemon unit, tightening an existing one, or a service eats the host. + +Sources (read 2026-11-10): https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html, +https://www.freedesktop.org/software/systemd/man/latest/systemd.resource-control.html, +https://www.freedesktop.org/software/systemd/man/latest/systemd-analyze.html. +This is service confinement, not a host audit - benchmarks belong to `fuse-security:security-expert`. + +--- + +## Method + +1. Start from [hardened.service](templates/hardened.service). +2. `systemd-analyze security myapp.service` - exposure score per setting. +3. Restart, exercise every code path, read `journalctl -u myapp` for `EPERM`/`EACCES`/`SIGSYS`. +4. Loosen only the one directive that blocks, with a comment saying why. + +Directives below appeared in the systemd version shown; RHEL 9 ships an older systemd than +RHEL 10 and Debian 13 - compare with `systemctl --version` (not verified per distro here). + +--- + +## Sandboxing Directives (`[Service]`) + +| Directive | Effect | Added in | +|-----------|--------|----------| +| `NoNewPrivileges=yes` | setuid/file caps cannot raise privileges | 187 | +| `ProtectSystem=strict` | whole FS read-only except `/dev`, `/proc`, `/sys`; open holes with `ReadWritePaths=` or `StateDirectory=` | 214 | +| `ProtectHome=yes` | `/home`, `/root`, `/run/user` empty; `read-only` / `tmpfs` variants | 214 | +| `PrivateTmp=yes` | private `/tmp` and `/var/tmp` | 209 | +| `PrivateDevices=yes` | minimal `/dev`, no raw devices | 209 | +| `ProtectKernelTunables=yes` | `/proc/sys`, `/sys` read-only | 232 | +| `ProtectKernelModules=yes` | no module loading | 232 | +| `ProtectKernelLogs=yes` | no kernel log access | 244 | +| `ProtectControlGroups=yes` | cgroup tree read-only | 232 | +| `ProtectClock=yes` | no clock changes | 245 | +| `ProtectHostname=yes` | no hostname changes | 242 | +| `RestrictNamespaces=yes` | no new namespaces | 233 | +| `RestrictRealtime=yes` | no realtime scheduling | 231 | +| `RestrictSUIDSGID=yes` | cannot create setuid/setgid files | 242 | +| `LockPersonality=yes` | locks execution domain | 235 | +| `MemoryDenyWriteExecute=yes` | no W+X memory - breaks JITs (Node, Java, LuaJIT) | 231 | +| `RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6` | socket family allow-list | 211 | +| `SystemCallArchitectures=native` | blocks foreign-ABI syscalls | 209 | +| `SystemCallFilter=@system-service` | syscall allow-list by group; list groups with `systemd-analyze syscall-filter` | - | +| `CapabilityBoundingSet=` | empty = no capabilities; `CAP_NET_BIND_SERVICE` for ports < 1024 | 229 | +| `ProtectProc=invisible` / `ProcSubset=pid` | hide other processes in `/proc` | 247 | +| `DynamicUser=yes` | transient UID; implies `ProtectSystem=strict`, `ProtectHome=read-only`, `RemoveIPC=` and a private `/tmp` unless `PrivateTmp=` is set | 232 | +| `UMask=0077` | file creation mask | 246 | + +Writable state the sandbox-friendly way: `StateDirectory=myapp` (`/var/lib/myapp`), +`LogsDirectory=`, `RuntimeDirectory=` - created and owned by systemd for the service user. +Secrets: `LoadCredential=name:/path` exposes them under `$CREDENTIALS_DIRECTORY`. + +Note from the man page: with `ProtectSystem=strict` plus `PrivateTmp=`, `/tmp` and `/var/tmp` stay writable; +`ReadWritePaths=` cannot make a read-only superblock writable. + +--- + +## Resource Control (cgroup v2) + +| Directive | Effect | +|-----------|--------| +| `MemoryHigh=` | throttle point - reclaim pressure before the hard limit | +| `MemoryMax=` | hard limit - OOM kill inside the unit above it | +| `MemorySwapMax=` | swap cap for the unit | +| `CPUQuota=` | e.g. `200%` = two CPUs worth | +| `CPUWeight=` / `IOWeight=` | relative share under contention | +| `TasksMax=` | process/thread cap - stops fork bombs | +| `IPAddressDeny=` / `IPAddressAllow=` | per-unit IP ACL (firewall design stays in devops-networking) | + +```bash +systemctl set-property myapp.service MemoryMax=2G # persistent drop-in +systemctl set-property --runtime myapp.service CPUQuota=50% # until reboot +systemd-cgtop # live per-unit usage +systemd-run --scope -p MemoryMax=1G -- /usr/local/bin/heavy-task # cap an ad-hoc command +``` + +--- + +## Verification + +```bash +systemd-analyze security myapp.service # exposure score, lower is better +systemctl show myapp -p MemoryMax -p MemoryHigh -p TasksMax +journalctl -u myapp -b --no-pager | grep -Ei 'denied|EPERM|EACCES|SIGSYS' +``` + +## Pitfalls + +- `ProtectSystem=strict` without `StateDirectory=`/`ReadWritePaths=` - the app cannot write its data +- `MemoryDenyWriteExecute=` on a JIT runtime - instant crash, read the journal +- `MemoryMax=` without `MemoryHigh=` - abrupt OOM kills instead of gradual throttling +- `DynamicUser=` with pre-existing files owned by a fixed UID - ownership mismatch + +## Not verified + +- Exact systemd version shipped by each distro release (packages.debian.org was behind a challenge page) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/systemd-units-journald.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/systemd-units-journald.md new file mode 100644 index 0000000..dd57d2d --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/systemd-units-journald.md @@ -0,0 +1,115 @@ +--- +name: systemd-units-journald +description: systemd service units, drop-ins, timers replacing cron, and journald retention +when-to-use: Writing or overriding a service, converting a cron job, reading or capping logs +keywords: systemd, unit, drop-in, systemctl edit, timer, OnCalendar, Persistent, journalctl, journald.conf +priority: high +related: systemd-hardening-resources.md, bash-scripting.md +--- + +# systemd Units, Timers and journald + +**Load when:** creating a service, overriding a packaged unit, replacing cron, or sizing the journal. + +Sources (read 2026-11-10, systemd 262 pages): https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html, +`systemd.service.html`, `systemd.timer.html`, `systemd.time.html`, `systemctl.html`, `journalctl.html`, +`journald.conf.html`, `systemd-analyze.html`. Shipped versions are older: check `systemctl --version` +against the "Added in version N" line of each directive. + +--- + +## Decision Rules + +| Need | Do | +|------|-----| +| Change a packaged unit | `systemctl edit ` (drop-in under `/etc/systemd/system/.d/`) - never edit `/usr/lib/systemd/system/` | +| Replace the whole unit | `systemctl edit --full ` - only when a drop-in cannot express it | +| Undo local overrides | `systemctl revert ` (removes drop-ins and `/etc` copies) | +| Periodic job | `.timer` + `.service` pair, not cron ([job.timer](templates/job.timer)) | +| One-off delayed/ad-hoc job | `systemd-run --on-calendar=... ` or `systemd-run --unit=name --wait ` | +| Stop a unit from ever starting | `systemctl mask` - destructive to availability, see below | + +--- + +## Service Essentials + +| Directive | Section | Notes | +|-----------|---------|-------| +| `Type=` | `[Service]` | `simple`/`exec` for foreground daemons, `oneshot` for jobs, `notify` if the daemon supports sd_notify | +| `ExecStart=` / `ExecStartPre=` | `[Service]` | absolute paths; a failing `ExecStartPre=` aborts the start | +| `Restart=` / `RestartSec=` | `[Service]` | `on-failure` for daemons; never on `oneshot` jobs driven by a timer | +| `StartLimitIntervalSec=` / `StartLimitBurst=` | `[Unit]` | stop restart storms | +| `After=` / `Wants=` / `Requires=` | `[Unit]` | ordering (`After=`) is separate from dependency (`Wants=`/`Requires=`) | +| `OnFailure=` | `[Unit]` | trigger an alert unit on failure | +| `EnvironmentFile=` | `[Service]` | `-/path` prefix = optional file; keep secrets out of the unit (see devops-secrets) | + +To clear a list-type setting in a drop-in, assign it empty first (`ExecStart=` then `ExecStart=/new/cmd`). + +--- + +## Timers (cron replacement) + +| Directive | Effect | +|-----------|--------| +| `OnCalendar=` | wall-clock schedule; `daily` = `*-*-* 00:00:00`, `weekly` = `Mon *-*-* 00:00:00` | +| `Persistent=true` | runs immediately at boot if a run was missed while powered off (only with `OnCalendar=`) | +| `RandomizedDelaySec=` | spreads load across a fleet; `FixedRandomDelay=true` keeps the offset stable per host | +| `AccuracySec=` | coalescing window (default 1 min) | +| `OnBootSec=` / `OnUnitActiveSec=` | monotonic schedules relative to boot / last activation | +| `Unit=` | defaults to the `.service` with the same name | + +```bash +systemd-analyze calendar 'Mon..Fri *-*-* 02:30:00' # normalise + next elapse +systemctl enable --now job.timer # enable the TIMER, not the service +systemctl list-timers --all # next/last run +systemctl clean --what=state job.timer # drop the Persistent= stamp +``` + +--- + +## journald + +| Setting (`/etc/systemd/journald.conf.d/*.conf`) | Effect | +|-------------------------------------------------|--------| +| `Storage=` | `persistent` / `volatile` / `auto` / `none`; the default is set at compile time - check before assuming | +| `SystemMaxUse=` / `SystemKeepFree=` | disk cap for `/var/log/journal` | +| `MaxRetentionSec=` | age cap | +| `RateLimitIntervalSec=` / `RateLimitBurst=` | per-service flood control | +| `ForwardToSyslog=` | forward to a syslog daemon if one is used | + +```bash +journalctl -u myapp -b -p warning --since "1 hour ago" --no-pager +journalctl -k -b -1 # kernel log of the previous boot +journalctl --list-boots +journalctl --disk-usage +journalctl --vacuum-size=1G # deletes archived journal files - propose first +systemd-analyze cat-config systemd/journald.conf # effective merged config +``` + +--- + +## Destructive Operations + +| Command | Exact effect | Rule | +|---------|--------------|------| +| `systemctl mask ` | links the unit to `/dev/null`; blocks manual start, enablement and dependency activation; `--runtime` = until reboot, `--now` also stops it | propose with the reason, wait for the go; undo with `unmask` | +| `systemctl revert ` | deletes local drop-ins and `/etc` unit copies | show `systemctl cat ` first | +| `journalctl --vacuum-*` | deletes archived journal files | confirm logs are shipped elsewhere first | + +--- + +## Verification + +```bash +systemd-analyze verify /etc/systemd/system/myapp.service # syntax + referenced units +systemctl daemon-reload && systemctl restart myapp +systemctl is-active myapp && systemctl status myapp --no-pager +systemctl cat myapp # shows the unit plus every drop-in in effect +systemctl show myapp -p Restart -p ExecMainStatus +``` + +## Pitfalls + +- Enabling the `.service` of a timer job instead of the `.timer` - the job runs once at boot only +- Forgetting `daemon-reload` after writing files by hand (`systemctl edit` does it for you) +- `Restart=always` on a crash-looping daemon without `StartLimit*` - fills the journal diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/52unattended-upgrades-local b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/52unattended-upgrades-local new file mode 100644 index 0000000..593c6f8 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/52unattended-upgrades-local @@ -0,0 +1,48 @@ +// Template: local unattended-upgrades policy (Debian / Ubuntu). +// Install: /etc/apt/apt.conf.d/52unattended-upgrades-local (sorts after the packaged +// 50unattended-upgrades, so package upgrades never conflict with these edits). +// Enable the periodic run in /etc/apt/apt.conf.d/20auto-upgrades: +// APT::Periodic::Update-Package-Lists "1"; +// APT::Periodic::Unattended-Upgrade "1"; +// Check: unattended-upgrade --dry-run --debug +// less /var/log/unattended-upgrades/unattended-upgrades.log +// systemctl list-timers 'apt-daily*' +// Sources: https://raw.githubusercontent.com/mvo5/unattended-upgrades/master/data/50unattended-upgrades.Debian +// https://manpages.debian.org/trixie/unattended-upgrades/unattended-upgrade.8.en.html +// https://documentation.ubuntu.com/server/how-to/software/automatic-updates/ +// https://manpages.debian.org/trixie/apt/apt.conf.5.en.html (#clear, list entries append) +// Note: a list entry here APPENDS to the list from 50unattended-upgrades; "#clear" erases the +// inherited list first so this file fully defines the allowed origins. + +// Debian: security archive only (pattern copied from the upstream Debian default). +// Values available on the host: "apt-cache policy". +#clear Unattended-Upgrade::Origins-Pattern; +Unattended-Upgrade::Origins-Pattern { + "origin=Debian,codename=${distro_codename}-security,label=Debian-Security"; +}; + +// Ubuntu instead (upstream Ubuntu default uses Allowed-Origins "origin:archive" pairs): +// #clear Unattended-Upgrade::Allowed-Origins; +// Unattended-Upgrade::Allowed-Origins { +// "${distro_id}:${distro_codename}-security"; +// }; + +// Python regular expressions of packages never upgraded automatically. +Unattended-Upgrade::Package-Blacklist { +// "linux-"; +}; + +// Mail a report (needs a working MTA / mailx on the host). +// Unattended-Upgrade::Mail "ops@example.invalid"; +// Unattended-Upgrade::MailReport "on-change"; + +Unattended-Upgrade::Remove-Unused-Kernel-Packages "true"; +Unattended-Upgrade::Remove-New-Unused-Dependencies "true"; +Unattended-Upgrade::Remove-Unused-Dependencies "false"; + +// Reboot only inside a maintenance window and with monitoring; default upstream is "false". +Unattended-Upgrade::Automatic-Reboot "false"; +// Unattended-Upgrade::Automatic-Reboot-WithUsers "false"; +// Unattended-Upgrade::Automatic-Reboot-Time "03:30"; + +Unattended-Upgrade::SyslogEnable "true"; diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/dnf-automatic.conf b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/dnf-automatic.conf new file mode 100644 index 0000000..4f6f376 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/dnf-automatic.conf @@ -0,0 +1,30 @@ +# Template: dnf-automatic policy (RHEL family, dnf 4). +# Install: /etc/dnf/automatic.conf (merge with the packaged file; keep unlisted keys). +# Enable: systemctl enable --now dnf-automatic.timer # behaves as this file says +# systemctl list-timers 'dnf*' +# dnf5 hosts (check "dnf --version"): the timer is dnf5-automatic.timer and the packaged +# defaults live in /usr/share/dnf5/dnf5-plugins/automatic.conf; override in /etc/dnf/automatic.conf. +# Reboot check after a run: needs-restarting -r ; echo $? (1 = reboot needed) +# Sources: https://dnf.readthedocs.io/en/latest/automatic.html +# https://dnf5.readthedocs.io/en/latest/dnf5_plugins/automatic.8.html +# https://dnf-plugins-core.readthedocs.io/en/latest/needs_restarting.html + +[commands] +# "security" = only updates with an issued security advisory; "default" = everything. +upgrade_type = security +# Download and install (apply_updates implies download_updates). +download_updates = yes +apply_updates = yes +# Extra random delay before downloading, in seconds (the timers already add up to 1 h). +random_sleep = 0 +# never | when-changed | when-needed (default: never). Use when-needed only with a +# maintenance window and monitoring. +reboot = never +# reboot_command = "shutdown -r +5 'Rebooting after applying package updates'" + +[emitters] +# stdio lands in the journal of the timer's service; add "email" or "motd" if wanted. +emit_via = stdio + +# [base] overrides options of dnf's main configuration (see the DNF Configuration Reference); +# leave it as packaged unless a specific dnf.conf option must differ for unattended runs. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/hardened.service b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/hardened.service new file mode 100644 index 0000000..c5db3a7 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/hardened.service @@ -0,0 +1,80 @@ +# Template: sandboxed long-running daemon. +# Install: /etc/systemd/system/myapp.service, then +# systemd-analyze verify /etc/systemd/system/myapp.service +# systemctl daemon-reload && systemctl enable --now myapp.service +# systemd-analyze security myapp.service +# Sources: https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html +# https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html +# https://www.freedesktop.org/software/systemd/man/latest/systemd.resource-control.html +# Each directive below needs the systemd version noted in ../systemd-hardening-resources.md; +# compare with `systemctl --version`. Loosen one directive at a time when the journal shows a denial. + +[Unit] +Description=myapp daemon +Documentation=https://example.invalid/myapp +Wants=network-online.target +After=network-online.target +StartLimitIntervalSec=300 +StartLimitBurst=5 + +[Service] +Type=exec +ExecStart=/usr/local/bin/myapp --config /etc/myapp/config.toml +Restart=on-failure +RestartSec=5s + +# Identity: transient user; state/logs/runtime dirs owned by it. +DynamicUser=yes +StateDirectory=myapp +LogsDirectory=myapp +RuntimeDirectory=myapp +ConfigurationDirectory=myapp +UMask=0077 + +# Optional environment file ("-" = may be absent). Secrets: prefer LoadCredential=. +EnvironmentFile=-/etc/myapp/env +# LoadCredential=api-token:/etc/myapp/api-token + +# Filesystem +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +PrivateDevices=yes + +# Kernel and system +NoNewPrivileges=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes +ProtectProc=invisible +ProcSubset=pid +LockPersonality=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +RestrictNamespaces=yes +# Remove for JIT runtimes (Node.js, Java, LuaJIT): +MemoryDenyWriteExecute=yes + +# Capabilities: none. For a port < 1024 use: +# CapabilityBoundingSet=CAP_NET_BIND_SERVICE +# AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet= + +# Network and syscalls +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +SystemCallArchitectures=native +# "relatively safe basic choice for the majority of system services" (systemd.exec): +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM + +# Resources (cgroup v2): throttle before the hard limit. +MemoryHigh=768M +MemoryMax=1G +TasksMax=256 +CPUWeight=100 + +[Install] +WantedBy=multi-user.target diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/job.service b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/job.service new file mode 100644 index 0000000..a1f7aa1 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/job.service @@ -0,0 +1,37 @@ +# Template: oneshot job started by job.timer (cron replacement). +# Install: /etc/systemd/system/job.service + /etc/systemd/system/job.timer, then +# systemd-analyze verify /etc/systemd/system/job.service /etc/systemd/system/job.timer +# systemctl daemon-reload && systemctl enable --now job.timer # enable the TIMER +# systemctl start job.service # one manual run to test +# journalctl -u job.service -n 50 --no-pager +# Sources: https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html +# https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html +# https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html + +[Unit] +Description=Nightly job (replaces the old crontab entry) +Wants=network-online.target +After=network-online.target +# Optional alert unit, e.g. a templated notifier: +# OnFailure=notify-failure@%n.service + +[Service] +Type=oneshot +# No Restart= on a timer-driven oneshot: the next timer run is the retry. +ExecStart=/usr/local/bin/job.sh --apply +User=job +Group=job +# Lower priority than interactive work. +Nice=10 +IOSchedulingClass=idle + +# Sandbox (see hardened.service for the full set and ../systemd-hardening-resources.md). +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +StateDirectory=job +ReadWritePaths=/srv/job-output + +# Kill a hung run instead of overlapping the next one. +TimeoutStartSec=2h diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/job.timer b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/job.timer new file mode 100644 index 0000000..599fdf1 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/job.timer @@ -0,0 +1,21 @@ +# Template: timer for job.service (same base name, so Unit= is implicit). +# Replaces a crontab line such as "30 2 * * * /usr/local/bin/job.sh". +# Check the schedule: systemd-analyze calendar '*-*-* 02:30:00' +# After enabling: systemctl list-timers job.timer +# Sources: https://www.freedesktop.org/software/systemd/man/latest/systemd.timer.html +# https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html + +[Unit] +Description=Run job.service nightly at 02:30 + +[Timer] +OnCalendar=*-*-* 02:30:00 +# Catch up a run missed while the host was off (only applies with OnCalendar=). +Persistent=true +# Spread a fleet over 15 minutes; FixedRandomDelay keeps each host's offset stable. +RandomizedDelaySec=15min +FixedRandomDelay=true +AccuracySec=1min + +[Install] +WantedBy=timers.target diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/journald-retention.conf b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/journald-retention.conf new file mode 100644 index 0000000..c8b5ee9 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/journald-retention.conf @@ -0,0 +1,22 @@ +# Template: journald drop-in - persistent logs with size and age caps. +# Install: /etc/systemd/journald.conf.d/10-retention.conf, then +# systemd-analyze cat-config systemd/journald.conf # effective merged config +# systemctl restart systemd-journald +# journalctl --disk-usage +# Source: https://www.freedesktop.org/software/systemd/man/latest/journald.conf.html +# Note: the compiled-in default for Storage= differs between builds; set it explicitly. + +[Journal] +# Keep logs across reboots (/var/log/journal), fall back to /run during early boot. +Storage=persistent +# Disk cap for the persistent journal, and free space to leave on the filesystem. +SystemMaxUse=2G +SystemKeepFree=1G +# Drop entries older than 30 days. +MaxRetentionSec=30day +Compress=yes +# Per-service flood protection: at most 10000 messages per 30s per service. +RateLimitIntervalSec=30s +RateLimitBurst=10000 +# Set to yes only if a syslog daemon (rsyslog) ships logs off-host. +ForwardToSyslog=no diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/script-skeleton.sh b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/script-skeleton.sh new file mode 100644 index 0000000..44e429d --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/script-skeleton.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Template: operational script skeleton (strict mode, traps, lock, dry-run). +# Sources: GNU Bash manual 5.3 (set, shopt, trap, BASH_COMMAND), mktemp(1), flock(1), +# the ShellCheck wiki pages SC2155, SC2086, SC2164. See ../bash-scripting.md. +# Usage: ./script-skeleton.sh [--apply] [--verbose] +# Without --apply nothing is changed (dry run is the default). + +set -Eeuo pipefail +# inherit_errexit exists from bash 4.4; ignore on older shells. +shopt -s inherit_errexit 2>/dev/null || true + +readonly SCRIPT_NAME="${0##*/}" +readonly LOCK_FILE="${LOCK_FILE:-/run/lock/${SCRIPT_NAME%.sh}.lock}" +DRY_RUN=1 +VERBOSE=0 +TARGET="" +TMPDIR_WORK="" + +log() { printf '%s %s: %s\n' "$(date -u +%FT%TZ)" "$SCRIPT_NAME" "$*" >&2; } +die() { log "ERROR: $*"; exit 1; } +debug() { if [[ $VERBOSE -eq 1 ]]; then log "DEBUG: $*"; fi; } + +# Print the command in dry-run mode, execute it otherwise. +run() { + if [[ $DRY_RUN -eq 1 ]]; then + log "DRY-RUN: $*" + else + debug "exec: $*" + "$@" + fi +} + +cleanup() { + local status=$? + if [[ -n $TMPDIR_WORK && -d $TMPDIR_WORK ]]; then + rm -rf -- "$TMPDIR_WORK" + fi + exit "$status" +} + +on_err() { + local line=$1 + log "failed at line ${line}: ${BASH_COMMAND}" +} + +trap cleanup EXIT +trap 'on_err $LINENO' ERR +trap 'exit 130' INT +trap 'exit 143' TERM + +usage() { + cat >&2 < + --apply perform changes (default: dry run, prints what would change) + --verbose debug logging +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + --apply) DRY_RUN=0 ;; + --verbose) VERBOSE=1 ;; + -h|--help) usage; exit 0 ;; + --) shift; break ;; + -*) usage; die "unknown option: $1" ;; + *) TARGET=$1 ;; + esac + shift + done + [[ -n $TARGET ]] || { usage; die "missing "; } +} + +acquire_lock() { + # flock(1) ships with util-linux; fail loudly instead of reporting a false "already running". + command -v flock >/dev/null 2>&1 || die "flock not found (install util-linux)" + exec 9>"$LOCK_FILE" || die "cannot open lock file $LOCK_FILE" + if ! flock -n 9; then + log "another instance holds $LOCK_FILE - exiting" + exit 0 + fi +} + +main() { + parse_args "$@" + acquire_lock + + TMPDIR_WORK=$(mktemp -d) + debug "workdir: $TMPDIR_WORK" + + # 1. Discovery (read-only): check current state before changing anything. + if [[ -e $TARGET ]]; then + log "target exists: $TARGET" + else + log "target missing: $TARGET" + fi + + # 2. Change (idempotent): only act when the state differs from the desired one. + # Replace with real work; keep every mutating command behind run(). + run touch -- "$TARGET" + + # 3. Verify: re-check the state that motivated the change. + if [[ $DRY_RUN -eq 0 ]]; then + [[ -e $TARGET ]] || die "verification failed: $TARGET not present" + fi + log "done (dry_run=${DRY_RUN})" +} + +main "$@" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/sshd-10-hardening.conf b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/sshd-10-hardening.conf new file mode 100644 index 0000000..5073207 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/sshd-10-hardening.conf @@ -0,0 +1,39 @@ +# Template: sshd drop-in - key-only logins for members of one group. +# Install: /etc/ssh/sshd_config.d/10-hardening.conf (low number: for most keywords the FIRST +# obtained value wins, so this file must sort before others that set the same keys). +# Before applying: +# 1. Keep a second root session open (rollback path). +# 2. Ensure your admin account has a working key AND is in the AllowGroups group: +# groupadd -f sshusers && usermod -aG sshusers +# 3. sshd -t # must print nothing +# 4. sshd -T | grep -Ei 'passwordauth|permitroot|allowgroups|authenticationmethods' +# 5. systemctl reload ssh (Debian/Ubuntu) | systemctl reload sshd (RHEL family) +# 6. Test a NEW login from another terminal before closing the spare session. +# Sources: https://man.openbsd.org/sshd_config , https://man.openbsd.org/sshd +# https://documentation.ubuntu.com/server/how-to/security/openssh-server/ + +PermitRootLogin no +PasswordAuthentication no +KbdInteractiveAuthentication no +PermitEmptyPasswords no +PubkeyAuthentication yes +AuthenticationMethods publickey + +# Only members of this group may log in (lock-out risk: see step 2). +AllowGroups sshusers + +MaxAuthTries 3 +LoginGraceTime 30 +ClientAliveInterval 300 +ClientAliveCountMax 2 + +# No forwarding on application servers; remove on a bastion that needs ProxyJump. +DisableForwarding yes +X11Forwarding no + +LogLevel VERBOSE + +# Example: allow password auth only from a management subnet (Match blocks go last). +# Match Address 10.10.0.0/24 +# PasswordAuthentication yes +# AuthenticationMethods publickey,password diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/sudoers-ops b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/sudoers-ops new file mode 100644 index 0000000..cf56095 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/templates/sudoers-ops @@ -0,0 +1,27 @@ +# Template: sudoers drop-in for an operations group. +# File name MUST NOT contain a "." or end in "~" - sudo silently skips such files in sudoers.d. +# Install (keep a second root session open): +# visudo -f /etc/sudoers.d/ops # edit through visudo, never a plain editor +# visudo -cf /etc/sudoers.d/ops # check this file +# visudo -c # check the whole tree +# chmod 0440 /etc/sudoers.d/ops +# sudo -l -U # effective rights, from the spare session +# Sources: https://www.sudo.ws/docs/man/sudoers.man/ , https://www.sudo.ws/docs/man/visudo.man/ + +# New pseudo-terminal per command: blocks terminal-injection and background processes that keep +# access to the user's terminal after the command ends (sudoers(5), use_pty). +Defaults:%ops use_pty + +# Read-only diagnostics without a password prompt. --no-pager is forced as the first argument: +# under elevated privileges the pager (less) is a security boundary - it can run shell commands +# (systemctl(1), $SYSTEMD_PAGERSECURE). Members must type the commands exactly in this form. +Cmnd_Alias OPS_DIAG = /usr/bin/journalctl --no-pager *, /usr/bin/systemctl --no-pager status *, /usr/bin/systemctl --no-pager list-units *, /usr/bin/systemctl --no-pager list-timers * + +# Service lifecycle for the application units only (password required). +Cmnd_Alias OPS_APP = /usr/bin/systemctl restart myapp.service, /usr/bin/systemctl reload myapp.service + +%ops ALL=(root) NOPASSWD: OPS_DIAG +%ops ALL=(root) OPS_APP + +# Paths differ per distro (/usr/bin vs /bin symlinks): check with "command -v systemctl" and adjust. +# Avoid "NOPASSWD: ALL" and wildcards on commands that accept arbitrary arguments (editors, pagers, shells). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/time-sync.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/time-sync.md new file mode 100644 index 0000000..3278a94 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/time-sync.md @@ -0,0 +1,106 @@ +--- +name: time-sync +description: Clock synchronisation with chrony or systemd-timesyncd, drift diagnosis, time zone +when-to-use: Clock skew errors (TLS, Kerberos, cluster quorum, log correlation), choosing or checking the NTP client +keywords: chrony, chronyc, tracking, sources, makestep, iburst, nts, timedatectl, set-ntp, timesync-status, systemd-timesyncd +priority: medium +related: systemd-units-journald.md +--- + +# Time Synchronisation + +**Load when:** certificates "not yet valid", Kerberos/etcd/Ceph complaining about skew, logs out of order across hosts, +or a new host is provisioned. Serving NTP to a network (firewall, VLANs) is **devops-networking** territory. + +Sources (read 2026-11-10): https://chrony-project.org/doc/latest/chrony.conf.html, +https://chrony-project.org/doc/latest/chronyc.html, +https://www.freedesktop.org/software/systemd/man/latest/timedatectl.html. + +--- + +## Decision Rules + +| Situation | Choice | +|-----------|--------| +| Server, VM, cluster node | chrony - full NTP client/server, NTS authentication, step + slew control | +| Minimal client, no serving | systemd-timesyncd is acceptable (syncs the local clock from a remote NTP server) | +| Two clients enabled | pick one; disable the other - they fight over the clock | +| Cluster (etcd, Ceph, Corosync, Kerberos) | all nodes on the same internal sources; monitor offset | + +Which daemon is present depends on the distro release and image - detect, do not assume: + +```bash +timedatectl # "System clock synchronized" + "NTP service" +systemctl is-active chronyd chrony systemd-timesyncd 2>/dev/null +``` + +--- + +## chrony + +Key `chrony.conf` directives (verified on chrony-project.org): + +| Directive | Effect | +|-----------|--------| +| `server iburst` | one source; `iburst` sends a burst of 4-8 requests at start for a faster first sync | +| `pool iburst` | a DNS pool of sources | +| `nts` (option on `server`/`pool`) | Network Time Security authentication | +| `makestep ` | step the clock if the offset exceeds threshold during the first `limit` updates (e.g. `makestep 1.0 3`); otherwise slew | +| `rtcsync` | keep the hardware RTC in sync from the kernel | + +```text +# drop-in example (path depends on distro packaging - check "confdir"/"sourcedir" in the main chrony.conf) +server ntp1.internal.example iburst +server ntp2.internal.example iburst +makestep 1.0 3 +``` + +```bash +chronyc tracking # offset, frequency, stratum, leap status +chronyc sources -v # per-source state, reachability, last sample +chronyc sourcestats # drift statistics per source +chronyc makestep # step NOW - see Destructive Operations +``` + +--- + +## systemd-timesyncd / timedatectl + +```bash +timedatectl set-ntp true # enable and start the first available sync service +timedatectl timesync-status # timesyncd only: server, offset, poll interval +timedatectl show-timesync # timesyncd properties +timedatectl set-timezone UTC # servers: UTC simplifies log correlation +``` + +--- + +## Destructive / Risky Operations + +| Command | Exact effect | Rule | +|---------|--------------|------| +| `chronyc makestep` / large `makestep` threshold | jumps the clock; timers, TOTP, DB replication and cluster leases can misbehave | on a cluster node, propose, wait for the go, one node at a time | +| `timedatectl set-time` | sets the clock manually (only with NTP off) | almost never on a server | +| Disabling the NTP client | clock drifts silently | never without a replacement | + +--- + +## Verification + +```bash +timedatectl | grep -E 'synchronized|NTP service' +chronyc tracking | grep -E 'System time|Leap status' # offset small, "Normal" +chronyc sources -v | head -20 # at least one '*' (selected) source +``` + +## Pitfalls + +- chrony and systemd-timesyncd both enabled +- VM snapshots/restores jump the guest clock - allow `makestep` at start, verify after restore +- Firewall blocking UDP/123 outbound - sources show unreachable (fix in devops-networking) + +## Not verified + +- The chrony service unit name per distro (`chronyd.service` vs `chrony.service`) and the default client per release - detect on the host +- Drop-in directory for chrony (`confdir`/`sourcedir` usage differs per packaging) +- That systemd-timesyncd cannot serve time to other hosts (its man page describes only client sync) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/users-sudo-ssh.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/users-sudo-ssh.md new file mode 100644 index 0000000..d7e1899 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-linux/references/users-sudo-ssh.md @@ -0,0 +1,115 @@ +--- +name: users-sudo-ssh +description: Users and groups, sudoers drop-ins with visudo, sshd server drop-ins and ssh client config, with lock-out protection +when-to-use: Creating/locking accounts, granting sudo, changing sshd, setting up jump hosts +keywords: useradd, usermod, chage, userdel, sudoers.d, visudo, sshd_config.d, sshd -t, sshd -T, ProxyJump, ControlMaster +priority: high +related: selinux-apparmor.md +--- + +# Users, sudo and SSH + +**Load when:** account lifecycle, sudo rights, or any sshd / ssh client change. + +Sources (read 2026-11-10): https://man7.org/linux/man-pages/man8/useradd.8.html, `usermod.8`, `userdel.8`, +https://man7.org/linux/man-pages/man1/chage.1.html, https://www.sudo.ws/docs/man/sudoers.man/, +https://www.sudo.ws/docs/man/visudo.man/, https://man.openbsd.org/sshd_config, https://man.openbsd.org/sshd, +https://man.openbsd.org/ssh_config, https://documentation.ubuntu.com/server/how-to/security/openssh-server/. + +## Lock-out Protocol (sshd, sudoers, PAM) + +1. Open **two** root sessions; work in one, keep the other idle as the rollback path. +2. Back up: `cp -a /etc/ssh/sshd_config.d /root/sshd_config.d.bak.$(date +%F)`. +3. Write the drop-in, validate (`sshd -t`, `visudo -c`). +4. Reload (sshd re-reads its config on SIGHUP: `systemctl reload ssh` on Debian/Ubuntu, `sshd` on RHEL). +5. From a **third, new** connection, log in and `sudo -v`. Only then close the spare session. +6. Fail -> revert the file from the spare session, validate, reload. + +## Users and Groups + +| Task | Command | +|------|---------| +| Human user with home | `useradd -m -s /bin/bash -G alice` | +| Service account | `useradd -r -s /usr/sbin/nologin svc-app` (or `DynamicUser=` in the unit) | +| Add to group | `usermod -aG docker alice` - without `-a`, `-G` replaces all supplementary groups | +| Lock password | `usermod -L alice` (prefixes `!` to the hash; keys may still work - also expire: `chage -E 0 alice`) | +| Aging info | `chage -l alice` | +| Inspect | `id alice`, `getent passwd alice`, `getent group sudo` | + +Admin group: `sudo` on Debian/Ubuntu, `wheel` on RHEL family (check `grep -E '^%(sudo|wheel)' /etc/sudoers`). + +## sudo + +- Main file `/etc/sudoers` includes `/etc/sudoers.d` (`@includedir`; `#includedir` is the form accepted by sudo before 1.9.1). +- Files in `sudoers.d` whose name **ends in `~` or contains a `.`** are skipped - `ops.conf` is silently ignored. +- Always edit through visudo: `visudo -f /etc/sudoers.d/ops`; check everything with `visudo -c`. +- Grant to groups, not users; list exact commands with absolute paths; avoid `NOPASSWD: ALL`. +- Commands that open a pager (`systemctl status`, `journalctl`) can spawn a shell from `less` - systemctl(1) + recommends `--no-pager` or pager secure mode when granting them via sudo; the template forces `--no-pager`. + +Template: [sudoers-ops](templates/sudoers-ops). + +```bash +visudo -cf /etc/sudoers.d/ops && chmod 0440 /etc/sudoers.d/ops +visudo -c # whole tree, including every include +sudo -l -U alice # effective rights +``` + +## sshd (server) + +Ubuntu ships `Include /etc/ssh/sshd_config.d/*.conf` at the very top of `sshd_config` (Ubuntu docs); +confirm on other distros with `grep -n '^Include' /etc/ssh/sshd_config`. Included files expand in +lexical order and for most keywords **the first obtained value wins**, so a low-numbered drop-in +overrides later files (check what else is in that directory - another drop-in may set the same key). + +| Keyword | Hardened value | Note | +|---------|----------------|------| +| `PermitRootLogin` | `no` (or `prohibit-password`, the default) | | +| `PasswordAuthentication` | `no` | default is `yes` | +| `KbdInteractiveAuthentication` | `no` | `ChallengeResponseAuthentication` is its deprecated alias | +| `AuthenticationMethods` | `publickey` | | +| `AllowGroups` | `sshusers` | lock-out risk if your user is not in it | +| `MaxAuthTries` / `LoginGraceTime` | `3` / `30` | | +| `DisableForwarding` | `yes` on non-jump hosts | overrides all forwarding options | +| `PerSourcePenalties` | review defaults | recent OpenSSH only - check `sshd -T` | + +Template: [sshd-10-hardening.conf](templates/sshd-10-hardening.conf). + +```bash +sshd -t # syntax + keys +sshd -T | grep -Ei 'permitrootlogin|passwordauth|allowgroups' # effective config +sshd -T -C user=alice,host=x,addr=10.0.0.5 # effective config after Match blocks +``` + +## ssh (client) - `~/.ssh/config` + +```text +Host bastion + HostName bastion.example.net + User ops + IdentitiesOnly yes + IdentityFile ~/.ssh/id_ed25519_ops +Host app-* + ProxyJump bastion + StrictHostKeyChecking accept-new + ControlMaster auto + ControlPath ~/.ssh/cm-%r@%h:%p + ControlPersist 10m + ServerAliveInterval 30 +``` + +`accept-new` adds unknown host keys but refuses changed ones. + +## Destructive Operations + +| Command | Exact effect | Rule | +|---------|--------------|------| +| `userdel -r alice` | deletes the account, its home directory and mail spool; files elsewhere stay orphaned | archive the home first, propose, wait for the go | +| `usermod -G grp alice` (no `-a`) | removes every other supplementary group, sudo included | always `-aG` | +| sshd/sudoers change | can lock every admin out | lock-out protocol above | + +## Not verified + +- Exact contents of the distro-shipped `sshd_config.d/` (cloud images may add files) - list them on the host +- Ubuntu socket-activated `ssh.socket` behaviour on port changes - Ubuntu docs show `systemctl restart ssh.service` +- `sudo`/`wheel` group defaults - read `/etc/sudoers` on the host diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/SKILL.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/SKILL.md new file mode 100644 index 0000000..c121ffd --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/SKILL.md @@ -0,0 +1,172 @@ +--- +name: devops-networking +description: Use when configuring or debugging Linux server and infrastructure networking - host firewall with nftables (iptables-nft, ufw, firewalld as front-ends), bridges/bonds/VLANs with netplan, ifupdown2, systemd-networkd or NetworkManager, static and policy routing, DNS (systemd-resolved, Unbound/BIND/CoreDNS, split-horizon), TLS and ACME certificates (certbot, acme.sh, lego, HTTP-01 vs DNS-01), reverse proxies (Traefik, Nginx, Caddy, HAProxy), WireGuard (and Tailscale/Headscale as a pointer) and layer-by-layer network troubleshooting. Not for Kubernetes networking, Gateway API or CNI (devops-kubernetes, devops-k8s-cluster), Docker networks (devops-docker), Proxmox SDN (devops-proxmox), Neutron (devops-openstack), or security audit and TLS grading (fuse-security:security-headers, security-expert). +versions: + nftables-wiki: "atomic reload via nft -f, check via nft -c (manpage updated 2026-02-07)" + systemd: "262 (resolved.conf docs)" + netplan: "1.1.2 (Ubuntu 24.04 package)" + traefik: "3.7.18 (3.7.17 per endoflife.date)" + caddy: "2.11.9" + nginx: "1.31.8 mainline, 1.30.6" + haproxy: "3.4.7 LTS" + certbot: "5.9.0" + lego: "5.6.0" + acme-sh: "3.1.6" + unbound: "1.26.2" + coredns: "1.15.0" + wireguard-tools: "1.0.20260223" + headscale: "0.29.7" + tailscale: "1.104.5" +user-invocable: true +references: + - references/safe-network-changes.md + - references/nftables-firewall.md + - references/interfaces-bridge-bond-vlan.md + - references/routing-policy.md + - references/dns.md + - references/tls-acme.md + - references/reverse-proxy.md + - references/wireguard.md + - references/troubleshooting-order.md + - references/templates/nftables.conf + - references/templates/nft-safe-apply.sh + - references/templates/wg0-server.conf + - references/templates/wg0-client.conf + - references/templates/traefik-static.yml + - references/templates/traefik-dynamic.yml + - references/templates/nginx-proxy.conf + - references/templates/Caddyfile + - references/templates/haproxy.cfg + - references/templates/netplan-vlan-bridge.yaml + - references/templates/interfaces-vlan-aware-bridge + - references/templates/unbound-split-horizon.conf +related-skills: + - devops-linux + - devops-opnsense + - devops-docker + - devops-kubernetes + - devops-k8s-cluster + - devops-proxmox + - devops-virtualization + - devops-openstack + - devops-observability + - devops-secrets + - devops-sre-ops + - devops-ansible +--- + + +Covers server networking on Linux hosts: the host firewall, interfaces (bridge, bond, VLAN), routing, DNS, TLS/ACME certificates, reverse proxies, WireGuard and a fixed troubleshooting order, with a lock-out-proof way to apply every change. Does not cover Kubernetes networking, Gateway API and CNI (devops-kubernetes, devops-k8s-cluster), Docker networks (devops-docker), Proxmox SDN (devops-proxmox), Neutron (devops-openstack) or security audit and TLS grading (fuse-security:security-headers, security-expert). + + +# DevOps Networking + +## Agent Workflow (MANDATORY) + +Before ANY network change, spawn in parallel, one `Task` call each: + +1. **fuse-ai-pilot:explore-codebase** - find existing firewall, netplan/interfaces, proxy and ACME configs in the repo or IaC +2. **fuse-ai-pilot:research-expert** - verify directives against the version in use (nftables wiki, nginx.org, caddyserver.com, doc.traefik.io, haproxy docs) +3. **mcp__context7__query-docs** - `/traefik/traefik`, `/websites/nginx_en`, `/websites/caddyserver_caddyfile`, `/haproxy/docs`, `/canonical/netplan`, `/websites/unbound_nlnetlabs_nl_en`, `/websites/eff-certbot_readthedocs_io_en_stable`, `/go-acme/lego` + +Then inspect the host read-only (`ip -br a`, `ip r`, `nft list ruleset`, `ss -tulpn`, `resolvectl status`, which network manager owns the config). After file changes, run **fuse-ai-pilot:sniper**. + +## Versions (checked 2026-11-10) + +See the frontmatter `versions` map; sources per reference under "Sources". Traefik 3.6 reached end of life 2026-08-16; Nginx 1.29 on 2026-05-13; HAProxy 3.4 is an LTS (EOL 2031-04-01). Tool versions on the target host decide which directives exist: check `nginx -v`, `nft --version`, `caddy version`, `haproxy -v`, `traefik version` first. + +## Overview + +| Area | Decision in one line | +|------|----------------------| +| **Firewall** | One `inet` table, `policy drop` on input/forward, SSH limited to admin networks; never `flush ruleset` on a container/VM host | +| **Interfaces** | netplan on Ubuntu, ifupdown2 on Proxmox/Debian hypervisors; VLAN-aware bridge for guests; LACP only if the switch is configured | +| **Routing** | Static route for one prefix; `ip rule` + table for multi-homing; forwarding needs sysctl + firewall + NAT | +| **DNS** | Unbound for recursive/internal names, BIND for authoritative zones, CoreDNS for plugin chains/cluster; systemd-resolved on clients | +| **TLS** | Proxy built-in ACME if Traefik/Caddy; certbot/lego/acme.sh otherwise; DNS-01 for wildcard; staging first | +| **Reverse proxy** | Caddy (simple), Traefik (dynamic), Nginx (classic), HAProxy (load balancing); fix proxy headers and trusted proxies first | +| **VPN** | WireGuard; Tailscale/Headscale when SSO/ACLs/NAT traversal are needed | +| **Troubleshooting** | link -> IP -> route -> firewall -> DNS -> TLS -> application, stop at the first failure | + +## Critical Rules + +1. **Lock-out is the main danger.** Any change to firewall, SSH port, routing, DNS resolver, interface config or tunnels on a remote host is proposed with ACTION / EFFECT / PROOF / ROLLBACK, then waits for the user's go ([safe-network-changes.md](references/safe-network-changes.md)) +2. **Rollback that fires without the operator** - `nft -c` then apply via `templates/nft-safe-apply.sh` (timed revert), `netplan try`, `systemd-run --on-active=` restore for ifupdown2/routes/resolver, a second session open, console access known +3. **Check mode before apply** - `nft -c -f`, `netplan generate`, `nginx -t`, `caddy validate`, `haproxy -c`, `unbound-checkconf` +4. **Read-only first** - identify the owning network manager and the current rules; never run two managers on one interface +5. **Exact effect for destructive commands** - `nft flush ruleset`, `ip link del`, `wg-quick down`, `certbot revoke|delete`, deleting `acme.json`: show what disappears +6. **One change at a time**, verified from a NEW session before the next +7. **Never invent a directive or flag** - cite the official page for the version in use; unknown = "not verified" +8. **Secrets stay out of output** - WireGuard private keys, DNS-provider API tokens, ACME account keys: names and paths only (`devops-secrets`) + +## Architecture + +``` +/etc/nftables.conf # host firewall (table inet firewall) +/etc/netplan/*.yaml | /etc/network/interfaces # interfaces, one manager only +/etc/wireguard/wg0.conf # chmod 600 +/etc/systemd/resolved.conf.d/ # client DNS +/etc/unbound/unbound.conf.d/ # internal resolver +/etc/nginx/conf.d | /etc/caddy/Caddyfile | /etc/traefik | /etc/haproxy # one proxy per public port +/etc/letsencrypt/live// # certbot lineages (or Traefik acme.json, Caddy data dir) +``` + +## Reference Guide + +### Concepts + +| Topic | Reference | When to Consult | +|-------|-----------|-----------------| +| **Safe changes** | [safe-network-changes.md](references/safe-network-changes.md) | Before ANY change that can cut your session | +| **nftables** | [nftables-firewall.md](references/nftables-firewall.md) | Host firewall, opening ports, NAT, Docker coexistence | +| **Interfaces** | [interfaces-bridge-bond-vlan.md](references/interfaces-bridge-bond-vlan.md) | Bridges, bonds, VLANs, which tool owns the config | +| **Routing** | [routing-policy.md](references/routing-policy.md) | Static routes, multi-homing, forwarding | +| **DNS** | [dns.md](references/dns.md) | Resolver config, Unbound/BIND/CoreDNS, split-horizon | +| **TLS/ACME** | [tls-acme.md](references/tls-acme.md) | Issue/renew/revoke certificates, challenge choice | +| **Reverse proxy** | [reverse-proxy.md](references/reverse-proxy.md) | Choosing and configuring a proxy, headers, HTTP/3 | +| **WireGuard** | [wireguard.md](references/wireguard.md) | Site-to-site, road-warrior, full vs split tunnel | +| **Troubleshooting** | [troubleshooting-order.md](references/troubleshooting-order.md) | Any "cannot connect" | + +### Templates + +| Template | When to Use | +|----------|-------------| +| [nftables.conf](references/templates/nftables.conf) | Default-deny server firewall: SSH from admin nets, HTTP(S), HTTP/3, WireGuard | +| [nft-safe-apply.sh](references/templates/nft-safe-apply.sh) | Apply a ruleset with a timed automatic revert | +| [wg0-server.conf](references/templates/wg0-server.conf) / [wg0-client.conf](references/templates/wg0-client.conf) | WireGuard hub and client pair | +| [traefik-static.yml](references/templates/traefik-static.yml) / [traefik-dynamic.yml](references/templates/traefik-dynamic.yml) | Traefik v3 entrypoints, ACME, file-provider routers and middlewares | +| [nginx-proxy.conf](references/templates/nginx-proxy.conf) | Nginx TLS proxy, h2/h3, proxy headers, WebSocket, rate limit | +| [Caddyfile](references/templates/Caddyfile) | Caddy automatic HTTPS reverse proxy with health checks | +| [haproxy.cfg](references/templates/haproxy.cfg) | HAProxy TLS frontend, health checks, per-IP rate limit | +| [netplan-vlan-bridge.yaml](references/templates/netplan-vlan-bridge.yaml) | Bond + VLAN + bridges on Ubuntu | +| [interfaces-vlan-aware-bridge](references/templates/interfaces-vlan-aware-bridge) | VLAN-aware bridge on a bond with ifupdown2 | +| [unbound-split-horizon.conf](references/templates/unbound-split-horizon.conf) | Internal zone answers plus a forward-zone | + +## Quick Reference + +```bash +ip -br a ; ip r ; ip rule # state +nft list ruleset ; ss -tulpn # firewall and listeners +nft -c -f /etc/nftables.conf # validate only +resolvectl status ; resolvectl query example.com +netplan try --timeout 120 # apply with auto-rollback (Ubuntu) +wg show # tunnel state +openssl s_client -connect host:443 -servername host &1 | grep -E 'HTTP/|subject|expire' +``` + +## Best Practices + +### DO +- Keep the firewall in one file under version control and reload it atomically +- Restrict SSH by source network, keep a second session and a console path while changing +- Use staging ACME endpoints before production; persist ACME state +- Set trusted proxies explicitly and make the backend trust only the proxy +- Open UDP 443 when serving HTTP/3 and the WireGuard port when running a tunnel + +### DON'T +- Run `netplan apply`, plain `nft -f` or `ifdown` on a remote host without a timed rollback +- `flush ruleset` where Docker, libvirt or kube-proxy own tables +- Expose the Traefik dashboard or API unauthenticated, or run an open DNS resolver +- Edit `/etc/resolv.conf` by hand on a resolved/NetworkManager host +- Put private keys, API tokens or `acme.json` in Git or in a chat diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/dns.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/dns.md new file mode 100644 index 0000000..6de161b --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/dns.md @@ -0,0 +1,79 @@ +--- +name: dns +description: DNS on servers - systemd-resolved client config, choosing Unbound, BIND or CoreDNS, split-horizon, resolver changes without lock-out +when-to-use: Fixing name resolution on a host, deploying an internal resolver or authoritative server, split DNS for an internal zone +keywords: systemd-resolved, resolvectl, resolved.conf, Unbound, BIND, CoreDNS, split-horizon, forward-zone, local-zone, stub listener, dig +priority: high +related: safe-network-changes.md, troubleshooting-order.md, tls-acme.md +--- + +# DNS + +**Load when:** a host cannot resolve, you choose a DNS server, or an internal zone must answer differently inside the LAN. Changing a host's resolver can silently break updates, ACME and package installs: [safe-network-changes.md](safe-network-changes.md). + +## 1. Client side: systemd-resolved + +```bash +resolvectl status # per-link servers, domains, DNSSEC, stub state +resolvectl query app.example.com # resolve through resolved, shows interface and protocol +resolvectl dns eth0 192.0.2.53 # runtime change for one link (lost at reboot/link reload) +resolvectl domain eth0 '~corp.example.com' # route only that domain to this link's servers +cat /etc/resolv.conf ; ls -l /etc/resolv.conf # symlink to the stub (127.0.0.53) or a static file? +``` + +Persistent config: `/etc/systemd/resolved.conf` or drop-ins `/etc/systemd/resolved.conf.d/*.conf`, section `[Resolve]`, keys `DNS=` (system servers, optional `:port`, `%ifname`, `#SNI`), `FallbackDNS=`, `Domains=` (a `~` prefix makes it a routing-only domain, not a search domain). Drop-ins sort lexicographically and the last value wins for single-value keys; local drop-ins belong in `/etc/systemd/resolved.conf.d/` with numbers 60-90 (resolved.conf(5), systemd 262). In netplan, `nameservers:` feeds the backend. + +Check order for "DNS broken": link up and route (troubleshooting-order.md) -> `resolvectl status` shows the expected server -> query that server directly with `dig @192.0.2.53 example.com` -> is the answer wrong (zone) or missing (server/firewall UDP+TCP 53)? + +## 2. Choosing a DNS server (pointer level) + +| Need | Pick | Why | +|------|------|-----| +| Recursive, caching, validating resolver for LAN/servers | **Unbound** (NLnet Labs) | Small, validating, `local-zone`/`local-data` for simple internal names, `forward-zone`, `stub-zone`, `auth-zone`, RPZ; not an authoritative server for complex zones | +| Authoritative server for real zones (secondaries, DNSSEC signing, zone transfers) | **BIND 9** (ISC) | Full-featured authoritative + recursive; the heavier config. Alternatives (PowerDNS, Knot) are out of this skill's scope | +| Resolver/authoritative inside Kubernetes or plugin-chained | **CoreDNS** | Plugin architecture (`forward`, `cache`, `file`, `kubernetes`); cluster DNS belongs to `devops-k8s-cluster` | +| Just a laptop/server client | systemd-resolved (above) | Nothing to run | + +Unbound docs state that complex authoritative data (referrals, wildcards, CNAME/DNAME, DNSSEC authoritative service) goes to a `stub-zone` pointing at a real authoritative server; a `forward-zone` sends queries to another recursive server (https://unbound.docs.nlnetlabs.nl/en/latest/manpages/unbound.conf.html). + +## 3. Split-horizon + +Same name, different answers for LAN vs Internet: + +1. Public authoritative zone (registrar/provider) has the public A records. +2. The LAN resolver (Unbound) carries `local-zone: "corp.example.com." transparent` plus `local-data` for internal hosts, or forwards the zone to an internal authoritative server. Template [unbound-split-horizon.conf](templates/unbound-split-horizon.conf). +3. Clients use the LAN resolver only (DHCP/netplan/resolved `DNS=`); `~corp.example.com` routing domain on VPN links sends just that zone through the tunnel. + +Pitfall: DoH/DoT in browsers bypasses the LAN resolver; internal names then fail for those clients. + +## 4. Running Unbound next to systemd-resolved + +Both want port 53. resolved listens on `127.0.0.53`; Unbound with `interface: 0.0.0.0` fails with "address in use". Bind Unbound to the LAN address or `127.0.0.1` (Unbound docs mention this workaround), and point resolved at it with `DNS=127.0.0.1`. Or disable the stub listener (`DNSStubListener=` in `[Resolve]`; key name not read today, see Not verified). + +```bash +unbound-checkconf # validate before reload +systemctl reload unbound ; dig @192.0.2.53 git.corp.example.com +short +``` + +## Effects to show before applying + +| Change | Exact effect | +|--------|--------------| +| Edit `/etc/resolv.conf` by hand | Overwritten by resolved/NetworkManager; if it is a symlink you edit the target | +| Point a host at a resolver that is down or unreachable | Every lookup stalls ~ seconds before fallback; `apt`, `curl`, ACME and monitoring fail while IP connectivity is fine | +| `systemctl restart systemd-resolved` | Flushes the cache; briefly interrupts lookups | +| Open Unbound `access-control: 0.0.0.0/0 allow` | Open resolver usable for amplification attacks | + +## Sources (read 2026-11-10) + +- https://www.freedesktop.org/software/systemd/man/latest/resolved.conf.html (systemd 262) +- https://www.freedesktop.org/software/systemd/man/latest/resolvectl.html +- https://unbound.docs.nlnetlabs.nl/en/latest/manpages/unbound.conf.html and `.../getting-started/configuration.html` via Context7 `/websites/unbound_nlnetlabs_nl_en` +- Unbound 1.26.2 (release-1.26.2 tag, 2026-10-21; "a number of security fixes": https://github.com/NLnetLabs/unbound/tags.atom); CoreDNS v1.15.0 (2026-10-21: https://github.com/coredns/coredns/releases.atom) + +## Not verified + +- BIND 9 current version and supported branches (ISC pages not read); CoreDNS Corefile syntax (not read, pointer only). +- `DNSStubListener=` and `resolvectl dns|domain` subcommand syntax (resolvectl page read only up to `query`). +- DNSSEC setup, zone transfers (AXFR/TSIG), and PowerDNS/Knot: out of the research done today. +- The DoH/browser bypass remark is operational knowledge, not sourced. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/interfaces-bridge-bond-vlan.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/interfaces-bridge-bond-vlan.md new file mode 100644 index 0000000..c251f1c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/interfaces-bridge-bond-vlan.md @@ -0,0 +1,99 @@ +--- +name: interfaces-bridge-bond-vlan +description: Bridges, bonds and VLANs on Linux hosts - which tool each distro uses (netplan, ifupdown2, systemd-networkd, NetworkManager), VLAN-aware bridge, LACP, safe apply +when-to-use: Configuring uplinks, hypervisor bridges, bonds or tagged VLANs; reading an unfamiliar host's network stack +keywords: netplan, ifupdown2, ifreload, systemd-networkd, NetworkManager, bridge-vlan-aware, 802.3ad, LACP, vmbr0, networkctl +priority: critical +related: safe-network-changes.md, routing-policy.md, troubleshooting-order.md +--- + +# Interfaces: Bridge, Bond, VLAN + +**Load when:** creating or changing L2/L3 interface config. Always [safe-network-changes.md](safe-network-changes.md) first: this is where lock-outs happen. + +## 1. Find which tool owns the config (read-only) + +```bash +ls /etc/netplan/ /etc/network/interfaces /etc/systemd/network/ 2>/dev/null +systemctl is-active systemd-networkd NetworkManager networking 2>/dev/null +networkctl list ; nmcli device status 2>/dev/null ; ip -br a ; ip -d link show type bridge +``` + +| Tool | Where it is the norm | Config | Safe apply | +|------|---------------------|--------|-----------| +| netplan (renderer networkd or NetworkManager) | Ubuntu Server (netplan originates at Canonical) | `/etc/netplan/*.yaml` | `netplan try` | +| ifupdown2 | Proxmox VE (recommended, default for new installs since 7.0); installable on Debian | `/etc/network/interfaces` | `ifreload -a` + your own timed restore | +| systemd-networkd | netplan backend; stand-alone on minimal systems | `/etc/systemd/network/*.network,*.netdev` | `networkctl reload` + timed restore | +| NetworkManager | desktops, RHEL-family servers | `nmcli`, keyfiles | `nmcli connection up` + timed restore | + +Only the first two rows were confirmed from docs today; distro defaults for Debian 13 and RHEL 10 are in "Not verified". Never run two managers on the same interface. + +## 2. Decision rules + +| Need | Choice | +|------|--------| +| Guests/VMs on several VLANs | One **VLAN-aware bridge** (`bridge-vlan-aware yes`, `bridge-vids 2-4094`); tag per guest NIC. Fewer devices than one bridge per VLAN | +| Host IP on a tagged VLAN | Put the VLAN on the least-abstracted device: `vmbr0.5` on a VLAN-aware bridge, or `eno1.5` / `bond0.5` (Proxmox docs) | +| Redundancy | Bond mode `802.3ad` (LACP) only if the switch ports are configured for it; otherwise `active-backup`. Proxmox example uses `bond-xmit-hash-policy layer2+3` | +| Several uplinks, no switch support | `active-backup` (no switch config needed) | +| Proxmox node | Use the GUI or `devops-proxmox`; Proxmox writes `interfaces.new` and applies live | + +## 3. Netplan (Ubuntu) + +Template [netplan-vlan-bridge.yaml](templates/netplan-vlan-bridge.yaml): bond + VLAN + two bridges. + +```bash +netplan generate # parse/validate, writes backend config, no apply +netplan try --timeout 120 # applies, auto-rolls back unless confirmed +netplan status # runtime view +``` + +- `netplan try` reverts automatically after the timeout (default 120 s) unless confirmed with ENTER or `SIGUSR1`/`SIGINT`; STP may need over a minute to settle; after a timeout or cancel, verify the revert actually happened (netplan-try(8) KNOWN ISSUES). +- `vlans:` entries need `id` (0-4094) and `link` naming a device defined in the same config; a bridge lists the VLAN device under `interfaces`. +- Default route: `routes: - to: default via: ` (netplan examples). +- Files in `/etc/netplan` must not be world-readable (mode 600). + +## 4. ifupdown2 (Proxmox/Debian) + +Template [interfaces-vlan-aware-bridge](templates/interfaces-vlan-aware-bridge). + +```bash +ifquery -a # shows parsed state (read-only) +cp /etc/network/interfaces /root/interfaces.bak +ifreload -a # applies live, no reboot (Proxmox docs) +``` + +- Avoid classic `ifup`/`ifdown` on bridges: `ifdown vmbrX` interrupts all guest traffic and a later `ifup` does not reconnect the guests (Proxmox docs). +- Device naming: `en*` predictable names, bridges `vmbr[N]`, bonds `bond[N]`, VLAN `.`. A systemd upgrade can rename NICs and break the config; pin the naming scheme or a `.link` file on hypervisors (Proxmox docs). + +## 5. Verification + +```bash +ip -br a ; ip -br link ; ip r # addresses, state, routes +bridge vlan show # per-port VLAN membership of a VLAN-aware bridge +cat /proc/net/bonding/bond0 # LACP partner, slave state, "MII Status: up" +ip r get 1.1.1.1 ; ping -c2 # path and reachability +``` + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| `netplan apply` over SSH | No rollback; wrong gateway = permanent lock-out until console | +| LACP configured on host, not on switch | Bond flaps or one link carries all, switch logs MAC flapping | +| NIC rename after upgrade | Config references a missing device; host boots with no network | +| Gateway not reachable on the new subnet | Route add fails silently in some managers; verify with `ip r get` | +| Two managers (NetworkManager + networkd) | Fight over the same interface after reboot | + +## Sources (read 2026-11-10) + +- https://manpages.ubuntu.com/manpages/noble/man8/netplan-try.8.html +- https://github.com/canonical/netplan/blob/main/doc/multi-nic-vm-host-with-bonds-and-vlans.md and `netplan-yaml.md` (vlans section) via Context7 `/canonical/netplan` +- https://pve.proxmox.com/wiki/Network_Configuration and https://pve.proxmox.com/pve-docs/chapter-sysadmin.html (via Context7 `/websites/pve_proxmox_pve-docs`) + +## Not verified + +- Default network manager for Debian 13, RHEL/Alma/Rocky 10 and Ubuntu 26.04 (assumed netplan on Ubuntu; not read). +- `netplan status`, `bridge vlan show`, `ifquery -a`, `networkctl reload`: standard commands, no page read today. +- systemd-networkd `.netdev`/`.network` syntax (not read); NetworkManager bridge/bond/VLAN `nmcli` syntax (not read). +- Netplan version (the netplan GitHub release feed returned 404); Ubuntu Noble ships netplan.io 1.1.2 (manpage footer). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/nftables-firewall.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/nftables-firewall.md new file mode 100644 index 0000000..5951790 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/nftables-firewall.md @@ -0,0 +1,103 @@ +--- +name: nftables-firewall +description: Host firewall with nftables - default-deny inet ruleset, atomic reload, coexisting with Docker/libvirt, iptables-nft, ufw and firewalld front-ends +when-to-use: Writing, reviewing or changing a host firewall; migrating from iptables; opening a port +keywords: nftables, nft -c, inet table, ct state, default drop, flush ruleset, iptables-nft, ufw, firewalld, nat, masquerade +priority: critical +related: safe-network-changes.md, wireguard.md, troubleshooting-order.md +--- + +# nftables Host Firewall + +**Load when:** touching packet filtering on a Linux host. Every apply follows [safe-network-changes.md](safe-network-changes.md). + +## Decision rules + +| Situation | Choice | +|-----------|--------| +| New host, nothing else manages netfilter | One `table inet firewall` with `policy drop` on `input` and `forward`; file `/etc/nftables.conf`; template [nftables.conf](templates/nftables.conf) | +| Host also runs Docker, Podman, libvirt, kube-proxy | Do NOT `flush ruleset`. Manage only your own table (below). Docker documents that editing its tables is lost on restart; its nftables backend is experimental, not for Swarm, and has no `DOCKER-USER` chain (https://docs.docker.com/engine/network/firewall-nftables/). Container port filtering: `devops-docker` | +| Team already uses ufw / firewalld | Keep the front-end; do not mix it with hand-written nft files on the same host. They generate netfilter rules for you; inspect with `nft list ruleset` | +| Old scripts call `iptables` | The iptables compatibility layer runs iptables syntax over the nf_tables kernel framework (nftables wiki). Do not mix legacy and nft-backed iptables on one host; check which backend `iptables --version` reports | +| Kubernetes node | The CNI / kube-proxy own netfilter; no host-wide default-drop without the cluster's documented ports (`devops-k8s-cluster`) | + +## Facts that matter + +- `nft -f file` loads the whole file as one atomic transaction: no half-configured moment, unlike a shell script of many `nft add` calls. Add `flush table X` (or `flush ruleset`) at the top, otherwise every reload duplicates rules. `flush table` does not flush sets; `flush ruleset` does (https://wiki.nftables.org/wiki-nftables/index.php/Atomic_rule_replacement). +- `nft -c -f file` (`--check`) validates without applying (nft manpage). +- The `inet` family sees IPv4 and IPv6 in one table; address matches stay family-specific (`ip saddr`, `ip6 saddr`): a set mixing both families does not type-check, so define two sets. +- Must-accept traffic in an IPv6 input chain: neighbour discovery (`nd-neighbor-solicit`, `nd-router-advert`, `nd-neighbor-advert`), otherwise IPv6 connectivity breaks (nftables wiki simple server ruleset). +- Variables: `define name = value`; `$name`. Includes: `include "/etc/nftables/*.nft"`; the file header is `#!/usr/sbin/nft -f` (https://wiki.nftables.org/wiki-nftables/index.php/Scripting). + +## Own-table variant (hosts with Docker / libvirt / k8s) + +Replace only `inet firewall`, never other tables. Idiom: declare the table (idempotent), delete it, define it again, all in one `nft -f` transaction. + +``` +table inet firewall +delete table inet firewall +table inet firewall { + chain inbound { type filter hook input priority 0; policy drop; ... } +} +``` + +Check with `nft -c -f` first. Hook priority 0 runs next to Docker's chains; both tables see the packet and a drop in any of them wins. If a container port is published, your input chain is not on that path (it is forwarded); filter it with the `forward` hook or the container engine's own mechanism. + +## NAT / forwarding (gateway, VPN hub) + +``` +table ip nat { + chain postrouting { + type nat hook postrouting priority 100; policy accept; + oifname "eth0" ip saddr 10.8.0.0/24 masquerade + } +} +``` + +Plus `sysctl -w net.ipv4.ip_forward=1` (persist in `/etc/sysctl.d/`), plus a `forward` rule allowing `iifname "wg0" oifname "eth0"`. Hook syntax `type nat hook postrouting priority 100` appears in the nftables wiki scripting examples (as `type filter` there); the `masquerade` statement is not covered by a page read today (see Not verified). + +## Boot persistence + +Distribution packages ship an `nftables.service` that loads the ruleset file at boot (`/etc/nftables.conf` on Debian/Ubuntu); enable with `systemctl enable nftables`. Persist only after the live apply is confirmed, otherwise a bad file locks you out at the next reboot. + +## Frequent changes + +| Task | How | +|------|-----| +| Open a port | Edit the file, `nft -c -f`, `nft-safe-apply.sh`; for a live one-off `nft add rule inet firewall inbound tcp dport 8443 accept` (lost at reload; add it to the file) | +| List with handles | `nft -a list ruleset` (`-a` shows handles) | +| Delete one rule | `nft delete rule inet firewall inbound handle ` - propose with the listing first | +| Count / trace | add `counter`; `nft monitor trace` needs `meta nftrace set 1` (not read today) | + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| `flush ruleset` on a container host | Deletes the engine's tables; container networking breaks until the engine restarts | +| SSH allowed by `ct state established` only | Existing session lives, new ones fail: test with a NEW session | +| `policy drop` on `output` | Breaks DNS/updates unless every flow is listed; leave output open unless required | +| IPv6 forgotten | Host reachable/exposed on v6 with no rules; `inet` table covers both | +| Rules saved but `nftables.service` disabled | Ruleset empty after reboot (all traffic accepted) | +| Mixed `ip saddr` set with v6 prefixes | Syntax error at load | + +## Verification + +`nft list ruleset` shows the expected chains and policy; `ss -tlnp` vs rules; external port scan from another host (`nmap` is out of this skill's scope - use `nc -vz host port`); a new SSH session; `journalctl -k | grep nft` if logging is enabled. + +## Sources (read 2026-11-10) + +- https://wiki.nftables.org/wiki-nftables/index.php/Atomic_rule_replacement +- https://wiki.nftables.org/wiki-nftables/index.php/Scripting +- https://wiki.nftables.org/wiki-nftables/index.php/Simple_ruleset_for_a_server +- https://wiki.nftables.org/wiki-nftables/index.php/What_is_nftables%3F +- https://www.netfilter.org/projects/nftables/manpage.html (page updated 07/02/2026) +- https://docs.docker.com/engine/network/firewall-nftables/ (via the research file, fetched earlier today) + +## Not verified + +- nftables tool version (no authoritative version page read). +- `nftables.service` and `/etc/nftables.conf` path per distribution (Debian wiki returned an anti-bot challenge). +- `masquerade` statement, NAT hook priorities, `nft monitor trace` / `meta nftrace`, `nft delete rule ... handle`: standard syntax not confirmed on a page read today. +- The `table X` / `delete table X` / `table X {...}` single-transaction idiom: widely used, not on a page read today; check with `nft -c -f` on the target. +- Interaction of your input chain with Docker's forward-path chains at equal hook priority (statement "a drop in any table wins" is netfilter behaviour, not on a page read today). +- ufw and firewalld behaviour (not read); which iptables backend a given distro defaults to. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/reverse-proxy.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/reverse-proxy.md new file mode 100644 index 0000000..358b8e7 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/reverse-proxy.md @@ -0,0 +1,104 @@ +--- +name: reverse-proxy +description: Choosing and configuring a reverse proxy - Traefik, Nginx, Caddy, HAProxy - TLS termination, HTTP/2 and HTTP/3, proxy headers, rate limiting, health checks +when-to-use: Putting services behind a proxy, picking the proxy, fixing wrong client IP/scheme, WebSockets, 502/504, enabling HTTP/3 +keywords: Traefik, Nginx, Caddy, HAProxy, TLS termination, X-Forwarded-For, proxy_set_header, http2, http3, quic, limit_req, health check, trusted_proxies +priority: high +related: tls-acme.md, nftables-firewall.md, troubleshooting-order.md +--- + +# Reverse Proxy + +**Load when:** a service is published over HTTP(S). Kubernetes ingress: `devops-kubernetes` (Gateway API); containers on a Docker host with labels: `devops-docker`. + +## 1. Choose + +| Need | Pick | Templates | +|------|------|-----------| +| Few services, automatic HTTPS, minimal config | **Caddy** (automatic HTTPS, `reverse_proxy`, health checks) | [Caddyfile](templates/Caddyfile) | +| Dynamic discovery (Docker labels, Kubernetes, file provider), middlewares | **Traefik** v3 | [traefik-static.yml](templates/traefik-static.yml), [traefik-dynamic.yml](templates/traefik-dynamic.yml) | +| Classic host, static sites + proxy, huge ecosystem, fine control | **Nginx** | [nginx-proxy.conf](templates/nginx-proxy.conf) | +| High-throughput L4/L7 load balancing, stick tables, advanced health checks | **HAProxy** (3.4 LTS) | [haproxy.cfg](templates/haproxy.cfg) | +| TCP/UDP load balancing (databases, SMTP) | HAProxy (`mode tcp`) or Nginx `stream` | pointer only | + +Run exactly one proxy per public port. Do not stack proxies without declaring trust (section 3). + +## 2. TLS termination + +- Terminate at the proxy, plain HTTP to the backend on a private/loopback address. If the hop crosses an untrusted network, re-encrypt (HAProxy `server ... ssl verify required ca-file ...`, Nginx `proxy_pass https://`). +- Minimum TLS 1.2: Nginx `ssl_protocols TLSv1.2 TLSv1.3;`, Traefik `tls.options..minVersion: VersionTLS12`, HAProxy `bind ... ssl-min-ver TLSv1.2`. Grading and cipher audit: `fuse-security:security-headers` / `security-expert`. +- Certificates: [tls-acme.md](tls-acme.md). HAProxy wants cert + key in one PEM. +- HSTS only after HTTPS works everywhere for that name; a long `max-age` is hard to undo. + +## 3. Proxy headers (the usual bug) + +The backend must see the client IP and the original scheme/host. + +| Header | Nginx | Caddy | Traefik | HAProxy | +|--------|-------|-------|---------|---------| +| Host | `proxy_set_header Host $host;` (default would be `$proxy_host`) | passed through unchanged | passed through | passed through | +| Client IP | `X-Real-IP $remote_addr`, `X-Forwarded-For $proxy_add_x_forwarded_for` | sets/augments `X-Forwarded-For` | sets `X-Forwarded-For` | `option forwardfor` | +| Scheme | `X-Forwarded-Proto $scheme` | sets `X-Forwarded-Proto` | sets `X-Forwarded-Proto` | `http-request set-header X-Forwarded-Proto https if { ssl_fc }` | + +- Caddy ignores incoming `X-Forwarded-*` by default to prevent spoofing; behind a CDN or another proxy declare it with the global `servers { trusted_proxies static }` (recommended over the per-proxy setting). +- Nginx: `proxy_set_header` directives are inherited from the previous level only if the current level defines none; redefining one in a `location` drops the server-level ones. +- The backend app must be told to trust the proxy (framework "trusted proxies" setting) and only from the proxy's address; trusting `X-Forwarded-For` from anywhere allows IP spoofing. +- WebSockets: Nginx needs `Upgrade` and `Connection` passed explicitly (hop-by-hop headers); `proxy_http_version 1.1` before nginx 1.29.7 (nginx.org websocket page). + +## 4. HTTP/2 and HTTP/3 + +- Nginx: `listen 443 ssl; http2 on;` (nginx.org example for ngx_http_v2_module; needs ALPN, OpenSSL 1.0.2+); HTTP/3: `listen 443 quic reuseport;` + `listen 443 ssl;` + `add_header Alt-Svc 'h3=":443"; ma=86400';`, build with `--with-http_v3_module`; the nginx doc recommends the same port for h3 and https. +- HTTP/3 uses **UDP 443**: open it in the firewall (templates/nftables.conf has it). Without it clients silently fall back to h2. +- Caddy: HTTP/3 handled through the global `servers { protocols h1 h2 h3 }` option (default set not confirmed today). HAProxy: `alpn h2,http/1.1` on `bind`. + +## 5. Rate limiting and health checks + +| Proxy | Rate limit | Health check | +|-------|-----------|--------------| +| Nginx | `limit_req_zone $binary_remote_addr zone=perip:10m rate=10r/s;` + `limit_req zone=perip burst=20 nodelay;` | passive: `max_fails` / `fail_timeout` on `server`; active checks are an NGINX Plus feature (not verified) | +| Traefik | `rateLimit` middleware: `average`, `period` (default 1s), `burst` | `loadBalancer.healthCheck: {path, interval, timeout}` | +| Caddy | no built-in rate limiter in the standard distribution (plugin; not verified) | `health_uri`, `health_interval`, `health_timeout` (active); `max_fails`, `fail_duration` (passive) | +| HAProxy | stick table + `http-request track-sc0 src` + `deny if { sc_http_req_rate(0) gt N }` | `option httpchk` / `http-check`, `check` + `inter rise fall` on `server` | + +Rate limiting by IP behind another proxy limits the proxy's IP unless the real client IP is restored first. + +## 6. Applying changes (all graceful, all with a check first) + +| Proxy | Check | Reload | +|-------|-------|--------| +| Nginx | `nginx -t` | `systemctl reload nginx` | +| Caddy | `caddy validate --config /etc/caddy/Caddyfile` | `caddy reload --config ...` or `systemctl reload caddy` | +| HAProxy | `haproxy -c -f /etc/haproxy/haproxy.cfg` | `systemctl reload haproxy` | +| Traefik | file provider `watch: true` hot-reloads the dynamic directory; static config needs a restart | `systemctl restart traefik` (brief connection reset: propose it) | + +## Pitfalls + +| Trap | Consequence | +|------|-------------| +| `proxy_pass http://app/;` vs without trailing slash | Path rewriting differs; test URLs with a query and a sub-path | +| Backend redirects to `http://` | Missing `X-Forwarded-Proto`; mixed content / redirect loops | +| 502 after deploy | Backend not listening on the address used; `ss -tlnp`, then `curl -v http://127.0.0.1:8080/healthz` from the proxy host | +| 504 | Upstream slow: check `proxy_read_timeout` (Nginx), backend load; not a network fault by default | +| HTTP-01 behind a blanket redirect | Challenge path must be served on port 80 before the redirect | +| Dashboard/API exposed | Traefik `api.insecure` is unauthenticated: never on a public interface | + +## Verification + +`curl -sv -o /dev/null https://app.example.com/ 2>&1 | grep -E 'HTTP/|subject|alt-svc'`; `curl --http3 -I https://app.example.com` (needs an HTTP/3-capable curl); check the app sees the real client IP in its log; `ss -tulpn | grep -E ':(80|443)\b'`. + +## Sources (read 2026-11-10) + +- nginx.org via Context7 `/websites/nginx_en`: ngx_http_proxy_module, ngx_http_v2_module, ngx_http_v3_module / quic.html, ngx_http_limit_req_module, websocket.html +- https://caddyserver.com/docs/caddyfile/directives/reverse_proxy , `/caddyfile/options`, `/caddyfile/directives/encode` via Context7 `/websites/caddyserver_caddyfile` +- Traefik v3.7.15 docs via Context7 `/traefik/traefik` (entryPoints, file provider, healthCheck, rateLimit, tls options) +- HAProxy docs via Context7 `/haproxy/docs` (bind ssl/alpn, redirect, stick-table actions, httpchk) +- Versions (endoflife.date API, 2026-11-10): Nginx mainline 1.31.8, 1.30.6 (1.29 EOL 2026-05-13); HAProxy 3.4.7 (LTS, EOL 2031-04-01), 3.3.17, 3.2.26 (LTS); Traefik 3.7.18 (GitHub releases feed; endoflife lists 3.7.17), 3.6 EOL 2026-08-16; Caddy 2.11.9 + +## Not verified + +- Which nginx line is "stable" (even minor 1.30 assumed from nginx convention; nginx.org news page not read). +- Caddy `lb_policy` names, built-in HTTP/3 default, absence of a standard rate-limit module; NGINX Plus-only active checks. +- HAProxy `deny_status`, `http-check send`, `option forwardfor`, `stick-table ... store http_req_rate` in a frontend: standard syntax, individual lines not confirmed. +- nginx reload keeps old workers on a bad config (documented behaviour, page not read today). +- `curl --http3` availability depends on the curl build. +- Traefik's default `X-Forwarded-*` behaviour and `forwardedHeaders.trustedIPs` (table cell is from general knowledge); HAProxy `option forwardfor` only adds the header in `mode http`. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/routing-policy.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/routing-policy.md new file mode 100644 index 0000000..61399c7 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/routing-policy.md @@ -0,0 +1,77 @@ +--- +name: routing-policy +description: Static and policy routing on Linux - routes, ip rule with custom tables, multi-homing, forwarding sysctl, persistence per manager +when-to-use: Adding a route, a second uplink, source-based routing, a gateway/VPN hub, or debugging "wrong path" traffic +keywords: ip route, ip rule, routing table, policy routing, multi-homing, ip_forward, rp_filter, ip route get, metric +priority: high +related: safe-network-changes.md, interfaces-bridge-bond-vlan.md, wireguard.md, troubleshooting-order.md +--- + +# Routing and Policy Routing + +**Load when:** the default path is wrong, a host has two uplinks, or you build a gateway. A bad default route cuts the session: [safe-network-changes.md](safe-network-changes.md). + +## Decision rules + +| Need | Choice | +|------|--------| +| One uplink, extra LAN/VPN prefix | Static route: `ip route add 10.20.0.0/16 via 192.0.2.254 dev eth0`; persist in the manager's config (netplan `routes:`, `interfaces` `up ip route ...`) | +| Two uplinks, answer on the interface the request came in | Policy routing: one table per uplink + `ip rule from lookup
` | +| Active/backup default | Two default routes with different `metric`; lowest wins; failover needs link or health detection (not automatic for a dead upstream) | +| Gateway / VPN hub | `net.ipv4.ip_forward=1` + firewall forward rules + NAT (nftables-firewall.md) | +| Container/VM host | Let the engine or hypervisor manage its own routes; do not hand-edit them | + +## Commands (syntax from ip-route(8) and ip-rule(8)) + +```bash +ip route show ; ip -6 route show ; ip rule show # read-only +ip route get 203.0.113.9 # the route the kernel WOULD use +ip route get 203.0.113.9 from 192.0.2.10 iif eth0 # as seen for a given source/iface + +# second uplink via table 100 (runtime only, lost at reboot) +ip route add default via 198.51.100.1 dev eth1 table 100 +ip rule add from 198.51.100.10/32 lookup 100 priority 1000 +ip route flush cache +``` + +Rule selectors include `from`, `to`, `fwmark`, `iif`, `oif`, `priority`; action `table TABLE_ID` or `lookup`. Routes accept `via`, `dev`, `metric`, `table` (man pages above). Table names map in `/etc/iproute2/rt_tables` (not read today). + +## Effect of mutating commands + +| Command | Exact effect | +|---------|--------------| +| `ip route replace default via X` | Replaces the main default route atomically; if X is unreachable every off-link connection, including your SSH session, stalls | +| `ip route del default` | Removes the default route immediately; same outage | +| `ip route flush ...` | Deletes all routes matching the selector; `flush table main` removes everything | +| `ip rule add ... priority N` | Takes effect for new lookups at once; a low priority number is evaluated before `main`, so a catch-all rule can hijack all traffic | + +Propose with ACTION / EFFECT / PROOF and schedule the restore (`systemd-run --on-active=120s ip route replace default via `). + +## Forwarding and reverse-path filter + +- Forwarding: `sysctl net.ipv4.ip_forward` (read), `sysctl -w net.ipv4.ip_forward=1` (runtime), file in `/etc/sysctl.d/` to persist (kernel parameters in depth: `devops-linux`). +- Asymmetric routing (second uplink) is dropped by strict reverse-path filtering: `net.ipv4.conf..rp_filter` (1 strict, 2 loose). Setting it is a security trade-off; propose it, do not apply it silently. + +## Persistence per manager + +| Manager | Where | +|---------|-------| +| netplan | `routes:` (`to`, `via`, `metric`, `table`) and `routing-policy:` per interface; examples "Configure source routing" in https://netplan.readthedocs.io/en/stable/examples/ | +| ifupdown2 | `up ip route add ...` / `post-up ip rule add ...` lines in the stanza | +| systemd-networkd | `[Route]` and `[RoutingPolicyRule]` sections in `.network` (not read today) | +| WireGuard `wg-quick` | adds routes from peers' `AllowedIPs`; default route case uses `ip-rule` (wg-quick(8)); `Table = off` disables | + +## Verification + +`ip route get [from ]` shows the chosen next hop; `ip rule show` ordering; `traceroute`/`mtr -n ` from the host (mtr: troubleshooting-order.md); a connection test from each source IP. + +## Sources (read 2026-11-10) + +- https://man7.org/linux/man-pages/man8/ip-rule.8.html and https://man7.org/linux/man-pages/man8/ip-route.8.html (synopsis and description only) +- https://git.zx2c4.com/wireguard-tools/about/src/man/wg-quick.8 + +## Not verified + +- Exact `sysctl` semantics of `rp_filter` values, `ip route flush cache` necessity on current kernels, `/etc/iproute2/rt_tables` format: standard practice, no page read. +- netplan `routing-policy:` keys and systemd-networkd `[RoutingPolicyRule]` (netplan examples page was nav-only through the fetch tool). +- Failover behaviour of equal-metric or multipath routes. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/safe-network-changes.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/safe-network-changes.md new file mode 100644 index 0000000..22b1b81 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/safe-network-changes.md @@ -0,0 +1,79 @@ +--- +name: safe-network-changes +description: Lock-out-proof procedure for any change to firewall, SSH port, routing, DNS resolver or interface config on a remote host +when-to-use: Before ANY command or file change that can cut the session you are working in +keywords: lock-out, nft -c, systemd-run, netplan try, ifreload, second session, rollback, console, propose then go +priority: critical +related: nftables-firewall.md, interfaces-bridge-bond-vlan.md, routing-policy.md, dns.md, wireguard.md +--- + +# Safe Network Changes (lock-out prevention) + +**Load when:** the change touches the path your own SSH session uses. Read-only inspection (`ip a`, `ip r`, `ss`, `nft list ruleset`, `resolvectl status`, `wg show`) needs no go; everything else follows this file. + +## The five rules + +1. **Propose, then the user's go.** Show ACTION / EFFECT / PROOF and wait. Never apply a firewall, routing, resolver, interface or SSH change on a remote host on your own initiative. +2. **A rollback that fires without the operator.** If the new config cuts you off, nobody is there to type the fix. Schedule the restore *before* the change (table below). +3. **A second session stays open** (and out-of-band console / IPMI / hypervisor console when it exists). Established SSH connections survive most firewall changes only because of `ct state established`; a new connection is the real test. Test from a NEW session. +4. **Validate before apply**, with the tool's check mode (`nft -c`, `netplan generate`, `nginx -t`, `caddy validate`, `haproxy -c`, `unbound-checkconf`). +5. **Change one thing, verify, then the next.** Never combine firewall + SSH port + route in one apply. + +## Format of every risky command + +``` +ACTION nft -f /etc/nftables.conf (via nft-safe-apply.sh, 120 s revert) +EFFECT default policy becomes drop on input; SSH only from 192.0.2.0/24, HTTP(S), UDP 51820 stay open +PROOF new SSH session from 192.0.2.0/24 succeeds; ss -tlnp unchanged; backup at /root/nft-before-*.conf +ROLLBACK systemd timer nft-revert reloads the backup after 120 s unless cancelled +WAITING for your go +``` + +## Rollback mechanisms + +| Change | Check first | Auto-revert | Confirm / cancel | +|--------|-------------|-------------|------------------| +| nftables ruleset | `nft -c -f file` | `templates/nft-safe-apply.sh`: backup (`flush ruleset` + `nft list ruleset`), `systemd-run --unit=nft-revert --on-active=120s nft -f backup`, then `nft -f file` | `systemctl stop nft-revert.timer` | +| netplan | `netplan generate` | `netplan try --timeout 120`: applies, rolls back unless confirmed (default 120 s; STP may need over a minute to settle; after a timeout verify the revert really happened) | press ENTER, or send SIGUSR1/SIGINT | +| ifupdown2 (`/etc/network/interfaces`) | `ifquery -a -c` is NOT a safe check; read the diff | no built-in timer: `cp interfaces interfaces.bak` then `systemd-run --unit=net-revert --on-active=180s sh -c 'cp interfaces.bak interfaces && ifreload -a'` | `systemctl stop net-revert.timer` | +| systemd-networkd / NetworkManager without netplan | `networkctl` / `nmcli` read-only | NetworkManager: `nmcli` changes are persistent; use the same `systemd-run` restore of the saved profile | stop the timer | +| sshd port / config | `sshd -t` | keep the old port also listening until the new one is proven (`Port 22` and `Port 2222` both), then remove the old one in a second change | second session on the new port | +| Routing (`ip route`, `ip rule`) | `ip route get ` after, not before | runtime-only changes vanish on reboot; still schedule `systemd-run --on-active=120s ip route replace default via ` | stop the timer | +| systemd-resolved / resolv.conf | `resolvectl query example.com` after | keep the old file (`cp -a`) and schedule its restore | stop the timer | +| Reverse proxy | `nginx -t` / `caddy validate` / `haproxy -c` | reload is graceful (old workers keep serving on a bad reload for nginx); keep the previous file | `systemctl reload` of the old file | + +`systemd-run ... --on-active=` creates a transient timer plus service (systemd-run(1), https://www.freedesktop.org/software/systemd/man/latest/systemd-run.html). The classic `at` command is an alternative on hosts that have `atd`; its man page was not read today. + +## Commands that need exact effect shown + +| Command | Exact effect | Do instead | +|---------|--------------|-----------| +| `nft flush ruleset` | Removes ALL tables of all families, including Docker/Podman/libvirt/kube-proxy tables. With a default-drop policy loaded the host is open until reload; with no ruleset all traffic is accepted | Replace only your own table (nftables-firewall.md) | +| `nft -f file` (plain, remote) | Atomic swap; a rule that drops SSH cuts new connections immediately | `nft-safe-apply.sh` | +| `ip link del ` | Removes the interface and its addresses/routes; a bridge or bond port deletion cuts every guest/VM behind it | `ip link set down` only when proven harmless; never on the uplink | +| `wg-quick down wg0` | Removes `wg0`, its routes and any `ip rule` override; over a VPN-only admin path this cuts your session | Schedule `wg-quick up wg0` via `systemd-run` first | +| `systemctl restart networking` / `ifdown vmbrX` | Drops all guest traffic on the bridge and may not reconnect guests on `ifup` (Proxmox warns about this) | `ifreload -a` | +| `certbot revoke` | Tells the CA the certificate is no longer valid; irreversible; clients get errors until a new one is served | Only for key compromise; issue the replacement first | +| `certbot delete --cert-name X` | Irreversible removal of the lineage files; does NOT revoke. Proxies still pointing at `/etc/letsencrypt/live/X/` fail to start/reload | `certbot certificates`, check which configs reference it, then propose | +| `rm -rf /etc/letsencrypt` / deleting `acme.json` | Loses the account key and issued certificates; re-issuing counts against CA rate limits | Back up first | + +## Verification after every change + +New SSH session works; `ss -tlnp` shows the expected listeners; external check of each published port; `journalctl -u --since "5 min ago"`; then cancel the revert timer. A command that exited 0 is not proof the service is reachable. + +## Sources (read 2026-11-10) + +- https://wiki.nftables.org/wiki-nftables/index.php/Atomic_rule_replacement +- https://www.netfilter.org/projects/nftables/manpage.html (`-c`, `--check`) +- https://manpages.ubuntu.com/manpages/noble/man8/netplan-try.8.html +- https://pve.proxmox.com/wiki/Network_Configuration +- https://www.freedesktop.org/software/systemd/man/latest/systemd-run.html +- https://git.zx2c4.com/wireguard-tools/about/src/man/wg-quick.8 +- https://eff-certbot.readthedocs.io/en/stable/using.html + +## Not verified + +- `at` / `atd` behaviour and man page. +- Exact unit naming of `systemd-run --unit=X --on-active=` (timer `X.timer`): standard behaviour, systemd-run page read only up to the synopsis. +- `ifquery -a -c` safety remark, NetworkManager rollback advice, sshd dual-port advice: operational practice, not from an official page read today. +- Revert-by-timer correctness was not executed on a live host (no host available to this agent). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/Caddyfile b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/Caddyfile new file mode 100644 index 0000000..d277a28 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/Caddyfile @@ -0,0 +1,37 @@ +# /etc/caddy/Caddyfile - automatic HTTPS reverse proxy (Caddy 2.11). +# Directives verified at caddyserver.com/docs (reverse_proxy, encode, global options) via Context7 +# /websites/caddyserver_caddyfile, 2026-11-10. +# Validate: caddy validate --config /etc/caddy/Caddyfile +# Reload: caddy reload --config /etc/caddy/Caddyfile (or systemctl reload caddy) + +{ + email admin@example.com + # While testing, use the staging CA to avoid rate limits: + # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory + + # Behind another proxy/CDN only: trust its address so X-Forwarded-* from it is kept. + # servers { + # trusted_proxies static 203.0.113.0/24 + # } +} + +app.example.com { + encode zstd gzip + + header Strict-Transport-Security "max-age=31536000" + + reverse_proxy 127.0.0.1:8080 127.0.0.1:8081 { + health_uri /healthz + health_interval 10s + health_timeout 3s + # Caddy sets X-Forwarded-For/-Proto/-Host itself; add Host only if the app needs the upstream's. + } +} + +# Wildcard needs DNS-01 and a Caddy build with the DNS provider module (xcaddy), e.g.: +# *.example.com { +# tls { +# dns cloudflare {env.CF_API_TOKEN} +# } +# ... +# } diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/haproxy.cfg b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/haproxy.cfg new file mode 100644 index 0000000..7c739c9 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/haproxy.cfg @@ -0,0 +1,39 @@ +# /etc/haproxy/haproxy.cfg - TLS-terminating HTTP proxy with health checks and a per-IP rate limit +# (HAProxy 3.4 LTS). Directives verified in the HAProxy docs through Context7 /haproxy/docs, 2026-11-10. +# Check: haproxy -c -f /etc/haproxy/haproxy.cfg +# Reload: systemctl reload haproxy (seamless; run the check first) +# The `crt` file is the certificate chain + private key concatenated in one PEM. + +global + log /dev/log local0 + maxconn 20000 + +defaults + mode http + log global + option httplog + timeout connect 5s + timeout client 30s + timeout server 30s + +frontend www + bind :80 + bind :443 ssl crt /etc/haproxy/certs/app.example.com.pem alpn h2,http/1.1 ssl-min-ver TLSv1.2 + http-request redirect scheme https unless { ssl_fc } + + # per-IP request rate limit through a stick table + stick-table type ip size 100k expire 30s store http_req_rate(10s) + http-request track-sc0 src + http-request deny deny_status 429 if { sc_http_req_rate(0) gt 200 } + + http-request set-header X-Forwarded-Proto https if { ssl_fc } + option forwardfor + default_backend app + +backend app + balance roundrobin + option httpchk + http-check send meth GET uri /healthz + default-server inter 3s rise 2 fall 3 + server app1 127.0.0.1:8080 check + server app2 127.0.0.1:8081 check diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/interfaces-vlan-aware-bridge b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/interfaces-vlan-aware-bridge new file mode 100644 index 0000000..9f2665f --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/interfaces-vlan-aware-bridge @@ -0,0 +1,38 @@ +# /etc/network/interfaces - ifupdown2, VLAN-aware bridge on a bond (Proxmox/Debian style). +# Syntax from https://pve.proxmox.com/pve-docs/chapter-sysadmin.html (Network Configuration), +# read via Context7 /websites/pve_proxmox_pve-docs, 2026-11-10. +# +# APPLY: ifupdown2 reloads live with `ifreload -a`. There is NO built-in timer: schedule the +# restore first (references/safe-network-changes.md), keep the console/IPMI and a second session. +# cp /etc/network/interfaces /root/interfaces.bak +# systemd-run --unit=net-revert --on-active=180s /bin/sh -c 'cp /root/interfaces.bak /etc/network/interfaces && ifreload -a' +# ifreload -a # then confirm from a NEW session: systemctl stop net-revert.timer +# On a Proxmox node edit via the GUI (writes interfaces.new) or follow devops-proxmox instead. +# Interface names, addresses, VLAN range are placeholders: REPLACE. + +auto lo +iface lo inet loopback + +iface eno1 inet manual +iface eno2 inet manual + +auto bond0 +iface bond0 inet manual + bond-slaves eno1 eno2 + bond-miimon 100 + bond-mode 802.3ad + bond-xmit-hash-policy layer2+3 + +auto vmbr0 +iface vmbr0 inet manual + bridge-ports bond0 + bridge-stp off + bridge-fd 0 + bridge-vlan-aware yes + bridge-vids 2-4094 + +# host management address on VLAN 5, tagged on the bridge +auto vmbr0.5 +iface vmbr0.5 inet static + address 10.10.10.2/24 + gateway 10.10.10.1 diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/netplan-vlan-bridge.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/netplan-vlan-bridge.yaml new file mode 100644 index 0000000..239ac1e --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/netplan-vlan-bridge.yaml @@ -0,0 +1,35 @@ +# /etc/netplan/60-bridge-vlan.yaml - bond + tagged VLAN + bridges for a hypervisor/host. +# Structure from https://github.com/canonical/netplan/blob/main/doc/multi-nic-vm-host-with-bonds-and-vlans.md +# (Context7 /canonical/netplan, 2026-11-10). `renderer: networkd` as in the netplan examples. +# +# APPLY ONLY WITH: netplan try --timeout 120 (rolls back on its own unless you press ENTER) +# Never `netplan apply` over SSH. Names, VLAN IDs and addresses below are placeholders: REPLACE. +# File mode must be 600 (netplan warns on world-readable files). + +network: + version: 2 + renderer: networkd + ethernets: + eno1: {} + eno2: {} + bonds: + bond0: + interfaces: [eno1, eno2] + parameters: + mode: 802.3ad # requires LACP configured on the switch ports + mii-monitor-interval: 1000 + vlans: + bond0.40: + id: 40 + link: bond0 + bridges: + br0: # untagged: management of the host + interfaces: [bond0] + addresses: [192.0.2.10/24] + routes: + - to: default + via: 192.0.2.1 + nameservers: + addresses: [192.0.2.53, 9.9.9.9] + br40: # VLAN 40 for guests; no host address + interfaces: [bond0.40] diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/nft-safe-apply.sh b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/nft-safe-apply.sh new file mode 100644 index 0000000..7295c2c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/nft-safe-apply.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Apply an nftables ruleset with an automatic revert if you do not confirm. +# Run it from a session you can lose; keep a SECOND SSH session open. +# usage: nft-safe-apply.sh /etc/nftables.conf [revert-seconds] +# confirm (inside the window): systemctl stop nft-revert.timer +set -euo pipefail + +NEW=${1:?usage: nft-safe-apply.sh [revert-seconds]} +WINDOW=${2:-120} +BACKUP=/root/nft-before-$(date +%Y%m%d-%H%M%S).conf + +# 1. syntax check, changes nothing (nft --check) +nft -c -f "$NEW" + +# 2. backup of the live ruleset, loadable as-is: `flush ruleset` first, +# otherwise reloading it would be merged on top of the new rules +{ echo 'flush ruleset'; nft list ruleset; } > "$BACKUP" +echo "backup: $BACKUP" + +# 3. schedule the revert BEFORE touching the firewall (systemd transient timer + service) +systemd-run --unit=nft-revert --on-active="${WINDOW}s" /usr/sbin/nft -f "$BACKUP" +echo "auto-revert in ${WINDOW}s" + +# 4. atomic apply (nft -f swaps old and new ruleset in one transaction) +nft -f "$NEW" +echo "applied. From ANOTHER terminal open a new SSH session now." +echo "If it works: systemctl stop nft-revert.timer (cancels the revert)" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/nftables.conf b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/nftables.conf new file mode 100644 index 0000000..df7d0b7 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/nftables.conf @@ -0,0 +1,66 @@ +#!/usr/sbin/nft -f +# Default-deny host firewall: SSH + HTTP(S) + WireGuard. +# Based on https://wiki.nftables.org/wiki-nftables/index.php/Simple_ruleset_for_a_server +# +# BEFORE USE +# 1. Set ssh_port, wan_if, wg_if, admin_nets to the real values (admin_nets: where SSH may come from). +# 2. Check: nft -c -f nftables.conf (syntax only, changes nothing) +# 3. Apply with the timed auto-revert: templates/nft-safe-apply.sh (never plain `nft -f` on a remote host). +# +# WARNING: `flush ruleset` deletes EVERY table, including tables owned by Docker, Podman, +# libvirt or kube-proxy. On such hosts use the own-table variant in references/nftables-firewall.md. + +define ssh_port = 22 +define wg_port = 51820 +define wan_if = "eth0" +define wg_if = "wg0" +define admin_nets4 = { 192.0.2.0/24 } # documentation ranges: REPLACE +define admin_nets6 = { 2001:db8:1::/48 } + +flush ruleset + +table inet firewall { + + chain inbound_ipv4 { + # ping, rate limited + icmp type echo-request limit rate 5/second accept + } + + chain inbound_ipv6 { + # neighbour discovery: without it IPv6 connectivity breaks + icmpv6 type { nd-neighbor-solicit, nd-router-advert, nd-neighbor-advert } accept + icmpv6 type echo-request limit rate 5/second accept + } + + chain inbound { + type filter hook input priority 0; policy drop; + + # established/related in, invalid out + ct state vmap { established : accept, related : accept, invalid : drop } + + iifname "lo" accept + + meta protocol vmap { ip : jump inbound_ipv4, ip6 : jump inbound_ipv6 } + + # SSH only from the admin networks + tcp dport $ssh_port ip saddr $admin_nets4 accept + tcp dport $ssh_port ip6 saddr $admin_nets6 accept + + # web + tcp dport { 80, 443 } accept + udp dport 443 accept # HTTP/3 (QUIC); remove if no HTTP/3 is served + + # WireGuard handshake/data from anywhere + udp dport $wg_port accept + + # uncomment to log what is dropped (rate limited) + # limit rate 5/minute log prefix "[nft] input drop: " counter drop + } + + chain forward { + type filter hook forward priority 0; policy drop; + # Router/VPN gateway: add `iifname $wg_if oifname $wan_if accept` here, plus NAT in a separate table. + } + + # no output chain: default policy is accept +} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/nginx-proxy.conf b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/nginx-proxy.conf new file mode 100644 index 0000000..c7f1d99 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/nginx-proxy.conf @@ -0,0 +1,56 @@ +# /etc/nginx/conf.d/app.conf - TLS-terminating reverse proxy with correct proxy headers. +# Directives verified at nginx.org (ngx_http_proxy_module, ngx_http_v2_module, ngx_http_v3_module, +# ngx_http_limit_req_module, websocket.html) through Context7 /websites/nginx_en, 2026-11-10. +# Test before reload: nginx -t then systemctl reload nginx (reload keeps old workers on error). + +# rate limit zone (http context: this file is included inside http{}) +limit_req_zone $binary_remote_addr zone=perip:10m rate=10r/s; + +# WebSocket support: only send "Connection: upgrade" when the client asked for an upgrade +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +upstream app_backend { + server 127.0.0.1:8080 max_fails=3 fail_timeout=10s; + keepalive 16; +} + +server { + listen 80; + listen [::]:80; + server_name app.example.com; + # ACME HTTP-01 webroot, then redirect everything else + location /.well-known/acme-challenge/ { root /var/www/acme; } + location / { return 301 https://$host$request_uri; } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + listen 443 quic reuseport; # HTTP/3: needs nginx built with ngx_http_v3_module + listen [::]:443 quic reuseport; + http2 on; + server_name app.example.com; + + ssl_certificate /etc/letsencrypt/live/app.example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/app.example.com/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + add_header Alt-Svc 'h3=":443"; ma=86400' always; + add_header Strict-Transport-Security "max-age=31536000" always; + + location / { + limit_req zone=perip burst=20 nodelay; + + proxy_pass http://app_backend; + proxy_http_version 1.1; # required for keepalive to the upstream on < 1.29.7 + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } +} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/traefik-dynamic.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/traefik-dynamic.yml new file mode 100644 index 0000000..d123d01 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/traefik-dynamic.yml @@ -0,0 +1,41 @@ +# /etc/traefik/dynamic/app.yml - dynamic configuration (file provider, hot-reloaded). +# Structure verified in Context7 /traefik/traefik v3.7.15 (file provider, service healthCheck, tls.options). + +http: + routers: + app: + rule: Host(`app.example.com`) + entryPoints: + - websecure + middlewares: + - secure-headers + - rate-limit + service: app + tls: + options: modern + + middlewares: + secure-headers: + headers: + stsSeconds: 31536000 + forceSTSHeader: true + rate-limit: + rateLimit: + average: 50 # requests per second (period default 1s) + burst: 100 + + services: + app: + loadBalancer: + servers: + - url: http://127.0.0.1:8080 + healthCheck: + path: /healthz + interval: 10s + timeout: 3s + +tls: + options: + modern: + minVersion: VersionTLS12 + sniStrict: true diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/traefik-static.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/traefik-static.yml new file mode 100644 index 0000000..690f960 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/traefik-static.yml @@ -0,0 +1,40 @@ +# /etc/traefik/traefik.yml - static configuration (Traefik v3.7). +# Keys verified in Context7 /traefik/traefik v3.7.15 (entrypoints, ACME reference, file provider). +# Run first with the Let's Encrypt STAGING caServer; remove that line only when issuance works +# (production rate limits are real). Mount /letsencrypt on persistent storage. + +entryPoints: + web: + address: ":80" + http: + redirections: + entryPoint: + to: websecure + scheme: https + permanent: true + websecure: + address: ":443" + http: + tls: + certResolver: le + +certificatesResolvers: + le: + acme: + email: admin@example.com + storage: /letsencrypt/acme.json # chmod 600 + caServer: https://acme-staging-v02.api.letsencrypt.org/directory # REMOVE for production + keyType: EC256 # default is RSA4096 + httpChallenge: + entryPoint: web + # Wildcard certificates need DNS-01 instead of httpChallenge: + # dnsChallenge: + # provider: cloudflare # credentials via the provider's environment variables + # resolvers: ["1.1.1.1:53", "9.9.9.9:53"] + +providers: + file: + directory: /etc/traefik/dynamic + watch: true + +# No `api:` block on purpose: never enable `api.insecure` (unauthenticated dashboard) on a public host. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/unbound-split-horizon.conf b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/unbound-split-horizon.conf new file mode 100644 index 0000000..ad99928 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/unbound-split-horizon.conf @@ -0,0 +1,21 @@ +# /etc/unbound/unbound.conf.d/split-horizon.conf - recursive resolver for the LAN with a +# private view of an internal zone (split horizon) and forwarding of one zone. +# Options verified in https://unbound.docs.nlnetlabs.nl/en/latest/manpages/unbound.conf.html +# (Context7 /websites/unbound_nlnetlabs_nl_en, 2026-11-10). +# Check: unbound-checkconf Reload: systemctl reload unbound (or `unbound-control reload`) +# systemd-resolved owns 127.0.0.53:53; bind Unbound to a different address/interface or disable +# the resolved stub listener (references/dns.md), else Unbound fails with "address in use". + +server: + interface: 192.0.2.53 # REPLACE with the resolver's LAN address + access-control: 127.0.0.0/8 allow + access-control: 192.0.2.0/24 allow # LAN only; never allow 0.0.0.0/0 (open resolver) + # Internal view: answers from local-data are authoritative for this zone and only for LAN clients + local-zone: "corp.example.com." transparent + local-data: "git.corp.example.com. IN A 192.0.2.20" + local-data: "nas.corp.example.com. IN A 192.0.2.30" + +# Send one internal zone to another recursive/authoritative server +forward-zone: + name: "lab.example.com." + forward-addr: 192.0.2.54 diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/wg0-client.conf b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/wg0-client.conf new file mode 100644 index 0000000..c24c819 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/wg0-client.conf @@ -0,0 +1,18 @@ +# /etc/wireguard/wg0.conf on a client (chmod 600). Counterpart of wg0-server.conf. +# +# SPLIT TUNNEL (default here): only the VPN subnet and the listed LAN go through the tunnel. +# FULL TUNNEL: AllowedIPs = 0.0.0.0/0, ::/0 -> wg-quick installs an ip-rule override of the +# default route; a wrong endpoint or a dead server then cuts the client off from the Internet, +# and on a REMOTE client it cuts your SSH session. Never test a full tunnel on a remote host +# without a scheduled `wg-quick down wg0` (see references/safe-network-changes.md). + +[Interface] +Address = 10.8.0.2/24 +PrivateKey = +# DNS = 10.8.0.1 # wg-quick runs `resolvconf -a tun.wg0 -m 0 -x`; needs resolvconf (see wg-quick(8)) + +[Peer] +PublicKey = +Endpoint = vpn.example.com:51820 +AllowedIPs = 10.8.0.0/24, 192.168.10.0/24 +PersistentKeepalive = 25 # only for a peer behind NAT that must receive traffic diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/wg0-server.conf b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/wg0-server.conf new file mode 100644 index 0000000..eeb1da4 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/templates/wg0-server.conf @@ -0,0 +1,26 @@ +# /etc/wireguard/wg0.conf on the server (chmod 600, owner root) +# Keys: umask 077; wg genkey | tee privatekey | wg pubkey > publickey +# Syntax of wg-quick keys: https://git.zx2c4.com/wireguard-tools/about/src/man/wg-quick.8 +# +# Road-warrior hub: clients get 10.8.0.0/24, server is 10.8.0.1. +# Open UDP 51820 in the firewall (templates/nftables.conf already does). + +[Interface] +Address = 10.8.0.1/24 +ListenPort = 51820 +PrivateKey = +# Table = off # uncomment to stop wg-quick from adding routes itself + +# Only if clients must reach the Internet / LAN through this host: +# sysctl -w net.ipv4.ip_forward=1 (persist in /etc/sysctl.d/), then a forward rule and +# masquerade in the firewall. Do NOT use iptables PostUp lines on an nftables-only host. + +[Peer] +# laptop +PublicKey = +AllowedIPs = 10.8.0.2/32 + +[Peer] +# phone +PublicKey = +AllowedIPs = 10.8.0.3/32 diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/tls-acme.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/tls-acme.md new file mode 100644 index 0000000..c460483 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/tls-acme.md @@ -0,0 +1,100 @@ +--- +name: tls-acme +description: TLS certificates with ACME - choosing certbot, acme.sh, lego or the proxy's built-in ACME; HTTP-01 vs DNS-01, wildcard, staging, renewal, revocation +when-to-use: Issuing, renewing or debugging certificates; choosing the challenge type; wildcard or internal-host certificates +keywords: ACME, Let's Encrypt, certbot, acme.sh, lego, HTTP-01, DNS-01, wildcard, staging, renew, deploy-hook, revoke +priority: high +related: reverse-proxy.md, dns.md, troubleshooting-order.md, safe-network-changes.md +--- + +# TLS and ACME + +**Load when:** a certificate must be obtained, renewed or replaced. Revocation and deletion are irreversible: show the exact effect first ([safe-network-changes.md](safe-network-changes.md)). + +## 1. Choose the client + +| Situation | Choice | +|-----------|--------| +| Traefik or Caddy in front | Use their built-in ACME (Traefik `certificatesResolvers`, Caddy automatic HTTPS); no second client on the same names | +| Nginx / Apache / HAProxy on a classic host | **certbot** (`nginx` or `webroot` plugin, DNS plugins), or **lego** / **acme.sh** if you want a single static binary / shell script | +| Wildcard, or host not reachable on port 80 | DNS-01 with a provider plugin (certbot DNS plugin, lego `--dns `, Caddy/Traefik DNS challenge) | +| Many hosts, central issuance | lego or certbot on one host, deploy with a hook; or one wildcard on a DMZ proxy | + +## 2. Challenge types + +| | HTTP-01 | DNS-01 | +|-|---------|--------| +| Needs | Port 80 reachable from the Internet to the host being validated | API credentials for the DNS zone (or a delegated `_acme-challenge` zone) | +| Wildcard | No | **Yes, the only way** with Let's Encrypt (certbot docs) | +| Works for internal-only hosts | No | Yes (record is public, host is not) | +| Risk | Port 80 must stay open; redirect must not break `/.well-known/acme-challenge/` | DNS API token in the host: scope it to TXT records of one zone, file mode 600 | +| Propagation | None | Wait for the TXT record; Traefik exposes `dnsChallenge.delayBeforeCheck` and `resolvers`; disabling the propagation check is flagged "NOT RECOMMENDED" (rate-limit risk) | + +Traefik's resolver also offers TLS-ALPN-01 (`tlsChallenge`). + +## 3. certbot + +```bash +certbot certonly --webroot -w /var/www/acme -d app.example.com --dry-run # test against staging, nothing stored +certbot certonly --webroot -w /var/www/acme -d app.example.com # real issuance +certbot certificates # list lineages, expiry, paths +certbot renew --dry-run # test the renewal path (staging) +certbot renew --deploy-hook "systemctl reload nginx" # hook runs only after a successful renewal +``` + +- Files land in `/etc/letsencrypt/live//` (`fullchain.pem`, `privkey.pem`); point servers at these paths, never copy them. +- `--pre-hook` / `--post-hook` run around every renewal attempt; `--deploy-hook` only after success. `--dry-run` skips deploy hooks unless `--run-deploy-hooks`. +- Authenticators: `nginx`, `apache`, `webroot`, `standalone` (needs free port 80), DNS plugins, `manual` (no auto-renewal without hook scripts). +- Automatic renewal is set by the distribution package (timer or cron); verify with `systemctl list-timers | grep -i certbot` (not read today). + +## 4. lego (v5 line) and acme.sh + +```bash +# lego: wildcard via DNS-01 (credentials as environment variables of the chosen provider) +CLOUDFLARE_EMAIL=... CLOUDFLARE_API_KEY=... lego run --dns cloudflare -d 'example.org' -d '*.example.org' +``` + +Command shape from the lego v5.6.0 docs (`lego run --dns -d `); `--email` appears in other provider examples. Renewal subcommand and flags: read https://go-acme.github.io/lego/ for your version before scripting (the fetch tool returned only a redirect today). acme.sh is a pure-shell client (3.1.6; release tags signed from 3.1.6 on); its issue/deploy commands were not read today. + +## 5. Safe defaults + +1. **Staging first**: `https://acme-staging-v02.api.letsencrypt.org/directory` (Traefik `caServer`, Caddy `acme_ca`, certbot `--dry-run`/`--staging`). Production has rate limits: https://letsencrypt.org/docs/rate-limits/ (not read today). +2. One ACME client per name. Two clients competing for the same hostname produce validation races. +3. Persist ACME state: Traefik `acme.json` (mode 600, on a volume), Caddy's data dir, `/etc/letsencrypt`. Losing it forces re-issuance and burns rate limit. +4. Reload, do not restart, after renewal (`nginx -s reload` / `systemctl reload`); test the chain with `openssl s_client` (below). +5. Monitor expiry independently of the renewer (`devops-observability`); renewal failures are silent until the cert expires. + +## 6. Checking a served certificate + +```bash +openssl s_client -connect app.example.com:443 -servername app.example.com /dev/null \ + | openssl x509 -noout -subject -issuer -dates -ext subjectAltName +openssl s_client -connect app.example.com:443 -servername app.example.com -showcerts &1 | grep -E 'SSL|subject|expire|issuer|HTTP/' +``` + +`-servername` sets SNI; without it a multi-site proxy serves the default certificate and misleads you. + +## 7. Revocation and deletion (exact effect) + +| Command | Effect | +|---------|--------| +| `certbot revoke --cert-name X` (or `--cert-path`) | Revokes at the CA; add the same `--staging`/`--server` flag used at issuance; use only on key compromise; clients holding it get errors | +| `certbot delete --cert-name X` | Irreversible; deletes lineage files; does NOT revoke; services referencing `live/X` break on next reload | +| Deleting `acme.json` / account key | Loses issued certs and the ACME account; re-issue counts against rate limits | + +Propose with ACTION / EFFECT / PROOF; issue and deploy the replacement before revoking. + +## Sources (read 2026-11-10) + +- https://eff-certbot.readthedocs.io/en/stable/using.html (plugin table, hooks, revoke, delete) via Context7 `/websites/eff-certbot_readthedocs_io_en_stable`; certbot 5.9.0 (https://github.com/certbot/certbot/releases.atom, 2026-10-29) +- https://letsencrypt.org/docs/challenge-types/ (updated 2026-02-12; "if unsure go with the client default or HTTP-01") +- Traefik ACME reference in Context7 `/traefik/traefik` v3.7.15 (caServer, keyType, dnsChallenge options) +- lego v5.6.0 (2026-10-28, https://github.com/go-acme/lego/releases.atom) via Context7 `/go-acme/lego`; acme.sh 3.1.6 (2026-09-20) + +## Not verified + +- Let's Encrypt rate-limit numbers and certificate lifetime policy (page not read); ZeroSSL/other CAs. +- lego `renew` flags and acme.sh command syntax; certbot's timer unit name per distro. +- `openssl x509 -ext` availability on old OpenSSL (needs 1.1.1+; not checked). +- Context7 IDs: certbot `/certbot/certbot` and `/websites/eff-certbot_readthedocs_io_en_stable`, lego `/go-acme/lego`, Traefik `/traefik/traefik` (resolved today). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/troubleshooting-order.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/troubleshooting-order.md new file mode 100644 index 0000000..42fd827 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/troubleshooting-order.md @@ -0,0 +1,63 @@ +--- +name: troubleshooting-order +description: Fixed-order network troubleshooting - link, IP, route, firewall, DNS, TLS, application - with the read-only command for each layer +when-to-use: "Cannot connect", timeouts, resets, name errors, certificate errors, intermittent loss; before proposing any fix +keywords: ip, ss, nft, dig, tcpdump, mtr, openssl s_client, curl -v, traceroute, nc, troubleshooting +priority: critical +related: safe-network-changes.md, nftables-firewall.md, dns.md, tls-acme.md, reverse-proxy.md +--- + +# Network Troubleshooting Order + +**Load when:** something is unreachable. Walk the layers in this order and stop at the first failure; every command here is read-only. No fix before the evidence; any fix then follows [safe-network-changes.md](safe-network-changes.md). + +State first: source host, destination (name AND IP), port, protocol, what changed and when. + +| # | Layer | Question | Commands | Failure means | +|---|-------|----------|----------|---------------| +| 1 | Link | Interface up, carrier, right speed, errors? | `ip -br link` ; `ip -s link show eth0` ; `ethtool eth0` ; `bridge link` ; `cat /proc/net/bonding/bond0` | Cable/switch/VLAN tag/bond; nothing above works | +| 2 | IP | Right address, mask, no duplicate? | `ip -br a` ; `ip neigh show` ; `arping -D -I eth0 ` | Wrong subnet; neighbour `FAILED`/`INCOMPLETE` = L2 problem or VLAN mismatch | +| 3 | Route | Which path would the kernel use? | `ip r` ; `ip rule` ; `ip route get [from ]` ; `ping -c3 ` | No route, wrong gateway, asymmetric policy routing (routing-policy.md) | +| 4 | Firewall | Is a rule dropping it, here or at the destination? | `nft list ruleset` ; `nft -a list chain inet firewall inbound` ; counters ; `ss -tlnp` on the destination | Dropped by policy, port not listening, or listening on 127.0.0.1 only | +| 5 | DNS | Does the name resolve to the intended IP, from the failing host? | `dig +short app.example.com` ; `dig @192.0.2.53 app.example.com` ; `resolvectl query app.example.com` | Wrong/stale record, wrong resolver, split-horizon mismatch (dns.md) | +| 6 | TLS | Handshake, chain, SNI, expiry? | `openssl s_client -connect host:443 -servername host -n 100` ; proxy logs | App down, 502/504 at the proxy (reverse-proxy.md) | + +## Tools: what each tells you + +| Tool | Use | Reading it | +|------|-----|-----------| +| `ss -tulpn` | Listening sockets and owning process | `127.0.0.1:port` = not reachable remotely; no line = nothing listening | +| `ss -tn state established '( dport = :443 )'` | Live connections | Many `SYN-SENT` = packets leave but no reply (firewall/route) | +| `nc -vz host port` | TCP connect test | `refused` = host reached, nothing listening; `timed out` = dropped or no route | +| `curl -v --connect-timeout 5 URL` | Connect, TLS, HTTP in one trace | Where it stops in the output = the failing layer | +| `dig +trace name` | Walks delegation from the root | Finds lame delegation / wrong NS at the registrar | +| `mtr -n -r -c 20 host` | Per-hop loss and latency | Loss at one hop that continues downstream = real; loss only at one hop = ICMP rate-limiting there | +| `tcpdump -ni eth0 host X and port Y` | Packets on the wire | SYN out, no SYN-ACK in = remote/path drop; SYN in, nothing out = local firewall/app | +| `openssl s_client -showcerts` | Chain actually served | Compare with the intended chain; always give `-servername` | + +`tcpdump` captures traffic: use a tight filter and a bounded count (`-c 50`), write to a file you delete after (`-w /tmp/x.pcap`), and never capture payloads of sensitive traffic casually. + +## Quick discriminators + +- **Timeout** (no answer): firewall drop or routing/L2 problem -> layers 1-4. **Refused** (RST): host reachable, no listener -> layer 7/4. +- **Works by IP, fails by name**: layer 5. **Works from LAN, fails from Internet**: NAT/port-forward/firewall at the edge or split-horizon. +- **Small packets work, large hang** (SSH banner OK then stall, TLS hang): MTU/PMTUD; test `ping -M do -s 1472 ` and lower until it passes. +- **Intermittent**: two backends/IPs, one bad (`dig` returns several A records; `curl --resolve host:443:IP`), or a flapping bond/link (`dmesg`, `/proc/net/bonding`). +- **Only one client fails**: its resolver, proxy setting or source-based rule, not the server. + +## Verification of a fix + +Re-run the failing check from the original source, then the layer above it; confirm the change with a counter or log, not only exit code 0. + +## Sources (read 2026-11-10) + +- https://man7.org/linux/man-pages/man8/ip-route.8.html (`ip route get`), https://man7.org/linux/man-pages/man8/ip-rule.8.html +- https://www.netfilter.org/projects/nftables/manpage.html (`nft -a`, list commands) +- https://www.freedesktop.org/software/systemd/man/latest/resolvectl.html (`resolvectl query`) +- https://www.wireguard.com/quickstart/ (`wg show`) + +## Not verified + +- Option details of `ss`, `ethtool`, `arping`, `dig +trace`, `mtr`, `tcpdump`, `nc`, `curl --connect-timeout/--resolve` and `ping -M do -s` (standard utilities; man pages not read today; flag sets differ between BSD/GNU/iputils builds). +- Interpretation tables are operational practice, not from an official page. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/wireguard.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/wireguard.md new file mode 100644 index 0000000..6d51659 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-networking/references/wireguard.md @@ -0,0 +1,80 @@ +--- +name: wireguard +description: WireGuard site-to-site and road-warrior setups with wg-quick, key handling, AllowedIPs routing, NAT/forwarding, MTU, plus Tailscale/Headscale pointer +when-to-use: Building or debugging a VPN between hosts, sites or clients; choosing WireGuard vs Tailscale/Headscale +keywords: WireGuard, wg-quick, wg0.conf, AllowedIPs, PersistentKeepalive, full tunnel, split tunnel, Tailscale, Headscale +priority: high +related: safe-network-changes.md, nftables-firewall.md, routing-policy.md, troubleshooting-order.md +--- + +# WireGuard + +**Load when:** creating, changing or stopping a tunnel. `wg-quick down` and any full-tunnel change can cut the session you work in: [safe-network-changes.md](safe-network-changes.md). + +## Decision rules + +| Need | Choice | +|------|--------| +| Few hosts/sites, full control, no third party | WireGuard + `wg-quick` + `systemd` (`wg-quick@wg0`) | +| Many devices, SSO/ACLs, NAT traversal handled for you | Tailscale (SaaS control plane) or Headscale (self-hosted control plane) - pointer only; both use WireGuard underneath; Headscale 0.29.7 requires Tailscale client >= 1.80.0 (release note 2026-10-30); Tailscale 1.104.5 | +| Site-to-site | One `[Peer]` per site with `AllowedIPs` = that site's LAN prefix(es) on both ends; routes follow from AllowedIPs | +| Road-warrior hub | Server with `/24` for clients, one `/32` per client in server `AllowedIPs` ([wg0-server.conf](templates/wg0-server.conf), [wg0-client.conf](templates/wg0-client.conf)) | +| Peer behind NAT that must be reachable | `PersistentKeepalive = 25` on that peer (WireGuard quickstart: sensible value 25 s; default off; do not enable without need) | + +## Core model + +- `AllowedIPs` is both the routing table (outgoing: which peer gets the packet) and the source filter (incoming: which source IPs a peer may use). Overlapping ranges between peers are a bug: the more specific prefix wins. +- `wg-quick` infers routes from every peer's `AllowedIPs` and adds them; a default route (`0.0.0.0/0`, `::/0`) triggers an `ip rule` override of the default gateway. `Table = off` disables route creation; `Table = ` uses another table (wg-quick(8)). +- Keys: `umask 077; wg genkey | tee privatekey | wg pubkey > publickey`. Config files `chmod 600`, owner root. Never paste private keys into tickets/chat; share only public keys. Preshared keys are optional extra symmetric protection (not read today). +- Interface names: `[a-zA-Z0-9_=+.-]{1,15}`. + +## Operations + +```bash +wg genkey | tee privatekey | wg pubkey > publickey # per host +systemctl enable --now wg-quick@wg0 # persistent (unit name per wireguard-tools packaging; not read) +wg show # peers, endpoint, latest handshake, transfer +wg-quick strip wg0 # config without wg-quick extras, for `wg setconf` +wg syncconf wg0 <(wg-quick strip wg0) # apply changes without tearing down the interface +``` + +`wg syncconf` is the way to add/remove a peer on a live tunnel without `wg-quick down/up` (not read today; see Not verified). `wg show` "latest handshake" older than ~2 minutes or absent = no working path. + +## Firewall and forwarding + +- Allow the UDP `ListenPort` on the server (templates/nftables.conf opens 51820). +- Gateway use: `net.ipv4.ip_forward=1`, a forward rule `wg0 -> uplink`, and masquerade for the client subnet (nftables-firewall.md). `PostUp`/`PostDown` run via bash and are commonly used for firewall rules; on an nftables-managed host prefer the ruleset file over `PostUp` lines, so reloads do not drop them. +- MTU: wg-quick derives it from the endpoint or default route; set `MTU =` only after measuring (`ping -M do -s `), a wrong MTU shows as "ping works, TLS/SSH hangs". + +## Effects to show before running + +| Command | Exact effect | +|---------|--------------| +| `wg-quick down wg0` | Removes `wg0`, its routes, the ip-rule override and DNS set by `DNS=` (`resolvconf -d`); runs `PreDown/PostDown`. If your SSH session rides the tunnel it dies | +| `wg-quick up wg0` with `AllowedIPs = 0.0.0.0/0` | All traffic, incl. your SSH session's replies, goes into the tunnel; a wrong endpoint = host cut off. Not on a remote host without a scheduled `wg-quick down wg0` | +| Editing `AllowedIPs` of a live peer | Changes routing at once; overlap steals traffic from another peer | +| Rotating a key | Peer must receive the new public key out-of-band before the old one is removed; otherwise the peer is dropped | +| `SaveConfig = true` | The file is overwritten at shutdown with the live state, discarding your manual edits | + +## Verification + +```bash +wg show ; ip -br a show wg0 ; ip r | grep wg0 +ping -c3 10.8.0.1 # across the tunnel +ip r get 192.168.10.5 # does the route use wg0? +tcpdump -ni eth0 udp port 51820 # does the handshake reach the server? +``` + +## Sources (read 2026-11-10) + +- https://www.wireguard.com/quickstart/ (commands, key generation, PersistentKeepalive) +- https://git.zx2c4.com/wireguard-tools/about/src/man/wg-quick.8 (config keys, Table, PostUp, SaveConfig, route inference) +- wireguard-tools latest tag v1.0.20260223 (https://github.com/WireGuard/wireguard-tools/tags.atom) +- Headscale v0.29.7 and Tailscale v1.104.5 (GitHub releases feeds) + +## Not verified + +- `wg syncconf`, `wg-quick strip` usage detail (strip confirmed in wg-quick(8) synopsis), `systemctl enable wg-quick@wg0` unit naming, preshared keys, kernel module version. +- Tailscale / Headscale configuration (pointer only); Headscale ACL syntax. +- MTU values for specific transports. +- IPv6 inside the tunnel (not covered by a page read). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/SKILL.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/SKILL.md new file mode 100644 index 0000000..ad97cfe --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/SKILL.md @@ -0,0 +1,190 @@ +--- +name: devops-observability +description: Use when designing, writing, reviewing or debugging monitoring, logging or tracing - Prometheus (scrape config, service discovery, relabeling, recording and alerting rules, retention, remote write, cardinality), PromQL and histograms, Alertmanager routing/grouping/inhibition/silences, exporters (node, blackbox, cAdvisor, kube-state-metrics), kube-prometheus-stack and Prometheus Operator CRDs (ServiceMonitor, PodMonitor, PrometheusRule), long-term storage choices (Thanos, Mimir, VictoriaMetrics), Grafana provisioning as code, Loki with Grafana Alloy (Promtail is end of life), LogQL, OpenTelemetry Collector pipelines, sampling, Tempo/Jaeger, alert design and observability troubleshooting. Not for SLO and error-budget method or incident response (devops-sre-ops), Kubernetes workloads (devops-kubernetes), host OS tuning (devops-linux) or SaaS vendor products. +versions: + prometheus: "3.16.0 (2026-11-04); LTS 3.13 (supported to 2027-07-31)" + alertmanager: "0.35.0 (2026-11-09)" + grafana: "13.3.0" + loki: "3.7.9" + grafana-alloy: "1.21.3 (2026-10-28)" + opentelemetry-collector: "0.165.0 (core 1.71.0)" + prometheus-operator: "0.95.1 (2026-11-05); kube-prometheus-stack chart 93.1.1 (main)" + exporters: "node_exporter 1.12.1, blackbox_exporter 0.29.1, cAdvisor 0.60.6, kube-state-metrics 2.20.0" +user-invocable: true +references: + - references/signals-and-method.md + - references/prometheus-config.md + - references/promql-essentials.md + - references/alertmanager.md + - references/exporters.md + - references/prometheus-operator-k8s.md + - references/long-term-storage.md + - references/grafana-as-code.md + - references/logs-loki-alloy.md + - references/logs-other-stacks.md + - references/tracing-opentelemetry.md + - references/alert-design.md + - references/observability-safe-ops.md + - references/troubleshooting-playbook.md + - references/templates/prometheus.yml + - references/templates/file-sd-nodes.yml + - references/templates/blackbox.yml + - references/templates/rules-alerts.yml + - references/templates/rules-recording.yml + - references/templates/alertmanager.yml + - references/templates/servicemonitor.yaml + - references/templates/prometheusrule.yaml + - references/templates/kube-prometheus-stack-values.yaml + - references/templates/grafana-datasources.yaml + - references/templates/grafana-dashboards-provider.yaml + - references/templates/grafana-alerting-routing.yaml + - references/templates/loki-config.yaml + - references/templates/alloy-host.alloy + - references/templates/alloy-k8s-logs.alloy + - references/templates/alloy-otlp.alloy + - references/templates/otel-collector.yaml + - references/templates/docker-compose-monitoring.yaml +related-skills: + - devops-sre-ops + - devops-kubernetes + - devops-k8s-cluster + - devops-gitops-packaging + - devops-docker + - devops-linux + - devops-networking + - devops-storage-backup + - devops-virtualization +--- + + +Covers the three observability signals for an IT/DevOps estate: Prometheus metrics +and rules, Alertmanager routing, exporters, the Kubernetes Operator stack, +Grafana as code, Loki logs collected by Grafana Alloy, OpenTelemetry tracing and +sampling, alert design, and troubleshooting the pipeline itself. Does not cover +SLO/error-budget design and incident response (devops-sre-ops), workload +manifests (devops-kubernetes) or SaaS vendor products. + + +# DevOps Observability + +## Agent Workflow (MANDATORY) + +Before ANY monitoring config or change, spawn in parallel, one `Task` call each: + +1. **fuse-ai-pilot:explore-codebase** - find existing `prometheus.yml`, rule files, Alertmanager/Loki/Alloy/Collector configs, Helm values, dashboards, ServiceMonitors +2. **fuse-ai-pilot:research-expert** - verify option names for the deployed version at prometheus.io, grafana.com/docs, opentelemetry.io (the OpenTelemetry Collector renames components between 0.x releases) +3. **mcp__context7__query-docs** - Prometheus, Grafana, Loki, OpenTelemetry docs + +Then discover read-only (running versions, `promtool check`, `amtool check-config`, `alloy validate`). After writing files, run **fuse-ai-pilot:sniper**. Live changes follow the agent's Operations Safety Protocol and [observability-safe-ops.md](references/observability-safe-ops.md). + +## Versions (checked 2026-11-10) + +Prometheus 3.16.0 is latest; **LTS 3.13** is the conservative choice. Alertmanager 0.35.0 has breaking flag/config changes (event recorder outputs need a `name`). **Promtail reached end of life on 2026-03-02**: collect logs with Grafana Alloy. The OpenTelemetry Collector renamed `otlp` to `otlp_grpc`, `otlphttp` to `otlp_http`, and added a `queue_batch` processor meant to replace `batch`; read its changelog before every 0.x bump. + +## Overview + +| Need | Decision in one line | +|------|----------------------| +| **What to instrument** | USE for resources, RED for services, golden signals at the edge; plus a blackbox probe of every user-facing URL | +| **Metrics store** | Prometheus (local TSDB, retention by time AND size flags); long-term via remote write to Mimir/Thanos/VictoriaMetrics only when needed | +| **Alerts** | Symptom-based, `for:` + `keep_firing_for`, severity `page`/`ticket`, every alert has `runbook_url`; routing and inhibition in Alertmanager | +| **Kubernetes** | kube-prometheus-stack; ServiceMonitor/PodMonitor/PrometheusRule; check the `release` label selectors and PVCs | +| **Logs** | Loki + Alloy; few static labels, rest in structured metadata; retention by the compactor | +| **Traces** | OpenTelemetry SDK -> Collector/Alloy (OTLP 4317/4318) -> Tempo; tail-sample errors and slow traces | +| **Dashboards/alerts as code** | Grafana provisioning files in Git; metric alerts live in Prometheus, not Grafana | + +## Critical Rules + +1. **Version first** - read the running versions and the config of the target before writing an option; renames and removals are frequent (OpenTelemetry Collector, Alertmanager, Prometheus 3.x) +2. **Validate offline before any reload** - `promtool check config|rules`, `amtool check-config`, `alloy validate`, `otelcol validate`; reloading production is a proposal in ACTION / EFFECT / PROOF / WAIT-FOR-GO format ([observability-safe-ops.md](references/observability-safe-ops.md)) +3. **Cardinality is the budget** - no unbounded label values (user IDs, trace IDs, pod names as Loki labels); use `sample_limit`/`label_limit`, `metric_relabel_configs`, recording rules +4. **Destructive verbs are proposed, not run** - admin-API series deletion, shrinking retention, Loki delete requests, removing PVCs/buckets, mass silences, `helm uninstall` +5. **Persistent storage or it is not monitoring** - emptyDir/no volume loses metrics and silences on restart +6. **Never print secrets** - receivers use `*_file` options; Grafana `secureJsonData`; no tokens in configs or logs +7. **Monitor the monitoring** - Watchdog alert to an external dead man's switch, scrape and notification-failure alerts +8. **Never invent a key or metric name** - cite the page for the version; unknown = "not verified" + +## Architecture + +``` +monitoring/ +├── prometheus.yml # global, scrape_configs, alerting, (remote_write) +├── rules/ # alerts and recording rules (promtool-checked) +├── alertmanager.yml # route tree, inhibit_rules, receivers (secrets as files) +├── blackbox.yml # probe modules +├── alloy/*.alloy # logs (Loki), host metrics, OTLP gateway +├── otel-collector.yaml # OpenTelemetry Collector gateway (if used instead of Alloy) +├── loki-config.yaml # storage, schema, compactor retention +└── grafana/ + ├── provisioning/{datasources,dashboards,alerting}/ + └── dashboards/*.json +``` + +## Reference Guide + +### Concepts + +| Topic | Reference | When to Consult | +|-------|-----------|-----------------| +| **Signals and method** | [signals-and-method.md](references/signals-and-method.md) | Deciding what to instrument first | +| **Prometheus config** | [prometheus-config.md](references/prometheus-config.md) | Scrape, SD, relabeling, rules, retention, remote write, cardinality | +| **PromQL** | [promql-essentials.md](references/promql-essentials.md) | Queries, histograms, native histograms, pitfalls | +| **Alertmanager** | [alertmanager.md](references/alertmanager.md) | Routing, grouping, inhibition, silences, receivers | +| **Exporters** | [exporters.md](references/exporters.md) | node, blackbox, cAdvisor, kube-state-metrics | +| **Operator / K8s** | [prometheus-operator-k8s.md](references/prometheus-operator-k8s.md) | CRDs, kube-prometheus-stack, missing targets | +| **Long-term storage** | [long-term-storage.md](references/long-term-storage.md) | Thanos, Mimir, VictoriaMetrics pointers | +| **Grafana** | [grafana-as-code.md](references/grafana-as-code.md) | Provisioning datasources, dashboards, alerting | +| **Loki and Alloy** | [logs-loki-alloy.md](references/logs-loki-alloy.md) | Collection, labels, LogQL, retention, Promtail migration | +| **Other log stacks** | [logs-other-stacks.md](references/logs-other-stacks.md) | OpenSearch/Elastic, Vector, Fluent Bit | +| **Tracing** | [tracing-opentelemetry.md](references/tracing-opentelemetry.md) | Collector pipelines, OTLP, sampling, Tempo/Jaeger | +| **Alert design** | [alert-design.md](references/alert-design.md) | Writing and pruning alerts | +| **Safe ops** | [observability-safe-ops.md](references/observability-safe-ops.md) | Any reload, restart, deletion, silence | +| **Troubleshooting** | [troubleshooting-playbook.md](references/troubleshooting-playbook.md) | Missing data, silent alerts, slow or full stores | + +### Templates + +| Template | When to Use | +|----------|-------------| +| [prometheus.yml](references/templates/prometheus.yml) | Server config with SD, relabeling, blackbox, limits | +| [file-sd-nodes.yml](references/templates/file-sd-nodes.yml) | File-based target list | +| [blackbox.yml](references/templates/blackbox.yml) | Probe modules | +| [rules-alerts.yml](references/templates/rules-alerts.yml) | Symptom alerts with runbook links, Watchdog | +| [rules-recording.yml](references/templates/rules-recording.yml) | Recording rules, level:metric:operations | +| [alertmanager.yml](references/templates/alertmanager.yml) | Route tree, inhibition, time intervals, receivers | +| [servicemonitor.yaml](references/templates/servicemonitor.yaml) | ServiceMonitor + PodMonitor | +| [prometheusrule.yaml](references/templates/prometheusrule.yaml) | PrometheusRule | +| [kube-prometheus-stack-values.yaml](references/templates/kube-prometheus-stack-values.yaml) | Helm values: retention, PVC, selectors | +| [grafana-datasources.yaml](references/templates/grafana-datasources.yaml) | Prometheus, Loki, Tempo, Alertmanager datasources | +| [grafana-dashboards-provider.yaml](references/templates/grafana-dashboards-provider.yaml) | Dashboard file provider | +| [grafana-alerting-routing.yaml](references/templates/grafana-alerting-routing.yaml) | Contact point + notification policy | +| [loki-config.yaml](references/templates/loki-config.yaml) | Single-binary Loki with compactor retention | +| [alloy-host.alloy](references/templates/alloy-host.alloy) | Host metrics + log files | +| [alloy-k8s-logs.alloy](references/templates/alloy-k8s-logs.alloy) | Pod logs on Kubernetes | +| [alloy-otlp.alloy](references/templates/alloy-otlp.alloy) | OTLP gateway in Alloy | +| [otel-collector.yaml](references/templates/otel-collector.yaml) | Collector gateway with tail sampling | +| [docker-compose-monitoring.yaml](references/templates/docker-compose-monitoring.yaml) | Single-host stack | + +## Quick Reference + +```bash +promtool check config prometheus.yml && promtool check rules rules-*.yml +amtool check-config alertmanager.yml +amtool config routes test --config.file=alertmanager.yml --tree severity=page team=sre +alloy validate /etc/alloy/config.alloy +otelcol-contrib validate --config=otel-collector.yaml +curl -s http://prometheus:9090/-/ready ; curl -s http://loki:3100/ready +``` + +## Best Practices + +### DO +- Pin image and chart versions; read the release notes of each jump (Alertmanager 0.35, OpenTelemetry 0.x, kube-prometheus-stack majors) +- Alert on user-visible symptoms; link a runbook; group and inhibit in Alertmanager +- Keep dashboards, rules, datasources and Alloy/Collector configs in Git; deliver via GitOps (devops-gitops-packaging) +- Bind monitoring ports to localhost or an authenticated proxy; use TLS between sites + +### DON'T +- Enable `--web.enable-admin-api` or `--web.enable-lifecycle` on an exposed port +- Use `irate` in alerts or label metrics/logs with unbounded values +- Run Promtail or a second host-metrics collector next to Alloy/node_exporter +- Duplicate the same alert in Prometheus and Grafana diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/alert-design.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/alert-design.md new file mode 100644 index 0000000..b724022 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/alert-design.md @@ -0,0 +1,60 @@ +# Alert Design + +**Load when:** writing a new alert, reviewing an alert set, or fixing alert fatigue. + +SLO burn-rate alerts, error budgets and the incident process belong to `devops-sre-ops`: this file covers the alert rule itself. + +## Rules of thumb (Prometheus alerting practices) + +1. **Alert on symptoms tied to user pain**, not on every possible cause. Keep as few alerts as possible; use dashboards to find the cause. +2. **Page only when a human must act now.** If nothing can be done, it is not a page. Everything else is a ticket (next business day) or a dashboard. +3. For serving systems alert on **latency and error rate as high in the stack as possible**; page on latency at one point only; page on user-visible errors, not on every downstream error that causes them. +4. Allow slack for blips: use `for:` and wide enough `rate` ranges; `keep_firing_for:` stops flapping and false resolves. +5. Batch jobs: alert when the job has not **succeeded recently enough** (at least two run periods), never on a single failure. +6. Capacity: alert on predicted exhaustion (`predict_linear`) well before the incident. +7. **Monitor the monitoring**: Prometheus rule failures, Alertmanager notification failures, scrape failures, plus an external blackbox check as fallback. + +## Required content of every alert + +| Field | Content | +|-------|---------| +| `alert` | CamelCase name describing the symptom (`WebHighErrorRatio`) | +| `expr` | Uses aggregated recorded series where possible; no `irate`; handles missing data on purpose | +| `for` | At least 2-3 evaluation intervals unless the signal is already smoothed | +| `labels.severity` | A small closed set (`page`, `ticket`, `none`); routing and inhibition depend on it | +| `labels.team` / `service` | Routing key | +| `annotations.summary` | One line with the affected instance/service and the value, templated (`{{ $labels.instance }}`, `{{ $value }}`) | +| `annotations.runbook_url` | Link to what to check and how to fix; no runbook = not ready to page | + +## Severity model + +| Severity | Meaning | Delivery | +|----------|---------|----------| +| `page` | User-facing impact now or imminent; act immediately | On-call pager, any hour | +| `ticket` | Needs action within days; no current user impact | Ticket/chat in business hours (`active_time_intervals`) | +| `none` | Informational, `Watchdog` | Dead man's switch / nowhere | + +## Fighting alert fatigue + +- Measure: count notifications per alertname per week (`ALERTS`, Alertmanager notification logs). Anything that fired more than a few times with no action gets fixed or deleted. +- Group (`group_by`) and inhibit (node down hides its services) in Alertmanager ([alertmanager.md](alertmanager.md)); do not solve noise with long silences. +- Do not alert on both cause and symptom unless the cause alert adds action value. +- Review alerts after every incident: add the missing symptom alert, delete the ones nobody used. +- Test rules before shipping: `promtool check rules`, unit tests with `promtool test rules`, and `amtool config routes test` for routing. + +## Watchdog / dead man's switch + +An always-true alert (`expr: vector(1)`) routed to an external service that alerts when it **stops** arriving. It proves the chain Prometheus -> Alertmanager -> network is alive ([rules-alerts.yml](templates/rules-alerts.yml), route in [alertmanager.yml](templates/alertmanager.yml)). + +## Sources (read 2026-11-10) + +- https://raw.githubusercontent.com/prometheus/docs/main/docs/practices/alerting.md +- https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/ +- https://sre.google/sre-book/monitoring-distributed-systems/ (symptoms vs causes chapter) + +## Not verified + +- `promtool test rules` was not run or read today (named from the Prometheus unit-testing feature, page not fetched). +- Severity vocabulary (`page`/`ticket`/`none`) is this skill's convention, not a Prometheus standard. +- "At least 2-3 evaluation intervals" for `for:` is a rule of thumb. +- The weekly notification-count method is a suggestion; no specific metric from Alertmanager was verified for it. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/alertmanager.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/alertmanager.md new file mode 100644 index 0000000..73d4082 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/alertmanager.md @@ -0,0 +1,72 @@ +# Alertmanager + +**Load when:** designing the routing tree, grouping, inhibition, silences or receivers; debugging "no notification" or "too many notifications". + +Template: [alertmanager.yml](templates/alertmanager.yml). Validate: `amtool check-config alertmanager.yml`. Test the tree offline: + +```bash +amtool config routes test --config.file=alertmanager.yml --tree --verify.receivers=oncall-pager severity=page team=sre +``` + +(`--verify.receivers` makes amtool exit 1 on a mismatch, usable in CI.) + +## Pipeline + +Prometheus evaluates rules and sends alerts; Alertmanager deduplicates, **groups**, **inhibits**, **silences** and routes them to receivers. Alertmanager has no memory of the rules: it only sees label sets. + +## Routing tree + +- The top `route` matches every alert (no matchers). Children are tried in order; the first match wins unless `continue: true`. +- `receiver`, `group_by`, `group_wait`, `group_interval`, `repeat_interval` are inherited from the parent. `mute_time_intervals` and `active_time_intervals` are **not** inherited. Route `labels` merge parent to child. +- Use `matchers` (e.g. `severity = "page"`, `team =~ "a|b"`). `match` / `match_re` are deprecated. +- Defaults: `group_wait` 30s, `group_interval` 5m, `repeat_interval` 4h. `repeat_interval` is checked at each `group_interval` tick, so keep it a multiple of it. +- `group_by: ['...']` disables grouping (one notification per alert): only for very low volume. + +## Grouping + +Group by what a human would handle together: `[alertname, cluster, service]`. `group_wait` is the delay for the first notification of a new group: long enough for inhibiting alerts to arrive, short enough for paging. If an alert resolves before `group_wait` ends, nothing is sent (flap protection). + +## Inhibition + +An `inhibit_rules` entry mutes the **target** when a **source** alert is firing and both have equal values for the `equal` labels. Keep source and target matchers disjoint (an alert matching both sides cannot be inhibited by itself). Typical: node down mutes service alerts of the same `instance`; page-level mutes ticket-level for the same `alertname`+`cluster`. If a label in `equal` is absent on both sides, the rule applies. + +## Silences + +Silences are created in the UI/API or with `amtool silence add`, matched by label matchers, with a start/end and a comment. They are stored in Alertmanager (persist `--storage.path`) and replicated in a cluster. + +```bash +amtool silence query --alertmanager.url=http://alertmanager:9093 +amtool silence add alertname="HostMemoryExhausted" instance=~"web-0[12].*" --duration=2h --comment="planned resize" --author=me@example.com +amtool silence expire +``` + +Creating a broad silence (empty or `.*` matchers) or expiring all silences mutes or re-opens the whole pipeline: ACTION / EFFECT / PROOF / WAIT-FOR-GO in [observability-safe-ops.md](observability-safe-ops.md). + +## Time intervals + +Top-level `time_intervals:` define named windows (`times`, `weekdays` as `monday:friday`, `days_of_month`, `months`, `years`, `location`; UTC if no `location`). Reference them from a route with `mute_time_intervals` (silent during the window) or `active_time_intervals` (notify only during the window). + +## Receivers + +Built-in integrations in v0.35: email, Slack, PagerDuty, Opsgenie, VictorOps, webhook, Pushover, Telegram, Discord, Microsoft Teams, Webex, WeChat, Rocket.Chat, Mattermost, Jira, Amazon SNS, incident.io. `send_resolved` default differs per integration. Secrets: prefer the `*_file` variants (`slack_api_url_file`, `api_url_file`, `routing_key_file`, `url_file`) over inline values. Anything else goes through `webhook_configs` to a bridge. + +## High availability + +HA clustering is enabled by default. Point **every** Prometheus at **all** Alertmanager instances (do not load-balance them): the instances gossip and deduplicate. + +## v0.35.0 breaking changes to know before upgrading + +`--enable-feature=auto-gomemlimit` became `--auto-gomemlimit`; the event recorder (`eventrecorder`) now requires a `name` on every output and uses the `events/v2` schema; the deprecated `types` package was removed. Read the release notes before the bump. + +## Sources (read 2026-11-10) + +- https://raw.githubusercontent.com/prometheus/alertmanager/v0.35.0/docs/configuration.md +- https://raw.githubusercontent.com/prometheus/alertmanager/v0.35.0/README.md (amtool section) +- https://github.com/prometheus/alertmanager/releases/tag/v0.35.0 (2026-11-09) + +## Not verified + +- The full integration list is read from the config-reference headings (`*_config` sections); features per integration were not compared. +- "Point every Prometheus at all Alertmanagers" is the standard HA guidance; the HA page was not fetched today. +- `amtool silence add` flags `--duration`, `--comment`, `--author` were not confirmed against `amtool --help` of v0.35.0 (only the README examples were read). +- Gossip/cluster flags (`--cluster.*`) not checked. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/exporters.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/exporters.md new file mode 100644 index 0000000..5dfd0d3 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/exporters.md @@ -0,0 +1,62 @@ +# Exporters + +**Load when:** adding host, container, probe or Kubernetes-object metrics, or tuning what an exporter exposes. + +Latest tags read on GitHub `/releases/latest` redirects, 2026-11-10: node_exporter v1.12.1, blackbox_exporter v0.29.1, cAdvisor v0.60.6, kube-state-metrics v2.20.0, Pushgateway v1.11.4. Release dates were not read. + +## node_exporter (hosts) + +- Linux host metrics (CPU, memory, filesystem, network, load, systemd, ...). Enabled-by-default collectors can be switched off with `--no-collector.`; to run only chosen ones: `--collector.disable-defaults --collector. ...`. +- Containerised: needs the host namespaces and a read-only root mount, and `--path.rootfs` pointing at it, otherwise it reports the container, not the host: + +```bash +docker run -d --net=host --pid=host -v "/:/host:ro,rslave" quay.io/prometheus/node-exporter:v1.12.1 --path.rootfs=/host +``` + +- `--collector.systemd` (with `--collector.systemd.unit-include` / `-exclude`) exposes unit states; `logind` is separate. +- **Textfile collector**: `--collector.textfile.directory=DIR` reads `*.prom` files: the way to publish cron/batch results ("last success timestamp") without a Pushgateway. Write files atomically (write temp, then rename). +- Transport security: `--web.config.file=web-config.yml` (TLS and basic auth) per the exporter-toolkit format. +- When Grafana Alloy runs on the host, `prometheus.exporter.unix` embeds the same collectors ([alloy-host.alloy](templates/alloy-host.alloy)): run **one** of node_exporter or Alloy's unix exporter per host. + +## blackbox_exporter (outside-in probes) + +Probers: `http`, `tcp`, `dns`, `icmp`, `grpc`, `unix`. Modules live in `blackbox.yml` ([template](templates/blackbox.yml)); Prometheus calls `/probe?target=URL&module=NAME`. Use the multi-target relabel pattern in [prometheus.yml](templates/prometheus.yml): `__address__` -> `__param_target`, then `__address__` -> exporter address. Debug one probe by hand with `&debug=true`. + +- The module default `preferred_ip_protocol` is `ip6`: set `ip4` explicitly on IPv4-only networks or probes fail with a misleading error. +- Probe timeout must be below the job's `scrape_timeout`. +- HTTP probes accept a `hostname` parameter (Host header and SNI), useful with DNS service discovery. +- Alert on `probe_success == 0` and on certificate expiry (`probe_ssl_earliest_cert_expiry - time()`). + +## cAdvisor (containers) + +Per-container CPU, memory, network, filesystem for Docker/containerd hosts. Run with `/`, `/var/run`, `/sys` and `/var/lib/docker` mounted read-only; image `ghcr.io/google/cadvisor` (older than v0.53.0: `gcr.io/cadvisor/cadvisor`). On Kubernetes the kubelet already embeds cAdvisor: scrape the kubelet's cAdvisor endpoint (kube-prometheus-stack does) instead of deploying another one. + +Cost control flags: `--store_container_labels=false` (do not turn container labels/env into metric labels) with `--whitelisted_container_labels`, `--docker_only=true`, `--disable_metrics=...` (default-disabled set includes `tcp`, `udp`, `process`, `sched`...), `--housekeeping_interval`. Process metrics need `--pid=host --privileged`. + +## kube-state-metrics (Kubernetes objects) + +Exposes object state from the API (pods, deployments, nodes, PVCs, jobs...), **not** resource usage (that is cAdvisor/kubelet and metrics-server). Defaults enable most resources (`--resources`). Labels and annotations of objects are **not** exposed by default: opt in with `--metric-labels-allowlist=pods=[app,team]` style values; `*` allows everything and has severe performance cost. Limit series with `--metric-allowlist` (exact names or regex) / `--metric-denylist`. kube-prometheus-stack deploys it as a subchart. + +## Pushgateway + +Only for service-level batch jobs that cannot be scraped. It never forgets pushed series until deleted, so stale metrics keep alerting; prefer the node_exporter textfile collector for host cron jobs. Not covered further. + +## Platform exporters (pointers) + +Proxmox (`pve-exporter`), Ceph (built-in manager Prometheus module), OpenStack and other platform exporters: see the owning sibling skill (`devops-virtualization`, `devops-openstack`) for deployment; scrape and alert patterns here still apply. + +## Sources (read 2026-11-10) + +- https://raw.githubusercontent.com/prometheus/node_exporter/v1.12.1/README.md +- https://raw.githubusercontent.com/prometheus/blackbox_exporter/v0.29.1/README.md and CONFIGURATION.md +- https://raw.githubusercontent.com/google/cadvisor/v0.60.6/docs/running.md and docs/runtime_options.md +- https://raw.githubusercontent.com/kubernetes/kube-state-metrics/v2.20.0/docs/developer/cli-arguments.md +- GitHub `/releases/latest` of each project (versions) + +## Not verified + +- Pushgateway behavior (series persist until deleted) and the kubelet-embedded cAdvisor statement are well-known facts, not re-read from official pages today. +- `probe_ssl_earliest_cert_expiry` metric name used in [rules-alerts.yml](templates/rules-alerts.yml) was not located in the exporter source today. +- `node_memory_MemAvailable_bytes`, `node_filesystem_*` metric names in the templates follow node_exporter naming; not re-checked per collector. +- Atomic-write advice for textfile files comes from the exporter README section that was not re-read in full. +- The Ceph manager module and `pve-exporter` were not looked at. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/grafana-as-code.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/grafana-as-code.md new file mode 100644 index 0000000..a87dfaa --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/grafana-as-code.md @@ -0,0 +1,65 @@ +# Grafana as Code + +**Load when:** provisioning datasources, dashboards or alerting resources from files; deciding where alert rules live. + +Version read: Grafana **13.3.0** (GitHub release page; docs from the v13.3.0 tag). Templates: [grafana-datasources.yaml](templates/grafana-datasources.yaml), [grafana-dashboards-provider.yaml](templates/grafana-dashboards-provider.yaml), [grafana-alerting-routing.yaml](templates/grafana-alerting-routing.yaml). + +## Principle + +Everything that defines behavior lives in Git and is loaded at startup: the UI is for exploring, not for authoring. A provisioned datasource cannot be edited in the UI (Save is replaced by Test). + +## Layout + +``` +/etc/grafana/provisioning/ +├── datasources/*.yaml # apiVersion: 1, datasources: [...] +├── dashboards/*.yaml # apiVersion: 1, providers: [...] -> point to JSON files on disk +├── alerting/*.yaml # contactPoints, policies, muteTimes, groups (alert rules) +└── plugins/*.yaml +``` + +Config file location of `grafana.ini` for Deb/RPM packages: `/etc/grafana/grafana.ini`. Any setting can be overridden by env var `GF_
_` (e.g. `GF_SECURITY_ADMIN_USER`). + +## Datasources + +- Each entry: `name`, `type` (`prometheus`, `loki`, `tempo`, `alertmanager`, ...), `access: proxy`, `url`, a stable `uid`, `jsonData`, `secureJsonData` (encrypted, for passwords/tokens), `editable: false`. +- `deleteDatasources` removes listed ones; `prune: true` removes provisioned datasources that disappeared from the files. +- Set the Prometheus `jsonData.timeInterval` to the real scrape interval, otherwise `$__rate_interval` is computed wrongly. +- Correlation as config: Prometheus `exemplarTraceIdDestinations`, Loki `derivedFields` (regex on the log line -> Tempo trace), Tempo `tracesToLogsV2` / `tracesToMetrics` / `serviceMap`. These make "metric -> trace -> log" a click. +- Environment interpolation `${VAR}` is applied to the files; a literal dollar must be written `$$` (as in the derived-field URL). + +## Dashboards + +A `providers[]` entry of `type: file` scans `options.path` every `updateIntervalSeconds`. Useful flags: `disableDeletion`, `allowUiUpdates` (false = UI saves do not overwrite the file), `foldersFromFilesStructure`. Files are dashboard JSON (with `uid`, no `id`) or the Kubernetes resource format (`apiVersion: dashboard.grafana.app/v1`, required for dashboards v2 / dynamic dashboards). Review dashboard JSON diffs like code; avoid giant exported panels with hard-coded datasource IDs: reference datasources by `uid`. + +On Kubernetes: sidecar ConfigMaps ([prometheus-operator-k8s.md](prometheus-operator-k8s.md)). + +## Alerting resources + +File provisioning can create, update or delete: alert rule groups (`groups[].rules[]` with `uid` <= 40 chars, `condition`, `data`, `for`, `noDataState`, `execErrState`), contact points (`contactPoints[].receivers[]`), notification policies (`policies`), mute timings (`muteTimes`), templates. Rules reference a datasource by `datasourceUid` and use expression nodes (`__expr__`). + +**Where should an alert rule live?** Metric alerts: Prometheus rules (`PrometheusRule` / rule files) + Alertmanager: versioned with the service, evaluated by the server that owns the data, testable with `promtool test rules`. Grafana-managed rules only when the data source is not Prometheus-compatible (logs via Loki, SQL, mixed queries). Do not run the same alert in both places. The Grafana `Alertmanager` datasource with `implementation: prometheus` shows the external Alertmanager's alerts inside Grafana; `handleGrafanaManagedAlerts: true` additionally lets Grafana-managed alerts be sent to that Alertmanager. + +Template for the Grafana alert-rule JSON model is intentionally absent: build the first rule in the UI, export it ("Export" in the alerting UI), then commit the exported provisioning file, instead of writing the query model by hand. + +## Ops notes + +- Grafana's own database holds users, orgs, annotations, and state: back it up (`devops-storage-backup`); provisioned config is reproducible, the DB is not. +- Change of datasource `uid` breaks every dashboard that references it: treat `uid` as an API. +- Restarting or redeploying production Grafana to apply provisioning is a proposal ([observability-safe-ops.md](observability-safe-ops.md)). + +## Sources (read 2026-11-10) + +- https://raw.githubusercontent.com/grafana/grafana/v13.3.0/docs/sources/administration/provisioning/index.md +- https://raw.githubusercontent.com/grafana/grafana/v13.3.0/docs/sources/alerting/set-up/provision-alerting-resources/file-provisioning/index.md +- https://grafana.com/docs/grafana/latest/datasources/prometheus/configure/ , .../tempo/configure-tempo-data-source/provision/ , .../alertmanager/ , Loki datasource configure page (v13.3.0 tag) +- https://raw.githubusercontent.com/grafana/grafana/v13.3.0/docs/sources/setup-grafana/configure-grafana/_index.md (`GF_` env rule) +- https://github.com/grafana/grafana/releases/tag/v13.3.0 + +## Not verified + +- Behavior of `handleGrafanaManagedAlerts` (only its existence in the Alertmanager datasource example was read). +- `$__rate_interval` dependence on the datasource scrape interval is standard Grafana behavior, not re-read today. +- Whether an "Export" action exists with that exact name in 13.3 alerting UI (the provisioning page mentions obtaining the `data` model through the API). +- `grafana-datasources.yaml` was not loaded into a running Grafana; only YAML parsing was checked. +- Terraform / Grafana operator provisioning paths were not covered. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/logs-loki-alloy.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/logs-loki-alloy.md new file mode 100644 index 0000000..4587d68 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/logs-loki-alloy.md @@ -0,0 +1,68 @@ +# Logs: Loki and Grafana Alloy + +**Load when:** collecting, labeling, querying, retaining or deleting logs; migrating off Promtail. + +Versions read 2026-11-10: **Loki 3.7.9** (tag), **Grafana Alloy v1.21.3** (2026-10-28). Templates: [loki-config.yaml](templates/loki-config.yaml), [alloy-host.alloy](templates/alloy-host.alloy), [alloy-k8s-logs.alloy](templates/alloy-k8s-logs.alloy). + +## Collector choice: Alloy, not Promtail + +**Promtail is end of life as of 2026-03-02** (Loki docs): no support, no updates, all feature work goes to Alloy. Grafana Agent (Static, Flow, Operator) reached EOL on 2025-11-01. Migrate with Alloy's converter: `alloy convert --source-format=promtail ` (`-f` short form; also `otelcol`, `prometheus`, `static`), or check a Promtail file with `alloy validate --config.format=promtail ` (the `--config.format` values accepted by `validate` are `alloy`, `otelcol`, `prometheus`, `promtail`, `static`). Other supported clients: OpenTelemetry Collector, Fluent Bit, Vector ([logs-other-stacks.md](logs-other-stacks.md)). + +## Alloy in 60 seconds + +Config is `.alloy` files of components wired by references: `prometheus.exporter.unix.host.targets`, `loki.write.default.receiver`. Validate with `alloy validate ` (exit 0 valid; checks syntax, missing components, name clashes, required and unknown properties); format with `alloy fmt`. The UI listens on `--server.http.listen-addr` (default localhost: bind `0.0.0.0:12345` in a container). Data (positions, WAL) lives under `--storage.path`: persist it, otherwise logs are re-read or lost on restart. + +Log pipeline components: + +| Step | Component | Notes | +|------|-----------|-------| +| Discover files | `local.file_match` or `loki.source.file` with `file_match` block | doublestar globs in `__path__`, `__path_exclude__` | +| Discover pods | `discovery.kubernetes` role `pod` | then `discovery.relabel` to shape labels | +| Read | `loki.source.file`, `loki.source.kubernetes` | the latter tails via the Kubernetes API (needs `__meta_kubernetes_namespace`, `_pod_name`, `_pod_container_name`, `_pod_uid` or the `__pod_*__` equivalents) | +| Transform | `loki.process` with `stage.json`, `stage.regex`, `stage.labels`, `stage.static_labels`, `stage.structured_metadata`, `stage.drop`, `stage.match`, `stage.limit` | stages run in order | +| Send | `loki.write` `endpoint { url = ".../loki/api/v1/push" }` | `batch_size` 1MiB, `batch_wait` 1s, `external_labels`, `max_streams` | + +## Label design (the main Loki cost lever) + +Labels build the index and define **streams**; chunks are per stream. + +- Static, bounded labels only: `cluster`, `namespace`, `app`, `container`, `env`, `job`, `host`. Aim for tens of values per dynamic label. +- Never label request IDs, trace IDs, user IDs, pod names that churn, or `level` on low-volume apps (splits one app into several small streams). Use a line filter instead: `{app="x"} |= "level=error"`, or store the value as **structured metadata** (`stage.structured_metadata`) which does not create streams. +- Guideline from the Loki docs: keep a tenant under about 100,000 active streams. +- Find offenders: `logcli series ... --analyze-labels`. Watch `loki_discarded_samples_total`. +- OTLP ingestion (`/otlp`) maps selected resource attributes to labels and the rest to structured metadata; `k8s.pod.name` and `service.instance.id` are no longer recommended as labels for new users. Tune with `limits_config.otlp_config`. Structured-metadata limits per line: 64KB / 128 entries by default. + +## LogQL basics + +```logql +{namespace="shop", app="web"} |= "error" != "healthz" # stream selector + line filters +{app="web"} | json | status >= 500 and duration > 1s # parse then label filter +{app="web"} | logfmt | line_format "{{.method}} {{.path}} {{.status}}" +sum by (app) (rate({namespace="shop"} |= "error" [5m])) # log rate (entries/s) +count_over_time({app="web"} |= "timeout" [1h]) +``` + +Rules: always start with a stream selector using labels; put cheap line filters (`|=`, `|~`) before parsers (`json`, `logfmt`, `pattern`, `regexp`); `offset` goes inside the range selector: `count_over_time({job="x"}[5m] offset 5m)`. Metric queries: `rate`, `count_over_time`, `bytes_rate`, `bytes_over_time`, `absent_over_time`, with `sum by`, `topk`. Alert from logs with the Loki ruler (`ruler.alertmanager_url`) or a Grafana-managed rule ([grafana-as-code.md](grafana-as-code.md)). + +## Retention and deletion + +- Retention is done by the **compactor**: `compactor.retention_enabled: true` + `compactor.delete_request_store` (required) + `limits_config.retention_period` (global) or `retention_stream` entries (label matchers only, `period` >= 24h) and per-tenant overrides. Requires a **24h index period** (TSDB schema v13 in the template). Not retroactive for logs already ingested. +- Chunks are swept after `retention_delete_delay` (template: 2h): the window to catch a retention mistake. Run the compactor as a **single instance** with persistent storage for marker files. The docs strongly recommend object-store versioning before enabling retention. +- Log deletion API: `POST /loki/api/v1/delete?query=&start=...&end=...` (204 = accepted; `X-Scope-OrgID` for the tenant). It exists only when `retention_enabled` is true and the tenant `deletion_mode` is not `disabled` (default `filter-and-delete`). Requests have a cancellation period (`delete_request_cancel_period`, default 24h): list with `GET /loki/api/v1/delete`, cancel with `DELETE /loki/api/v1/delete?request_id=...`. Treat every delete and every retention shortening as destructive: [observability-safe-ops.md](observability-safe-ops.md). +- Production index store is **TSDB**; BoltDB Shipper is being removed in Loki 4.0. + +## Sources (read 2026-11-10) + +- https://grafana.com/docs/loki/latest/send-data/promtail/ and .../setup/migrate/migrate-to-alloy/ +- https://raw.githubusercontent.com/grafana/alloy/v1.21.3/docs/sources/reference/ (cli/validate, components loki.*, local.file_match, discovery.*, prometheus.*) +- https://raw.githubusercontent.com/grafana/loki/v3.7.9/docs/sources/ (operations/storage/retention.md, logs-deletion.md, get-started/labels/bp-labels.md, send-data/otel, reference/loki-http-api.md, query/*) +- https://raw.githubusercontent.com/grafana/loki/v3.7.9/cmd/loki/loki-local-config.yaml +- https://github.com/grafana/alloy/releases/tag/v1.21.3 and https://github.com/grafana/loki/releases/tag/v3.7.9 + +## Not verified + +- Which Promtail options the converter cannot translate (the `convert` page confirms `--source-format promtail` exists; its limitations table was not read). +- Whether `loki.source.kubernetes` copies all non-`__` target labels onto log entries (only the required target labels and an example were read). +- The `logcli series --analyze-labels` flag is quoted from the best-practices page; `logcli` itself was not run. Note: the delete API says cancellation is possible only until the request is picked up for processing, while the logs-deletion page mentions `force=true` for requests already processing. +- `limits_config.otlp_config` and structured-metadata defaults come from the OTLP page of v3.7.9; not tested. +- The Loki template was never started; no offline linter was verified. Alloy templates were not run through `alloy validate` (binary not installed). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/logs-other-stacks.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/logs-other-stacks.md new file mode 100644 index 0000000..c7f594d --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/logs-other-stacks.md @@ -0,0 +1,43 @@ +# Other Log Stacks (pointers) + +**Load when:** the user already runs, or must integrate with, OpenSearch/Elastic, or asks for Vector or Fluent Bit instead of Alloy. Default recommendation stays Loki + Alloy ([logs-loki-alloy.md](logs-loki-alloy.md)); this file only helps you route the work. + +## When another stack is the right answer + +| Situation | Stack | Why | +|-----------|-------|-----| +| Full-text search across log content, security/analytics use, existing Elasticsearch/OpenSearch skills | OpenSearch (+ OpenSearch Dashboards) or Elastic | Inverted index over message content; Loki indexes labels only and scans content at query time | +| Cheap long retention of high-volume operational logs queried mostly by label + time | Loki | Small index, chunks in object storage | +| Existing Fluent Bit / Vector deployment | Keep the agent, change the sink | Do not replace a working collector without a reason | + +Index-everything stacks cost more storage and operations effort (shard sizing, lifecycle/retention policies, JVM heap): size them before adopting. + +## Collectors and pipelines + +- **Grafana Alloy** (default): Loki, Prometheus and OTLP in one binary ([logs-loki-alloy.md](logs-loki-alloy.md)). +- **Fluent Bit** (CNCF graduated project under Fluentd): a telemetry agent for logs, metrics, traces and profiles, with native OTLP ingestion and delivery and Prometheus integration, low resource use. Docs: https://docs.fluentbit.io/manual/ . It can ship to Loki, OpenSearch/Elastic and OTLP endpoints through its output plugins; check the plugin docs for your sink. +- **Vector**: pipeline model of sources, transforms (VRL, the Vector Remap Language) and sinks, with agent and aggregator deployment roles, buffering and end-to-end acknowledgements. Docs: https://vector.dev/docs/ . +- **OpenSearch Data Prepper**: ingestion service for OpenSearch with pipelines, processors (grok, parse JSON, aggregate...) and sinks (OpenSearch, S3, OTLP, Prometheus, ...). Docs: https://docs.opensearch.org/latest/data-prepper/ . +- **Elastic**: Elastic Agent / Beats into Elasticsearch, or OTLP into Elastic Observability. Docs: https://www.elastic.co/docs/solutions/observability/get-started . +- **OpenTelemetry Collector** can be the common gateway for any of them ([tracing-opentelemetry.md](tracing-opentelemetry.md)); Loki accepts OTLP natively at `/otlp`. + +## Routing guidance + +- Same label discipline applies everywhere: Kubernetes metadata (`namespace`, `app`, `container`) as fields; no unbounded values as index keys. +- Mask secrets and personal data at the collector (transform/redact stage) before any sink; logs are a common leak path. +- Security analytics on logs (SIEM rules, audits): outside this skill; use `fuse-security`. +- Deleting indices, data streams or snapshots in OpenSearch/Elastic is destructive: ACTION / EFFECT / PROOF / WAIT-FOR-GO ([observability-safe-ops.md](observability-safe-ops.md)). + +## Sources (read 2026-11-10) + +- https://docs.fluentbit.io/manual/about/what-is-fluent-bit +- https://vector.dev/docs/ (navigation and concept headings) +- https://docs.opensearch.org/latest/data-prepper/ (navigation: pipelines, processors, sinks) +- https://www.elastic.co/docs/solutions/observability/get-started (title and scope) + +## Not verified + +- No version of OpenSearch, Elastic, Vector, Fluent Bit or Data Prepper was checked today: do not quote versions from this file. +- Fluent Bit output plugin names for Loki/OpenSearch/Elastic and Vector sink names were not read. +- The Loki-versus-inverted-index comparison and the cost remark are architectural generalizations from the Loki label documentation, not benchmarks. +- No config templates are provided for these stacks on purpose (nothing could be validated). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/long-term-storage.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/long-term-storage.md new file mode 100644 index 0000000..e2b097f --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/long-term-storage.md @@ -0,0 +1,47 @@ +# Long-Term Storage (pointers) + +**Load when:** one Prometheus no longer meets retention, durability, HA or multi-cluster query needs. This file is deliberately pointer-level: pick an option, then read that project's official docs. + +## First decide whether you need it + +Prometheus local TSDB is a single-node database ("not clustered or replicated", per its storage page) but with proper sizing holds years of data on one disk. Stay local if: one cluster, retention fits the disk, losing the disk is acceptable or you back up with snapshots. Reach for long-term storage when you need: retention beyond a disk, durability off the node, one query endpoint across clusters/replicas, or multi-tenant isolation. + +## Options + +| Option | Model | Ingest path | Pick when | Docs | +|--------|-------|-------------|-----------|------| +| **Thanos** (latest tag read: v0.42.5) | Sidecar or Receive next to Prometheus, object storage for blocks, Query layer over all | Sidecar uploads TSDB blocks; Receive accepts remote write | You keep Prometheus as is and add global view + cheap retention in S3/GCS | https://thanos.io/ | +| **Grafana Mimir** (mimir-3.2.4) | Horizontally scalable multi-tenant TSDB, Prometheus-compatible | Remote write to `/api/v1/push` | Many tenants/clusters, big scale, you already run the Grafana stack | https://grafana.com/docs/mimir/latest/ | +| **VictoriaMetrics** (v1.156.0) | Single-node or cluster; `vmagent` for scrape/forward; MetricsQL superset | Remote write or `vmagent` | Resource-efficient drop-in store; simple single-node is often enough | https://docs.victoriametrics.com/ | + +Mimir 3.x documents several supported architectures starting with 3.0 and a monolithic mode (`-target`) for evaluation. + +## Prometheus-side wiring + +- Remote write: `remote_write` with `url`, `name`, `write_relabel_configs` and `queue_config` ([prometheus-config.md](prometheus-config.md)). Start with all series, then filter once you know what dashboards and alerts need. +- Set unique `external_labels` per Prometheus (`cluster`, `replica`): Thanos/Mimir use the replica label for HA deduplication. +- Alerting and recording rules still evaluate on the local Prometheus unless you run the backend's ruler. +- Remote-write 2.0 (`io.prometheus.write.v2.Request`) sends metadata, start timestamps and native histograms by default; confirm the backend supports it before switching `protobuf_message`. +- `chunk_encoding.floats: xor2` can make blocks unreadable by Thanos sidecar uploads or older tools: leave it off when feeding LTS systems that do not support it. +- Grafana Alloy can be the remote-write client instead of Prometheus (`prometheus.remote_write`, Mimir example URL `http://mimir:9009/api/v1/push`, tenant header `X-Scope-OrgID`). + +## Capacity and cost levers + +Series count (cardinality) drives cost everywhere: control it before you ship data off the node. Downsampling, compaction and retention knobs are product-specific: read the product docs; they are not described here. + +Deleting tenants, buckets or object-storage data of a long-term store is irreversible: ACTION / EFFECT / PROOF / WAIT-FOR-GO ([observability-safe-ops.md](observability-safe-ops.md)). Backups of the object store belong to `devops-storage-backup`. + +## Sources (read 2026-11-10) + +- https://prometheus.io/docs/prometheus/latest/storage/ (single-node limits, Thanos sidecar-compatible block caveat via config reference) +- https://grafana.com/docs/mimir/latest/get-started/about-grafana-mimir-architecture/ +- https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/ (navigation: vmagent, Prometheus, Thanos, Mimir pages exist) +- https://raw.githubusercontent.com/grafana/alloy/v1.21.3/docs/sources/reference/components/prometheus/prometheus.remote_write.md (Mimir URL and tenant header example) +- GitHub `/releases/latest` of thanos-io/thanos, grafana/mimir, VictoriaMetrics/VictoriaMetrics + +## Not verified + +- Thanos component behavior (sidecar, receive, store gateway, compactor) was not read from thanos.io; the table row describes it from general knowledge. +- VictoriaMetrics `vmagent` roles and MetricsQL claims come from site navigation only. +- Mimir ingest URL `/api/v1/push` is confirmed only through the Alloy example, not the Mimir HTTP API page. +- No sizing numbers are given: none were read. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/observability-safe-ops.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/observability-safe-ops.md new file mode 100644 index 0000000..7bf8c70 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/observability-safe-ops.md @@ -0,0 +1,154 @@ +# Safe Operation of Monitoring Systems + +**Load when:** about to change, reload, restart, delete or silence anything on a live Prometheus, Alertmanager, Loki, Grafana or collector. Read-only commands (queries, `promtool check`, `amtool check-config`, `amtool silence query`, `GET` APIs, `kubectl get`) need no go. + +Monitoring is the system you use during an incident: a mistake here blinds you exactly when it matters. Follow the agent's Operations Safety Protocol: discover read-only, validate offline, **show the block below, wait for the go**, apply, verify, roll back. + +## Proposal format (one block per action) + +``` +ACTION +EFFECT +PROOF +WAIT-FOR-GO waiting for your explicit go on this action only +``` + +## Catalog + +### 1. Reload Prometheus or Alertmanager configuration + +``` +ACTION curl -X POST http://prometheus.example:9090/-/reload (or: kill -HUP ) +EFFECT Re-reads prometheus.yml and ALL rule files; applied only if every file is well-formed. A valid but wrong + rule/relabel change takes effect immediately: alerts can start or stop firing, targets can disappear. + Needs --web.enable-lifecycle for the HTTP form (anyone who can reach the port can reload or POST /-/quit). +PROOF promtool check config prometheus.yml -> exit 0; promtool check rules -> exit 0; + diff of the change reviewed; the previous config copy kept for rollback. +WAIT-FOR-GO yes +``` + +Alertmanager: validate with `amtool check-config alertmanager.yml` and the routing test from [alertmanager.md](alertmanager.md); reload by `SIGHUP`/its reload endpoint. After any reload verify: Status -> Config shows the new hash, `up` targets unchanged, no spike in `prometheus_config_last_reload_successful == 0`. + +### 2. Restart a production Prometheus + +``` +ACTION systemctl restart prometheus | kubectl --context CTX -n monitoring rollout restart sts/prometheus-... +EFFECT Scraping stops; WAL is replayed on start (minutes to tens of minutes on large heads) and during replay the + server is not ready: dashboards and alerts are blind. In a single-replica setup there is an alerting gap. + emptyDir storage = all data lost. +PROOF storage is a persistent volume; WAL size; a second replica or an external blackbox check covers the gap. +WAIT-FOR-GO yes +``` + +### 3. Delete series through the admin API + +``` +ACTION curl -X POST -g 'http://prometheus:9090/api/v1/admin/tsdb/delete_series?match[]=&start=&end=' + (then, optionally) curl -X POST http://prometheus:9090/api/v1/admin/tsdb/clean_tombstones +EFFECT Marks the matching samples deleted (tombstones); omitting start AND end deletes all history of the + matched series. clean_tombstones physically removes them. Not undoable without a backup. + The admin API is OFF unless --web.enable-admin-api is set (enabling it needs a restart and exposes delete + and snapshot to anyone who can reach the port). +PROOF Same selector run as an instant query to list exactly the series; count; a snapshot taken first + (POST /api/v1/admin/tsdb/snapshot) or blocks copied; time range written down. +WAIT-FOR-GO yes +``` + +### 4. Shrink retention or size limit + +``` +ACTION change --storage.tsdb.retention.time / --storage.tsdb.retention.size, restart +EFFECT Data older than the new limit (or oldest blocks beyond the size) is deleted by the next background + cleanup (up to ~2 hours later). Irreversible. A too-small size limit also deletes recent blocks. +PROOF du of the data dir; current oldest timestamp (prometheus_tsdb_lowest_timestamp_seconds); + confirmation that remote storage/backup holds what is about to go. +WAIT-FOR-GO yes +``` + +### 5. Loki: shorten retention, delete log entries, remove tenant data + +``` +ACTION edit limits_config.retention_period / retention_stream | POST /loki/api/v1/delete?query=...&start=...&end=... (X-Scope-OrgID: ) +EFFECT Compactor marks matching chunks and a sweeper deletes them after retention_delete_delay (and a delete + request after delete_request_cancel_period, default 24h). The Loki docs say retention changes are + "not retroactive" for already-ingested logs; do not rely on that to protect data when shortening: treat + a shorter period as a deletion. Deleting a whole tenant means a delete request over its full range or + deleting its objects in the bucket: unrecoverable without object versioning. +PROOF Query the same selector/time range first (counts); list pending requests with GET /loki/api/v1/delete; + object-store versioning enabled; the cancel path known (DELETE /loki/api/v1/delete?request_id=...). +WAIT-FOR-GO yes +``` + +### 6. Mass silences, expiring all silences + +``` +ACTION amtool silence add alertname=~".+" --duration=2h --comment="..." | amtool silence expire $(amtool silence query -q) +EFFECT The first mutes every alert for the duration (real incidents go unnoticed, pagers stay quiet); + the second removes every silence and releases all muted notifications at once (a flood). +PROOF amtool silence query output; the exact matchers and the number of alerts currently matched + (the UI preview or amtool alert query with the same matchers); end time and an author/comment set. +WAIT-FOR-GO yes (a narrow, time-boxed silence for one alert is ordinary; empty, wildcard or all-alerts matchers are not) +``` + +### 7. Remove monitoring storage (PVC, volume, bucket) + +``` +ACTION kubectl --context CTX -n monitoring delete pvc | docker volume rm | delete bucket/prefix +EFFECT Metrics/logs/silences/dashboards held on it are gone. With reclaimPolicy Delete the PV and disk go too. + Grafana's database (users, annotations) and Alertmanager's silences are state, not config. +PROOF kubectl get pvc,pv (reclaim policy, bound pod); backup/snapshot verified restorable; list of what the + volume contains; nothing still mounts it. +WAIT-FOR-GO yes (volume handling details: devops-kubernetes storage reference, devops-storage-backup) +``` + +### 8. Helm install/upgrade/uninstall of kube-prometheus-stack, CRD changes + +``` +ACTION helm upgrade kps prometheus-community/kube-prometheus-stack -n monitoring --version -f values.yaml +EFFECT Rolling restart of Prometheus/Alertmanager/Grafana; CRDs are not upgraded by Helm by default, so a new + operator against old CRDs can break; changed default rules change what alerts. Uninstall removes + workloads; whether PVCs survive depends on the chart/storage settings. +PROOF helm diff / --dry-run=server output; CRD versions listed (kubectl get crd | grep monitoring.coreos.com); + chart upgrade notes for the major read; values diff against upstream values. +WAIT-FOR-GO yes +``` + +### 9. Grafana provisioning with deletion semantics + +`deleteDatasources`, `prune: true`, dashboard `disableDeletion: false` and `deleteRules`/`deleteContactPoints` remove objects when files change. + +``` +ACTION deploy provisioning files containing delete/prune settings +EFFECT Datasources/alert resources vanish; dashboards bound to a removed datasource break. +PROOF list of current objects (API or UI export) compared with the files; Grafana DB backup. +WAIT-FOR-GO yes +``` + +### 10. Changing identity labels (`external_labels`, replica label, job names, metric renames) + +``` +EFFECT Series become new series (history split), alert grouping and inhibition `equal` labels stop matching, + HA deduplication in Thanos/Mimir breaks, dashboards and recording rules return empty. +PROOF grep of every rule/dashboard/route using the label; a plan for the overlap period. +WAIT-FOR-GO yes +``` + +## After any applied action + +Verify with data, not with the exit code: `up` count per job unchanged, `prometheus_config_last_reload_successful` = 1, Alertmanager receiving the `Watchdog`, Grafana datasource Test OK, Loki `/ready`, a known log line found. If verification fails, roll back to the last good file/snapshot before trying another change. + +## Sources (read 2026-11-10) + +- https://prometheus.io/docs/prometheus/latest/management_api/ (reload, quit, `--web.enable-lifecycle`) +- https://raw.githubusercontent.com/prometheus/prometheus/v3.16.0/docs/querying/api.md (admin API: snapshot, delete_series, clean_tombstones, `--web.enable-admin-api`) +- https://prometheus.io/docs/prometheus/latest/storage/ (retention flags, cleanup delay, snapshots as backup) +- https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/ (`promtool check rules`, SIGHUP, all-or-nothing apply) +- https://raw.githubusercontent.com/grafana/loki/v3.7.9/docs/sources/operations/storage/retention.md , logs-deletion.md , reference/loki-http-api.md +- https://raw.githubusercontent.com/prometheus/alertmanager/v0.35.0/README.md (`amtool silence` examples) + +## Not verified + +- `prometheus_config_last_reload_successful` and `prometheus_tsdb_lowest_timestamp_seconds` metric names are from experience; not located in the docs read. +- Alertmanager reload endpoint (`/-/reload`) exists in practice; its page was not fetched. Use `SIGHUP` if unsure. +- Helm uninstall behavior for the chart's PVCs and WAL replay duration claims are not sourced. +- Whether deleting a Loki tenant has a dedicated API in v3.7.9 was not found in the HTTP API page; the described paths are the two read ones. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/prometheus-config.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/prometheus-config.md new file mode 100644 index 0000000..4e9ed15 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/prometheus-config.md @@ -0,0 +1,88 @@ +# Prometheus Server Configuration + +**Load when:** writing or reviewing `prometheus.yml`, rule files, retention, remote write, or sizing a Prometheus. + +Template: [prometheus.yml](templates/prometheus.yml). Validate with `promtool check config prometheus.yml` and `promtool check rules ` (exit 0 = valid, 1 = error); `promtool` ships in the Prometheus archive. + +## Release line + +Prometheus cuts a minor every 6 weeks; non-LTS minors stop getting fixes after that. **LTS = 3.13** (released 2026-07-01, supported to 2027-07-31); latest is 3.16.0 (2026-11-04). Pick LTS for a server you rarely touch, latest for new features. LTS does not cover experimental feature flags. Check the release notes before every minor upgrade: 3.16.0 for example changes the default `--web.external-url` to https when TLS is on. + +## File layout + +``` +global: scrape_interval (default 1m), scrape_timeout (10s, <= interval), evaluation_interval (1m), + external_labels, sample_limit / label_limit / target_limit (0 = unlimited) +rule_files: globs of rule files +scrape_config_files: globs of extra scrape-config files (split big configs) +scrape_configs: one job per kind of target +alerting: alertmanagers + alert_relabel_configs +remote_write / remote_read, storage (tsdb, exemplars), otlp, tracing +``` + +Retention, listen address and the data path are **command-line flags**, not file keys. + +## Service discovery (pick by where targets live) + +| Mechanism | Key | Use for | +|-----------|-----|---------| +| Static | `static_configs` | Few fixed targets | +| File | `file_sd_configs` | Anything automated (Ansible/Terraform writes a file); changes apply without reload ([file-sd-nodes.yml](templates/file-sd-nodes.yml)) | +| Kubernetes | `kubernetes_sd_configs` role `node`, `service`, `pod`, `endpoints`, `endpointslice` | In-cluster; prefer the Operator CRDs ([prometheus-operator-k8s.md](prometheus-operator-k8s.md)) | +| DNS / Consul / EC2 / Docker / Swarm / HTTP / OpenStack | `dns_sd_configs`, `consul_sd_configs`, ... | As the platform dictates | + +`endpoints` is documented as superseded by `endpointslice`. + +## Relabeling in one page + +- `relabel_configs`: runs on the **target** before the scrape. Discovery metadata is in `__meta_*` labels; labels starting `__` are removed afterwards. Use `__tmp` prefix for scratch labels. +- `metric_relabel_configs`: runs on **samples** after the scrape, before ingestion. Does not touch auto-generated series such as `up`. This is the tool to drop expensive series. +- `alert_relabel_configs`: on alerts before they go to Alertmanager. `write_relabel_configs`: before remote write. +- Actions: `replace` (default), `keep`, `drop`, `labelmap`, `labeldrop`, `labelkeep`, `hashmod`, `lowercase`, `uppercase`, `keepequal`, `dropequal`. Regexes are RE2 and **anchored** on both ends. +- Common pattern: `keep` only pods annotated for scraping; `replace` to set `__address__` from an annotation; `labeldrop` of noisy labels (check the series stay unique). + +## Guardrails against cardinality blow-ups + +- `sample_limit`, `label_limit`, `target_limit` (global or per job): a scrape that exceeds the limit **fails entirely**. This protects the TSDB and surfaces the culprit via `up == 0`. +- `label_name_length_limit`, `label_value_length_limit` and `body_size_limit` exist next to them (all default `0` = off). +- Find the heavy metrics: `topk(10, count by (__name__) ({__name__=~".+"}))` (expensive: run off-peak) and the TSDB status page / `prometheus_tsdb_head_series`. +- Fix at the source first (stop exporting the label), then `metric_relabel_configs`. + +## Rules + +- Rule files hold `groups:`; each group has `name`, optional `interval`, `limit`, `query_offset`, and `rules` of `record:` or `alert:` kind. Rules in a group run **sequentially**; a group that overruns its interval skips evaluations (`rule_group_iterations_missed_total`). +- Alert fields: `for` (pending time), `keep_firing_for` (damps flapping), `labels`, `annotations` (templated with `$labels`, `$value`, `$externalLabels`). Templates: [rules-alerts.yml](templates/rules-alerts.yml), [rules-recording.yml](templates/rules-recording.yml). +- Reload rules and config: `SIGHUP`, or `POST /-/reload` if started with `--web.enable-lifecycle`. Changes apply only if **all** files are well-formed. A production reload is a proposal ([observability-safe-ops.md](observability-safe-ops.md)). + +## Storage and retention + +- Retention flags: `--storage.tsdb.retention.time` (default `15d` when neither flag is set) and `--storage.tsdb.retention.size`; whichever triggers first wins. Only persistent blocks are deleted; WAL counts toward the size. Cleanup can lag up to two hours. +- Capacity: `needed_disk = retention_seconds * samples_per_second * bytes_per_sample` (1-2 bytes/sample typical). Reduce series first, scrape interval second. +- Local filesystem only: NFS and EFS are **not supported** (corruption risk). +- Backups: use the snapshot API (needs `--web.enable-admin-api`), or copy blocks while excluding `chunks_head/`, `wal/`, `wbl/`. +- Out-of-order samples: `storage.tsdb.out_of_order_time_window` (config file, default `0s`). +- `chunk_encoding.floats: xor2` gives better compression but older Prometheus versions and downstream LTS tools may not read it: do not enable it if you might downgrade or upload blocks to Thanos. + +## Remote write + +`remote_write[]` takes `url`, optional `name`, `write_relabel_configs`, `queue_config` (defaults: `capacity` 10000, `max_shards` 50, `min_shards` 1, `max_samples_per_send` 2000, `batch_send_deadline` 5s), `protobuf_message` (default `prometheus.WriteRequest`; `io.prometheus.write.v2.Request` = Remote Write 2.0, check that the receiver supports it first). Filter with `write_relabel_configs` to send only what the long-term store needs. Watch `prometheus_remote_storage_*` metrics for lag. Backends: [long-term-storage.md](long-term-storage.md). + +## Native histograms (opt-in) + +`scrape_native_histograms` (global or per job) defaults to `false`. When true, the default `scrape_protocols` order puts `PrometheusProto` first. Related: `convert_classic_histograms_to_nhcb`, `always_scrape_classic_histograms`. Query side: [promql-essentials.md](promql-essentials.md). Enable per job, verify the dashboards, then widen. + +## Sources (read 2026-11-10) + +- https://raw.githubusercontent.com/prometheus/prometheus/v3.16.0/docs/configuration/configuration.md +- https://prometheus.io/docs/prometheus/latest/storage/ +- https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/ and .../alerting_rules/ +- https://prometheus.io/docs/prometheus/latest/management_api/ +- https://prometheus.io/docs/introduction/release-cycle/ +- https://github.com/prometheus/prometheus/releases/tag/v3.16.0 and raw CHANGELOG.md (3.16.0 entries) + +## Not verified + +- LTS patch number: the research doc lists 3.13.5; the 3.16.0 changelog read here stops at 3.13.2 for that line, so the current 3.13.x patch was not confirmed. +- The "4x scrape interval" minimum for `rate()` ranges is common guidance, not a statement from the pages read. +- `promtool` and `amtool` are not installed on the authoring host: no template was linted with them. +- Defaults of `--storage.tsdb.retention.size` right-sizing percentages (page truncated before the recommendation). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/prometheus-operator-k8s.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/prometheus-operator-k8s.md new file mode 100644 index 0000000..4dd4fd8 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/prometheus-operator-k8s.md @@ -0,0 +1,66 @@ +# Prometheus Operator and kube-prometheus-stack + +**Load when:** monitoring a Kubernetes cluster, writing ServiceMonitor / PodMonitor / PrometheusRule objects, or installing/upgrading the stack. + +Versions read 2026-11-10: Prometheus Operator **v0.95.1** (release 2026-11-05); Helm chart **kube-prometheus-stack 93.1.1** (appVersion v0.95.1, `kubeVersion >=1.25.0-0`, `main` branch Chart.yaml; published-release listing not checked). Workload basics: `devops-kubernetes`; chart/GitOps delivery: `devops-gitops-packaging`. + +## CRDs (group `monitoring.coreos.com`) + +| Kind | apiVersion | Purpose | +|------|-----------|---------| +| `Prometheus`, `Alertmanager`, `ThanosRuler` | `v1` | The servers the operator runs | +| `ServiceMonitor` | `v1` | Scrape the endpoints behind a **Service** | +| `PodMonitor` | `v1` | Scrape Pods directly (no Service) | +| `Probe` | `v1` | Blackbox probes of static targets or Ingresses | +| `PrometheusRule` | `v1` | Recording and alerting rules (same `groups` shape as a rule file) | +| `ScrapeConfig` | `v1alpha1` | Raw scrape config incl. non-Kubernetes SD (file, DNS, Consul, static...) | +| `AlertmanagerConfig` | `v1alpha1`, `v1beta1` | Namespaced routing/receivers merged into Alertmanager | + +## How selection works (the usual "my target is missing") + +1. The `Prometheus` object selects monitors with `serviceMonitorSelector`, `podMonitorSelector`, `probeSelector`, `scrapeConfigSelector`, `ruleSelector`, and namespaces with the matching `*NamespaceSelector`. +2. kube-prometheus-stack sets those selectors to the Helm release label unless `prometheus.prometheusSpec.SelectorNilUsesHelmValues: false` (chart default `true`). So a ServiceMonitor without `release: ` is silently ignored. Either label your objects or set the flags to `false` (see [kube-prometheus-stack-values.yaml](templates/kube-prometheus-stack-values.yaml)). +3. A ServiceMonitor `selector` matches **Service** labels; `endpoints[].port` is the **Service port name**. A PodMonitor `podMetricsEndpoints[].port` is the **container port name**. A number-only port needs `targetPort` (ServiceMonitor). +4. `namespaceSelector` defaults to the monitor's own namespace; use `matchNames` or `any: true`. + +Templates: [servicemonitor.yaml](templates/servicemonitor.yaml) (ServiceMonitor + PodMonitor), [prometheusrule.yaml](templates/prometheusrule.yaml). + +Useful spec fields (v0.95.1 API reference): ServiceMonitor `jobLabel`, `targetLabels`, `podTargetLabels`, `selectorMechanism`, `sampleLimit`, `targetLimit`, `labelLimit`, `keepDroppedTargets`, `scrapeProtocols`, `fallbackScrapeProtocol`, `scrapeClass`, `bodySizeLimit`, `attachMetadata`, `serviceDiscoveryRole`, and native-histogram switches `scrapeNativeHistograms`, `scrapeClassicHistograms`, `convertClassicHistogramsToNHCB`; per endpoint `port`, `targetPort`, `path`, `scheme`, `params`, `interval`, `scrapeTimeout`, `honorLabels`, `relabelings`, `metricRelabelings`, `tlsConfig`, `authorization`, `basicAuth`, `bearerTokenSecret`. Rule groups accept `name`, `labels`, `interval`, `query_offset`, `limit`, `rules`. + +## Installing and upgrading kube-prometheus-stack + +- Chart sub-charts: kube-state-metrics, prometheus-node-exporter, grafana, prometheus-windows-exporter. +- **Persistence**: chart defaults leave `prometheus.prometheusSpec.storageSpec` and `alertmanager.alertmanagerSpec.storage` as `{}` = emptyDir. Metrics and silences are lost on reschedule: set a `volumeClaimTemplate`. +- Retention: `prometheusSpec.retention` (chart default `10d`), `retentionSize` (default empty); Alertmanager `retention` default `120h`. +- **CRDs are not upgraded by Helm**: `crds.enabled` installs them once; the chart offers `crds.upgradeJob.enabled` (documented as preview, off by default). Read the chart's upgrade notes for each major before `helm upgrade`; chart majors are frequent. +- Control-plane scrapes (`kubeEtcd`, `kubeControllerManager`, `kubeScheduler`, `kubeProxy`) are enabled by default but fail on managed clusters or kubeadm defaults bound to localhost: disable or fix endpoints/ports per cluster (`devops-k8s-cluster`). +- Default rules (`defaultRules.create`) ship many alerts; tune per group, never copy-delete blindly. +- `helm upgrade`, CRD apply, PVC changes and deletions are proposals: [observability-safe-ops.md](observability-safe-ops.md). + +## Dashboards as code on Kubernetes + +Grafana sidecar (`grafana.sidecar.dashboards.enabled`) loads ConfigMaps labeled `grafana_dashboard: "1"` from all namespaces (`searchNamespace: ALL`). Keep the JSON in Git and ship it as ConfigMaps through GitOps. Details: [grafana-as-code.md](grafana-as-code.md). + +## Verify + +```bash +kubectl --context "$CTX" get crd | grep monitoring.coreos.com +kubectl --context "$CTX" -n monitoring get prometheus,alertmanager,servicemonitor,podmonitor,prometheusrule +kubectl --context "$CTX" -n monitoring port-forward svc/ 9090 # then Status -> Targets / Rules +``` + +If a target is absent: check selectors (point 1-2), the Service port name, the `release` label, and RBAC of the Prometheus ServiceAccount for the target namespace. + +## Sources (read 2026-11-10) + +- https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.95.1/Documentation/api-reference/api.md +- https://raw.githubusercontent.com/prometheus-community/helm-charts/main/charts/kube-prometheus-stack/Chart.yaml and values.yaml +- https://github.com/prometheus-operator/prometheus-operator/releases/tag/v0.95.1 + +## Not verified + +- Which chart version is the latest *published* release (only `main` was read). +- Chart default for `kubeEtcd`/`kubeScheduler`... failing on managed clusters is a field-experience statement; the values only show `enabled: true`. +- Default of `namespaceSelector` (own namespace) and the `Probe` target kinds are standard Operator behavior, not re-read in the API reference today. +- `AlertmanagerConfig` v1beta1 field differences from v1alpha1 were not compared. +- RBAC details for cross-namespace scraping were not read. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/promql-essentials.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/promql-essentials.md new file mode 100644 index 0000000..8d3df7e --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/promql-essentials.md @@ -0,0 +1,70 @@ +# PromQL Essentials and Pitfalls + +**Load when:** writing dashboard queries, alert expressions or recording rules, or debugging "the graph looks wrong". + +## Counters, gauges, rates + +| Need | Use | Not | +|------|-----|-----| +| Per-second rate of a counter, for alerts and recording rules | `rate(x_total[5m])` | `irate` | +| Spiky per-second rate for a zoomed-in graph | `irate(x_total[5m])` (uses the **last two points only**) | alerting on it | +| Total increase over a window | `increase(x_total[1h])` | `rate * 3600` by hand | +| Trend of a gauge | `deriv(g[1h])`, `predict_linear(g[6h], 24*3600)` | `rate` on a gauge | + +- `rate` handles counter resets and **extrapolates** to the window edges, so `increase` over short windows can return non-integers. +- The range must contain at least two samples; use a range of several scrape intervals (rule of thumb: 4x) so one missed scrape does not blank the result. +- `rate` of a metric that mixes float and native-histogram samples in the window is dropped with a warn annotation; the same rule applies to `irate`. +- Apply `rate` first, then aggregate: `sum by (job) (rate(x_total[5m]))`, never `rate(sum(x_total)[5m])` (a reset in one series corrupts the sum). + +## Aggregation and joins + +- `by (...)` keeps the listed labels; `without (...)` drops them. Prefer `without` in recording rules so new labels survive. +- Vector matching: `a / on (instance) b`, `a * on (instance) group_left (version) info_metric`. Many-to-one without `group_left` errors with "multiple matches". +- Ratios: divide **after** rate and aggregation on both sides, with the same grouping: `sum by (job)(rate(errors_total[5m])) / sum by (job)(rate(requests_total[5m]))`. +- Missing data: `up == 0` fires on a failed scrape; a metric that disappears needs `absent(metric{job="x"})`; `absent_over_time(...)` for gaps. An alert on `sum(...) < 1` stays silent when the series is gone. +- Use `unless`, `and`, `or` for set logic (see `HostFilesystemFillingUp` in [rules-alerts.yml](templates/rules-alerts.yml)). + +## Histograms + +**Classic histogram** (`_bucket{le=...}`, `_sum`, `_count`): + +``` +histogram_quantile(0.95, sum by (job, le) (rate(http_request_duration_seconds_bucket[5m]))) +``` + +`le` MUST stay in the `by` clause; without it the result is wrong or empty. Accuracy is bounded by bucket boundaries: the quantile is interpolated inside a bucket, so put boundaries around your SLO threshold. Quantiles cannot be averaged or summed across instances: aggregate buckets, then compute the quantile. + +**Native histogram** (opt-in, `scrape_native_histograms: true`, see [prometheus-config.md](prometheus-config.md)): no `le`, no `_bucket`: + +``` +histogram_quantile(0.95, sum by (job) (rate(http_request_duration_seconds[5m]))) +histogram_fraction(0, 0.2, rate(http_request_duration_seconds[1h])) # share of requests <= 200 ms +histogram_avg(rate(http_request_duration_seconds[5m])) +``` + +`rate` on a native histogram yields a histogram; `histogram_count` / `histogram_sum` extract components. A selector can mix classic and native series as long as names do not collide. During a migration, `convert_classic_histograms_to_nhcb` converts classic histograms to native histograms with custom buckets, and `always_scrape_classic_histograms` keeps the classic series alongside. + +## Pitfalls checklist + +1. Wrong `rate` window (shorter than 2 scrapes) gives empty graphs. +2. `irate` in an alert flaps on every spike. +3. `avg` of per-instance percentages is not the fleet percentage: sum numerators and denominators. +4. Label churn (pod names, build IDs) multiplies series; aggregate them away in recording rules. +5. Grafana `$__rate_interval` is the right range variable for `rate` in dashboards; a fixed `[5m]` ignores the dashboard's resolution. +6. `offset` and `@` modifiers apply to the selector, not to the whole expression. +7. Staleness: a series stops being returned 5 minutes after its last sample by default lookback; `last_over_time(x[1h])` bridges gaps deliberately. +8. Unquoted UTF-8 metric names need `--enable-feature=promql-unquoted-utf8-names`; otherwise quote them: `{"http.server.duration"}`. + +## Sources (read 2026-11-10) + +- https://raw.githubusercontent.com/prometheus/prometheus/v3.16.0/docs/querying/functions.md (`rate`, `irate`, `histogram_quantile`, `histogram_fraction` sections) +- https://raw.githubusercontent.com/prometheus/prometheus/v3.16.0/docs/feature_flags.md (UTF-8 names flag) +- https://raw.githubusercontent.com/prometheus/prometheus/v3.16.0/docs/configuration/configuration.md (native histogram settings) + +## Not verified + +- Items 5-7 and the join syntax (`group_left`, `absent_over_time`, `last_over_time`, the 5-minute lookback) are standard PromQL/Grafana behavior not re-read from the pages above today. +- `histogram_avg` exists in functions.md (heading seen) but its argument semantics were not read. +- The "4x scrape interval" figure is a rule of thumb. +- The quoted-name form `{"http.server.duration"}` (only the unquoted form under the feature flag appears in the page read). +- Quantile interpolation behavior is described from the Prometheus histograms practice page, which was not fetched today. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/signals-and-method.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/signals-and-method.md new file mode 100644 index 0000000..9ed1c78 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/signals-and-method.md @@ -0,0 +1,52 @@ +# Signals and What to Instrument First + +**Load when:** designing monitoring for a new service or host, deciding what to collect, or auditing why an outage was not seen. + +## The three signals + +| Signal | Answers | Cost shape | Default backend in this skill | +|--------|---------|------------|-------------------------------| +| Metrics | "Is it healthy, how much, how fast, trend?" | Cheap per series, expensive with label cardinality | Prometheus | +| Logs | "What exactly happened in this event?" | Volume (bytes ingested) | Loki (collector: Grafana Alloy) | +| Traces | "Where did this one request spend its time across services?" | Volume; needs sampling | Tempo or Jaeger via OpenTelemetry | + +Order of work: metrics first (alerting and trends), logs second (evidence), traces last (only when several services call each other). Correlate with shared labels (`service`, `namespace`) and a `trace_id` in log lines. + +## Which method for which target + +| Method | Use for | Questions per resource/service | +|--------|---------|--------------------------------| +| USE (Brendan Gregg) | Resources: CPU, memory, disk, NIC, queues, hypervisor | Utilization, Saturation, Errors | +| RED (Tom Wilkie) | Request-driven services: APIs, web, gRPC | Rate, Errors, Duration | +| Four golden signals (Google SRE book) | User-facing systems, when you can measure only four things | Latency, Traffic, Errors, Saturation | + +USE is hardware- and resource-oriented; RED was created for microservices, because USE "doesn't really apply to services" (Grafana blog on RED). Golden signals = RED plus saturation. + +## Instrument in this order + +1. **Is it up** - `up` per scrape target, plus an outside-in blackbox probe of every user-facing URL (HTTP status, TLS expiry). This catches what internal metrics cannot (blackbox_exporter, see [exporters.md](exporters.md)). +2. **The golden signals at the edge** - request rate, 5xx ratio and latency histogram at the reverse proxy / ingress / gateway: one place covers all services behind it. +3. **Saturation of shared resources** - disk fill prediction, memory available, CPU run-queue, connection pools, queue depth, certificate expiry, backup freshness. +4. **Per-service RED** - client libraries with a histogram for request duration; labels limited to `method`, `route` template (never the raw URL), `status_class`. +5. **Business and batch signals** - "last success timestamp" gauges for batch jobs (alert on age, not on a single failure). +6. **Monitor the monitoring** - Prometheus rule-evaluation failures, Alertmanager notification failures, an always-firing `Watchdog` alert routed to an external dead man's switch ([alert-design.md](alert-design.md)). + +## Naming and label rules (Prometheus practices) + +- Base units in the name (`_seconds`, `_bytes`), counters end in `_total`. +- Never use labels for unbounded values (user IDs, email addresses, raw paths): every distinct value is a new series. +- Recording rule names follow `level:metric:operations`. + +SLOs, error budgets and incident handling are not covered here: use `devops-sre-ops`. + +## Sources (read 2026-11-10) + +- https://sre.google/sre-book/monitoring-distributed-systems/ (four golden signals sentence) +- https://www.brendangregg.com/usemethod.html (USE definition) +- https://grafana.com/blog/the-red-method-how-to-instrument-your-services/ (RED, relation to USE) +- https://prometheus.io/docs/practices/rules/ and the Prometheus naming practices page (label cardinality warning, rule naming) + +## Not verified + +- The exact wording of "Errors" and "Saturation" definitions on the SRE book page (only the sentence naming the four signals was read). +- The advice "route template, never raw URL" is standard practice derived from the cardinality warning, not a quoted rule. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/alertmanager.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/alertmanager.yml new file mode 100644 index 0000000..8492910 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/alertmanager.yml @@ -0,0 +1,69 @@ +# Alertmanager configuration (keys checked in configuration.md of v0.35.0). +# Validate: amtool check-config alertmanager.yml +# Test routing: amtool config routes test --config.file=alertmanager.yml --tree --verify.receivers=oncall-pager severity=page team=sre +# Secrets are read from files (api_url_file, routing_key_file, url_file): mount them, never inline them. +global: + resolve_timeout: 5m + slack_api_url_file: /etc/alertmanager/secrets/slack_api_url + +templates: + - /etc/alertmanager/templates/*.tmpl + +route: + receiver: team-default # the root route matches every alert and carries no matchers + group_by: [alertname, cluster, service] + group_wait: 30s # default 30s: lets inhibiting alerts arrive first + group_interval: 5m # default 5m + repeat_interval: 4h # default 4h; keep it a multiple of group_interval + routes: + - matchers: [alertname = "Watchdog"] + receiver: deadmans-switch + group_wait: 0s + repeat_interval: 1m + - matchers: [severity = "page"] + receiver: oncall-pager + continue: false + - matchers: [severity = "ticket"] + receiver: team-tickets + active_time_intervals: [business-hours] # route only notifies inside the window (not inherited by children) + +inhibit_rules: + # A node-level outage hides the per-service symptoms that come from the same host. + - name: node-down-hides-service-alerts + source_matchers: [alertname = "TargetDown", job = "node"] + target_matchers: [severity =~ "page|ticket"] + equal: [instance] + # A page-level alert hides the ticket-level one for the same alert name and cluster. + - name: page-hides-ticket + source_matchers: [severity = "page"] + target_matchers: [severity = "ticket"] + equal: [alertname, cluster] + +time_intervals: + - name: business-hours + time_intervals: + - weekdays: ["monday:friday"] + times: + - start_time: "08:00" + end_time: "18:00" + location: "Europe/Zurich" + +receivers: + - name: team-default + slack_configs: + - channel: "#alerts" + send_resolved: true + title: '{{ .CommonLabels.alertname }} ({{ .Status }})' + text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ if .Annotations.runbook_url }} - {{ .Annotations.runbook_url }}{{ end }}{{ "\n" }}{{ end }}' + - name: team-tickets + webhook_configs: + - url_file: /etc/alertmanager/secrets/ticket_webhook_url + send_resolved: true + - name: oncall-pager + pagerduty_configs: + - routing_key_file: /etc/alertmanager/secrets/pagerduty_routing_key + send_resolved: true + - name: deadmans-switch + webhook_configs: + - url_file: /etc/alertmanager/secrets/deadmans_switch_url + send_resolved: false diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/alloy-host.alloy b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/alloy-host.alloy new file mode 100644 index 0000000..c85723b --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/alloy-host.alloy @@ -0,0 +1,66 @@ +// Grafana Alloy v1.21 host agent: node metrics -> Prometheus remote write, log files -> Loki. +// Components and arguments checked against the Alloy v1.21.3 reference. +// Validate: alloy validate /etc/alloy/config.alloy +// Run: alloy run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy + +// ---- metrics --------------------------------------------------------------- +prometheus.exporter.unix "host" { + // Built-in node_exporter collectors; no separate node_exporter process needed. + include_exporter_metrics = true +} + +discovery.relabel "host" { + targets = prometheus.exporter.unix.host.targets + + rule { + target_label = "instance" + replacement = constants.hostname + } +} + +prometheus.scrape "host" { + targets = discovery.relabel.host.output + forward_to = [prometheus.remote_write.default.receiver] + scrape_interval = "30s" + job_name = "node" +} + +prometheus.remote_write "default" { + external_labels = { + cluster = "homelab", + } + + endpoint { + // Prometheus needs --web.enable-remote-write-receiver for this URL; Mimir uses /api/v1/push. + url = "http://prometheus:9090/api/v1/write" + } +} + +// ---- logs ------------------------------------------------------------------ +local.file_match "varlog" { + path_targets = [ + {"__path__" = "/var/log/*.log", "job" = "varlog", "host" = constants.hostname}, + ] + sync_period = "10s" +} + +loki.source.file "varlog" { + targets = local.file_match.varlog.targets + forward_to = [loki.process.drop_noise.receiver] +} + +loki.process "drop_noise" { + // Drop debug-level lines at the edge; keep the label set small (job, host). + stage.drop { + expression = "(?i)level=debug" + drop_counter_reason = "debug_line" + } + + forward_to = [loki.write.default.receiver] +} + +loki.write "default" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/alloy-k8s-logs.alloy b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/alloy-k8s-logs.alloy new file mode 100644 index 0000000..82a05d2 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/alloy-k8s-logs.alloy @@ -0,0 +1,54 @@ +// Alloy v1.21 on Kubernetes: pod logs via the Kubernetes API -> Loki (no node file mounts needed). +// Needs RBAC to list/watch pods and read pods/log (see the Alloy Helm chart or loki.source.kubernetes docs). +// Validate: alloy validate config.alloy +discovery.kubernetes "pods" { + role = "pod" +} + +discovery.relabel "pod_logs" { + targets = discovery.kubernetes.pods.targets + + // Keep the stream label set low-cardinality: namespace, app, container. + // pod name is NOT promoted to a label (it churns); filter on it with a line filter or structured metadata. + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + target_label = "app" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + // Example exclusion: do not collect logs of one namespace (adapt or remove). + rule { + source_labels = ["__meta_kubernetes_namespace"] + regex = "kube-system" + action = "drop" + } +} + +loki.source.kubernetes "pods" { + targets = discovery.relabel.pod_logs.output + forward_to = [loki.process.pods.receiver] +} + +loki.process "pods" { + // JSON application logs: pull `level` out and keep it as structured metadata, not as a stream label. + stage.json { + expressions = { level = "level" } + } + stage.structured_metadata { + values = { level = "" } + } + + forward_to = [loki.write.default.receiver] +} + +loki.write "default" { + endpoint { + url = "http://loki.monitoring.svc:3100/loki/api/v1/push" + } +} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/alloy-otlp.alloy b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/alloy-otlp.alloy new file mode 100644 index 0000000..3ca1301 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/alloy-otlp.alloy @@ -0,0 +1,63 @@ +// Alloy v1.21 as an OTLP gateway: apps send OTLP; metrics -> Prometheus remote write, +// logs -> Loki, traces -> an OTLP/HTTP backend (Tempo). Components checked in the Alloy v1.21.3 reference. +// Validate: alloy validate config.alloy +otelcol.receiver.otlp "default" { + grpc {} // :4317 + http {} // :4318 + + output { + metrics = [otelcol.processor.memory_limiter.default.input] + logs = [otelcol.processor.memory_limiter.default.input] + traces = [otelcol.processor.memory_limiter.default.input] + } +} + +// First in the chain: refuses data instead of letting the process be OOM-killed. +otelcol.processor.memory_limiter "default" { + check_interval = "1s" + limit_percentage = 80 + + output { + metrics = [otelcol.processor.batch.default.input] + logs = [otelcol.processor.batch.default.input] + traces = [otelcol.processor.batch.default.input] + } +} + +// Batch after anything that drops data (sampling), before exporters. +otelcol.processor.batch "default" { + output { + metrics = [otelcol.exporter.prometheus.default.input] + logs = [otelcol.exporter.loki.default.input] + traces = [otelcol.exporter.otlphttp.tempo.input] + } +} + +otelcol.exporter.prometheus "default" { + forward_to = [prometheus.remote_write.default.receiver] +} + +prometheus.remote_write "default" { + endpoint { + url = "http://prometheus:9090/api/v1/write" + } +} + +otelcol.exporter.loki "default" { + forward_to = [loki.write.default.receiver] +} + +loki.write "default" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} + +otelcol.exporter.otlphttp "tempo" { + client { + endpoint = "http://tempo:4318" + tls { + insecure = true // lab only: use TLS between networks + } + } +} diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/blackbox.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/blackbox.yml new file mode 100644 index 0000000..f9f4635 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/blackbox.yml @@ -0,0 +1,13 @@ +# blackbox_exporter modules (fields checked in CONFIGURATION.md of v0.29.1). +# Try a module by hand: curl "http://blackbox:9115/probe?target=https://example.com&module=http_2xx&debug=true" +# Probe timeout must stay below the scrape_timeout of the Prometheus job that calls it. +modules: + http_2xx: + prober: http + timeout: 5s + http: + method: GET # valid_status_codes defaults to 2xx + preferred_ip_protocol: ip4 # the exporter default is ip6 + tcp_connect: + prober: tcp + timeout: 5s diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/docker-compose-monitoring.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/docker-compose-monitoring.yaml new file mode 100644 index 0000000..a5c3954 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/docker-compose-monitoring.yaml @@ -0,0 +1,98 @@ +# Single-host monitoring stack for a lab or small server (Compose file, no "version:" key). +# Put next to this file: prometheus.yml, rules-alerts.yml, rules-recording.yml, alertmanager.yml, blackbox.yml, +# loki-config.yaml, alloy-host.alloy, grafana-datasources.yaml, grafana-dashboards-provider.yaml, +# file-sd-nodes.yml (rename as shown below) and a .env with GRAFANA_ADMIN_PASSWORD (never commit it). +# Validate: docker compose -f docker-compose-monitoring.yaml config +# Alloy (alloy-host.alloy) already ships host metrics through remote write, so this stack does NOT run +# node_exporter: do not add both for the same host or every host series is duplicated. +# grafana-datasources.yaml also declares Tempo, which this stack does not run: delete that block or add Tempo. +# Ports bind to 127.0.0.1: put a reverse proxy with TLS and auth in front for remote access. +services: + prometheus: + image: prom/prometheus:v3.16.0 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=30d + - --storage.tsdb.retention.size=20GB + - --web.enable-lifecycle # allows POST /-/reload; the admin API stays OFF + - --web.enable-remote-write-receiver # receives Alloy remote write + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./rules-alerts.yml:/etc/prometheus/rules/rules-alerts.yml:ro + - ./rules-recording.yml:/etc/prometheus/rules/rules-recording.yml:ro + - ./file-sd-nodes.yml:/etc/prometheus/targets/node-example.yml:ro + - prometheus-data:/prometheus + ports: ["127.0.0.1:9090:9090"] + restart: unless-stopped + + alertmanager: + image: prom/alertmanager:v0.35.0 + command: + - --config.file=/etc/alertmanager/alertmanager.yml + - --storage.path=/alertmanager + volumes: + - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - ./secrets:/etc/alertmanager/secrets:ro # slack_api_url, pagerduty_routing_key, ... (files, mode 0400) + - alertmanager-data:/alertmanager # silences and notification log survive restarts + ports: ["127.0.0.1:9093:9093"] + restart: unless-stopped + + blackbox: + image: prom/blackbox-exporter:v0.29.1 + command: ["--config.file=/config/blackbox.yml"] + volumes: + - ./blackbox.yml:/config/blackbox.yml:ro + restart: unless-stopped + + cadvisor: + image: ghcr.io/google/cadvisor:v0.60.6 + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + restart: unless-stopped + + loki: + image: grafana/loki:3.7.9 + command: ["-config.file=/etc/loki/loki-config.yaml"] + volumes: + - ./loki-config.yaml:/etc/loki/loki-config.yaml:ro + - loki-data:/loki + ports: ["127.0.0.1:3100:3100"] + restart: unless-stopped + + alloy: + image: grafana/alloy:v1.21.3 + command: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /etc/alloy/config.alloy + volumes: + - ./alloy-host.alloy:/etc/alloy/config.alloy:ro + - /var/log:/var/log:ro + - alloy-data:/var/lib/alloy/data + restart: unless-stopped + + grafana: + image: grafana/grafana:13.3.0 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env} + volumes: + - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:ro + - ./grafana-dashboards-provider.yaml:/etc/grafana/provisioning/dashboards/provider.yaml:ro + - ./dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + ports: ["127.0.0.1:3000:3000"] + depends_on: [prometheus, loki] + restart: unless-stopped + +volumes: + prometheus-data: + alertmanager-data: + loki-data: + alloy-data: + grafana-data: diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/file-sd-nodes.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/file-sd-nodes.yml new file mode 100644 index 0000000..96f271c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/file-sd-nodes.yml @@ -0,0 +1,13 @@ +# File-based service discovery target file, referenced by file_sd_configs in prometheus.yml. +# Format: list of static configs. Edits are detected by disk watch, no Prometheus reload needed. +- targets: + - "web-01.example.com:9100" + - "web-02.example.com:9100" + labels: + env: prod + role: web +- targets: + - "db-01.example.com:9100" + labels: + env: prod + role: db diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/grafana-alerting-routing.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/grafana-alerting-routing.yaml new file mode 100644 index 0000000..4ac82af --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/grafana-alerting-routing.yaml @@ -0,0 +1,22 @@ +# Grafana-managed alerting: contact point + notification policy as code +# (mount at /etc/grafana/provisioning/alerting/; format from the Grafana 13.3 file-provisioning page). +# Use this only for Grafana-managed alerts. Metric alerts belong in Prometheus rules + Alertmanager. +apiVersion: 1 + +contactPoints: + - orgId: 1 + name: forward-to-alertmanager + receivers: + - uid: fwd-am + type: prometheus-alertmanager + disableResolveMessage: false + settings: + url: http://alertmanager:9093 + +policies: + - orgId: 1 + receiver: forward-to-alertmanager + group_by: [grafana_folder, alertname] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/grafana-dashboards-provider.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/grafana-dashboards-provider.yaml new file mode 100644 index 0000000..ff5c9c4 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/grafana-dashboards-provider.yaml @@ -0,0 +1,16 @@ +# Grafana dashboard provider, mount at /etc/grafana/provisioning/dashboards/ (Grafana 13.3 docs). +# Dashboard JSON files live under options.path. Export from the UI with "Export for sharing externally", +# strip the "id" field, keep a stable "uid", and review the JSON diff in Git like any other code. +apiVersion: 1 + +providers: + - name: git-managed + orgId: 1 + folder: "" + type: file + disableDeletion: true # removing a file does not delete the dashboard from the database + updateIntervalSeconds: 30 + allowUiUpdates: false # UI edits are not saved over the file: change Git, not the UI + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: true diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/grafana-datasources.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/grafana-datasources.yaml new file mode 100644 index 0000000..267ee92 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/grafana-datasources.yaml @@ -0,0 +1,67 @@ +# Grafana datasource provisioning, mount at /etc/grafana/provisioning/datasources/ (Grafana 13.3 docs). +# Secrets: use ${ENV_VAR} interpolation or secureJsonData; escape a literal dollar as $$. +apiVersion: 1 + +# prune: true # uncomment to delete provisioned datasources that disappear from the files + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + httpMethod: POST + timeInterval: 30s # must equal the Prometheus scrape_interval + manageAlerts: true + prometheusType: Prometheus + exemplarTraceIdDestinations: + - datasourceUid: tempo + name: trace_id + + - name: Loki + type: loki + uid: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + maxLines: 1000 + derivedFields: + - datasourceUid: tempo + matcherRegex: "trace_id=(\\w+)" + name: TraceID + url: "$${__value.raw}" + urlDisplayLabel: View trace + + - name: Tempo + type: tempo + uid: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: "-1h" + spanEndTimeShift: "1h" + tags: [{ key: "service.name", value: "service_name" }] + filterByTraceID: true + tracesToMetrics: + datasourceUid: prometheus + tags: [{ key: "service.name", value: "service" }] + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true + + - name: Alertmanager + type: alertmanager + uid: alertmanager + access: proxy + url: http://alertmanager:9093 + editable: false + jsonData: + implementation: prometheus diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/kube-prometheus-stack-values.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/kube-prometheus-stack-values.yaml new file mode 100644 index 0000000..f50e967 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/kube-prometheus-stack-values.yaml @@ -0,0 +1,55 @@ +# Helm values for prometheus-community/kube-prometheus-stack (chart main branch 93.1.1, appVersion v0.95.1). +# Every key below exists in the chart's values.yaml read on 2026-11-10; re-diff against the chart version you install: +# helm show values prometheus-community/kube-prometheus-stack --version > upstream-values.yaml +# Dry run: helm upgrade --install kps prometheus-community/kube-prometheus-stack -n monitoring \ +# --version -f kube-prometheus-stack-values.yaml --dry-run=server +# The real install/upgrade is a proposal (see observability-safe-ops.md). +crds: + enabled: true # chart installs the CRDs; Helm never upgrades CRDs by itself + upgradeJob: + enabled: false # preview feature in the chart: read its comments before enabling + +defaultRules: + create: true # the chart's built-in PrometheusRules; tune rule groups, do not delete alerts blindly + +prometheus: + prometheusSpec: + retention: 15d + retentionSize: "40GB" # keep it ~10% under the PVC size so the volume never fills + # Pick up ServiceMonitor/PodMonitor/PrometheusRule/Probe objects from ALL namespaces and with ANY labels. + # With the default (true) only objects labeled release= are selected. + serviceMonitorSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false + ruleSelectorNilUsesHelmValues: false + probeSelectorNilUsesHelmValues: false + scrapeConfigSelectorNilUsesHelmValues: false + walCompression: true + # The chart default is an emptyDir: metrics are LOST when the pod is rescheduled. Always set a PVC. + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: fast-ssd # your StorageClass + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi + +alertmanager: + alertmanagerSpec: + retention: 120h # silences and notification log retention + storage: # default is emptyDir: silences vanish on restart without a PVC + volumeClaimTemplate: + spec: + storageClassName: fast-ssd + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 2Gi + # Provide the real routing tree via a Secret or the alertmanager.config key + # (start from alertmanager.yml in this folder); never put receiver secrets in this file. + +grafana: + enabled: true + sidecar: + dashboards: + enabled: true # ConfigMaps labeled grafana_dashboard: "1" become dashboards (dashboards as code) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/loki-config.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/loki-config.yaml new file mode 100644 index 0000000..c88e84e --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/loki-config.yaml @@ -0,0 +1,52 @@ +# Loki 3.7 single-binary, filesystem storage. Derived from cmd/loki/loki-local-config.yaml (v3.7.9) +# plus the compactor retention settings from operations/storage/retention.md. +# Single node only: for object storage and scale-out follow the Loki "install" docs. +# Run: loki -config.file=/etc/loki/loki-config.yaml +# No offline linter was verified for Loki: check by starting it in a scratch container and reading /ready. +auth_enabled: false # single tenant; with true every request needs X-Scope-OrgID + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: info + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h # retention requires a 24h index period + +limits_config: + retention_period: 744h # 31 days; not retroactive for logs already ingested + # retention_stream: # per-stream override, period must be >= 24h + # - selector: '{namespace="dev"}' + # priority: 1 + # period: 72h + +compactor: + working_directory: /loki/retention + retention_enabled: true # without it the compactor only compacts + delete_request_store: filesystem # required when retention is enabled + retention_delete_delay: 2h # window to notice a retention mistake before chunks are swept + +ruler: + alertmanager_url: http://alertmanager:9093 + +analytics: + reporting_enabled: false diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/otel-collector.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/otel-collector.yaml new file mode 100644 index 0000000..97df7d4 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/otel-collector.yaml @@ -0,0 +1,71 @@ +# OpenTelemetry Collector (contrib distribution, release v0.165.0) gateway pipeline. +# Validate: otelcol-contrib validate --config=otel-collector.yaml (also: otelcol-contrib components) +# Component type names follow the renames in collector v0.144.0: `otlp` -> `otlp_grpc`, +# `otlphttp` -> `otlp_http` (the old names still work as deprecated aliases). +# +# Backends: +# metrics -> Prometheus OTLP receiver (start Prometheus with --web.enable-otlp-receiver; +# it expects CUMULATIVE metrics unless the delta feature flags are enabled) +# logs -> Loki native OTLP ingestion at /otlp +# traces -> Tempo OTLP/HTTP on 4318 +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 # the Collector default is localhost: open it on purpose, behind a firewall + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: # first processor in every pipeline + check_interval: 1s + limit_percentage: 80 + spike_limit_percentage: 20 + batch: # after any processor that drops data (sampling) + timeout: 5s + send_batch_size: 8192 + # Tail sampling keeps all errors and slow traces, 10% of the rest. It is STATEFUL: every span of a trace + # must reach the same Collector instance (use a load-balancing exporter tier when scaling out). + tail_sampling: + decision_wait: 10s + num_traces: 50000 + policies: + - name: errors + type: status_code + status_code: { status_codes: [ERROR] } + - name: slow + type: latency + latency: { threshold_ms: 1000 } + - name: baseline + type: probabilistic + probabilistic: { sampling_percentage: 10 } + +exporters: + otlp_http/prometheus: + metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics + otlp_http/loki: + endpoint: http://loki:3100/otlp + otlp_http/tempo: + endpoint: http://tempo:4318 + tls: + insecure: true # lab only + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +service: + extensions: [health_check] + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_http/prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_http/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, tail_sampling, batch] + exporters: [otlp_http/tempo] diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/prometheus.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/prometheus.yml new file mode 100644 index 0000000..ff11c91 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/prometheus.yml @@ -0,0 +1,97 @@ +# Prometheus 3.x server configuration (verified keys: configuration.md of v3.16.0). +# Validate: promtool check config prometheus.yml +# Reload: SIGHUP, or POST /-/reload when started with --web.enable-lifecycle +# (a production reload is a proposal: see observability-safe-ops.md). +global: + scrape_interval: 30s # default 1m + scrape_timeout: 10s # must not exceed scrape_interval + evaluation_interval: 30s # default 1m + external_labels: # identify this server on alerts and remote write + cluster: homelab + replica: prom-a # distinct per HA replica; drop it before dedup in Thanos/Mimir + # Guardrails against cardinality blow-ups: a target over the limit FAILS its scrape. + sample_limit: 50000 + label_limit: 64 + # scrape_native_histograms: true # opt in only after reading promql-essentials.md + +rule_files: + - /etc/prometheus/rules/*.yml + +alerting: + alertmanagers: + - static_configs: + - targets: ["alertmanager:9093"] + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + + - job_name: alertmanager + static_configs: + - targets: ["alertmanager:9093"] + + # Hosts: file-based service discovery. Each file holds a list of + # {targets: [...], labels: {...}}; edits are picked up without a reload. + - job_name: node + file_sd_configs: + - files: + - /etc/prometheus/targets/node-*.yml + refresh_interval: 5m + relabel_configs: + - source_labels: [__address__] + regex: "([^:]+)(:[0-9]+)?" + target_label: instance + replacement: "$1" + + - job_name: cadvisor + static_configs: + - targets: ["cadvisor:8080"] + metric_relabel_configs: + # Drop series nobody queries before they reach the TSDB. + - source_labels: [__name__] + regex: "container_(tasks_state|memory_failures_total)" + action: drop + + # Blackbox probes (multi-target pattern from the blackbox_exporter README). + - job_name: blackbox-http + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + - targets: + - https://example.com + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox:9115 # the exporter's own host:port + + - job_name: blackbox-exporter + static_configs: + - targets: ["blackbox:9115"] + + - job_name: loki + static_configs: + - targets: ["loki:3100"] + + - job_name: alloy + static_configs: + - targets: ["alloy:12345"] + +# Long-term storage (uncomment and fill in; see long-term-storage.md). +# remote_write: +# - url: https://mimir.example.com/api/v1/push +# name: mimir +# queue_config: +# max_shards: 50 +# capacity: 10000 +# write_relabel_configs: +# - source_labels: [__name__] +# regex: "go_.*" +# action: drop + +# Retention is a COMMAND-LINE FLAG, not part of this file: +# --storage.tsdb.retention.time=30d --storage.tsdb.retention.size=40GB diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/prometheusrule.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/prometheusrule.yaml new file mode 100644 index 0000000..e7cab1e --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/prometheusrule.yaml @@ -0,0 +1,31 @@ +# Prometheus Operator PrometheusRule (apiVersion monitoring.coreos.com/v1, operator v0.95.1). +# spec.groups[] has the same shape as a Prometheus rule file: name, interval, limit, query_offset, rules. +# promtool does not read this wrapper: copy spec.groups into a plain rule file to lint it. +# Server check: kubectl --context CTX apply --dry-run=server -f prometheusrule.yaml +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: web-slo-symptoms + namespace: shop + labels: + release: kube-prometheus-stack # must match the Prometheus ruleSelector +spec: + groups: + - name: web-requests + interval: 30s + rules: + - record: job:http_requests:rate5m + expr: sum by (job) (rate(http_requests_total{job="web"}[5m])) + - record: job:http_requests_errors:ratio_rate5m + expr: | + sum by (job) (rate(http_requests_total{job="web", code=~"5.."}[5m])) + / + sum by (job) (rate(http_requests_total{job="web"}[5m])) + - alert: WebHighErrorRatio + expr: job:http_requests_errors:ratio_rate5m{job="web"} > 0.05 + for: 10m + labels: + severity: page + annotations: + summary: "web returns 5xx for a ratio {{ $value }} of requests" + runbook_url: "https://runbooks.example.com/WebHighErrorRatio" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/rules-alerts.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/rules-alerts.yml new file mode 100644 index 0000000..8e3def4 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/rules-alerts.yml @@ -0,0 +1,86 @@ +# Alerting rules file for Prometheus rule_files (syntax: alerting_rules page, v3.16.0). +# Validate: promtool check rules rules-alerts.yml +# Style: alert on SYMPTOMS first, causes as ticket-level; every alert carries a runbook_url. +# Replace https://runbooks.example.com/ with your runbook base before use. +groups: + - name: monitoring-meta + rules: + - alert: Watchdog + # Always firing: routed to a dead man's switch that pages when it STOPS arriving. + expr: vector(1) + labels: + severity: none + annotations: + summary: "Alerting pipeline heartbeat" + + - alert: TargetDown + expr: up == 0 + for: 5m + labels: + severity: ticket + annotations: + summary: "Target {{ $labels.instance }} (job {{ $labels.job }}) is down" + description: "Scrape of {{ $labels.instance }} has failed for 5 minutes." + runbook_url: "https://runbooks.example.com/TargetDown" + + - alert: PrometheusRuleEvaluationSkipped + expr: increase(prometheus_rule_group_iterations_missed_total[10m]) > 0 + for: 10m + labels: + severity: ticket + annotations: + summary: "Rule group {{ $labels.rule_group }} skips evaluations on {{ $labels.instance }}" + runbook_url: "https://runbooks.example.com/PrometheusRuleEvaluationSkipped" + + - alert: AlertmanagerNotificationsFailing + expr: rate(alertmanager_notifications_failed_total[10m]) > 0 + for: 10m + labels: + severity: page + annotations: + summary: "Alertmanager {{ $labels.instance }} fails to deliver to {{ $labels.integration }}" + runbook_url: "https://runbooks.example.com/AlertmanagerNotificationsFailing" + + - name: hosts + rules: + - alert: HostFilesystemFillingUp + # Cause-based but actionable: predicts exhaustion 24h ahead. + expr: | + predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"}[6h], 24 * 3600) < 0 + and node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes < 0.4 + for: 1h + labels: + severity: ticket + annotations: + summary: "{{ $labels.mountpoint }} on {{ $labels.instance }} fills within 24h" + runbook_url: "https://runbooks.example.com/HostFilesystemFillingUp" + + - alert: HostMemoryExhausted + expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.05 + for: 10m + labels: + severity: ticket + annotations: + summary: "{{ $labels.instance }} has under 5% memory available" + runbook_url: "https://runbooks.example.com/HostMemoryExhausted" + + - name: probes + rules: + - alert: EndpointDown + # Symptom: the user-facing check fails from the outside. + expr: probe_success{job="blackbox-http"} == 0 + for: 3m + labels: + severity: page + annotations: + summary: "{{ $labels.instance }} fails its HTTP probe" + runbook_url: "https://runbooks.example.com/EndpointDown" + + - alert: TLSCertificateExpiringSoon + expr: probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time() < 14 * 24 * 3600 + for: 1h + labels: + severity: ticket + annotations: + summary: "Certificate of {{ $labels.instance }} expires in under 14 days" + runbook_url: "https://runbooks.example.com/TLSCertificateExpiringSoon" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/rules-recording.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/rules-recording.yml new file mode 100644 index 0000000..987408b --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/rules-recording.yml @@ -0,0 +1,21 @@ +# Recording rules, naming level:metric:operations (prometheus.io/docs/practices/rules). +# Validate: promtool check rules rules-recording.yml +# Aggregate with sum/rate BEFORE recording; never record an average of averages. +groups: + - name: node-aggregates + interval: 1m + rules: + - record: instance:node_cpu_utilisation:rate5m + expr: 1 - avg without (cpu) (rate(node_cpu_seconds_total{mode="idle"}[5m])) + + - record: instance:node_memory_utilisation:ratio + expr: 1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes + + - record: instance:node_network_receive_bytes:rate5m + expr: sum without (device) (rate(node_network_receive_bytes_total{device!~"lo|veth.*|docker.*|br-.*"}[5m])) + + - name: probe-aggregates + interval: 1m + rules: + - record: job:probe_success:ratio_avg1h + expr: avg by (job) (avg_over_time(probe_success[1h])) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/servicemonitor.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/servicemonitor.yaml new file mode 100644 index 0000000..f7f9672 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/templates/servicemonitor.yaml @@ -0,0 +1,49 @@ +# Prometheus Operator ServiceMonitor + PodMonitor (apiVersion monitoring.coreos.com/v1, operator v0.95.1). +# Check before applying: kubectl --context CTX apply --dry-run=server -f servicemonitor.yaml +# With kube-prometheus-stack the Prometheus only selects objects carrying the Helm release label +# unless prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues is false (see kube-prometheus-stack-values.yaml). +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: web + namespace: shop + labels: + release: kube-prometheus-stack # must match the Prometheus serviceMonitorSelector +spec: + selector: # selects Services, not Pods + matchLabels: + app.kubernetes.io/name: web + namespaceSelector: + matchNames: [shop] + jobLabel: app.kubernetes.io/name + targetLabels: [app.kubernetes.io/version] + sampleLimit: 20000 # a scrape over the limit fails: protects the TSDB + labelLimit: 64 + endpoints: + - port: metrics # NAME of the Service port, not a number + path: /metrics + interval: 30s + scrapeTimeout: 10s + scheme: http + metricRelabelings: + - sourceLabels: [__name__] + regex: "go_gc_.*" + action: drop +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: web-sidecar + namespace: shop + labels: + release: kube-prometheus-stack +spec: + selector: # selects Pods directly (no Service needed) + matchLabels: + app.kubernetes.io/name: web + namespaceSelector: + matchNames: [shop] + podMetricsEndpoints: + - port: sidecar-metrics # NAME of the container port + path: /metrics + interval: 30s diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/tracing-opentelemetry.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/tracing-opentelemetry.md new file mode 100644 index 0000000..96ffe43 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/tracing-opentelemetry.md @@ -0,0 +1,79 @@ +# Tracing and OpenTelemetry + +**Load when:** instrumenting services, building an OpenTelemetry Collector or Alloy pipeline, choosing sampling, wiring Tempo/Jaeger. + +Versions read 2026-11-10: OpenTelemetry Collector releases **v0.165.0** (core v1.71.0; `otelcol`, `otelcol-contrib`, `otelcol-k8s`, `otelcol-otlp` distributions), **Grafana Tempo v3.1.1**, **Jaeger v2.23.0** (latest-tag redirects; dates not read). Templates: [otel-collector.yaml](templates/otel-collector.yaml), [alloy-otlp.alloy](templates/alloy-otlp.alloy). + +## Model + +A trace is the path of one request; spans have a parent, timing and attributes. Apps use an OpenTelemetry SDK (or auto-instrumentation) and export **OTLP** to a local Collector/Alloy, which processes and forwards to the backend. Default OTLP ports: gRPC **4317**, HTTP **4318**. Always put `trace_id` into log lines so Loki `derivedFields` can link logs to traces ([grafana-as-code.md](grafana-as-code.md)). + +## Collector configuration structure + +`receivers` / `processors` / `exporters` / `connectors` / `extensions` are **defined** in their sections and only **used** if listed in `service.pipelines.` (and `service.extensions`). Processor order in the list is the execution order. A component type can be named `type/name` to have several instances. A processor shared by two pipelines is instantiated once per pipeline. + +```bash +otelcol-contrib validate --config=otel-collector.yaml # also: otelcol-contrib components (lists what this build contains) +``` + +Check the distribution before writing config: `otelcol` (core) lacks many contrib components. `tail_sampling`, `probabilistic_sampler` are in `otelcol-contrib` and `otelcol-k8s`. + +### Receiver defaults and exposure + +The Collector binds endpoints to `localhost` by default; the OpenTelemetry examples use `0.0.0.0` for convenience. Opening `0.0.0.0` exposes an unauthenticated ingest port: restrict by network policy/firewall, or add auth and TLS ([security best practices](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md)). + +### Processors that belong in almost every pipeline + +1. `memory_limiter` first (`check_interval`, and `limit`/`limit_percentage`, `spike_limit[_percentage]`): refuses data before the process is OOM-killed. Alloy: `otelcol.processor.memory_limiter` (`check_interval` required). +2. Context-dependent enrichment (e.g. `k8sattributes`, whose profiles signal is `development` stability; check its status for your signals before relying on it). +3. Sampling, then `batch` last (Alloy docs: batch goes after any processor that drops data). + +### Breaking changes you will meet in 0.x releases + +- `otlp` exporter was renamed **`otlp_grpc`** and `otlphttp` renamed **`otlp_http`** in v0.144.0 (old names remain deprecated aliases). Current OpenTelemetry docs use `otlp_grpc`. Loki's page still says `otlphttp`: both work today; prefer the new names in new files. +- The **`queue_batch`** processor (introduced v0.158.0 to replace the legacy `batch` processor; promoted to beta for traces, metrics, logs in v0.165.0) uses the same options as exporter `sending_queue`. Exporters also have `sending_queue::batch`; v0.165.0 warns at startup when a pipeline combines the `batch` processor with exporter batching. Do not enable both. +- `prometheusremotewrite` exporter is now **`prometheus_remote_write`** with the URL under `http.endpoint` (old type still an alias). Non-cumulative monotonic, histogram and summary OTLP metrics are dropped by that exporter. +- Read the changelog of every 0.x you jump over: breaking changes land in minors. Pin the image tag; never `latest` in production. + +## Sending OTLP to the backends + +| Signal | Path | Notes | +|--------|------|-------| +| Metrics | Prometheus OTLP receiver `POST /api/v1/otlp/v1/metrics` (start with `--web.enable-otlp-receiver`) | Cumulative expected; delta needs `--enable-feature=otlp-deltatocumulative` (or `otlp-native-delta-ingestion`, mutually exclusive) | +| Metrics (alt) | `prometheus_remote_write` exporter to Mimir/Prometheus | Cumulative only | +| Logs | Loki native OTLP at `http://loki:3100/otlp` via the HTTP exporter | `allow_structured_metadata: true` required (default on since Loki 3.0) | +| Traces | Tempo OTLP gRPC 4317 / HTTP 4318 | Tempo 3.x config differs from 2.x: read the Tempo configuration page for your version before copying old configs | + +Metrics from traces: the `spanmetrics` connector turns spans into RED metrics (status alpha at last read); Tempo's metrics-generator does the same server-side. + +## Sampling + +- **Head sampling**: decision at the start, e.g. probabilistic 5% by trace ID. Cheap, stateless, but cannot keep "all traces with an error". Collector: `probabilistic_sampler` (`sampling_percentage` required; `mode` `hash_seed` default, `proportional`, `equalizing`; the same `hash_seed` on all collectors of a tier). +- **Tail sampling**: decision after the trace is assembled (errors, latency, attributes, rates). Collector: `tail_sampling` (`decision_wait` default 30s, `num_traces` default 50000, `policies`: `status_code`, `latency`, `probabilistic`, `string_attribute`, `numeric_attribute`, `always_sample`, `and`, `drop`, `not`...). **Stateful**: all spans of one trace must reach the same instance; scaling out requires trace-ID-aware load balancing in front. It breaks context-dependent processors placed after it, so place `k8sattributes` before. Monitor the sampler's own memory. +- Combine: head-sample hard at the edge to protect the pipeline, tail-sample before the backend. +- Sample only when volume justifies it (OpenTelemetry suggests considering it from roughly 1000 traces/s); with little traffic keep 100%. + +## Backends + +- **Tempo** (Grafana): object-storage-backed trace store, queried by trace ID or TraceQL, integrates with Grafana ([grafana-datasources.yaml](templates/grafana-datasources.yaml): `tracesToLogsV2`, `tracesToMetrics`, `serviceMap`). Tempo 3.x config documents distributor, ingest, block-builder, live-store, metrics-generator, query-frontend, querier and backend scheduler/worker sections. No Tempo server template is shipped here: it could not be validated; start from the official docs for your version. +- **Jaeger** v2 (built on the OpenTelemetry Collector architecture; confirm in its docs) is an alternative UI/backend. Both accept OTLP; do not run both unless you need to migrate. + +## Sources (read 2026-11-10) + +- https://raw.githubusercontent.com/open-telemetry/opentelemetry.io/main/content/en/docs/collector/configuration.md +- https://raw.githubusercontent.com/open-telemetry/opentelemetry-collector/main/CHANGELOG.md (v1.71.0/v0.165.0, v0.158.0, v0.144.0 entries) +- https://opentelemetry.io/docs/concepts/sampling/ +- https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor , .../probabilisticsamplerprocessor , .../exporter/prometheusremotewriteexporter READMEs +- https://raw.githubusercontent.com/grafana/alloy/v1.21.3/docs/sources/reference/components/otelcol/ (receiver.otlp, processor.batch, processor.memory_limiter, exporter.otlphttp, exporter.prometheus, exporter.loki) +- https://raw.githubusercontent.com/grafana/loki/v3.7.9/docs/sources/shared/otel.md +- https://raw.githubusercontent.com/prometheus/prometheus/v3.16.0/docs/feature_flags.md and querying/api.md (OTLP receiver) +- https://grafana.com/docs/tempo/latest/configuration/ ; GitHub latest-release redirects for tempo and jaeger + +## Not verified + +- `otelcol-contrib validate` was not run (not installed). `otel-collector.yaml` was only parsed as YAML; the exact YAML key for the `otlp_http` exporter `metrics_endpoint` and `tls.insecure` follows the Alloy equivalents and the collector's HTTP client settings, not a page read for the Collector. +- Jaeger v2 architecture statement is from memory; the Jaeger docs were not fetched. +- Whether `batch` is formally deprecated in favor of `queue_batch` (only "new processor to replace the legacy batchprocessor" and the startup warning were read). +- Tempo 3.x server/Helm install steps and TraceQL syntax were not read. +- `spanmetrics` connector stability was read as `alpha` from a README header that may vary per signal. +- Exact Prometheus flag semantics for delta OTLP ingestion beyond the two flag names. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/troubleshooting-playbook.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/troubleshooting-playbook.md new file mode 100644 index 0000000..3b83c8b --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-observability/references/troubleshooting-playbook.md @@ -0,0 +1,88 @@ +# Observability Troubleshooting Playbook + +**Load when:** a target is down, a metric/log/trace is missing, an alert did not fire or did not notify, or the monitoring stack itself is slow or full. Everything here is read-only; any fix that changes state goes through [observability-safe-ops.md](observability-safe-ops.md). + +Order: **find which hop of the pipeline breaks**, then fix that hop. Pipeline: app -> exporter/SDK -> scrape/collector -> storage -> rule/query -> Alertmanager -> receiver -> human. + +## 1. A scrape target is down (`up == 0`) + +1. Prometheus UI Status -> Targets: read the **error text** of that target. +2. Reproduce from the Prometheus host/pod: `curl -sS -m 10 http://target:port/metrics | head`. Same network path, same TLS/auth. +3. Match the symptom: + +| Error / symptom | Likely cause | Check | +|-----------------|--------------|-------| +| connection refused / no route | wrong port, service not listening, firewall, NetworkPolicy | `ss -ltn` on target; for K8s see `devops-kubernetes` network policy | +| `context deadline exceeded` | scrape slower than `scrape_timeout` (default 10s) | `scrape_duration_seconds{job=...}`; raise timeout or fix the exporter | +| x509 / TLS errors | wrong CA/SAN/expired cert | `openssl s_client`, `tlsConfig` / `tls_config` | +| 401 / 403 | missing token/basic auth | job `authorization` / `basic_auth`; RBAC on K8s | +| scrape fails right after a deploy, `sample_limit` | cardinality jump exceeded `sample_limit` / `label_limit` (whole scrape rejected) | `scrape_samples_post_metric_relabeling`; find the new label | +| target absent from the list | relabel `keep/drop`, SD selector mismatch (K8s: ServiceMonitor selector/`release` label, Service port **name**) | [prometheus-operator-k8s.md](prometheus-operator-k8s.md) | + +4. Blackbox probe failing: `curl "http://blackbox:9115/probe?target=...&module=...&debug=true"`; check `preferred_ip_protocol` (default `ip6`). + +## 2. A metric or series is missing or wrong + +- Query it raw (no functions) over a short range. Empty? Then check: `metric_relabel_configs` drop rules, exporter flags that disable the collector (`--no-collector.*`), cAdvisor `--disable_metrics`, kube-state-metrics allowlists (labels are off by default), native-histogram vs classic naming ([promql-essentials.md](promql-essentials.md)). +- Present but graph blank: `rate` range shorter than two scrapes; wrong `by (..., le)`; Grafana datasource `timeInterval` not matching the scrape interval. +- Gap in a recorded series: the rule group missed iterations (`rule_group_iterations_missed_total`): group too slow for its interval. +- Duplicate or doubled values: two collectors for the same host (node_exporter and Alloy unix exporter), or two Prometheus replicas without a replica label deduplication. + +## 3. An alert did not fire + +1. Run the alert `expr` in the graph at the incident time: does it return data? (`ALERTS{alertname="X"}` shows pending/firing history.) +2. Status -> Rules: is the rule loaded, last evaluation error? Pending longer than `for`? +3. Data missing makes `sum(...) < N` silent: use `absent()`/`up` companions. +4. Rule file not in `rule_files` globs, or a reload that failed (the reload applies only if all files parse): check the reload-success metric and the server log. + +## 4. An alert fires but nobody is notified + +1. Is it in Alertmanager? UI Alerts, or `amtool alert query --alertmanager.url=...`. Not there: Prometheus `alerting.alertmanagers` wrong, `alert_relabel_configs` dropping it, or Alertmanager unreachable. +2. Which route does it match? `amtool config routes test --config.file=... `; fix severity/team labels. +3. Muted? Check silences (`amtool silence query`), inhibition rules (`equal` labels), `mute_time_intervals` / `active_time_intervals` (not inherited by child routes), and `group_wait` / `group_interval` (the first notice is delayed by `group_wait`). +4. Delivery: `alertmanager_notifications_failed_total` by `integration`; receiver secret files readable; webhook endpoint answering; `send_resolved` expectations. +5. Cluster: all Prometheus servers must point to all Alertmanager peers. + +## 5. Prometheus is slow, big or full + +- Series count: `prometheus_tsdb_head_series`; top metrics by series; cardinality fixes first ([prometheus-config.md](prometheus-config.md)). +- Slow dashboards: add recording rules; check `query_log_file` and the global `query_max_*` limits (all default `0` = off). +- Disk: retention time/size flags; WAL and `chunks_head` count in the size; blocks compact up to 10% of retention or 31 days; on-disk size can briefly exceed `retention.size`. NFS/EFS is unsupported: move to local disk. +- Corruption after a crash: back up the data dir, then per the Prometheus docs remove the corrupted block directories or WAL as a last resort (data for that range is lost). +- Memory growth after a deploy: a new high-cardinality label; compare head series before/after. + +## 6. Loki / Alloy + +- Alloy: component health on the UI (`:12345`), `alloy validate` on the config, positions under `--storage.path` persisted. Nothing arrives: check `loki.write` URL path `/loki/api/v1/push`, `local.file_match` globs and file permissions, `loki.process` `stage.drop` too broad. +- Loki rejects data: `loki_discarded_samples_total` by reason (for structured metadata: `structured_metadata_too_large`, `structured_metadata_too_many`); check stream and label limits and the label design in [logs-loki-alloy.md](logs-loki-alloy.md). Retention not applied: `compactor.retention_enabled`, `delete_request_store`, 24h index period. +- Slow queries: always start with a stream selector, filter lines before parsing, narrow the time range. + +## 7. Traces / OpenTelemetry + +- No traces: is the app exporting to 4317/4318 on the right host? Collector `health_check` extension (port 13133 in the template) up? Pipeline lists the receiver and the exporter? (`otelcol-contrib validate`). +- Partial traces after enabling tail sampling: spans of one trace reaching different Collector instances; `decision_wait` shorter than the trace duration. +- Exporter errors after an upgrade: renamed components (`otlp_grpc`, `otlp_http`, `prometheus_remote_write`), deprecated aliases removed ([tracing-opentelemetry.md](tracing-opentelemetry.md)). +- Metrics from OTLP missing in Prometheus: cumulative vs delta temporality; `--web.enable-otlp-receiver` set. + +## 8. Grafana + +- Datasource "Test" fails: URL from the Grafana container's point of view (not localhost), proxy `access`, credentials via `secureJsonData`. +- Provisioned object not appearing: file mounted at the provisioning path, YAML `apiVersion: 1`, Grafana server log at startup; a UI edit on a provisioned dashboard is not saved when `allowUiUpdates: false`. +- Dashboard shows "No data" only on long ranges: retention or `query_max_*` limits; use recording rules. + +## Sources (read 2026-11-10) + +- https://prometheus.io/docs/prometheus/latest/storage/ and alerting_rules/recording_rules pages +- https://raw.githubusercontent.com/prometheus/prometheus/v3.16.0/docs/configuration/configuration.md (limits, `query_*` globals) +- https://raw.githubusercontent.com/prometheus/blackbox_exporter/v0.29.1/README.md , CONFIGURATION.md +- https://raw.githubusercontent.com/prometheus/alertmanager/v0.35.0/docs/configuration.md and notify/metrics.go (`notifications_failed_total` with `integration` label) +- https://raw.githubusercontent.com/grafana/loki/v3.7.9/docs/sources/send-data/otel/_index.md (discard reasons), operations/storage/retention.md +- https://raw.githubusercontent.com/prometheus/prometheus/v3.16.0/rules/group.go (`rule_group_iterations_missed_total`) + +## Not verified + +- Metric names `scrape_duration_seconds`, `scrape_samples_post_metric_relabeling`, `prometheus_tsdb_head_series`, `prometheus_config_last_reload_successful` are standard Prometheus self-metrics not individually confirmed in the sources above (`scrape_samples_post_metric_relabeling` appears in the feature-flags page text). +- `amtool alert query` subcommand was not confirmed against v0.35.0. +- Alloy debugging via the `:12345` UI is the default-port convention from the Docker install page; component-level debug pages were not read. +- Collector self-metrics (`otelcol_*`) names for dropped/refused spans were not checked for span signals. +- Grafana 13.3 UI wording ("Test", "Save") for provisioned datasources is quoted from the Tempo page, not tested. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/SKILL.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/SKILL.md new file mode 100644 index 0000000..a608fb8 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/SKILL.md @@ -0,0 +1,146 @@ +--- +name: devops-openstack +description: Use when operating a private OpenStack cloud - mapping how a request flows through Keystone, Nova, Neutron/OVN, Glance, Cinder and Placement, picking a release and its SLURP upgrade path, choosing and running Kolla-Ansible, OpenStack-Ansible or OpenStack-Helm, sizing a small production control plane, day-2 work with the `openstack` CLI and `clouds.yaml` (projects, quotas, flavors, images, networks, routers, security groups, volumes, instances), series upgrades, or troubleshooting "No valid host", port binding failures, volume attach errors and RabbitMQ / MariaDB-Galera health. Not for Ceph internals (devops-storage-backup), the Terraform/OpenTofu OpenStack provider (devops-iac), generic Ansible (devops-ansible), Kubernetes itself (devops-kubernetes) or public clouds. +versions: + openstack: "2026.2 Hibiscus (current, released 2026-09-30)" + openstack-slurp: "2026.1 Gazpacho" + openstack-next: "2027.1 Indri (SLURP, planned 2027-03-24)" + kolla-ansible: "23.0.0 (2026.2)" + openstack-ansible: "33.1.0 (2026.1); 2026.2 not released on 2026-11-10" + openstack-helm: "2026.2.0" + python-openstackclient: "10.3.1 (2026.2 series)" +user-invocable: true +references: + - references/service-map.md + - references/release-upgrade-slurp.md + - references/deployment-choice-sizing.md + - references/kolla-ansible.md + - references/openstack-ansible.md + - references/openstack-helm.md + - references/networking-neutron-ovn.md + - references/storage-cinder-ceph.md + - references/operations-cli.md + - references/troubleshooting.md + - references/templates/clouds.yaml + - references/templates/kolla-globals.yml + - references/templates/project-bootstrap.md + - references/templates/heat-single-server.yaml + - references/templates/pre-upgrade-checklist.md +related-skills: + - devops-storage-backup + - devops-iac + - devops-ansible + - devops-kubernetes + - devops-k8s-cluster + - devops-linux + - devops-networking + - devops-virtualization + - devops-observability + - devops-databases + - devops-secrets + - devops-sre-ops +--- + + +Covers running a private OpenStack cloud end to end: the service map and request +flow, the release model (two releases a year, SLURP skip-level upgrades), the +choice and use of Kolla-Ansible, OpenStack-Ansible or OpenStack-Helm, small +production control-plane sizing, day-2 operations with the `openstack` CLI, +series upgrades and troubleshooting per deployment tool. +Does not cover Ceph internals (devops-storage-backup), the Terraform/OpenTofu +provider (devops-iac), Ansible in general (devops-ansible), Kubernetes itself +(devops-kubernetes, devops-k8s-cluster), public clouds, or vendor +distributions (Red Hat OpenStack Services on OpenShift, Canonical Sunbeam): +for those, read the vendor's own documentation. + + +# DevOps OpenStack + +## Agent Workflow (MANDATORY) + +Before advising or changing anything, run in parallel (one `Task` call each): + +1. **fuse-ai-pilot:explore-codebase** - find existing `globals.yml`, `passwords.yml`, inventories, `openstack_user_config.yml`, `clouds.yaml`, Helm values overrides +2. **fuse-ai-pilot:research-expert** - re-read https://releases.openstack.org/ and the deployment tool docs for the target series (versions move every six months) +3. **mcp__context7__query-docs** - library `/websites/openstack` for the exact command or config key + +After writing config or templates, run **fuse-ai-pilot:sniper**. + +## Overview + +| Area | What this skill gives | +|------|-----------------------| +| **Service map** | Which service owns what, and the path of `openstack server create` through Keystone → Nova → Placement → Glance/Neutron/Cinder | +| **Releases** | Current, SLURP and next series; which upgrade jumps are tested | +| **Deployment** | Kolla-Ansible vs OpenStack-Ansible vs OpenStack-Helm, plus a 3-controller baseline | +| **Day-2** | `clouds.yaml`, read-only discovery, project bootstrap, quotas, images, networks, volumes, instances | +| **Upgrades** | Backup, prechecks, per-tool upgrade commands, verification | +| **Troubleshooting** | Logs per tool, the four classic failure chains, control-plane health | + +## Critical Rules + +1. **Read-only discovery first on a live cloud**: `openstack service list`, `openstack compute service list`, `openstack network agent list`, `openstack volume service list` - then plan, change, verify ([operations-cli.md](references/operations-cli.md)). +2. **Destructive commands are proposed, never run on your own**: `openstack server delete`, `volume delete`, `network delete`, `router delete`, `project delete`, `project cleanup`, `kolla-ansible destroy`, `kolla-ansible stop`, `nova-manage db purge`. Show the exact effect and the object list, then wait for the user's go. +3. **Upgrade only along a tested path**: adjacent series, or SLURP to SLURP. Never skip from a non-SLURP release ([release-upgrade-slurp.md](references/release-upgrade-slurp.md)). +4. **Back up before any series upgrade or DB-touching action**: deployment config (`/etc/kolla` or `/etc/openstack_deploy`) and the MariaDB data. Nova cannot downgrade its database. +5. **Odd-sized quorum**: MariaDB-Galera and RabbitMQ need 3 (or 5) controllers; 2 gives no HA. +6. **Never invent a key or flag**: every `globals.yml` key and CLI option in this skill was checked against the 2026.2 docs or source; anything else is verified first or reported as unknown. +7. **Secrets stay out of output**: `passwords.yml`, `user_secrets.yml`, `clouds.yaml` passwords and application credential secrets are never printed, committed or pasted (devops-secrets). + +## Reference Guide + +### Concepts + +| Topic | Reference | When to consult | +|-------|-----------|-----------------| +| **Service map** | [service-map.md](references/service-map.md) | Which service does what, request flow, what to enable | +| **Releases & SLURP** | [release-upgrade-slurp.md](references/release-upgrade-slurp.md) | Picking a series, planning an upgrade path | +| **Deployment choice & sizing** | [deployment-choice-sizing.md](references/deployment-choice-sizing.md) | New cloud, choosing a tool, node layout | +| **Kolla-Ansible** | [kolla-ansible.md](references/kolla-ansible.md) | `globals.yml`, deploy, reconfigure, upgrade, MariaDB backup | +| **OpenStack-Ansible** | [openstack-ansible.md](references/openstack-ansible.md) | `openstack_user_config.yml`, playbooks, major upgrades | +| **OpenStack-Helm** | [openstack-helm.md](references/openstack-helm.md) | OpenStack on an existing Kubernetes cluster | +| **Neutron / OVN** | [networking-neutron-ovn.md](references/networking-neutron-ovn.md) | Provider vs tenant networks, OVN, floating IPs | +| **Cinder / Ceph** | [storage-cinder-ceph.md](references/storage-cinder-ceph.md) | Volume backends, external Ceph wiring, RGW as object store | +| **Operations CLI** | [operations-cli.md](references/operations-cli.md) | `clouds.yaml`, discovery, day-2 commands, destructive ops | +| **Troubleshooting** | [troubleshooting.md](references/troubleshooting.md) | Logs, No valid host, port binding, attach, RabbitMQ/Galera | + +### Templates + +| Template | When to use | +|----------|-------------| +| [clouds.yaml](references/templates/clouds.yaml) | Admin + project + application-credential entries | +| [kolla-globals.yml](references/templates/kolla-globals.yml) | Minimal 2026.2 Kolla-Ansible production `globals.yml` (OVN, external Ceph) | +| [project-bootstrap.md](references/templates/project-bootstrap.md) | New project: user, role, quotas, network, router, security group, first server | +| [heat-single-server.yaml](references/templates/heat-single-server.yaml) | Heat stack: network, router, port, server, floating IP, volume | +| [pre-upgrade-checklist.md](references/templates/pre-upgrade-checklist.md) | Gate before any series upgrade | + +## Quick Reference + +```bash +export OS_CLOUD=mycloud-admin # entry in clouds.yaml +openstack token issue -f value -c expires # auth works +openstack service list # catalog +openstack compute service list # nova-* up/enabled per host +openstack network agent list # OVN controller / metadata agents alive +openstack volume service list # cinder-scheduler / cinder-volume up +openstack quota show --usage # quota vs consumption +openstack server event list # what Nova did to an instance +``` + +→ Full sequences in [operations-cli.md](references/operations-cli.md) and [project-bootstrap.md](references/templates/project-bootstrap.md). + +## Best Practices + +### DO +- Pin the deployment tool to the series branch or tag (`stable/2026.2`, OSA tag `33.1.0`) and install `python-openstackclient` with the series upper constraints +- Keep the whole deployment config in git; diff it against the new sample files at each upgrade +- Use application credentials in automation instead of user passwords +- Use OVN (`neutron_plugin_agent: "ovn"` in Kolla, the OSA default `ml2.ovn`) for new clouds +- Schedule MariaDB backups and test a restore before you need one + +### DON'T +- Run `kolla-ansible upgrade` with `--limit` (the docs report known bugs with it) +- Put host-specific variables in Kolla `globals.yml` (highest precedence, applies to every host) +- Upgrade a non-SLURP release by two series at once +- Expect `enable_swift` in Kolla 2026.2 - the key is not in the sample `globals.yml`; use Ceph RGW (`enable_ceph_rgw`) +- Treat `openstack project delete` as cleanup - it removes the project, not its servers, volumes or networks (`project cleanup --dry-run` first) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/deployment-choice-sizing.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/deployment-choice-sizing.md new file mode 100644 index 0000000..9b0ae01 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/deployment-choice-sizing.md @@ -0,0 +1,86 @@ +# Choosing a deployment tool and sizing a small production cloud + +## Why it matters + +The deployment tool decides how you upgrade, where logs live and what the team +must know for the next five years. Switching tools later is a migration +project, not a config change. + +## The three upstream tools + +| | Kolla-Ansible | OpenStack-Ansible (OSA) | OpenStack-Helm (OSH) | +|---|---|---|---| +| Runs services as | Docker or Podman containers (`kolla_container_engine`) built by Kolla | Python venvs on hosts or in LXC containers, systemd units | Pods on an existing Kubernetes cluster, Helm charts | +| Config entry point | `/etc/kolla/globals.yml`, `passwords.yml`, Ansible inventory | `/etc/openstack_deploy/openstack_user_config.yml`, `user_variables.yml`, `user_secrets.yml` | Helm values + `helm osh get-values-overrides` feature sets | +| Host OS (current series) | Debian 13, Rocky Linux 10, Ubuntu 24.04 (2026.2 support matrix) | 2026.1 source install: Ubuntu 24.04 / 26.04, Debian 12 / 13, CentOS Stream 10, Rocky 10 (compatibility matrix) | Ubuntu Resolute images and hosts for 2026.2; Kubernetes `>=1.33,<=1.35` tested | +| Logs | `/var/log/kolla/` (Docker volume `kolla_logs`) | systemd-journald, `journalctl` on the metal host | `kubectl logs` | +| Upgrade | `kolla-ansible upgrade` (one command) | `scripts/run-upgrade.sh` or the manual playbook sequence | `helm upgrade` per chart | +| Current version | 23.0.0 for 2026.2 | 33.1.0 for 2026.1 (2026.2 not released) | 2026.2.0 | +| Kubernetes needed | no | no | yes - you also own the cluster lifecycle | + +Sources: https://docs.openstack.org/kolla-ansible/2026.2/user/support-matrix.html, +https://docs.openstack.org/openstack-ansible/2026.1/admin/upgrades/compatibility-matrix.html, +https://docs.openstack.org/openstack-helm/latest/readme.html. + +## Decision rules + +1. **Small team, first production cloud, wants the current series** → Kolla-Ansible. One CLI, single-command upgrade, upstream production architecture guide. +2. **Team already strong in Ansible, wants services as host packages/venvs, journald logging, can trail the coordinated release by some weeks** → OpenStack-Ansible. +3. **Organisation already runs Kubernetes in production and has people who can debug Helm, CNI and storage under OpenStack** → OpenStack-Helm. Note that the OSH Kubernetes test range (≤ 1.35) lags the newest Kubernetes minors (devops-kubernetes for versions). +4. **A vendor support contract is required** → a vendor distribution; out of scope here, use the vendor docs. +5. Never choose a tool for one feature; check the target service is supported by it (e.g. Kolla 2026.2 has no Swift key, uses Ceph RGW). + +## Node roles (Kolla inventory groups as the vocabulary) + +From https://docs.openstack.org/kolla-ansible/2026.2/admin/production-architecture-guide.html: + +| Role | Runs | Count | +|------|------|-------| +| deployment | the tool's CLI, config repo | 1 (can be a VM) | +| control | APIs, schedulers, MariaDB, RabbitMQ, Memcached | **odd number**, 3 recommended, 5 for two failures | +| network | Neutron/OVN gateway, HAProxy + keepalived (VIP) | often collocated with control in small clouds | +| compute | nova-compute, libvirt, OVN controller | as many as needed | +| storage | cinder-volume (LVM) - not needed with external Ceph | 0+ | +| monitoring | Prometheus / central logging | optional (devops-observability) | + +Quorum: MariaDB-Galera and RabbitMQ need a majority; 2 instances give no HA +(one failure stops the other), 3 tolerate one failure, more than 5 rarely helps. + +## Small production baseline + +- 3 converged control+network nodes behind one internal VIP (`kolla_internal_vip_address`) and optionally an external VIP with TLS (`kolla_enable_tls_external`). +- N compute nodes; plan host capacity for one compute failure (evacuation room). +- External Ceph cluster for Glance, Cinder, Nova ephemeral and RGW (devops-storage-backup owns Ceph). +- Separate networks: management/API (`api_interface`), tunnel (`tunnel_interface`), provider/external without IP (`neutron_external_interface`), storage. +- `/var/lib/docker` (Kolla) on fast disk with room: MariaDB, RabbitMQ and logs live in Docker volumes; a loaded 130-node cluster with central logging and debug produced 30-50 GB of logs per day (Kolla production guide). +- Local container registry for multinode Kolla (recommended by https://docs.openstack.org/kolla-ansible/2026.2/user/multinode.html). + +Evaluation floor (Kolla all-in-one quickstart, not production): 2 NICs, 8 GB +RAM, 40 GB disk (https://docs.openstack.org/kolla-ansible/2026.2/user/quickstart.html). + +## Sizing method (derive, do not guess) + +1. Compute capacity = sum of flavors expected × instance count, divided by the CPU/RAM allocation ratios you set in Nova; add one host of headroom. +2. Control nodes: measure on a staging deploy (`docker stats` / `systemd-cgtop`) under a load test, then add margin; MariaDB and RabbitMQ memory grows with compute count and API rate. +3. Storage: Ceph sizing belongs to devops-storage-backup; give it the volume/image/ephemeral totals. + +## Verify the chosen layout before deploy + +```bash +kolla-ansible prechecks -i multinode # Kolla +openstack-ansible openstack.osa.setup_infrastructure --syntax-check # OSA (run from /opt/openstack-ansible) +helm osh wait-for-pods openstack # OSH, after each chart +``` + +## Not verified + +- No upstream page gives CPU/RAM figures for production control nodes; the sizing method above is a measurement procedure, not a sourced number. +- OSH production readiness statements beyond the "Versions supported" table were not read. + +## Sources + +- https://docs.openstack.org/kolla-ansible/2026.2/admin/production-architecture-guide.html +- https://docs.openstack.org/kolla-ansible/2026.2/user/quickstart.html +- https://docs.openstack.org/openstack-ansible/2026.1/ +- https://docs.openstack.org/openstack-helm/latest/readme.html +- https://docs.openstack.org/2026.2/deploy/ diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/kolla-ansible.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/kolla-ansible.md new file mode 100644 index 0000000..7e573e9 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/kolla-ansible.md @@ -0,0 +1,114 @@ +# Kolla-Ansible (2026.2: 23.0.0) + +## Why it matters + +Kolla-Ansible deploys every OpenStack service as a container from one config +directory (`/etc/kolla`) and one CLI. Its safety depends on three files staying +consistent: `globals.yml`, `passwords.yml` and the inventory. + +## Install the tool for the series + +From https://docs.openstack.org/kolla-ansible/2026.2/user/quickstart.html: + +```bash +python3 -m venv /opt/kolla-venv && source /opt/kolla-venv/bin/activate +pip install -U pip +pip install git+https://opendev.org/openstack/kolla-ansible@stable/2026.2 +sudo mkdir -p /etc/kolla && sudo chown $USER:$USER /etc/kolla +cp -r /opt/kolla-venv/share/kolla-ansible/etc_examples/kolla/* /etc/kolla +cp /opt/kolla-venv/share/kolla-ansible/ansible/inventory/multinode . +kolla-ansible install-deps # Ansible Galaxy requirements +kolla-genpwd # fills /etc/kolla/passwords.yml +``` + +## Configuration rules + +- `globals.yml` keys used in the template were checked against the 2026.2 sample + https://opendev.org/openstack/kolla-ansible/src/branch/stable/2026.2/etc/kolla/globals.yml → + [kolla-globals.yml](templates/kolla-globals.yml). +- `kolla_base_distro`: `debian`, `rocky` or `ubuntu`; match the host OS (Debian 13, Rocky 10, Ubuntu 24.04 supported). +- Leave `openstack_release` unset: the tool deploys the series embedded in it; overriding is for a local registry with its own tag policy. +- aarch64: images are not multi-arch, set `openstack_tag_suffix: "-aarch64"`. +- Extra per-service options go in `/etc/kolla/globals.d/*.yml` (all files are added automatically) and service config overrides in `/etc/kolla/config//...` (e.g. `/etc/kolla/config/neutron/openvswitch_agent.ini`). +- **Variable precedence**: `globals.yml` is passed as extra vars, so it beats everything. Host- or group-specific values (`api_interface` per host, `neutron_external_interface` per group) belong in inventory `host_vars/` / `group_vars/`, never in `globals.yml` (https://docs.openstack.org/kolla-ansible/2026.2/user/multinode.html). +- Several keepalived clusters on one L2 segment: give each a unique `keepalived_virtual_router_id` (0-255). +- RabbitMQ does not work with IP addresses: every controller's `api_interface` IP must resolve to its hostname on all RabbitMQ hosts. + +## Deploy sequence + +```bash +kolla-ansible bootstrap-servers -i multinode # Docker/Podman + host deps +kolla-ansible prechecks -i multinode +kolla-ansible deploy -i multinode +kolla-ansible validate-config -i multinode # only meaningful after the first deploy +kolla-ansible post-deploy -i multinode # writes /etc/kolla/clouds.yaml (+ admin openrc) +pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/2026.2 +``` + +Copy `/etc/kolla/clouds.yaml` to `~/.config/openstack/` or `/etc/openstack`, +or point `OS_CLIENT_CONFIG_FILE` at it. `init-runonce` is a demo script that +creates networks/images; do not run it on a production cloud. + +## Day-2 commands (all take `-i INVENTORY`) + +| Command | Effect | +|---------|--------| +| `reconfigure [-t ]` | regenerate config and restart changed services | +| `genconfig` | generate config without restarting (not applied) | +| `deploy-containers` | check/update containers without generating config | +| `pull` | pull images for the current tag | +| `prune-images --yes-i-really-really-mean-it` | removes orphaned images (guarded) | +| `stop --yes-i-really-really-mean-it` | **stops** containers - outage; guarded flag is required | +| `destroy --yes-i-really-really-mean-it [--include-images]` | **removes containers and volumes = the cloud and its databases**; propose only with a verified backup and the user's go | +| `mariadb-backup [--incremental]` | Mariabackup into the `mariadb_backup` volume (needs `enable_mariabackup: true` + `reconfigure -t mariadb`) | +| `mariadb-recovery` | restart a fully stopped Galera cluster | +| `rabbitmq-reset-state` | wipes RabbitMQ queues/exchanges (used in queue-type migrations) | + +Flags verified in `kolla_ansible/cli/commands.py` on `stable/2026.2`; command +list from https://docs.openstack.org/kolla-ansible/2026.2/user/operating-kolla.html. + +## Series upgrade (e.g. 2026.1 → 2026.2) + +1. Read the 2026.2 release notes (upgrade section) and back up `/etc/kolla` (git) plus a MariaDB backup. +2. `pip3 install --upgrade git+https://opendev.org/openstack/kolla-ansible@stable/2026.2` +3. SLURP jump only: upgrade `ansible-core` to the range the target supports (2026.2 docs example: `'ansible-core>=2.20,<2.21.99'`). +4. `kolla-ansible install-deps`; diff your inventory against the new sample inventories. +5. Merge passwords: + ```bash + cp /etc/kolla/passwords.yml passwords.yml.old + cp /opt/kolla-venv/share/kolla-ansible/etc/kolla/passwords.yml passwords.yml.new + kolla-genpwd -p passwords.yml.new + kolla-mergepwd --old passwords.yml.old --new passwords.yml.new --final /etc/kolla/passwords.yml + ``` +6. `kolla-ansible pull` then `kolla-ansible prechecks`. +7. `kolla-ansible upgrade -i multinode` - recreates containers from new images and runs schema migrations. +8. Verify (below). Never pass `--limit` to the upgrade: the docs report known bugs (e.g. with parts of Nova). + +Within a series: update the package, `pull`, then `deploy` again. + +## MariaDB backup and restore + +- Enable once: `enable_mariabackup: true`, then `kolla-ansible reconfigure -i multinode -t mariadb`. +- Back up: `kolla-ansible mariadb-backup -i multinode` (full) / `--incremental`. Kolla does not schedule it; the docs suggest daily full kept two weeks, hourly incremental kept one day (cron on the deployment host). +- Copy the `mariadb_backup` Docker volume off the node; a backup on the same disk is not a backup. +- Restore is manual (prepare with `mariabackup --prepare` in a `mariadb-server` image container, stop MariaDB with `kolla-ansible stop -t mariadb --yes-i-really-really-mean-it`, copy back, then `kolla-ansible mariadb-recovery -e mariadb_recover_inventory_name=`). Follow https://docs.openstack.org/kolla-ansible/2026.2/admin/mariadb-backup-and-restore.html step by step; the restore deletes `/var/lib/mysql/*` - propose, then the user's go. + +## Verify + +```bash +docker ps --filter health=unhealthy # on each node; podman ps if Podman +kolla-ansible validate-config -i multinode +openstack compute service list && openstack network agent list && openstack volume service list +``` + +## Not verified + +- Whether the `mariadb` client binary exists in every 2026.2 `mariadb-server` image (OSA docs use `mariadb`; older images ship `mysql`). + +## Sources + +- https://docs.openstack.org/kolla-ansible/2026.2/user/quickstart.html +- https://docs.openstack.org/kolla-ansible/2026.2/user/operating-kolla.html +- https://docs.openstack.org/kolla-ansible/2026.2/user/multinode.html +- https://docs.openstack.org/kolla-ansible/2026.2/admin/mariadb-backup-and-restore.html +- https://opendev.org/openstack/kolla-ansible/src/branch/stable/2026.2/kolla_ansible/cli/commands.py diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/networking-neutron-ovn.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/networking-neutron-ovn.md new file mode 100644 index 0000000..e523399 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/networking-neutron-ovn.md @@ -0,0 +1,111 @@ +# Neutron networking with OVN + +## Why it matters + +Most "instance unreachable" tickets are a mismatch between the physical wiring +(provider interface, VLANs, MTU) and what Neutron was told. Decide the network +model before the first deploy; changing the mechanism driver later is a +migration. + +## Concepts + +| Term | Meaning | Who creates it | +|------|---------|----------------| +| Provider network | maps to a physical network (`flat` or `vlan`) via a physnet name; used for external/floating IPs or direct VM attachment | admin only | +| Tenant (self-service) network | overlay network (Geneve with OVN) inside a project | project members | +| Router | connects tenant subnets to an external provider network, does SNAT and floating IPs | project members (gateway on external net) | +| Floating IP | public address from the external network, NATed to a port | project members | +| Security group | stateful per-port firewall; default group allows egress only and ingress from the same group | project members | + +## Mechanism driver choice + +- Kolla-Ansible: `neutron_plugin_agent` accepts `openvswitch` (default) or `ovn`. Set `"ovn"` for new clouds. +- OpenStack-Ansible 2026.1: role default `neutron_plugin_type: "ml2.ovn"`. +- OpenStack-Helm: docs example uses Open vSwitch; OVN needs its own charts/overrides. + +Sources: https://docs.openstack.org/kolla-ansible/2026.2/reference/networking/neutron.html, +https://opendev.org/openstack/openstack-ansible-os_neutron/src/branch/stable/2026.1/defaults/main.yml. + +## Kolla-Ansible keys (all present in the 2026.2 sample `globals.yml`) + +```yaml +neutron_plugin_agent: "ovn" +neutron_external_interface: "eth1" # no IP on it; Kolla plugs it into br-ex +# several provider networks: lists of the same length, zipped together +# neutron_external_interface: "eth1,eth2" +# neutron_bridge_name: "br-ex1,br-ex2" +# neutron_physical_networks: "physnet1,physnet2" # default physnet1..N +enable_neutron_provider_networks: true # external bridge on computes too (VMs directly on VLANs) +neutron_ovn_distributed_fip: true # floating IP traffic leaves from the compute (needs external bridge on computes) +``` + +- Default physnet name is `physnet1` (or `physnet1..N`); your `openstack network create --provider-physical-network` must match it. +- With OVN, Kolla does **not** enable distributed floating IPs by default: FIP and SNAT traffic goes through gateway chassis (network nodes) unless `neutron_ovn_distributed_fip` is set. +- Large clouds: OVN southbound relays are computed as `ceil(ovn-controller hosts / ovn_sb_db_relay_compute_per_relay)` (default 50 per relay); override with `ovn_sb_db_relay_count`. +- Internal DNS on ports: `neutron_dns_integration: "yes"` and `neutron_dns_domain` (must end with a dot, not `openstacklocal`). + +## Create the external network (admin, once) + +```bash +openstack network create --external --provider-network-type flat \ + --provider-physical-network physnet1 public +openstack subnet create --network public --subnet-range 203.0.113.0/24 \ + --gateway 203.0.113.1 --no-dhcp \ + --allocation-pool start=203.0.113.100,end=203.0.113.200 public-subnet +# VLAN instead of flat: --provider-network-type vlan --provider-segment 100 +``` + +## Tenant topology (per project) + +```bash +openstack network create app-net +openstack subnet create --network app-net --subnet-range 10.20.0.0/24 \ + --dns-nameserver 192.0.2.53 app-subnet +openstack router create app-router +openstack router set --external-gateway public app-router +openstack router add subnet app-router app-subnet +openstack security group create app-sg +openstack security group rule create --ingress --protocol tcp --dst-port 22 \ + --remote-ip 198.51.100.0/24 app-sg +openstack floating ip create public +openstack server add floating ip +``` + +All commands and options verified in the python-openstackclient 2026.2 command +reference (https://docs.openstack.org/python-openstackclient/2026.2/cli/commands.html). + +## MTU + +Geneve encapsulation adds a larger header than VXLAN; if the underlay MTU is +1500, tenant MTU drops below it. Raise the underlay MTU (jumbo frames on +`tunnel_interface`) or accept the smaller tenant MTU, and test with a +don't-fragment ping between two instances on different computes +(OVN troubleshooting page, MTU section). + +## Verify + +```bash +openstack network agent list -c "Agent Type" -c Host -c Alive -c State +openstack port list --server --long +openstack router show app-router -c external_gateway_info -c status +``` + +OVN agents (`OVN Controller agent`, `OVN Controller Gateway agent`, +`OVN Metadata agent`) must be `Alive :-)` and `UP` on every host. + +## Pitfalls + +- An IP left on `neutron_external_interface` becomes unusable once Kolla plugs it into the bridge; never use the SSH/management interface as provider interface. +- Duplicated OVN agents after a host upgrade: the OVS `system-id` changed and stale `Chassis` / `Chassis_Private` records remain in the southbound DB. The fix (`ovn-sbctl destroy Chassis ` and `... Chassis_Private `) deletes records: list them first, propose, then the user's go (https://docs.openstack.org/neutron/2026.2/admin/ovn/troubleshooting.html). +- Geneve needs kernel support (`lsmod | grep openvswitch` should list geneve). + +## Not verified + +- Exact `Agent Type` strings per OVN version beyond the troubleshooting page example (`OVN Controller Gateway agent`, `OVN Metadata agent`); `OVN Controller agent` (non-gateway host) is from general knowledge. +- Default security group rules (egress allowed, ingress only from the same group) are general knowledge, not re-read for 2026.2; check with `openstack security group rule list default`. + +## Sources + +- https://docs.openstack.org/kolla-ansible/2026.2/reference/networking/neutron.html +- https://docs.openstack.org/neutron/2026.2/admin/ovn/troubleshooting.html +- https://docs.openstack.org/python-openstackclient/2026.2/cli/command-objects/network.html diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/openstack-ansible.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/openstack-ansible.md new file mode 100644 index 0000000..dc8e7bc --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/openstack-ansible.md @@ -0,0 +1,118 @@ +# OpenStack-Ansible (2026.1: 33.1.0) + +## Why it matters + +OSA installs services from source into venvs, on metal or in LXC containers, +driven by playbooks in the `openstack.osa` collection. It trails the +coordinated release: on 2026-11-10 the newest OSA series is **2026.1 Gazpacho** +(tag 33.1.0); 2026.2 has a deliverable file with no release yet +(https://opendev.org/openstack/releases/src/branch/master/deliverables/hibiscus/openstack-ansible.yaml). + +## Layout + +| Path | Content | +|------|---------| +| `/opt/openstack-ansible` | the OSA git checkout, pinned to a tag | +| `/etc/openstack_deploy/openstack_user_config.yml` | hosts per group, networks (`cidr_networks`, `provider_networks`), VIPs | +| `/etc/openstack_deploy/user_variables.yml` | global and role overrides | +| `/etc/openstack_deploy/user_secrets.yml` | generated secrets - never print | +| `/etc/openstack_deploy/conf.d/*.yml` | per-service host groups | +| `/usr/local/bin/openstack-ansible.rc` | environment for ad-hoc `ansible` runs | + +Group names use underscores since 2026.1 (`galera_all`, `rabbitmq_all`); OSA sets +`ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS=always` and warns on dashes. + +## Deploy sequence + +From https://docs.openstack.org/project-deploy-guide/openstack-ansible/2026.1/ (deployment host, configure, run playbooks pages): + +```bash +git clone -b 33.1.0 https://opendev.org/openstack/openstack-ansible /opt/openstack-ansible +cd /opt/openstack-ansible && scripts/bootstrap-ansible.sh +cp -a /opt/openstack-ansible/etc/openstack_deploy /etc/openstack_deploy +cd /etc/openstack_deploy && cp openstack_user_config.yml.example openstack_user_config.yml +cd /opt/openstack-ansible # edit the files above first +./scripts/pw-token-gen.py --file /etc/openstack_deploy/user_secrets.yml +openstack-ansible openstack.osa.setup_infrastructure --syntax-check +openstack-ansible openstack.osa.setup_hosts +openstack-ansible openstack.osa.setup_infrastructure +openstack-ansible openstack.osa.setup_openstack +``` + +The deploy guide clones 33.0.1; use the latest 2026.1 tag (33.1.0, the tag the +2026.1 major-upgrade guide checks out). Decide `install_method` (source, the +default, or distro packages) in `user_variables.yml` before the first run: the +configure page states OSA cannot convert from one method to the other later. + +## Networking default + +`os_neutron` role default `neutron_plugin_type: "ml2.ovn"` on `stable/2026.1` +(https://opendev.org/openstack/openstack-ansible-os_neutron/src/branch/stable/2026.1/defaults/main.yml): +new OSA clouds are OVN unless you override it. + +## Verify the database cluster after `setup_infrastructure` + +```bash +cd /opt/openstack-ansible +ansible galera_container -m shell \ + -a "mariadb -h localhost -e 'show status like \"%wsrep_cluster_%\";'" +``` + +Every node must report `wsrep_cluster_size` = number of Galera nodes and +`wsrep_cluster_status` = `Primary`. + +## Logs + +Since Train, services log to systemd-journald; container journals are visible +on the host under `/var/log/journal/`, so `journalctl` on the metal host reads +every service. `/openstack/log/` mappings no longer exist. Journals persist +across reboots (`openstack_host_keep_journals: true` by default) +(https://docs.openstack.org/openstack-ansible/2026.1/admin/maintenance-tasks/logging.html). + +```bash +journalctl -u nova-scheduler --since "15 min ago" # unit names follow the service; check with systemctl list-units +``` + +## Major upgrade (2025.2 Flamingo or 2025.1 Epoxy → 2026.1 Gazpacho) + +From https://docs.openstack.org/openstack-ansible/2026.1/admin/upgrades/major-upgrades.html +- the 2026.1 series migrates from the adjacent release or from the previous SLURP: + +1. Preflight health checks; read release notes of the target **and skipped** series. +2. `cd /opt/openstack-ansible && git checkout 33.1.0` +3. Back up: `tar zcf /openstack/backup-openstack-ansible-2025.2.tar.gz /etc/openstack_deploy /etc/ansible/ /usr/local/bin/openstack-ansible.rc` +4. Either `./scripts/run-upgrade.sh`, or the manual steps: + ```bash + unset ANSIBLE_INVENTORY + ./scripts/bootstrap-ansible.sh + openstack-ansible openstack.osa.upgrade.deploy_config_changes + openstack-ansible openstack.osa.setup_hosts --limit '!galera_all:!rabbitmq_all' -e package_state=latest + openstack-ansible openstack.osa.setup_hosts -e 'lxc_container_allow_restarts=false' --limit 'galera_all:rabbitmq_all' + openstack-ansible openstack.osa.setup_infrastructure -e 'galera_upgrade=true' -e 'rabbitmq_upgrade=true' -e package_state=latest + openstack-ansible openstack.osa.tools.galera_cluster_rolling_restart + openstack-ansible openstack.osa.setup_openstack -e package_state=latest + ``` +5. Before the infrastructure step: RabbitMQ must be ≥ 3.13 (minor upgrade documented as `openstack-ansible openstack.osa.rabbitmq_server -e rabbitmq_upgrade=true -e rabbitmq_package_version=3.13.7-1`) and mirrored queues migrated to quorum queues. +6. Magnum: the Cluster API driver is integrated in `os_magnum` from 2026.1; follow the Magnum CAPI migration doc. +7. Ceph: `setup_infrastructure` / `setup_openstack` include Ceph playbooks; `upgrade_ceph_packages: true` upgrades Ceph packages too. Keep the Ceph client version with `ceph_stable_release` if needed (Ceph cluster upgrades: devops-storage-backup). + +`-e package_state=latest` upgrades only the packages the roles manage; it is not a full `apt upgrade` / `dnf upgrade`. + +## Pitfalls + +- Non-trusted certificates for RabbitMQ are not possible with newer amqp versions (upgrade guide warning). +- Running `setup_infrastructure` without the `--limit` exclusions restarts Galera/RabbitMQ containers in an uncontrolled way during upgrades. +- Do not track `master`: the docs warn it is unstable. + +## Not verified + +- The exact systemd unit names per service on a 2026.1 host (check with `systemctl list-units | grep -E 'nova|neutron|cinder'`). +- OSA 2026.2 release date. + +## Sources + +- https://docs.openstack.org/openstack-ansible/2026.1/ +- https://docs.openstack.org/openstack-ansible/2026.1/admin/upgrades/major-upgrades.html +- https://docs.openstack.org/openstack-ansible/2026.1/admin/upgrades/compatibility-matrix.html +- https://docs.openstack.org/project-deploy-guide/openstack-ansible/2026.1/run-playbooks.html +- https://docs.openstack.org/openstack-ansible/2026.1/admin/maintenance-tasks/logging.html diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/openstack-helm.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/openstack-helm.md new file mode 100644 index 0000000..02bf948 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/openstack-helm.md @@ -0,0 +1,112 @@ +# OpenStack-Helm (2026.2.0) + +## Why it matters + +OpenStack-Helm (OSH) runs OpenStack as Helm releases on a Kubernetes cluster you +already operate. You gain Kubernetes rolling updates and scheduling; you also +inherit two control planes to upgrade and debug. Kubernetes itself is out of +OSH scope (devops-kubernetes, devops-k8s-cluster). + +## Tested combinations + +From https://docs.openstack.org/openstack-helm/latest/readme.html (read 2026-11-10): + +| OpenStack | Host OS | Image OS | Kubernetes | +|-----------|---------|----------|------------| +| 2025.2 Flamingo | Ubuntu Noble | Ubuntu Noble | >=1.33, <=1.35 | +| 2026.1 Gazpacho | Ubuntu Noble | Ubuntu Noble | >=1.33, <=1.35 | +| 2026.2 Hibiscus | Ubuntu Resolute | Ubuntu Resolute | >=1.33, <=1.35 | + +Decision rule: run OSH only on a Kubernetes minor inside that range; a cluster on +1.36/1.37 is outside what OSH tests. + +## Prerequisites (OSH install guide) + +- Kubernetes cluster, `kubectl` and `helm` configured. The upstream `deploy-env` Ansible role builds a lab cluster only ("not production-ready"). +- `openstack` namespace. +- External access: OSH charts **no longer ship Ingress templates**; expose services with Gateway API objects passed through `.Values.extraObjects` (examples in `values_overrides//gateway.yaml`). OSH recommends Envoy Gateway + MetalLB. Legacy Ingress still works if you add the objects yourself. +- Ceph deployed and enabled for OSH (storage: devops-storage-backup). +- Helm repo and plugin: + +```bash +helm repo add openstack-helm https://tarballs.opendev.org/openstack/openstack-helm +helm plugin install https://opendev.org/openstack/openstack-helm-plugin +``` + +Source: https://docs.openstack.org/openstack-helm/latest/install/before_starting.html, +https://docs.openstack.org/openstack-helm/latest/install/prerequisites.html. + +## Values overrides by feature set + +```bash +export OPENSTACK_RELEASE=2026.2 +export FEATURES="${OPENSTACK_RELEASE} ubuntu_resolute" # the docs example uses "2026.1 ubuntu_noble" +export OVERRIDES_DIR=$(pwd)/overrides +OVERRIDES_URL=https://opendev.org/openstack/openstack-helm/raw/branch/master/values_overrides +for chart in rabbitmq mariadb memcached openvswitch libvirt keystone heat glance cinder placement nova neutron horizon; do + helm osh get-values-overrides -d -u ${OVERRIDES_URL} -p ${OVERRIDES_DIR} -c ${chart} ${FEATURES} +done +``` + +`helm osh get-values-overrides` looks up `.yaml`, `.yaml` and +`-.yaml` per chart. Check the `ubuntu_resolute` override files exist +for your charts before relying on the 2026.2 feature name (the docs page shows +the 2026.1 example). + +## Deploy order + +Backends first, then Keystone, then the rest; wait for pods between steps: + +```bash +helm upgrade --install rabbitmq openstack-helm/rabbitmq --namespace=openstack \ + $(helm osh get-values-overrides -p ${OVERRIDES_DIR} -c rabbitmq ${FEATURES}) +helm osh wait-for-pods openstack +# repeat for: mariadb, memcached, keystone, heat, glance, cinder, openvswitch, libvirt, +# placement, nova, neutron, horizon (libvirt waits for the Neutron agent pods) +``` + +The docs example sets `--set pod.replicas.server=1` on RabbitMQ and MariaDB: +that is a lab value. Production keeps three replicas for quorum. + +Two override traps: +- Helm merges maps but **replaces lists**: two values files that both define `extraObjects` as a list silently keep only the last one. Use the map form. +- The docs Neutron example uses Open vSwitch (`openvswitch_agent`, `auto_bridge_add`); OVN on OSH needs the OVN chart and overrides - read the Neutron multiple-SDN spec in the OSH docs before choosing. + +## Upgrades + +OSH assumes every upgrade goes through Helm: chart templates, ConfigMaps/Secrets +(hash annotations trigger rolling updates) and image tags. Default rolling +update in charts: `max_unavailable: 1`, `max_surge: 3`, `revision_history: 3` +(https://docs.openstack.org/openstack-helm/latest/devref/upgrades.html). + +Sequence per series: +1. Back up MariaDB (OSH docs: "Backing up a PVC", database troubleshooting pages) and the values files (git). +2. Re-download overrides for the new `OPENSTACK_RELEASE`, diff them. +3. `helm upgrade` backends, then Keystone, Placement before Nova, then the remaining charts; `helm osh wait-for-pods openstack` after each. +4. Watch PodDisruptionBudgets: rolling update values can conflict with them. + +Rollback is `helm rollback ` for templates, but database +schema migrations are not rolled back - the DB backup remains the real rollback. + +## Troubleshooting entry points + +```bash +kubectl -n openstack get pods -o wide | grep -v -E 'Running|Completed' +kubectl -n openstack logs deploy/nova-scheduler --since=15m # names depend on the chart +kubectl -n openstack get jobs | grep -v '1/1' # failed db-sync / bootstrap jobs +helm -n openstack history nova +``` + +## Not verified + +- Existence of `ubuntu_resolute` values override files for every chart (feature name inferred from the 2026.2 Host OS column). +- Exact Deployment names inside each chart (`deploy/nova-scheduler` is an assumption; list with `kubectl get deploy`). +- OVN deployment steps on OSH. + +## Sources + +- https://docs.openstack.org/openstack-helm/latest/ +- https://docs.openstack.org/openstack-helm/latest/install/openstack.html +- https://docs.openstack.org/openstack-helm/latest/install/kubernetes.html +- https://docs.openstack.org/openstack-helm/latest/devref/upgrades.html +- https://github.com/openstack/openstack-helm/tags diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/operations-cli.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/operations-cli.md new file mode 100644 index 0000000..2789441 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/operations-cli.md @@ -0,0 +1,131 @@ +# Day-2 operations with the `openstack` CLI + +## Why it matters + +The `openstack` client (python-openstackclient, 10.3.1 in the 2026.2 series) +is the one interface that works the same on Kolla, OSA and OSH clouds. Every +live change follows: **read-only discovery → plan → change → verify**, and +every destructive command is shown with its effect before the user's go. + +## Install and authenticate + +```bash +python3 -m venv ~/osc && source ~/osc/bin/activate +pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/2026.2 +pip install osc-placement # adds `openstack resource provider ...` (needed for scheduling issues) +export OS_CLOUD=mycloud-admin # or --os-cloud mycloud-admin on each command +openstack token issue -f value -c expires +``` + +`clouds.yaml` lookup order (openstacksdk config docs): current directory, +`~/.config/openstack/` (Linux), `/etc/openstack/`; first file found wins. +`OS_CLIENT_CONFIG_FILE` inserts an explicit path first. Secrets can live in a +separate `secure.yaml` (same search rules, takes precedence), or use +`auth_type: v3applicationcredential` with `application_credential_id` / +`application_credential_secret` → [clouds.yaml](templates/clouds.yaml). +Kolla writes an admin `clouds.yaml` with `kolla-ansible post-deploy`. + +TLS: openstacksdk verifies certificates by default; use `cacert:` for a private +CA. `verify: False` disables verification and wins over `cacert` - do not ship it. + +## Read-only discovery (run before any change) + +```bash +openstack service list +openstack endpoint list --interface public +openstack compute service list # Host, Status (enabled/disabled), State (up/down) +openstack network agent list +openstack volume service list +openstack hypervisor list --long +openstack resource provider list # osc-placement +openstack quota list --compute # admin overview +openstack server list --all-projects --long +openstack versions show +``` + +Record the output in the change log; it is the "before" state you verify against. + +## Projects, users, roles, quotas + +```bash +openstack project create --domain default --description "Team A" team-a +openstack user create --domain default --password-prompt alice +openstack role add --project team-a --user alice member # default roles: admin, member, reader +openstack quota set --instances 20 --cores 40 --ram 81920 \ + --volumes 20 --gigabytes 1000 --floating-ips 5 \ + --networks 5 --routers 2 --ports 100 --secgroups 20 team-a +openstack quota show --usage team-a +``` + +Keystone provides `admin`, `member` and `reader` by default since Rocky +(https://docs.openstack.org/keystone/2026.2/admin/service-api-protection.html). +For automation create an application credential in the target project instead of +sharing a password: `openstack application credential create `. + +## Flavors, images, keypairs + +```bash +openstack flavor create --vcpus 2 --ram 4096 --disk 40 --public m1.medium +openstack flavor create --vcpus 8 --ram 32768 --disk 0 --private --project team-a team-a.large # disk 0 = boot from volume +openstack image create --disk-format raw --container-format bare \ + --file noble.raw --property hw_disk_bus=virtio --public ubuntu-24.04 +openstack keypair create --public-key ~/.ssh/id_ed25519.pub alice-key +``` + +## Networks and instances + +Tenant network, router, security group, floating IP: see +[networking-neutron-ovn.md](networking-neutron-ovn.md). Then: + +```bash +openstack server create --flavor m1.medium --image ubuntu-24.04 \ + --boot-from-volume 40 --network app-net --security-group app-sg \ + --key-name alice-key --wait web01 +openstack server show web01 -c status -c addresses -c fault +openstack server event list web01 +openstack console log show --lines 50 web01 +``` + +`--boot-from-volume ` creates the root volume from the image; it is **not** +deleted with the server unless `--delete-on-termination` is given. + +## Maintenance on a compute host + +```bash +openstack compute service set --disable --disable-reason "maint CHG-123" nova-compute +openstack server list --all-projects --host +openstack server migrate --live-migration --wait # scheduler picks the target +openstack server migration list --server +openstack compute service set --enable nova-compute +``` + +`openstack server evacuate` rebuilds instances of a **dead** host elsewhere; run +it only when the host is confirmed down (fenced), never on a host that may still +run the guests. + +## Destructive commands: exact effect, then the user's go + +| Command | Effect | Check before proposing | +|---------|--------|------------------------| +| `openstack server delete [--wait] ` | deletes the instance and its ephemeral disk; boot volumes survive unless created with delete-on-termination | `server show`, `server volume list` | +| `openstack server delete --force` | force delete (CLI help: "Force delete server(s)") | why the normal delete failed | +| `openstack volume delete [--cascade\|--force] ` | deletes data (and snapshots with `--cascade`) | attachments, snapshots, backups | +| `openstack network delete ` / `router delete ` | fails while ports/interfaces exist; success cuts every attached workload | `port list --network`, `router show` | +| `openstack project cleanup --dry-run --project

` then `--auto-approve` | lists, then deletes **all** resources of the project | dry-run output attached to the proposal | +| `openstack project delete

` | deletes the Keystone project; run `project cleanup` first so resources are not orphaned | dry-run cleanup is empty | +| `nova-manage db archive_deleted_rows --until-complete` / `nova-manage db purge --before ` | archives soft-deleted rows to shadow tables / deletes shadow rows permanently | DB backup taken | + +Never pipe a `list` into a `delete` loop without showing the list first. + +## Not verified + +- That `openstack project delete` leaves Nova/Neutron/Cinder resources orphaned is general knowledge (the existence of `project cleanup` implies it; no 2026.2 page read states it). +- The `hw_disk_bus` image property name was not re-checked in the 2026.2 Glance metadata docs. + +## Sources + +- https://docs.openstack.org/python-openstackclient/2026.2/cli/commands.html +- https://docs.openstack.org/openstacksdk/2026.2/user/config/configuration.html +- https://docs.openstack.org/osc-placement/2026.2/cli/index.html +- https://docs.openstack.org/nova/2026.2/cli/nova-manage.html +- https://docs.openstack.org/kolla-ansible/2026.2/user/quickstart.html diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/release-upgrade-slurp.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/release-upgrade-slurp.md new file mode 100644 index 0000000..a125073 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/release-upgrade-slurp.md @@ -0,0 +1,98 @@ +# Release model, SLURP and upgrade paths + +## Why it matters + +OpenStack ships two coordinated releases a year (spring `YYYY.1`, autumn +`YYYY.2`). Upgrades are only tested along specific paths; leaving them means +running untested schema migrations on production data. + +## Series status on 2026-11-10 + +Source: https://releases.openstack.org/ and +https://raw.githubusercontent.com/openstack/releases/master/data/series_status.yaml + +| Series | Name | SLURP | Status | Released | Next phase | +|--------|------|-------|--------|----------|------------| +| 2027.1 | Indri | yes | development | planned 2027-03-24 | maintained 2027-03-24 | +| **2026.2** | **Hibiscus** | no | **maintained (current)** | 2026-09-30 | end of life, estimated 2028-04-26 | +| 2026.1 | Gazpacho | **yes** | maintained | 2026-04-01 | unmaintained, estimated 2027-10-27 | +| 2025.2 | Flamingo | no | maintained | 2025-10-01 | end of life, estimated 2027-04-28 | +| 2025.1 | Epoxy | yes | unmaintained | 2025-04-02 | end of life TBD | +| 2024.2 | Dalmatian | no | end of life (2026-04-29) | 2024-10-02 | - | + +Deployment tools for those series (deliverable files in +https://opendev.org/openstack/releases/src/branch/master/deliverables, read 2026-11-10): + +| Tool | 2026.2 Hibiscus | 2026.1 Gazpacho | +|------|-----------------|-----------------| +| Kolla-Ansible | 23.0.0 (branch `stable/2026.2`) | 22.2.0 | +| OpenStack-Ansible | **not released yet** (trailing deliverable, no version in the file) | 33.1.0 | +| OpenStack-Helm | tag 2026.2.0 (versions follow the series) | tag 2026.1.0 | +| python-openstackclient | 10.3.1 | 9.0.1 | + +Kolla-Ansible and OpenStack-Ansible are "trailing" deliverables: they ship +weeks after the coordinated release. On 2026-11-10 an OpenStack-Ansible shop +cannot be on 2026.2 yet. + +## SLURP: what is actually guaranteed + +From the TC resolution +https://governance.openstack.org/tc/resolutions/20220210-release-cadence-adjustment.html: + +- Every other release is a **SLURP** (Skip Level Upgrade Release Process) release; currently the `.1` releases (2025.1, 2026.1, 2027.1). +- Upgrades are tested between **adjacent** releases and between **two consecutive SLURP** releases. +- **Non-SLURP → non-SLURP is not tested**; from a non-SLURP release the only path is the next release (a SLURP). +- Jumping more than one SLURP cycle (SLURP A → SLURP E) still needs the intermediate SLURP (C) as a step. +- Deprecations and removals happen only in SLURP releases, so a skipped non-SLURP release never carries a required change - but read its release notes anyway (OpenStack-Ansible asks for it explicitly). +- The community supports the latest SLURP and the one before it. + +## Decision rules + +| You run | Want yearly upgrades | Want every release | +|---------|----------------------|--------------------| +| 2025.1 Epoxy (unmaintained) | → 2026.1 Gazpacho (skip-level) now | → 2025.2 → 2026.1 → 2026.2 | +| 2025.2 Flamingo | → 2026.1 Gazpacho (adjacent), then 2027.1 | → 2026.1 → 2026.2 | +| 2026.1 Gazpacho | stay, → 2027.1 Indri when released (skip-level) | → 2026.2 Hibiscus | +| 2026.2 Hibiscus | only → 2027.1 Indri (adjacent) | → 2027.1 | + +- New cloud today: 2026.2 if Kolla-Ansible or OpenStack-Helm, 2026.1 if OpenStack-Ansible (2026.2 not released). Prefer a SLURP (2026.1) if the team can only absorb one upgrade a year. +- Never stay on an `unmaintained` series without saying so: no more coordinated fixes. + +## Upgrade discipline (any tool) + +1. Read the release notes of the target series **and** of any skipped series (upgrade notes first). +2. Run [pre-upgrade-checklist.md](templates/pre-upgrade-checklist.md): healthy services, backups, config diff. +3. Upgrade the deployment tool first, then infrastructure (MariaDB, RabbitMQ), then OpenStack services. Nova requires Placement to be upgraded before any Nova service (https://docs.openstack.org/nova/2026.2/admin/upgrades.html). +4. Nova readiness check before restarting on new code: `nova-status upgrade check` - exit 0 all good, 1 warning (may be OK), 2 failure (stop), 255 unexpected error (https://docs.openstack.org/nova/2026.2/cli/nova-status.html). +5. Nova supports N and N-2 `nova-compute` coexistence since 2023.1; the conductor refuses to start next to an older compute - finish compute upgrades before the next series. +6. Nova does not support database downgrade: the DB backup is the only rollback. + +Per-tool commands: [kolla-ansible.md](kolla-ansible.md), +[openstack-ansible.md](openstack-ansible.md), [openstack-helm.md](openstack-helm.md). + +## RabbitMQ is the usual skip-level trap + +- Kolla-Ansible: RabbitMQ upgrades are restricted to one minor version at a time unless the path is listed in `rabbitmq_allowed_upgrades` (default allows 3.13 → 4.0/4.1/4.2 and 4.0 → 4.1/4.2); an invalid path fails in the `rabbitmq-version-check` task. A SLURP upgrade may need an intermediate RabbitMQ upgrade first (https://docs.openstack.org/kolla-ansible/2026.2/reference/message-queues/rabbitmq.html). +- OpenStack-Ansible 2026.1: RabbitMQ must be 3.13 or later before the infrastructure upgrade, and mirrored (HA) queues must be migrated to quorum queues first (https://docs.openstack.org/openstack-ansible/2026.1/admin/upgrades/major-upgrades.html). + +## Verify after an upgrade + +```bash +openstack versions show # API versions per service +openstack compute service list # every service up, new version on all hosts +nova-status upgrade check # inside the nova-api container / host +openstack server create ... # canary instance on each AZ, then delete after the user's go +``` + +## Not verified + +- The OpenStack-Ansible 2026.2 release date (not announced in the deliverable file; "I don't know"). +- Exact Kolla-Ansible 23.0.0 publication date (final release commit in openstack/releases on 2026-10-13). + +## Sources + +- https://releases.openstack.org/ +- https://releases.openstack.org/hibiscus/index.html +- https://governance.openstack.org/tc/resolutions/20220210-release-cadence-adjustment.html +- https://opendev.org/openstack/releases (deliverables/hibiscus, deliverables/gazpacho) +- https://github.com/openstack/openstack-helm/tags diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/service-map.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/service-map.md new file mode 100644 index 0000000..5f42e56 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/service-map.md @@ -0,0 +1,98 @@ +# Service map and request flow + +## Why it matters + +Every OpenStack failure surfaces in one service but is caused in another: a +"No valid host" from Nova is often a Placement inventory problem, a server stuck +in `BUILD` is often a Neutron port that never bound. Knowing who calls whom tells +you which log to open first. + +## Services and their 2026.2 Hibiscus versions + +Versions from https://releases.openstack.org/hibiscus/index.html (read 2026-11-10). + +| Service | Project | Owns | 2026.2 version | +|---------|---------|------|----------------| +| Identity | Keystone | users, projects, domains, roles, tokens, service catalog | 30.0.0 | +| Compute | Nova | instances, flavors, scheduling, cells, live migration | 34.0.0 | +| Placement | Placement | resource providers, inventories, allocations, traits | 16.0.0 | +| Networking | Neutron | networks, subnets, ports, routers, floating IPs, security groups | 29.0.0 | +| Image | Glance | images and their backends (file, Ceph RBD, S3) | 33.0.0 | +| Block storage | Cinder | volumes, snapshots, backups, volume types | 29.0.0 | +| Dashboard | Horizon (Skyline is the alternative UI: `skyline-apiserver`, `skyline-console`) | web UI | 27.0.0 | +| Orchestration | Heat | stacks from HOT templates | 27.0.1 | +| Load balancer | Octavia (+ `ovn-octavia-provider` 11.0.0) | load balancers, listeners, pools | 19.0.0 | +| DNS | Designate | zones, recordsets, Neutron DNS integration | 23.0.0 | +| Key manager | Barbican | secrets, certificates, encryption keys | 23.0.0 | +| Containers | Magnum | Kubernetes clusters as a service (Cluster API driver) | 23.0.0 | +| Object storage | Swift, or Ceph RGW exposing a Swift-compatible endpoint | objects, containers | Swift 2.38.3 | + +Infrastructure under all of them: MariaDB (Galera cluster), RabbitMQ (RPC and +notifications, quorum queues by default since RabbitMQ 4.0), Memcached (token and +API caches), HAProxy + keepalived (API VIP) - Kolla-Ansible docs +https://docs.openstack.org/kolla-ansible/2026.2/admin/production-architecture-guide.html +and https://docs.openstack.org/kolla-ansible/2026.2/reference/message-queues/rabbitmq.html. + +## What to enable + +| Need | Minimum set | +|------|-------------| +| VMs with networking | Keystone, Nova, Placement, Neutron, Glance (Kolla enables these plus Heat and Horizon via `enable_openstack_core`) | +| Persistent disks | + Cinder (with a backend: Ceph RBD, LVM, NFS...) | +| Tenant load balancers | + Octavia (OVN provider avoids amphora VMs, L4 only - check the provider limits in the Octavia docs before promising L7) | +| DNS names for ports / floating IPs | + Designate | +| Object storage | Ceph RGW (Kolla: `enable_ceph_rgw`) or Swift (OSA has Swift roles; Kolla 2026.2 has no `enable_swift` key) | +| Kubernetes for tenants | + Magnum, Barbican, Octavia | + +Every extra service adds endpoints, database schemas and upgrade steps; enable +it when a user needs it, not "for later". + +## Request flow: `openstack server create` + +1. **Client → Keystone**: the CLI reads `clouds.yaml`, gets a token and the service catalog. +2. **Client → Nova API**: request with flavor, image, network(s), key, security groups. +3. **Nova API → Glance / Neutron / Cinder**: validates the image, the networks and (boot-from-volume) the volume source. +4. **Nova conductor → Nova scheduler → Placement**: Placement returns allocation candidates matching the flavor's resources and traits. Empty answer → `NoValidHost` ("Got no allocation candidates from the Placement API" in the scheduler log, source `nova/scheduler/manager.py`). +5. **Scheduler filters and weighers** reduce the candidates; all removed → "Filtering removed all hosts for the request" (source `nova/filters.py`), then `NoValidHost`. +6. **Allocation claimed in Placement**, request sent over RabbitMQ to the chosen `nova-compute`. +7. **nova-compute → Neutron**: creates/binds the port to the host (`binding:host_id`); OVN programs the logical port. Bind failure → `PortBindingFailed` ("Binding failed for port ..., please check neutron logs"). +8. **nova-compute → Glance or Cinder**: fetches the image or attaches the root volume (os-brick, iSCSI/RBD). +9. **nova-compute → libvirt**: defines and starts the domain; Neutron reports the port active. +10. **Instance `ACTIVE`**; metadata and DHCP are served by OVN (metadata agent) on the compute host. + +Exception names and messages verified in +https://opendev.org/openstack/nova/src/branch/stable/2026.2/nova/exception.py. + +## Where each step breaks (first log to open) + +| Symptom | Step | First place to look | +|---------|------|---------------------| +| 401 / auth errors | 1 | Keystone log, `clouds.yaml` `auth_url`, project/domain names | +| `ERROR` immediately, fault "No valid host was found" | 4-5 | nova-scheduler log, `openstack resource provider list` | +| Stuck `BUILD` then `ERROR`, fault mentions binding | 7 | neutron-server log, `openstack network agent list` | +| `ERROR` with image/volume fault | 8 | nova-compute log on the host, cinder-volume log | +| `ACTIVE` but unreachable | 9-10 | security groups, router/floating IP, OVN metadata agent | + +→ Detailed chains in [troubleshooting.md](troubleshooting.md). + +## Verify + +```bash +openstack catalog list # every enabled service has public/internal endpoints +openstack endpoint list --interface public +openstack compute service list # nova-scheduler, nova-conductor, nova-compute: State up +openstack network agent list # OVN agents Alive +openstack volume service list # cinder-scheduler, cinder-volume up +``` + +## Not verified + +- Octavia OVN provider feature limits (L4-only claim is general knowledge, read the Octavia provider driver matrix before relying on it). +- Skyline version numbers for 2026.2 (listed on the series page, not extracted). + +## Sources + +- https://releases.openstack.org/hibiscus/index.html +- https://docs.openstack.org/2026.2/admin/ (service admin guides index) +- https://docs.openstack.org/kolla-ansible/2026.2/admin/production-architecture-guide.html +- https://opendev.org/openstack/nova/src/branch/stable/2026.2/nova/scheduler/manager.py diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/storage-cinder-ceph.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/storage-cinder-ceph.md new file mode 100644 index 0000000..4f57d60 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/storage-cinder-ceph.md @@ -0,0 +1,111 @@ +# Block, image and object storage: Cinder, Glance, Ceph RBD and RGW + +## Why it matters + +Where Glance images, Cinder volumes and Nova ephemeral disks live decides +whether live migration works, whether boot-from-volume is a fast clone or a +full copy, and what a storage outage takes down. Ceph internals (pools, CRUSH, +cephadm, upgrades) belong to devops-storage-backup; this file covers the +OpenStack side of the wiring. + +## Backend decision + +| Backend | Use when | Cost | +|---------|----------|------| +| External Ceph RBD for Glance + Cinder + Nova | any production cloud with 3+ storage nodes | Ceph to operate; copy-on-write clones, live migration without block copy | +| LVM + iSCSI (`enable_cinder_backend_lvm`) | lab or single storage node | no HA; the storage node is a SPOF | +| NFS (`enable_cinder_backend_nfs`) | existing NAS | performance and locking depend on the NAS | +| Glance `file` | all-in-one only | Kolla starts **one** `glance_api` container when `file` is the backend (multinode guide) | + +Kolla 2026.2 sample `globals.yml` also lists Pure, Lightbits, VAST, Quobyte +backends and S3 for Glance and Cinder backup (`glance_backend_s3`, +`cinder_backup_driver: s3`). + +## Kolla-Ansible with an external Ceph cluster + +Keys verified in the 2026.2 sample `globals.yml`; file paths from +https://docs.openstack.org/kolla-ansible/2026.2/reference/storage/external-ceph-guide.html: + +```yaml +glance_backend_ceph: "yes" +cinder_backend_ceph: "yes" +nova_backend_ceph: "yes" +# defaults shown - change only if your Ceph users/pools differ +# ceph_glance_user: "glance" ceph_glance_pool_name: "images" +# ceph_cinder_user: "cinder" ceph_cinder_pool_name: "volumes" +# ceph_cinder_backup_user: "cinder-backup" ceph_cinder_backup_pool_name: "backups" +# ceph_nova_user: "{{ ceph_cinder_user }}" ceph_nova_pool_name: "vms" +``` + +Files to place on the deployment host (one minimal `ceph.conf` per service, +e.g. from `ceph config generate-minimal-conf`, leading tabs removed): + +| Service | Files | +|---------|-------| +| Glance | `/etc/kolla/config/glance/ceph.conf`, `/etc/kolla/config/glance/ceph.client.glance.keyring` | +| Cinder | `/etc/kolla/config/cinder/ceph.conf`, `/etc/kolla/config/cinder/cinder-volume/ceph.client.cinder.keyring`, `/etc/kolla/config/cinder/cinder-backup/ceph.client.cinder.keyring` (+ the backup user keyring) | +| Nova | `/etc/kolla/config/nova/ceph.conf`, `/etc/kolla/config/nova/ceph.client.cinder.keyring` | + +Pools and cephx users with the right capabilities are created on the Ceph side +(Ceph docs "Block Devices and OpenStack", https://docs.ceph.com/en/latest/rbd/rbd-openstack/). +If Nova and Cinder use separate keys, set `ceph_nova_user: nova` before an upgrade +(Kolla operating guide note). + +Apply: `kolla-ansible reconfigure -i multinode -t glance,cinder,nova` (or a full +`deploy` on a new cloud). + +## Object storage + +- Kolla 2026.2 has no `enable_swift` key in its sample `globals.yml`. Use Ceph RGW: `enable_ceph_rgw: true` registers Swift-compatible endpoints in Keystone (user `ceph_rgw`, port `ceph_rgw_port` default 6780) and load-balances RGW through HAProxy. +- `ceph_rgw_swift_compatibility` and `ceph_rgw_swift_account_in_url` must match the RGW-side settings (`rgw_swift_account_in_url`); a mismatch gives hard-to-diagnose 403s from Swift clients such as Horizon. +- OpenStack-Ansible documents RGW as a drop-in Swift replacement ("Using Ceph Object Gateway (radosgw) as a drop-in replacement for Swift" in its user guide) and still ships Swift roles. + +## Day-2 commands + +```bash +openstack volume service list # cinder-volume per backend host@backend: up +openstack volume type list --long +openstack volume create --size 20 --type data01 +openstack server add volume data01 # --device is a hint only +openstack volume attachment list +openstack server volume list +openstack image create --disk-format raw --container-format bare \ + --file ubuntu-24.04.raw --private ubuntu-24.04 # raw avoids conversion for RBD clones +``` + +Image format with Ceph: use `raw` images so Glance → Cinder/Nova can clone in +RBD; `qcow2` forces a download and conversion. Upstream Ceph RBD/OpenStack +guide states this; verify against your Ceph release docs. + +## Destructive operations (propose, then the user's go) + +| Command | Effect | +|---------|--------| +| `openstack volume delete ` | deletes the volume and its data; fails if attached or with snapshots | +| `openstack volume delete --cascade ` | also deletes its snapshots | +| `openstack volume delete --force ` | forced removal regardless of state - can leave a dangling attachment on the compute | +| `openstack image delete ` | removes the image; instances booted from RBD clones still depend on the parent in Ceph | + +Before proposing: `openstack volume show -c status -c attachments`, +`openstack volume snapshot list --volume `. + +## Verify + +```bash +openstack volume service list +openstack volume create --size 1 canary && openstack volume show canary -c status # available +# then propose: openstack volume delete canary +``` + +## Not verified + +- The raw-vs-qcow2 cloning statement was not re-read in the current Ceph docs. +- The exact key name for a separate Cinder backup user keyring in Kolla 2026.2 (guide lists the cinder keyring under `cinder-backup/`; read the guide section if `ceph_cinder_backup_user` differs). +- `openstack image delete` behaviour with RBD clone parents is general knowledge. + +## Sources + +- https://docs.openstack.org/kolla-ansible/2026.2/reference/storage/external-ceph-guide.html +- https://opendev.org/openstack/kolla-ansible/src/branch/stable/2026.2/etc/kolla/globals.yml +- https://docs.openstack.org/python-openstackclient/2026.2/cli/command-objects/volume.html +- https://docs.openstack.org/openstack-ansible/2026.1/ diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/clouds.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/clouds.yaml new file mode 100644 index 0000000..81e3528 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/clouds.yaml @@ -0,0 +1,44 @@ +# clouds.yaml - place in ~/.config/openstack/ or /etc/openstack/ (first file found wins), +# or point OS_CLIENT_CONFIG_FILE at it. Select an entry with OS_CLOUD= or --os-cloud . +# Secrets: keep passwords and application credential secrets in secure.yaml (same search +# path, same structure, takes precedence) and never commit either file with values. +# Keys verified in https://docs.openstack.org/openstacksdk/2026.2/user/config/configuration.html +clouds: + # Cloud admin (what `kolla-ansible post-deploy` generates in /etc/kolla/clouds.yaml) + mycloud-admin: + auth: + auth_url: https://api.cloud.example.com:5000 + username: admin + password: CHANGE_ME_IN_SECURE_YAML + project_name: admin + user_domain_name: Default + project_domain_name: Default + region_name: RegionOne + interface: public + identity_api_version: 3 + cacert: /etc/ssl/certs/cloud-ca.pem # private CA; never use verify: False in production + + # A project member using a password + team-a: + auth: + auth_url: https://api.cloud.example.com:5000 + username: alice + project_name: team-a + user_domain_name: Default + project_domain_name: Default + region_name: RegionOne + interface: public + identity_api_version: 3 + cacert: /etc/ssl/certs/cloud-ca.pem + + # Automation: application credential (scoped to one project, revocable, no password) + team-a-ci: + auth_type: v3applicationcredential + auth: + auth_url: https://api.cloud.example.com:5000 + application_credential_id: REPLACE_WITH_ID + application_credential_secret: CHANGE_ME_IN_SECURE_YAML + region_name: RegionOne + interface: public + identity_api_version: 3 + cacert: /etc/ssl/certs/cloud-ca.pem diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/heat-single-server.yaml b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/heat-single-server.yaml new file mode 100644 index 0000000..59957ef --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/heat-single-server.yaml @@ -0,0 +1,125 @@ +heat_template_version: 2021-04-16 # "wallaby" - latest HOT version listed in the 2026.2 HOT spec + +description: > + One private network behind a router on an existing external network, a security + group allowing SSH from one CIDR, one server on a port, a floating IP and a data volume. + Resource types and properties verified in + https://docs.openstack.org/heat/2026.2/template_guide/openstack.html + Validate: openstack orchestration template validate -t heat-single-server.yaml + Preview: openstack stack create --dry-run -t heat-single-server.yaml --parameter key_name=alice-key demo + Create: openstack stack create --wait -t heat-single-server.yaml --parameter key_name=alice-key demo + Delete (deletes every resource below, including the volume data): openstack stack delete demo + +parameters: + key_name: + type: string + description: Existing Nova keypair + constraints: + - custom_constraint: nova.keypair + image: + type: string + default: ubuntu-24.04 + constraints: + - custom_constraint: glance.image + flavor: + type: string + default: m1.medium + constraints: + - custom_constraint: nova.flavor + external_network: + type: string + default: public + constraints: + - custom_constraint: neutron.network + subnet_cidr: + type: string + default: 10.30.0.0/24 + dns_nameserver: + type: string + default: 192.0.2.53 + ssh_cidr: + type: string + description: CIDR allowed to SSH in + default: 198.51.100.0/24 + volume_size: + type: number + default: 20 + +resources: + net: + type: OS::Neutron::Net + + subnet: + type: OS::Neutron::Subnet + properties: + network: { get_resource: net } + cidr: { get_param: subnet_cidr } + ip_version: 4 + dns_nameservers: + - { get_param: dns_nameserver } + + router: + type: OS::Neutron::Router + properties: + external_gateway_info: + network: { get_param: external_network } + + router_interface: + type: OS::Neutron::RouterInterface + properties: + router: { get_resource: router } + subnet: { get_resource: subnet } + + secgroup: + type: OS::Neutron::SecurityGroup + properties: + rules: + - direction: ingress + protocol: tcp + port_range_min: 22 + port_range_max: 22 + remote_ip_prefix: { get_param: ssh_cidr } + - direction: ingress + protocol: icmp + remote_ip_prefix: { get_param: ssh_cidr } + + port: + type: OS::Neutron::Port + properties: + network: { get_resource: net } + fixed_ips: + - subnet: { get_resource: subnet } + security_groups: + - { get_resource: secgroup } + + server: + type: OS::Nova::Server + properties: + image: { get_param: image } + flavor: { get_param: flavor } + key_name: { get_param: key_name } + networks: + - port: { get_resource: port } + + floating_ip: + type: OS::Neutron::FloatingIP + depends_on: router_interface + properties: + floating_network: { get_param: external_network } + port_id: { get_resource: port } + + data_volume: + type: OS::Cinder::Volume + properties: + size: { get_param: volume_size } + + data_attachment: + type: OS::Cinder::VolumeAttachment + properties: + instance_uuid: { get_resource: server } + volume_id: { get_resource: data_volume } + +outputs: + floating_ip: + description: Public address of the server + value: { get_attr: [floating_ip, floating_ip_address] } diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/kolla-globals.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/kolla-globals.yml new file mode 100644 index 0000000..ba76628 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/kolla-globals.yml @@ -0,0 +1,75 @@ +--- +# /etc/kolla/globals.yml - minimal production baseline for Kolla-Ansible 23.x (OpenStack 2026.2 Hibiscus) +# 3 control+network nodes, N computes, OVN, external Ceph, TLS on the external VIP. +# Every key below exists in the 2026.2 sample: +# https://opendev.org/openstack/kolla-ansible/src/branch/stable/2026.2/etc/kolla/globals.yml +# Host- or group-specific values (interface names that differ per host) go in inventory +# host_vars/ or group_vars/, NOT here: globals.yml is passed as extra vars and wins over all. + +# Dummy variable kept from the upstream sample +workaround_ansible_issue_8743: true + +# --- Images --- +kolla_base_distro: "rocky" # debian | rocky | ubuntu - match the host OS +kolla_container_engine: docker # docker | podman +# openstack_release: leave unset - the tool deploys its own series +docker_registry: "registry.internal.example.com:4000" # local registry, recommended for multinode +docker_registry_insecure: false + +# --- VIPs and names --- +kolla_internal_vip_address: "10.10.10.254" # unused IP on the api_interface network +kolla_internal_fqdn: "api-int.cloud.example.com" +kolla_external_vip_address: "203.0.113.10" +kolla_external_fqdn: "api.cloud.example.com" +keepalived_virtual_router_id: "51" # unique per L2 segment (0-255) +openstack_region_name: "RegionOne" + +# --- Interfaces (defaults; override per host in inventory if names differ) --- +network_interface: "bond0" # default for the management-type networks below +api_interface: "bond0.10" # management/API network, internal only +tunnel_interface: "bond0.20" # Geneve overlay; jumbo frames recommended +kolla_external_vip_interface: "bond0.30" # public API network +neutron_external_interface: "bond1" # provider/external network, NO IP address on it + +# --- Networking --- +neutron_plugin_agent: "ovn" # openvswitch (default) | ovn +neutron_ovn_distributed_fip: false # true = FIP traffic leaves from computes (needs external bridge there) +enable_neutron_provider_networks: false # true = VMs attach directly to provider VLANs on computes + +# --- TLS --- +kolla_enable_tls_external: "yes" # needs kolla_external_vip_interface set +kolla_enable_tls_internal: "no" +kolla_copy_ca_into_containers: "yes" # when the certificates come from a private CA +# certificates expected in /etc/kolla/certificates/ (haproxy.pem, haproxy-internal.pem) + +# --- Services --- +enable_openstack_core: true # glance, keystone, neutron, nova, heat, horizon +enable_cinder: true +enable_cinder_backup: true +enable_octavia: false # enable with neutron_plugin_agent ovn -> OVN provider driver agent +enable_designate: false +enable_barbican: false +enable_magnum: false +enable_skyline: false +enable_ceph_rgw: false # Swift-compatible object store via external Ceph RGW +enable_mariabackup: true # then: kolla-ansible mariadb-backup (schedule it yourself) +enable_prometheus: false # monitoring stack: see devops-observability +enable_central_logging: false + +# --- External Ceph (keyrings + ceph.conf under /etc/kolla/config/{glance,cinder,nova}/) --- +glance_backend_ceph: "yes" +glance_backend_file: "no" +cinder_backend_ceph: "yes" +cinder_backup_driver: "ceph" +nova_backend_ceph: "yes" +ceph_glance_user: "glance" +ceph_glance_pool_name: "images" +ceph_cinder_user: "cinder" +ceph_cinder_pool_name: "volumes" +ceph_cinder_backup_user: "cinder-backup" +ceph_cinder_backup_pool_name: "backups" +ceph_nova_pool_name: "vms" + +# --- Compute --- +nova_compute_virt_type: "kvm" # qemu only for nested labs +nova_console: "novnc" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/pre-upgrade-checklist.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/pre-upgrade-checklist.md new file mode 100644 index 0000000..6821e90 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/pre-upgrade-checklist.md @@ -0,0 +1,51 @@ +# Template: pre-upgrade checklist (series upgrade) + +Copy into the change ticket and fill every line with evidence (command output or +path). Any unchecked line = no upgrade. The user's go is given on the completed +checklist; a later change gets its own checklist. + +## Change identity + +- Cloud / region: `` · Deployment tool: Kolla-Ansible | OpenStack-Ansible | OpenStack-Helm +- From series: `<2026.1 Gazpacho>` → To series: `<2026.2 Hibiscus>` +- [ ] Path is supported: adjacent, or SLURP → SLURP (never non-SLURP → non-SLURP) - [release-upgrade-slurp.md](../release-upgrade-slurp.md) +- [ ] Target tool version exists for the target series (Kolla 23.x for 2026.2; OSA has no 2026.2 release on 2026-11-10) - checked at https://releases.openstack.org/ on `` +- [ ] Release notes read: target series **and every skipped series**, upgrade notes first (links: `<...>`) +- [ ] Tool-specific prerequisites listed (Kolla: `ansible-core` range for SLURP, RabbitMQ allowed path; OSA: RabbitMQ ≥ 3.13, quorum queues; OSH: Kubernetes in the tested range) + +## Health before (all green, outputs attached) + +- [ ] `openstack compute service list` - every service `enabled`/`up` (or the disabled ones explained) +- [ ] `openstack network agent list` - every agent alive +- [ ] `openstack volume service list` - every service `up` +- [ ] Galera: `wsrep_cluster_size` = node count, `wsrep_cluster_status` = `Primary` on every node +- [ ] RabbitMQ: `rabbitmqctl cluster_status` lists all nodes running; `rabbitmq-diagnostics check_local_alarms` clean +- [ ] `nova-status upgrade check` on the **current** code: exit 0 (1 = read and justify, 2 = stop) +- [ ] Previous upgrade fully finished: `nova-manage db online_data_migrations` (it migrates data, not read-only) exited 0 after the last upgrade - 1 = run again, 2 = intervention needed (https://docs.openstack.org/nova/2026.2/cli/nova-manage.html) +- [ ] No instance in `ERROR`/`BUILD`/`MIGRATING` that matters: `openstack server list --all-projects --status ERROR` +- [ ] Free disk on control nodes for new images and DB growth (`df -h /var/lib/docker` or equivalent) + +## Backups (restorable, off the node) + +- [ ] Deployment config committed in git and tagged: `/etc/kolla` (globals.yml, passwords.yml stored encrypted), or `/etc/openstack_deploy` + `/etc/ansible` + `/usr/local/bin/openstack-ansible.rc` tarball, or Helm values files +- [ ] MariaDB full backup taken and copied off-node (Kolla: `kolla-ansible mariadb-backup -i `; `mariadb_backup` volume) - path: `<...>` +- [ ] Restore of that backup tested on a scratch node within the last `` days (date + evidence) +- [ ] Ceph / external storage untouched by this change, or its own backup plan referenced (devops-storage-backup) + +## Rehearsal and plan + +- [ ] Upgrade rehearsed on staging with the same tool version and config; duration measured: `` +- [ ] Config diff against the new sample files reviewed (Kolla inventory/globals; OSA `deploy_config_changes` contents; OSH new values overrides) +- [ ] Passwords merged (Kolla `kolla-mergepwd`), new secrets generated +- [ ] Maintenance window and user notice sent; API downtime expectation stated +- [ ] Canary plan: one instance create/attach/delete per AZ after the upgrade +- [ ] Rollback stated honestly: Nova (and most services) cannot downgrade the database → rollback = restore DB backup + previous tool version + previous images + +## After the upgrade (verify, attach outputs) + +- [ ] `openstack versions show` and `openstack compute service list` show the new version on all hosts +- [ ] `nova-status upgrade check` exit 0 on the new code +- [ ] Agents and volume services up; Galera Primary; RabbitMQ clean +- [ ] Canary per AZ: create, attach volume, floating IP reachable - deletion proposed, then the user's go +- [ ] Release-note follow-ups done (online data migrations, deprecated options removed) +- [ ] Report: versions before/after, duration, incidents, evidence links diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/project-bootstrap.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/project-bootstrap.md new file mode 100644 index 0000000..a52be48 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/templates/project-bootstrap.md @@ -0,0 +1,94 @@ +# Template: bootstrap a new project with the `openstack` CLI + +Run as cloud admin (`OS_CLOUD=mycloud-admin`), then switch to the project entry +for the tenant part. Every command and option is in the python-openstackclient +2026.2 reference (https://docs.openstack.org/python-openstackclient/2026.2/cli/commands.html). +Replace the `<...>` values; nothing here deletes anything. + +## 0. Read-only discovery (keep the output as the "before" state) + +```bash +#!/usr/bin/env bash +set -euo pipefail +export OS_CLOUD=mycloud-admin +openstack token issue -f value -c expires +openstack service list +openstack compute service list +openstack network agent list +openstack volume service list +openstack network list --external # name of the external network, e.g. "public" +openstack flavor list +openstack image list +openstack project list +``` + +Stop and report if a service is `down`, an agent is not alive, or the external +network does not exist. + +## 1. Identity and quotas (admin) + +```bash +PROJECT= +USER= +openstack project create --domain default --description "" "$PROJECT" +openstack user create --domain default --password-prompt "$USER" +openstack role add --project "$PROJECT" --user "$USER" member +openstack quota set --instances 20 --cores 40 --ram 81920 \ + --volumes 20 --gigabytes 1000 --floating-ips 5 \ + --networks 5 --routers 2 --ports 100 --secgroups 20 "$PROJECT" +openstack quota show --usage "$PROJECT" +``` + +Optional project-private flavor: + +```bash +openstack flavor create --vcpus 4 --ram 8192 --disk 0 --private --project "$PROJECT" "$PROJECT.medium" +``` + +## 2. Tenant network (as the project user: `OS_CLOUD=`) + +```bash +export OS_CLOUD= +openstack network create app-net +openstack subnet create --network app-net --subnet-range 10.20.0.0/24 \ + --dns-nameserver app-subnet +openstack router create app-router +openstack router set --external-gateway public app-router +openstack router add subnet app-router app-subnet +openstack security group create app-sg +openstack security group rule create --ingress --protocol tcp --dst-port 22 \ + --remote-ip app-sg +openstack security group rule create --ingress --protocol icmp --remote-ip app-sg +openstack keypair create --public-key ~/.ssh/id_ed25519.pub "$USER-key" +``` + +## 3. First server (canary) + +```bash +openstack server create --flavor --image \ + --boot-from-volume 20 --network app-net --security-group app-sg \ + --key-name "$USER-key" --wait canary01 +FIP=$(openstack floating ip create public -f value -c floating_ip_address) +openstack server add floating ip canary01 "$FIP" +``` + +## 4. Verify + +```bash +openstack server show canary01 -c status -c addresses -c fault # ACTIVE, no fault +openstack server event list canary01 +openstack console log show --lines 30 canary01 # cloud-init finished +ssh -o ConnectTimeout=10 @"$FIP" true && echo "reachable" +openstack quota show --usage "$PROJECT" # admin cloud +``` + +## 5. Cleanup of the canary (propose, then the user's go) + +Effect: deletes the instance; its 20 GB root volume survives (`--boot-from-volume` +without `--delete-on-termination`), the floating IP stays allocated to the project. + +```bash +openstack server delete --wait canary01 +openstack volume list --status available # identify the orphaned root volume, then propose its deletion +openstack floating ip delete "$FIP" +``` diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/troubleshooting.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/troubleshooting.md new file mode 100644 index 0000000..ccf677e --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-openstack/references/troubleshooting.md @@ -0,0 +1,124 @@ +# Troubleshooting: logs, failure chains, control-plane health + +## Method + +1. Read-only first: `openstack compute service list`, `network agent list`, `volume service list`, `server show -c fault`, `server event list `. +2. Follow the request flow ([service-map.md](service-map.md)) to the service that failed; open its log with the request ID (`req-...`) from `server event list`. +3. One hypothesis, one change, verify; never stack fixes. + +## Where the logs are + +| Tool | Location | Command | +|------|----------|---------| +| Kolla-Ansible | `/var/log/kolla//` on the host (= Docker volume `kolla_logs`, `/var/lib/docker/volumes/kolla_logs/_data`); most containers do **not** log to stdout | `tail -f /var/log/kolla/nova/nova-scheduler.log`; `docker ps -a` for exited containers | +| OpenStack-Ansible | systemd-journald, container journals passed to the metal host under `/var/log/journal/` | `journalctl -u --since "30 min ago"` | +| OpenStack-Helm | pod stdout | `kubectl -n openstack logs -c ` | +| Kolla central logging | OpenSearch Dashboards on the VIP when `enable_central_logging: true` | browser, credentials in `passwords.yml` | + +Sources: https://docs.openstack.org/kolla-ansible/2026.2/user/troubleshooting.html, +https://docs.openstack.org/openstack-ansible/2026.1/admin/maintenance-tasks/logging.html. +Kolla warns that Ansible can leak secrets to system logs; restrict access to +logs and OpenSearch. + +## Chain 1: "No valid host was found" + +The exception is `NoValidHost` ("No valid host was found. %(reason)s"). Two origins +in the scheduler (source: `nova/scheduler/manager.py`, `nova/filters.py` on `stable/2026.2`): + +| Scheduler log line | Meaning | Check | +|--------------------|---------|-------| +| `Got no allocation candidates from the Placement API. This could be due to insufficient resources or a temporary occurrence as compute nodes start up.` | Placement found no provider with enough VCPU/MEMORY_MB/DISK_GB or the required traits | `openstack resource provider list`, `openstack resource provider usage show `, `openstack resource provider inventory list `, `openstack allocation candidate list --resource VCPU=2 --resource MEMORY_MB=4096` | +| `Filter returned 0 hosts` then `Filtering removed all hosts for the request ...` | a scheduler filter rejected everything | the named filter: AZ, aggregate metadata, server group (anti-)affinity, image properties, compute disabled | + +Usual causes, in order: compute service `disabled` or `down`; flavor larger than +any host's free capacity after allocation ratios; leftover allocations from +failed migrations (Nova admin guide "Orphaned resource allocations"); +`--disk` flavor on a boot-from-volume-only host; AZ or aggregate mismatch. +Repair tools exist (`nova-manage placement heal_allocations`, `nova-manage +placement audit [--delete]`) - `audit --delete` removes allocations: propose +with the audit output first. + +## Chain 2: port binding failure + +Nova raises `PortBindingFailed` ("Binding failed for port , please check +neutron logs for more information."). + +```bash +openstack port show -c binding_vif_type -c binding_host_id -c status +openstack network agent list --host +``` + +`binding_vif_type` = `binding_failed` (constant in neutron-lib portbindings) means +no mechanism driver could bind on that host. Check, in order: the OVN controller +agent on that compute is alive; the network's physnet exists on that host +(`neutron_physical_networks` / bridge mappings); the network type is enabled; +the host name Nova uses matches the agent `Host` exactly. Stale duplicated OVN +agents after a host upgrade: see [networking-neutron-ovn.md](networking-neutron-ovn.md). + +## Chain 3: volume attach fails + +1. `openstack volume show -c status -c attachments` - must be `available` (or `in-use` with multiattach). +2. `openstack volume service list` - the `cinder-volume` for that backend `up`. +3. nova-compute log on the instance's host (os-brick errors: iSCSI login, RBD auth/keyring, multipath). +4. Ceph backend: the `nova` side keyring `/etc/kolla/config/nova/ceph.client.cinder.keyring` matches `ceph_cinder_user` ([storage-cinder-ceph.md](storage-cinder-ceph.md)). +5. Attach after detach of the same volume can fail on a stale device name; the Cinder admin guide's fix is a different device name or, after repeated cycles, restarting the KVM host (https://docs.openstack.org/cinder/2026.2/admin/ts-failed-attach-vol-after-detach.html). + +A volume stuck in `attaching`/`detaching`: do not `volume set --state` blindly; +compare with `openstack volume attachment list --volume-id ` and the guest, +then propose the state reset. + +## Chain 4: RabbitMQ health + +Symptoms: services flapping `down` in `compute service list`, timeouts +"MessagingTimeout" in logs, builds hanging. + +```bash +docker exec rabbitmq rabbitmqctl cluster_status # Kolla; all nodes in running_nodes +docker exec rabbitmq rabbitmq-diagnostics check_running +docker exec rabbitmq rabbitmq-diagnostics check_local_alarms # memory/disk alarms block publishers +``` + +Commands from https://www.rabbitmq.com/docs/monitoring and +https://www.rabbitmq.com/docs/clustering. Queues are quorum queues by default +with RabbitMQ 4.0; streams are used for fanout unless +`om_enable_rabbitmq_stream_fanout: false`. Streams declared while a node was +down are not replicated to it; Kolla ships +`contrib/ops/rabbitmq/rabbitmq-repair-stream-replicas.sh` for a healthy cluster +only. `kolla-ansible rabbitmq-reset-state` wipes queues - outage, propose only. + +## Chain 5: MariaDB-Galera health + +```bash +# Kolla: user root_shard_0, password = database_password in /etc/kolla/passwords.yml +docker exec -it mariadb mariadb -u root_shard_0 -p -e "SHOW STATUS LIKE 'wsrep_cluster_%';" +# OSA, from /opt/openstack-ansible: +ansible galera_container -m shell -a "mariadb -h localhost -e 'show status like \"%wsrep_cluster_%\";'" +``` + +Healthy: `wsrep_cluster_size` = node count and `wsrep_cluster_status` = `Primary` +on every node. A non-Primary component or size 1 means split or lost quorum. +Whole cluster stopped: Kolla `kolla-ansible mariadb-recovery -i `; OSA has +`openstack.osa.tools.galera_cluster_rolling_restart` for controlled restarts. +Never bootstrap a node with stale data as the new primary without comparing +sequence numbers - propose with evidence. + +## After the fix + +Re-run the read-only discovery set, a canary `server create` on the affected +host/AZ, and record root cause + evidence. + +## Not verified + +- `rabbitmq-diagnostics` being present in the Kolla 2026.2 RabbitMQ image (the role healthcheck uses a `healthcheck_rabbitmq` script). +- The `mariadb` client binary name in the Kolla 2026.2 image (fallback `mysql`). +- The `MessagingTimeout` log string (oslo.messaging exception name, general knowledge). +- `openstack allocation candidate list` may need `OS_PLACEMENT_API_VERSION` set (the osc-placement example exports 1.10); the minimum microversion per option was not checked. + +## Sources + +- https://docs.openstack.org/kolla-ansible/2026.2/user/troubleshooting.html +- https://docs.openstack.org/nova/2026.2/admin/support-compute.html +- https://docs.openstack.org/nova/2026.2/cli/nova-manage.html +- https://opendev.org/openstack/neutron-lib/src/branch/master/neutron_lib/api/definitions/portbindings.py +- https://docs.openstack.org/kolla-ansible/2026.2/reference/message-queues/rabbitmq.html +- https://docs.openstack.org/project-deploy-guide/openstack-ansible/2026.1/run-playbooks.html diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/SKILL.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/SKILL.md new file mode 100644 index 0000000..15f163c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/SKILL.md @@ -0,0 +1,141 @@ +--- +name: devops-opnsense +description: Use when operating an OPNsense firewall/router - interfaces, VLANs and bridges, firewall rules and NAT, DNS (Unbound) and DHCP (Dnsmasq, Kea), WireGuard/OpenVPN/IPsec, multi-WAN gateway groups, high availability (CARP, pfsync, config sync), reverse proxy and IDS plugins, firmware updates, major upgrades, snapshots and rollback, config.xml backup and restore, the REST API and automation (Ansible, OpenTofu), running OPNsense as a VM, and troubleshooting. Not for Linux host nftables or generic networking (devops-networking), Proxmox itself (devops-proxmox), or security audit (fuse-security:security-expert). +versions: + opnsense-community: "26.7 series, latest listed 26.7.8 (2026-10-30), FreeBSD 15.1 base" + opnsense-business: "26.10 series listed" + release-cadence: "two majors per year, minors about every two weeks" + dhcp: "Dnsmasq default, Kea for large HA, ISC DHCP end of life" + ansible-collection: "oxlorg.opnsense (third party, 26.1.11 tag shown in README)" + terraform-provider: "browningluke/opnsense ~> 0.16 (third party, pre-1.0)" +user-invocable: true +references: + - references/safe-opnsense-changes.md + - references/upgrade-rollback.md + - references/firewall-nat-interfaces.md + - references/dns-dhcp.md + - references/vpn.md + - references/multiwan.md + - references/ha-carp.md + - references/plugins-proxy-ids.md + - references/backup-restore-api.md + - references/console-vm-pfsense.md + - references/troubleshooting-order.md + - references/templates/api-calls.sh + - references/templates/config-backup.sh + - references/templates/opentofu-opnsense.tf + - references/templates/requirements.yml + - references/templates/wireguard-checklist.md + - references/templates/carp-ha-checklist.md + - references/templates/pre-upgrade-checklist.md + - references/templates/rollback-checklist.md +related-skills: + - devops-networking + - devops-proxmox + - devops-virtualization + - devops-ansible + - devops-iac + - devops-secrets + - devops-storage-backup + - devops-observability + - devops-sre-ops + - devops-linux +--- + + +Covers day-to-day and risky operations on an OPNsense firewall: network objects, rules and NAT, DNS/DHCP, VPN, multi-WAN, HA, plugins, upgrades and rollback, backup, API and automation, VM deployment and troubleshooting, with a lock-out-proof way to apply every risky change. Does not cover Linux host firewalls and generic networking (devops-networking), Proxmox administration (devops-proxmox), or security audits (fuse-security:security-expert). + + +# DevOps OPNsense + +## Agent Workflow (MANDATORY) + +Before ANY change, spawn in parallel, one `Task` call each: + +1. **fuse-ai-pilot:explore-codebase** - find existing OPNsense IaC (Terraform/OpenTofu, Ansible), exported `config.xml` and runbooks +2. **fuse-ai-pilot:research-expert** - verify menu paths, API endpoints and plugin names against docs.opnsense.org for the installed release +3. **mcp__context7__query-docs** - library docs if the automation uses a provider or collection + +Then inspect read-only (GUI views, `GET` API calls) and confirm the installed version first. After file changes (templates, IaC), run **fuse-ai-pilot:sniper**. + +## Versions (checked 2026-11-10) + +See the frontmatter `versions` map and the "Sources" of each reference. Community Edition 26.7 series on FreeBSD 15.1; Business Edition series 26.10 is listed. Menu paths and API endpoints change between series: check the release page of the installed version before quoting a path. + +## Overview + +| Area | Decision in one line | +|------|----------------------| +| **Rules** | MVC rules are the default; order is Floating > group > interface; `quick` is first-match; NAT is evaluated before filter rules | +| **DHCP** | Dnsmasq for small/medium, Kea for large HA, ISC is end of life | +| **VPN** | WireGuard for simple tunnels, IPsec connections (swanctl) for interop, OpenVPN 2.7 shipped in 26.7 | +| **Multi-WAN** | Monitor IPs + gateway group tiers + policy rule on the traffic that should fail over | +| **HA** | CARP VIPs + pfsync on a dedicated link + XMLRPC sync from master only; same version on both nodes | +| **Upgrade** | Major upgrades are offline: console, snapshot (ZFS) or VM snapshot, config backup first | +| **API** | Basic auth with key/secret, read first, then write, then apply step, then read back | +| **Automation** | Third-party Ansible collection and Terraform provider, pre-production grade | + +## Critical Rules + +1. **Lock-out is the main danger.** Every operation in the table of [safe-opnsense-changes.md](references/safe-opnsense-changes.md) is proposed as ACTION / EFFECT / PROOF / ROLLBACK, console path and rollback stated first, then WAIT-FOR-GO +2. **Out-of-band access before any change** to interfaces, rules on the management path, restores, upgrades or HA actions +3. **Backup before change**: download `config.xml`, snapshot (ZFS) or VM snapshot; keep copies off-box and encrypted +4. **Read-only first**: GUI views, `GET` API calls, `pfctl -s ...`; never `pfctl -d` or a state flush on a remote box without the format +5. **One change at a time**, verified from a NEW session before the next +6. **Never invent a menu path, endpoint, command or flag**: unknown = "not verified" (every reference ends with that list) +7. **Secrets stay out of output and Git**: API key/secret and WireGuard keys only through environment variables or a secret store (`devops-secrets`) +8. **HA**: change the master, sync to the backup; a backup node never holds sync settings + +## Reference Guide + +### Concepts + +| Topic | Reference | When to Consult | +|-------|-----------|-----------------| +| **Safe changes** | [safe-opnsense-changes.md](references/safe-opnsense-changes.md) | Before ANY change that can cut access or lose state | +| **Upgrades** | [upgrade-rollback.md](references/upgrade-rollback.md) | Release line, minor/major upgrades, snapshots, rollback | +| **Rules and NAT** | [firewall-nat-interfaces.md](references/firewall-nat-interfaces.md) | Interfaces, rule order, aliases, NAT modes | +| **DNS/DHCP** | [dns-dhcp.md](references/dns-dhcp.md) | Unbound, Dnsmasq, Kea, relay | +| **VPN** | [vpn.md](references/vpn.md) | WireGuard, IPsec, OpenVPN | +| **Multi-WAN** | [multiwan.md](references/multiwan.md) | Gateways, groups, failover | +| **HA** | [ha-carp.md](references/ha-carp.md) | CARP, pfsync, config sync | +| **Plugins** | [plugins-proxy-ids.md](references/plugins-proxy-ids.md) | Caddy/HAProxy/nginx, ACME, Suricata, CrowdSec | +| **Backup and API** | [backup-restore-api.md](references/backup-restore-api.md) | Backups, REST API, Ansible, OpenTofu | +| **Console, VM, pfSense** | [console-vm-pfsense.md](references/console-vm-pfsense.md) | Console menu, VM install, migrating from pfSense | +| **Troubleshooting** | [troubleshooting-order.md](references/troubleshooting-order.md) | Any "cannot connect" | + +### Templates + +| Template | When to Use | +|----------|-------------| +| [api-calls.sh](references/templates/api-calls.sh) | Read-only API calls with key/secret from environment | +| [config-backup.sh](references/templates/config-backup.sh) | Pull and encrypt the config off-box | +| [opentofu-opnsense.tf](references/templates/opentofu-opnsense.tf) | OpenTofu/Terraform provider example (pre-1.0) | +| [requirements.yml](references/templates/requirements.yml) | Install the Ansible collection | +| [wireguard-checklist.md](references/templates/wireguard-checklist.md) | WireGuard instance and peer checklist | +| [carp-ha-checklist.md](references/templates/carp-ha-checklist.md) | HA cluster setup and failover checklist | +| [pre-upgrade-checklist.md](references/templates/pre-upgrade-checklist.md) | Before a firmware upgrade | +| [rollback-checklist.md](references/templates/rollback-checklist.md) | Recovering from a failed change or upgrade | + +## Quick Reference + +```bash +pfctl -s nat # NAT/reflection rules (read-only, on the firewall) +less /tmp/rules.debug # generated ruleset (on the firewall) +curl -sS -u "$OPNSENSE_API_KEY:$OPNSENSE_API_SECRET" "https://$OPNSENSE_HOST/api/firewall/alias/get" +``` +Boot loader menu (press Space during boot): option 8 = Boot Environments (ZFS snapshots). Console menu: option 12 = update / major upgrade. + +## Best Practices + +### DO +- Keep the web GUI and SSH on all interfaces unless a loopback alias design is proven +- Use a dedicated least-privilege API user and rotate its key +- Keep both HA nodes on the same version and the same interface assignments +- Prefer explicit manual NAT over automatic reflection helpers + +### DON'T +- Deploy ISC DHCP, or leave a plugin owning ports 80/443 while the GUI uses them +- Run `pfctl -d`, reset states, restore a config or start a major upgrade without the ACTION/EFFECT/PROOF/ROLLBACK format and the user's go +- Treat the third-party Ansible collection or Terraform provider as production-stable +- Paste API secrets, WireGuard private keys or `config.xml` into chat or Git diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/backup-restore-api.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/backup-restore-api.md new file mode 100644 index 0000000..cc9e471 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/backup-restore-api.md @@ -0,0 +1,43 @@ +--- +name: backup-restore-api +description: OPNsense configuration backup and restore, REST API usage and automation (Ansible collection, Terraform provider) +when-to-use: Backing up, restoring, scripting or automating an OPNsense firewall +keywords: config.xml, history, API key secret, basic auth, reconfigure, oxlorg.opnsense, terraform-provider-opnsense +priority: high +related: safe-opnsense-changes.md, upgrade-rollback.md +--- + +# Backup, Restore, API, Automation + +## Backup and restore (backups.html) +- System > Configuration > Backups: download (optional password, optional RRD data), restore complete or partial. Prefer a complete restore; partial restores "can lead to unexpected behavior". +- Restore on another machine with different NIC names: no automatic reboot, you are prompted to fix interface assignments. Console settings of the backup are NOT applied unless you untick "Exclude console settings from import". +- Defaults tool = factory reset then shutdown. +- History: each saved config is kept (count is configurable), diffs in unified format, revert per entry restarts no service (reboot may be needed). +- Restore/reset = lock-out format in safe-opnsense-changes.md. Store backups off-box, encrypted (`devops-secrets`, `devops-storage-backup`). Backup plugins (Git, Nextcloud, etc.): not verified. +- The live config file is `/conf/config.xml` (path not re-read today). Template: templates/config-backup.sh. + +## REST API (development/api.html) +- URL: `https:///api////[/...]`. `GET` reads; `POST` creates, updates or executes; body and response are `application/json`. +- Auth: HTTP basic auth with API key as username and secret as password (curl `-u "$key:$secret"`). Create the key on a dedicated user with the minimum effective privileges (the key owner's ACL decides what is reachable). +- Find parameters by watching the GUI: it uses the same `/api/` calls (e.g. POST `core/service/search` with `{"current":1,"rowCount":7,"sort":{},"searchPhrase":""}`). +- Firewall aliases: controller `firewall/alias` offers `add_item`, `del_item`, `get_item`, `search_item`, `import`, `export`, `reconfigure` (core firewall API page). +- Pattern: read first (GET/search) -> write (POST add/set) -> apply (aliases and groups: POST `firewall/alias/reconfigure` or `firewall/group/reconfigure`; filter rules: POST `firewall/filter/apply`) -> read back. A write without the apply step stays unapplied. The apply step for interfaces is not verified; apply on interfaces/firewall is lock-out class and the API offers no automatic rollback. +- Templates: templates/api-calls.sh (credentials from environment variables only). + +## Automation +| Route | Maintainer / state | Notes | +|-------|--------------------|-------| +| Ansible collection `oxlorg.opnsense` | O-X-L (repo github.com/O-X-L/ansible-opnsense; formerly ansibleguy) - third party, not OPNsense project | Uses the API via python `httpx` or `requests`; install `ansible-galaxy collection install oxlorg.opnsense`; module list not read | +| Terraform/OpenTofu provider `browningluke/opnsense` | Third party (browningluke); README: under active development, schemas break until v1.0, "not recommended ... in production"; README shows `~> 0.16` | Credentials via `OPNSENSE_API_KEY` / `OPNSENSE_API_SECRET`; uses core API; example resource `opnsense_firewall_filter` | +Treat both as pre-production helpers; apply with plan/check mode first (`devops-iac`, `devops-ansible`). Templates: templates/opentofu-opnsense.tf, templates/requirements.yml. + +## Not verified +- Apply endpoints other than alias reconfigure; API key creation menu path; Ansible module names; Terraform provider latest version; backup plugins. + +## Sources +- https://docs.opnsense.org/manual/backups.html (2026-11-10) +- https://docs.opnsense.org/development/api.html (2026-11-10) +- https://docs.opnsense.org/development/api/core/firewall.html (2026-11-10) +- https://github.com/O-X-L/ansible-opnsense (2026-11-10) +- https://github.com/browningluke/terraform-provider-opnsense (2026-11-10) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/console-vm-pfsense.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/console-vm-pfsense.md new file mode 100644 index 0000000..1011128 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/console-vm-pfsense.md @@ -0,0 +1,34 @@ +--- +name: console-vm-pfsense +description: OPNsense console and shell inspection, running it as a VM (Proxmox/KVM), and the pfSense difference +when-to-use: Working on the box console, sizing a VM, migrating from pfSense +keywords: console menu, pfctl, virtio, QEMU guest agent, offloading, Proxmox, pfSense +priority: medium +related: safe-opnsense-changes.md, troubleshooting-order.md +--- + +# Console, VM, pfSense + +## Console and shell +- Boot loader menu (press Space during boot): option 8 = Boot Environments (snapshots.html). Console menu: option 12 = firmware upgrade (updates.html). Other option numbers: not verified. +- Read-only inspection verified in docs: `pfctl -s nat` (NAT/reflection rules), `/tmp/rules.debug` (generated ruleset). GUI equivalents: Firewall > Diagnostics (States, Sessions, Statistics). +- `configctl`, `pluginctl`, packet capture (Interfaces > Diagnostics) and log paths were not read today: do not state flags. Mutating `pfctl` commands (`-d`, `-F`) are lock-out class. + +## Running as a VM (virtuals.html) +- Minimum 3 GB RAM, 8 GB disk recommended; disable all hardware offloading in Interfaces > Settings. +- KVM: install plugin `os-qemu-guest-agent` (tick "Show community plugins"); both i440FX and Q35 chipsets work. VMware: `os-vmware`, VMXNET3 NIC. Hyper-V: no extra tools, Secure Boot off. +- VM-level snapshots replace ZFS snapshots for recovery (snapshots.html). +- virtio NIC choice and Proxmox-specific settings (firewall flag, VLAN-aware bridge, CPU type): not read here. See `devops-proxmox`. +- CARP with a physical master and VM backup breaks state sync (different interface names): ha-carp.md. + +## pfSense difference (short) +OPNsense is a separate FreeBSD-based project with a fixed fortnightly update schedule and its own plugin repository, MVC/API-driven GUI and its own config format. Migration: a pfSense `config.xml` is not restored as an OPNsense backup. Whether an official import tool exists: not verified (the docs how-to URL tried returned 404). Rebuild rules and NAT by hand or via the API/IaC, then compare behavior with the rule order in firewall-nat-interfaces.md. + +## Not verified +- `configctl`, `pluginctl`, packet capture and log locations; Proxmox virtio specifics; pfSense import tooling. + +## Sources +- https://docs.opnsense.org/manual/virtuals.html (2026-11-10) +- https://docs.opnsense.org/manual/snapshots.html (2026-11-10) +- https://docs.opnsense.org/manual/updates.html (2026-11-10) +- https://docs.opnsense.org/manual/firewall_settings.html (2026-11-10) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/dns-dhcp.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/dns-dhcp.md new file mode 100644 index 0000000..8e2c314 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/dns-dhcp.md @@ -0,0 +1,36 @@ +--- +name: dns-dhcp +description: OPNsense DNS (Unbound) and DHCP servers (Dnsmasq, Kea, ISC end of life, DHCP relay) +when-to-use: Choosing or changing the DHCP/DNS service, static reservations, HA DHCP +keywords: Unbound, Dnsmasq, Kea, ISC DHCP, dhcrelay, reservations, control agent, DDNS +priority: medium +related: multiwan.md, ha-carp.md, safe-opnsense-changes.md +--- + +# DNS and DHCP + +## DHCP servers (dhcp.html, read 2026-11-10) +| Server | Status in the docs | Use | +|--------|--------------------|-----| +| Dnsmasq DNS & DHCP | Default for DHCPv4, DHCPv6 and Router Advertisements out of the box; couples DNS and DHCP | Small/medium, under 1000 unique clients | +| Kea DHCP | ISC successor, DHCPv4/v6, no DNS or RA built in | Medium/large HA setups (over 1000 clients) or API-driven configuration | +| ISC DHCP | End of life, no updates or security patches; migrate to Kea or Dnsmasq | Do not deploy | +| DHCrelay | Relay to a server on another network; binds port 67, use strict interface binding to coexist | Segments without local server | + +- Reservations: Dnsmasq inside the pool; ISC and Kea outside the pool. +- Kea: dynamic lease registration into Unbound is not implemented, only static reservations are synchronized at an Unbound restart. The Control Agent (REST) is required for Kea HA peers; DDNS (D2) needed for RFC2136 updates. Kea HA example: all config on the master, then System > High Availability > Status > "Synchronize and reconfigure all" with Kea in "Services to synchronize" (kea.html). +- Dnsmasq needs a recursive resolver to forward to (Unbound or upstream); HA for Dnsmasq DHCP is split pools (dnsmasq.html). + +## Unbound (unbound.html) +- Standard DNS service since 17.7, enabled by default on new installs. Recommended listen and outbound interfaces: "All". Options include DNSSEC, DNS64 (needs NAT64), host/domain overrides and DHCP lease registration (ISC leases and static mappings). +- DoT upstream forwarding and blocklists: the Unbound page was read only in its opening sections; the DNS-over-TLS and blocklist settings were not read. See "Not verified". + +## Not verified +- DoT and blocklist setting names and menu paths; override menu paths. +- Whether the 26.7 wizard still defaults to Dnsmasq for DHCP (dhcp.html says default; dnsmasq.html says the wizard configures Dnsmasq for DHCP). + +## Sources +- https://docs.opnsense.org/manual/dhcp.html (2026-11-10) +- https://docs.opnsense.org/manual/kea.html (2026-11-10) +- https://docs.opnsense.org/manual/dnsmasq.html (2026-11-10) +- https://docs.opnsense.org/manual/unbound.html (2026-11-10) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/firewall-nat-interfaces.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/firewall-nat-interfaces.md new file mode 100644 index 0000000..76ea4b1 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/firewall-nat-interfaces.md @@ -0,0 +1,48 @@ +--- +name: firewall-nat-interfaces +description: OPNsense interfaces and devices, firewall rule order, aliases, states and NAT modes +when-to-use: Writing or debugging rules, NAT, port forwards, reflection, bridges +keywords: floating rules, quick, MVC rules, os-firewall-legacy, alias, state, port forward, outbound NAT, 1:1, reflection, bridge +priority: high +related: safe-opnsense-changes.md, multiwan.md, troubleshooting-order.md +--- + +# Interfaces, Firewall Rules, NAT + +## Interfaces and devices +- Virtual devices live under Interfaces > Devices: bridge (L2, optional RSTP/STP, no DHCP across it: use DHCP relay), plus VLAN and LAGG devices on the same page family (other-interfaces.html; VLAN/LAGG sections were not read in detail). +- Assignments must be identical on both HA nodes (opt1 must be the same network on both) (carp.html). +- Disable all hardware offloading in Interfaces > Settings for virtual installs (virtuals.html). + +## Rules (firewall.html) +- Two implementations: **Firewall Rules** = default MVC with API support; **Firewall Rules [legacy]** = optional plugin `os-firewall-legacy`. Existing legacy rules keep working and can be migrated with Firewall > Migration Assistant. 26.7 notes say rules now default to MVC/API. +- The "Firewall > Automation" screen is a reference/testbed UI for the firewall API, unrelated to rules managed by the core system; rules not visible there are not returned by that API (api core firewall page). +- Actions: pass, block (silent, for untrusted), reject (TCP RST / UDP ICMP unreachable). +- Stateful by default; after a policy change reset states in Firewall > Diagnostics > States (drops live flows: format in safe-opnsense-changes.md). Default 10% of RAM reserved for states (Firewall > Settings). +- **Order:** system rules first, then Floating, then interface-group rules, then interface rules. Priorities 200000 / 300000 / 400000. `quick` is the default: first match wins; without quick, last match wins (the default deny relies on that). +- **NAT is processed before filter rules.** A port forward whose filter-rule association is "Pass" leaves no other rule to apply to it. +- Floating = any number of interfaces or one inverted interface; group = a group interface; interface rule = one interface. +- Inspect the final ruleset in `/tmp/rules.debug`. +- Aliases: managed under Firewall > Aliases; API controller `firewall/alias` has `add_item`, `del_item`, `get_item`, `search_item`, `import`, `export`, `reconfigure`. + +## NAT (nat.html) +| Mode | Where | Note | +|------|-------|------| +| Destination NAT (port forward) | Firewall > NAT > Destination NAT | Also used for transparent proxies; cannot NAT to `::1` or link-local | +| One-to-one | Firewall > NAT > One-to-one | Two IPs mapped 1:1 | +| Source NAT (outbound) | Firewall > NAT > Source NAT (Outbound) | Modes: automatic (default), manual, hybrid, disabled (for transparent bridges) | +| NPTv6 | Firewall > NAT > NPTv6 | IPv6 prefix translation | +- Reflection options are in Firewall > Settings > Advanced; no firewall rules are generated for reflection, add them manually. Inspect auto reflection rules with `pfctl -s nat`. Docs advise explicit manual NAT over the automatic reflection helpers. +- Disabling pf disables NAT. NAT is not a security measure. + +## Not verified +- VLAN and LAGG menu paths and LAGG protocols; exact anti-lockout rule behavior beyond "hidden rule on LAN". +- MVC rule field names for API writes (see backup-restore-api.md). + +## Sources +- https://docs.opnsense.org/manual/firewall.html (2026-11-10) +- https://docs.opnsense.org/manual/nat.html (2026-11-10) +- https://docs.opnsense.org/manual/firewall_settings.html (2026-11-10) +- https://docs.opnsense.org/manual/other-interfaces.html (2026-11-10) +- https://docs.opnsense.org/development/api/core/firewall.html (2026-11-10) +- https://docs.opnsense.org/releases/CE_26.7.html (2026-11-10) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/ha-carp.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/ha-carp.md new file mode 100644 index 0000000..a8cd381 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/ha-carp.md @@ -0,0 +1,45 @@ +--- +name: ha-carp +description: OPNsense high availability - CARP virtual IPs, pfsync, XMLRPC config sync, maintenance and upgrade care +when-to-use: Building or operating a two-node firewall cluster +keywords: CARP, vhid, skew, pfsync, XMLRPC sync, preempt, HA status, sync compatibility +priority: high +related: safe-opnsense-changes.md, dns-dhcp.md, vpn.md, upgrade-rollback.md +--- + +# High Availability (CARP) + +## Concepts (carp.html, hacarp.html) +- CARP = IP protocol 112, multicast advertisements; each virtual IP has a unique vhid shared across nodes; lower advertised skew wins (master uses 0). Every interface must be able to receive CARP packets (pass rule). +- Virtual IPs of type CARP are required for automatic failover. Each node also has its own real address per network plus the shared virtual IP. +- **pfsync** replicates the state table over a dedicated interface (security: state injection; performance). Since 24.7 a "Sync compatibility" selector exists: both nodes must run the same version. +- **XMLRPC sync** (System > High Availability > Settings) pushes config on demand from the master to the backup; services to synchronize include Kea. +- Interface assignments must be identical on both nodes. Different NIC drivers (physical master + VM backup) prevent state sync because interface names differ; workaround is a LAGG. +- Preempt: default is enabled-behaviour (option "Disable preempt" unchecked), so CARP interfaces act as a group. +- "Disconnect dialup interfaces": a CARP backup drops PPP interfaces and reconnects on becoming master. + +## Operating rules +- Configure the master first, commit, then update the backup from System > High Availability > Status ("Synchronize and reconfigure all"); the backup stays in a known-good state meanwhile. +- A backup node must never hold configuration-synchronization settings, or it can overwrite the master. Use a different theme per node. +- Optional cron "HA update and reconfigure backup" runs only if all CARP interfaces are MASTER. +- WireGuard: "Depend on (CARP)" per instance. + +## Failover, maintenance, upgrade (lock-out/state-loss format) +``` +ACTION put the master in CARP persistent maintenance / disable CARP (exact menu: not verified) +EFFECT traffic moves to the backup; states survive only if pfsync is healthy and versions match +PROOF backup shows MASTER on all CARP VIPs; new sessions pass; state count rising on the backup +ROLLBACK re-enable CARP on the master (console available), config unchanged on backup +WAIT-FOR-GO +``` +HA upgrade order (backup first, then master, or the reverse) is not stated in the pages read: not verified. Keep versions equal after the upgrade because of sync compatibility. + +## Not verified +- Maintenance-mode menu names and HA upgrade order. +- pfsync/CARP sysctl tunables. + +## Sources +- https://docs.opnsense.org/manual/hacarp.html (2026-11-10) +- https://docs.opnsense.org/manual/how-tos/carp.html (2026-11-10) +- https://docs.opnsense.org/manual/kea.html (2026-11-10) +- https://docs.opnsense.org/manual/how-tos/wireguard-s2s.html (2026-11-10) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/multiwan.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/multiwan.md new file mode 100644 index 0000000..01a804f --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/multiwan.md @@ -0,0 +1,28 @@ +--- +name: multiwan +description: OPNsense multi-WAN - gateways, monitor IPs, gateway groups, policy routing and failover +when-to-use: Adding a second uplink, failover or load balancing +keywords: gateway group, tier, trigger level, monitor IP, policy based routing, reply-to, route-to +priority: medium +related: firewall-nat-interfaces.md, ha-carp.md, safe-opnsense-changes.md +--- + +# Multi-WAN + +Failover procedure from the how-to (multiwan.html): +1. Gateway monitoring: enable it, set a monitor IP per gateway (System > Gateways > Configuration). +2. Gateway group (System > Gateways > Group): members with tiers (Tier 1 primary, Tier 2 backup) and a trigger level: Member Down (100% loss), Packet Loss, High Latency, or Packet Loss or High Latency. +3. DNS: one DNS server per gateway (System > Settings > General). +4. Policy based routing: set the Gateway field of the LAN pass rule to the group. It skips normal routing, so add an explicit earlier rule for traffic to internal networks and VPNs, or it leaves via the group too. +5. A rule for DNS traffic aimed at the firewall itself, since the group rule would route it the wrong way. +- On static-address interfaces the interface gateway setting drives automatic source NAT and return-path rules (`reply-to`, `route-to`). +- 26.7 adds gateway groups via MVC/API; 26.7.8 notes mention ECMP route support in the development version only. + +Deleting a gateway or group, or changing the rule you connect through: lock-out format (safe-opnsense-changes.md). + +## Not verified +- Weighted load balancing options; gateway group API endpoints. + +## Sources +- https://docs.opnsense.org/manual/how-tos/multiwan.html (2026-11-10) +- https://docs.opnsense.org/releases/CE_26.7.html (2026-11-10) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/plugins-proxy-ids.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/plugins-proxy-ids.md new file mode 100644 index 0000000..a359756 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/plugins-proxy-ids.md @@ -0,0 +1,37 @@ +--- +name: plugins-proxy-ids +description: OPNsense plugins - reverse proxy (Caddy, HAProxy, nginx), ACME, IDS/IPS (Suricata), CrowdSec pointers +when-to-use: Publishing services behind the firewall, certificates, intrusion detection +keywords: os-caddy, os-haproxy, os-nginx, os-acme-client, Suricata, Netmap, CrowdSec, plugin tiers +priority: medium +related: firewall-nat-interfaces.md, safe-opnsense-changes.md +--- + +# Plugins: Reverse Proxy, Certificates, IDS/IPS + +## Plugin model (firmware.html) +Plugins install from System > Firmware > Plugins; tier 1 and 2 are shown by default, lower tiers (e.g. bind, c-icap, freeradius) need "Show community plugins". Support tiers: https://docs.opnsense.org/support.html (not read: do not claim a tier for a given plugin). + +Plugins with API documentation pages (development/api.html plugin list): Acmeclient, Caddy, Haproxy, Nginx, Crowdsec, Quagga, Bind, Dnscryptproxy, Netbird, Cloudflared and others. Presence on that list proves the plugin exists in the OPNsense plugins repository, not its support tier. + +## Reverse proxy +- **Caddy** (`os-caddy`): documented how-to. It uses ports 80 and 443, so the web GUI must move (System > Settings > Administration, e.g. TCP port 8443, tick "disable web GUI redirect rule") and WAN rules must allow HTTP and HTTPS to "This Firewall". On LAN a hidden anti-lockout rule keeps GUI access; other interfaces need explicit rules. Caddy obtains and renews certificates automatically (Let's Encrypt, ZeroSSL). + The GUI port move is a lock-out operation: use the format in safe-opnsense-changes.md. +- **HAProxy** and **nginx**: plugins exist (API pages); configuration details were not read. +- **ACME client** (`Acmeclient` API page exists): setup not read. Use it for certificates when not using Caddy's automatic HTTPS. +- Generic proxy choice and TLS theory: `devops-networking`. + +## IDS/IPS (ips.html) +Suricata-based, uses Netmap for performance, can run as IDS or IPS. Interface choice matters: on WAN you see post-NAT addresses only. Rule download and policy details not read. + +## CrowdSec +Plugin API page exists (`Crowdsec`). Its setup was not read: pointer only. + +## Not verified +- Tier of each plugin; HAProxy/nginx/ACME procedures; CrowdSec and Suricata rule management. + +## Sources +- https://docs.opnsense.org/manual/firmware.html (2026-11-10) +- https://docs.opnsense.org/development/api.html (2026-11-10) +- https://docs.opnsense.org/manual/how-tos/caddy.html (2026-11-10) +- https://docs.opnsense.org/manual/ips.html (2026-11-10) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/safe-opnsense-changes.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/safe-opnsense-changes.md new file mode 100644 index 0000000..6429138 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/safe-opnsense-changes.md @@ -0,0 +1,60 @@ +--- +name: safe-opnsense-changes +description: Lock-out-proof procedure for any OPNsense change that can cut management access or lose state +when-to-use: Before ANY change on the interface you are connected through, any restore, upgrade, HA action or API reconfigure +keywords: lock-out, anti-lockout, console, snapshot, config history, ACTION EFFECT PROOF ROLLBACK, WAIT-FOR-GO +priority: critical +related: backup-restore-api.md, upgrade-rollback.md, ha-carp.md, multiwan.md +--- + +# Safe OPNsense Changes + +**Load when:** the change touches the path your own session uses, or loses state. Read-only inspection (GUI views, `GET` API calls, `pfctl -s ...`, logs) needs no go. + +## Rules + +1. **Out-of-band path first.** Know and test a console path (VGA, serial, IPMI, hypervisor console) before the change. Major upgrades run offline: "no web or SSH server is running" during them (updates.html). +2. **Rollback stated first, and it must fire without you.** Prepare one of: a config backup downloaded off the box, a ZFS snapshot (ZFS installs only), a hypervisor VM snapshot (virtual install), or the config history entry to revert to. A GUI "revert" of a config history item restarts no service; a reboot may be needed for a clean state (backups.html). +3. **Propose, then wait for the user's go.** Never apply a listed risky operation on your own initiative. +4. **Second session + new-connection test.** Established states survive many rule changes; the real test is a new connection from a new session. +5. **One change, verify, next.** Never combine interface IP + rule + gateway changes in one apply. + +## Format of every risky operation + +``` +ACTION +EFFECT +PROOF +ROLLBACK +WAIT-FOR-GO +``` + +## Operations that need this format + +| Operation | Why it can lock you out or lose state | Out-of-band and rollback to state first | +|-----------|---------------------------------------|-----------------------------------------| +| Change LAN / management interface or its IP | The address you connect to disappears | Console menu to reassign/reset interface IP (console menu option names: not verified); VM or ZFS snapshot | +| Rule change on the interface you connect through | Block rule before your pass rule drops new sessions | Console access; config history revert (restarts nothing: reboot or re-apply) | +| Disable anti-lockout rule | Docs: on LAN a hidden anti-lockout rule keeps GUI access; on other interfaces rules must be explicit (caddy how-to) | Never disable on the only management path without console | +| Restrict web GUI / SSH "Listen interfaces" | Docs warn misconfiguration "likely lead to a non-accessible web interface and/or missing ssh access" | Leave on default "all"; or bind a loopback alias reachable through the firewall | +| Change GUI TCP port / enable redirect off | GUI moves off 443 | Add the explicit rule first, test new port from a new session | +| `pfctl -d`, flushing states (Firewall > Diagnostics > States reset) | `pfctl -d` disables pf and with it NAT (nat.html); state reset drops every live connection incl. yours | Do not disable pf on a remote box; states reset only after the pass rule is proven | +| Restore a `config.xml` | Whole config replaced; interface names may not match: "you will be prompted to fix the interface assignments"; console settings excluded by default (backups.html) | Download current config first; VM snapshot; console | +| Factory reset (System > Configuration > Defaults) | Config erased, firewall shuts down (backups.html) | Off-box backup, console | +| Firmware major upgrade | Offline install, no web/SSH during it (updates.html) | Console/serial, ZFS snapshot or VM snapshot first | +| HA failover / maintenance mode / config sync | Active role moves; sync overwrites the backup; a backup node must never hold sync settings (hacarp.html) | See ha-carp.md | +| Delete a gateway, VPN instance or gateway group | Routes, rules using it and tunnels vanish; tunnels may be your admin path | Export config; keep the VPN path plus a non-VPN path | +| API apply: `POST firewall/filter/apply` (rules), `POST firewall/alias/reconfigure` or `firewall/group/reconfigure` | Reloads the live ruleset immediately; no timed rollback exists in the core API | Same as GUI apply: console path and config backup first | + +## Not verified +- Console menu option numbers other than 8 (boot environments, snapshots.html) and 12 (upgrade, updates.html). +- A timed auto-revert mechanism for rule changes on OPNsense (none read in the docs today; use snapshots/backup + console). + +## Sources +- https://docs.opnsense.org/manual/updates.html (read 2026-11-10) +- https://docs.opnsense.org/manual/backups.html (read 2026-11-10) +- https://docs.opnsense.org/manual/settingsmenu.html (read 2026-11-10) +- https://docs.opnsense.org/manual/nat.html (read 2026-11-10) +- https://docs.opnsense.org/manual/hacarp.html (read 2026-11-10) +- https://docs.opnsense.org/manual/how-tos/caddy.html (read 2026-11-10) +- https://docs.opnsense.org/manual/snapshots.html (read 2026-11-10) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/api-calls.sh b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/api-calls.sh new file mode 100644 index 0000000..e04c3d7 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/api-calls.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# OPNsense API examples. Credentials come from the environment, never inline. +# export OPNSENSE_HOST=fw.example.net +# export OPNSENSE_API_KEY=... (from a dedicated least-privilege user) +# export OPNSENSE_API_SECRET=... +# Read-only calls first. The write/apply block is commented: propose it with +# ACTION / EFFECT / PROOF / ROLLBACK and wait for the user's go. +set -euo pipefail +: "${OPNSENSE_HOST:?set OPNSENSE_HOST}" "${OPNSENSE_API_KEY:?}" "${OPNSENSE_API_SECRET:?}" + +api() { # api METHOD path [json-body] + local method="$1" path="$2" body="${3:-}" + if [ -n "$body" ]; then + curl -sS --fail-with-body -X "$method" -u "${OPNSENSE_API_KEY}:${OPNSENSE_API_SECRET}" \ + -H 'Content-Type: application/json' -d "$body" "https://${OPNSENSE_HOST}/api/${path}" + else + curl -sS --fail-with-body -X "$method" -u "${OPNSENSE_API_KEY}:${OPNSENSE_API_SECRET}" \ + "https://${OPNSENSE_HOST}/api/${path}" + fi +} + +# Read-only: list services (same call the GUI makes) +api POST core/service/search '{"current":1,"rowCount":20,"sort":{},"searchPhrase":""}' + +# Read-only: firewall alias settings +api GET firewall/alias/get + +# Apply step for aliases only, after an add_item call (write class: wait for the go) +# api POST firewall/alias/reconfigure diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/carp-ha-checklist.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/carp-ha-checklist.md new file mode 100644 index 0000000..c0d77fb --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/carp-ha-checklist.md @@ -0,0 +1,12 @@ +# CARP / HA checklist (docs.opnsense.org hacarp + carp how-to, read 2026-11-10) + +- [ ] Both nodes on the same OPNsense version ("Sync compatibility" for pfsync) +- [ ] Interface assignments identical on both nodes; same NIC driver type (or LAGG) +- [ ] Pass rules for CARP (protocol 112) on every CARP interface +- [ ] Real address per node per network + one CARP VIP per network, unique vhid, master skew 0 +- [ ] Dedicated pfsync interface with its own rule +- [ ] Config sync (XMLRPC) set on the master ONLY; the backup holds no sync settings +- [ ] Services to synchronize reviewed (e.g. Kea) +- [ ] Order: change the master, then System > High Availability > Status > synchronize to the backup +- [ ] Proof: all CARP VIPs MASTER on one node and BACKUP on the other; state table filling on the backup +- [ ] Failover test announced and approved (ACTION / EFFECT / PROOF / ROLLBACK / WAIT-FOR-GO), console on both nodes diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/config-backup.sh b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/config-backup.sh new file mode 100644 index 0000000..7923c3c --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/config-backup.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Pull the OPNsense config off-box and keep a dated, encrypted copy. +# Read-only on the firewall. Host, user and recipient are placeholders. +set -euo pipefail +FW_HOST="${FW_HOST:?set FW_HOST}" # e.g. fw.example.net +FW_USER="${FW_USER:-backup-admin}" # key-based SSH user +GPG_RECIPIENT="${GPG_RECIPIENT:?set GPG_RECIPIENT}" +DEST="${DEST:-$HOME/opnsense-backups}" +umask 077 +mkdir -p "$DEST" +stamp="$(date -u +%Y%m%dT%H%M%SZ)" +# /conf/config.xml is the live config path (not re-read in the docs today) +scp -q "${FW_USER}@${FW_HOST}:/conf/config.xml" "$DEST/config-${stamp}.xml" +gpg --yes --encrypt --recipient "$GPG_RECIPIENT" -o "$DEST/config-${stamp}.xml.gpg" "$DEST/config-${stamp}.xml" +rm -f "$DEST/config-${stamp}.xml" +echo "saved $DEST/config-${stamp}.xml.gpg" diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/opentofu-opnsense.tf b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/opentofu-opnsense.tf new file mode 100644 index 0000000..989b22b --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/opentofu-opnsense.tf @@ -0,0 +1,41 @@ +# Provider is third party and pre-1.0 (breaking schema changes expected; its README says not for production). +# Credentials: export OPNSENSE_API_KEY and OPNSENSE_API_SECRET; never write them here. +terraform { + required_providers { + opnsense = { + source = "browningluke/opnsense" + version = "~> 0.16" + } + } +} + +provider "opnsense" { + uri = "https://fw.example.net" +} + +resource "opnsense_firewall_filter" "allow_https" { + enabled = true + description = "Allow inbound HTTPS to the web server" + + interface = { + interface = ["wan"] + } + + filter = { + action = "pass" + direction = "in" + protocol = "TCP" + + source = { + net = "any" + } + + destination = { + net = "192.0.2.10" + port = "https" + } + + log = true + } +} +# Always: plan, review, then apply. A rule on the interface you manage through = lock-out format. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/pre-upgrade-checklist.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/pre-upgrade-checklist.md new file mode 100644 index 0000000..2b198e6 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/pre-upgrade-checklist.md @@ -0,0 +1,10 @@ +# Pre-upgrade checklist (updates.html, snapshots.html, read 2026-11-10) + +- [ ] Release page of the target series read (removed features, plugin changes) +- [ ] Config backup downloaded and stored off-box +- [ ] ZFS snapshot named (e.g. known-good) OR VM snapshot taken +- [ ] Console/serial/IPMI access tested; maintenance window agreed +- [ ] Release type is Production +- [ ] HA: version plan for both nodes written; never leave the nodes on different versions +- [ ] Plugins up to date; third-party plugins checked +- [ ] ACTION / EFFECT / PROOF / ROLLBACK / WAIT-FOR-GO written, user's go received diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/requirements.yml b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/requirements.yml new file mode 100644 index 0000000..f882845 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/requirements.yml @@ -0,0 +1,3 @@ +# Third-party collection (O-X-L), API based; needs python httpx or requests on the controller. +collections: + - name: oxlorg.opnsense diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/rollback-checklist.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/rollback-checklist.md new file mode 100644 index 0000000..f39e7ee --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/rollback-checklist.md @@ -0,0 +1,8 @@ +# Rollback checklist (snapshots.html, backups.html, read 2026-11-10) + +- [ ] Decide which: ZFS snapshot (GUI System > Snapshots, or boot menu option 8), VM snapshot, config history revert, or full config restore +- [ ] ZFS: activate the known-good snapshot and reboot; at boot press Space, option 8, option 2 to cycle, ENTER +- [ ] Config history revert restarts no service: reboot or re-apply for a clean state +- [ ] Config restore on different hardware: fix interface assignments when prompted; console settings are excluded by default +- [ ] Proof: GUI reachable, gateways up, new sessions pass, HA roles correct +- [ ] Record what failed before retrying (never the same change twice) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/wireguard-checklist.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/wireguard-checklist.md new file mode 100644 index 0000000..57628b7 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/templates/wireguard-checklist.md @@ -0,0 +1,12 @@ +# WireGuard on OPNsense checklist (docs.opnsense.org wireguard how-tos, read 2026-11-10) + +- [ ] Plugin `os-wireguard` installed if the release still ships it as a plugin (verify on the box) +- [ ] Out-of-band/console path known; the admin path is NOT only through this tunnel +- [ ] Instance: unique tunnel subnet in CIDR (never /32 or /128), listen port set, MTU 1420 (1412 for PPPoE) +- [ ] Instance DNS server left blank +- [ ] Keys generated with the cogwheel; private key never pasted in chat or Git +- [ ] Peer: public key, unique tunnel IP in allowed IPs (plus the remote LAN for site-to-site) +- [ ] WAN rule allows the listen port (UDP); rules on the WireGuard interface allow intended traffic +- [ ] HA: "Depend on (CARP)" set on each instance +- [ ] Proof: handshake seen, ping through tunnel, new session to the target LAN +- [ ] Rollback: disable the instance (do not delete) first; config backup downloaded diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/troubleshooting-order.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/troubleshooting-order.md new file mode 100644 index 0000000..066d671 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/troubleshooting-order.md @@ -0,0 +1,31 @@ +--- +name: troubleshooting-order +description: Fixed troubleshooting order for "cannot connect" on an OPNsense firewall +when-to-use: Any connectivity or rule problem behind or through OPNsense +keywords: interface, gateway, rules, NAT, states, DNS, VPN, HA +priority: medium +related: firewall-nat-interfaces.md, multiwan.md, ha-carp.md +--- + +# Troubleshooting Order + +Stop at the first failure. Read-only checks only until the cause is known. + +1. **Link and interface:** assignment, status and IP in Interfaces > Overview; on HA, assignments identical on both nodes. +2. **Gateway:** System > Gateways status (monitor IP up?), gateway group tier/trigger (multiwan.md). +3. **Rules:** Firewall > Log Files > live view; rule order Floating > group > interface; quick vs last-match; read `/tmp/rules.debug` when in doubt (firewall-nat-interfaces.md). +4. **NAT:** port forward with filter association "Pass" bypasses later rules; outbound mode; reflection needs manual rules; `pfctl -s nat` (read-only). +5. **States:** Firewall > Diagnostics > Sessions/States; stale states after a rule change need a reset (lock-out/disruption format). +6. **DNS/DHCP:** which DHCP server runs (Dnsmasq/Kea; ISC is end of life); Unbound listen interfaces; Kea does not register dynamic leases in Unbound (dns-dhcp.md). +7. **VPN:** instance enabled, listen port open on WAN, allowed IPs/children match, HA "Depend on (CARP)" (vpn.md). +8. **HA:** both nodes same version (sync compatibility), pfsync interface, CARP status, backup node holds no sync settings (ha-carp.md). +9. **Service/plugin:** System > Diagnostics > Services; plugin port conflicts (Caddy vs web GUI on 443). + +## Not verified +- Exact menu names of the live log view and capture tool on 26.7. + +## Sources +- https://docs.opnsense.org/manual/firewall.html (2026-11-10) +- https://docs.opnsense.org/manual/how-tos/multiwan.html (2026-11-10) +- https://docs.opnsense.org/manual/hacarp.html (2026-11-10) +- https://docs.opnsense.org/development/api.html (2026-11-10) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/upgrade-rollback.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/upgrade-rollback.md new file mode 100644 index 0000000..6d5723b --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/upgrade-rollback.md @@ -0,0 +1,62 @@ +--- +name: upgrade-rollback +description: OPNsense release line, cadence, editions, minor and major upgrades, snapshots, opnsense-update and bootstrap +when-to-use: Planning or running a firmware update, a major upgrade or a rollback +keywords: 26.7, FreeBSD 15.1, opnsense-update, opnsense-bootstrap, snapshots, ZFS, boot environment, console option 12 +priority: high +related: safe-opnsense-changes.md, backup-restore-api.md +--- + +# Release Line, Upgrades, Snapshots + +## Release facts (read 2026-11-10) + +| Fact | Value | Source | +|------|-------|--------| +| Cadence | Two majors a year, minor updates about every two weeks; version = year.month, minors add a third number | updates.html | +| Community Edition latest series | 26.7 "Xenial Xenops"; latest patch listed 26.7.8 (2026-10-30); previous series 26.1 | releases.html, CE_26.7.html | +| FreeBSD base of 26.7 | FreeBSD 15.1 (also OpenSSL 3.5, OpenVPN 2.7, PHP 8.5, Python 3.13) | CE_26.7.html | +| Business Edition | Separate series list: latest 26.10, then 26.4, 25.10 | releases.html | +| Release type setting | System > Firmware > Settings: Production (default, keep) or Development | updates.html | +| 26.7 changes that affect operations | firewall rules default to MVC/API; interface assignments and gateway groups via MVC/API; outbound NAT to source NAT migration assistant; Kea DDNS, custom options, dynamic prefix delegation | CE_26.7.html | +| 26.7.8 notes | IKEv1 deprecation in strongSwan is more visible; certificate storage limited to a single certificate per entry; ML-DSA OpenSSH host keys regenerate once | CE_26.7.html | + +Edition differences beyond the series numbering (support, features, price) were not read: see "Not verified". + +## Minor updates +System > Firmware > Status > Check for updates, then install (updates.html). Read the changelog first; services may restart. + +## Major upgrades (high risk) +- Recommended over VGA or serial console; installed offline, no web or SSH meanwhile (updates.html). +- Console menu option 12 asks for the newest version or the next major (type e.g. "19.1"); the box downloads sets, reboots, installs, reboots again. +- Before: config backup off-box, snapshot (below), read the series release page for removed features, check plugins and their migrations. + +``` +ACTION console menu option 12, target major +EFFECT offline upgrade, two reboots, no GUI/SSH during it; firewall down for the duration +PROOF GUI reachable, version shown, gateways up, a new session through each rule path works +ROLLBACK console boot menu option 8 to the known-good snapshot (ZFS) or VM snapshot restore +WAIT-FOR-GO +``` + +## Snapshots / boot environments +- Require ZFS; UFS cannot. Create in System > Snapshots (name e.g. `known-good`) before the risky change. Activate the snapshot in the GUI then reboot, or at boot press Space, option 8 (Boot Environments), option 2 to cycle, ENTER to boot (snapshots.html). +- On a virtual install the docs point to hypervisor VM snapshots instead. + +## CLI tools (opnsense/update README) +- `opnsense-update(8)`: single CLI for packages plus base and kernel sets, signature-verified; major version upgrade supported. +- `opnsense-bootstrap(8)`: reinstalls a running system in place (factory-reset class, can restore file consistency), or turns a stock FreeBSD into OPNsense: example in README is a FreeBSD 15.1-RELEASE image with `sh ./opnsense-bootstrap.sh.in -r 26.7`. It can wipe the config directory but does not by default. Treat as destructive: ACTION/EFFECT/PROOF/ROLLBACK format. +- `opnsense-revert`: reverts a package to its state in a given OPNsense release, as long as the package mirrors still carry that package (update README). Flags were not read: do not state any flag. + +## Not verified +- `opnsense-revert` syntax and scope; `opnsense-update` flags. +- Community vs Business edition feature and support differences. +- HA upgrade order beyond the generic rule in ha-carp.md. + +## Sources +- https://docs.opnsense.org/manual/updates.html (2026-11-10) +- https://docs.opnsense.org/releases.html (2026-11-10) +- https://docs.opnsense.org/releases/CE_26.7.html (2026-11-10) +- https://docs.opnsense.org/manual/snapshots.html (2026-11-10) +- https://docs.opnsense.org/manual/virtuals.html (2026-11-10) +- https://raw.githubusercontent.com/opnsense/update/master/README.md (2026-11-10) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/vpn.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/vpn.md new file mode 100644 index 0000000..2f849bd --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-opnsense/references/vpn.md @@ -0,0 +1,41 @@ +--- +name: vpn +description: OPNsense VPN - WireGuard, OpenVPN, IPsec connections (swanctl) and HA behaviour +when-to-use: Building or changing a site-to-site or road-warrior tunnel +keywords: WireGuard instance, peer generator, IPsec connections, swanctl, strongSwan, IKEv1, OpenVPN 2.7 +priority: medium +related: safe-opnsense-changes.md, ha-carp.md, firewall-nat-interfaces.md +--- + +# VPN + +## Overview (vpnet.html) +SSL VPNs (OpenVPN), IPsec, and WireGuard/ZeroTier via plugins are the listed technologies. + +## WireGuard +- Install plugin `os-wireguard` (System > Firmware > Plugins) per the site-to-site how-to; menu then at VPN > WireGuard (Instances, Peers, Peer generator). Whether it is now in core in 26.7 was not read. +- Instance: listen port (51820 or another unique port), MTU 1420 default (1412 for PPPoE; 80 bytes less than WAN MTU), tunnel address in CIDR as a unique subnet, never a /32 or /128. Leave the instance DNS server blank, or WireGuard overwrites OPNsense DNS config. +- Public key is generated by the cogwheel; save, reopen, copy public key. Peer generator exists for clients; with Unbound, set the client DNS to the tunnel address. +- In HA, set "Depend on (CARP)" on each instance (wireguard-s2s.html). +- You still need firewall rules on WAN (listen port) and on the WireGuard interface group. Step details beyond the first screens were not read. + +## IPsec (vpnet.html) +- Configured as Connections (swanctl.conf format, strongSwan). Menu items: Connections, Pre-Shared Keys, Key Pairs, Mobile & Advanced Settings, Status Overview, Lease Status, SAD, SPD, Virtual Tunnel Interfaces (`if_ipsec`), Log File. +- Old Phase 1 = connection settings; Phase 2 = children (CHILD_SA). +- 26.7.8 notes: IKEv1 deprecation in strongSwan is more visible; plan to move away from IKEv1. + +## OpenVPN +- 26.7 ships OpenVPN 2.7. The "instances" configuration model and its menu path were not read (the how-to URL tried returned 404). + +## Lock-out rule +Deleting or editing the VPN instance you manage through = ACTION/EFFECT/PROOF/ROLLBACK/WAIT-FOR-GO with a non-VPN path kept ([safe-opnsense-changes.md](safe-opnsense-changes.md)). Binding the GUI/SSH to a VPN interface is unsupported (settingsmenu.html). + +## Not verified +- OpenVPN instances procedure and options; IPsec connection field names; whether os-wireguard is still a plugin in 26.7. + +## Sources +- https://docs.opnsense.org/manual/vpnet.html (2026-11-10) +- https://docs.opnsense.org/manual/how-tos/wireguard-client.html (2026-11-10) +- https://docs.opnsense.org/manual/how-tos/wireguard-s2s.html (2026-11-10) +- https://docs.opnsense.org/releases/CE_26.7.html (2026-11-10) +- https://docs.opnsense.org/manual/settingsmenu.html (2026-11-10) diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/SKILL.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/SKILL.md new file mode 100644 index 0000000..943c680 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/SKILL.md @@ -0,0 +1,157 @@ +--- +name: devops-proxmox +description: Use when installing, operating or automating Proxmox VE 9.x - repositories, clusters and quorum (corosync, QDevice), HA and the Cluster Resource Scheduler, storage.cfg (LVM-thin, ZFS, Ceph, NFS, PBS), VLAN-aware bridges and SDN, the PVE firewall, VMs (qm) and containers (pct), cloud-init templates, pvesh/pveum and API tokens, backup jobs to Proxmox Backup Server, 8 to 9 upgrades (pve8to9), or importing VMs from ESXi. Not for ZFS/Ceph/PBS internals or restore drills (devops-storage-backup), plain KVM/libvirt/Incus/Packer (devops-virtualization), Terraform/OpenTofu providers (devops-iac) or generic nftables/DNS/TLS (devops-networking). +versions: + proxmox-ve: "9.2" + proxmox-ve-docs: "9.2.13" + proxmox-backup-server: "4.2" + ceph-on-pve: "20.2 Tentacle (default), 19.2 Squid" +user-invocable: true +references: + - references/install-repositories.md + - references/cluster-quorum.md + - references/ha-crs.md + - references/storage-model.md + - references/networking-sdn.md + - references/firewall.md + - references/guests-qm-pct.md + - references/api-access.md + - references/backup-pbs.md + - references/upgrade-migration.md + - references/safe-change-protocol.md + - references/templates/cloud-init-template.md + - references/templates/vlan-aware-bridge.md + - references/templates/api-token-least-privilege.md + - references/templates/backup-job-pbs.md + - references/templates/pre-upgrade-checklist.md +related-skills: + - devops-storage-backup + - devops-virtualization + - devops-networking + - devops-linux + - devops-iac + - devops-ansible + - devops-observability + - devops-secrets + - devops-sre-ops +--- + + +Covers Proxmox VE 9.x as a platform - repositories, clusters, HA/CRS, the storage +model as PVE sees it, networking and SDN, the firewall, VM/CT lifecycle with +cloud-init, API automation and least-privilege access, backup jobs to PBS, major +upgrades and imports - always via read-only discovery, dry run, one go, safety +net, verification and a ready rollback. Does not cover ZFS/Ceph/PBS internals +(devops-storage-backup), generic KVM/libvirt (devops-virtualization), IaC +providers (devops-iac) or general nftables/DNS/TLS (devops-networking). + + +# Proxmox VE + +## Agent Workflow (MANDATORY) + +Before ANY Proxmox change, spawn in parallel, one `Task` call each: + +1. **fuse-ai-pilot:explore-codebase** - find local IaC/Ansible/scripts touching PVE (`pvesh`, `qm`, `/etc/pve`, provider configs) +2. **fuse-ai-pilot:research-expert** - re-read https://pve.proxmox.com/wiki/Roadmap (newer release? its known issues) and the man page of every command you will run +3. **mcp__context7__query-docs** (`/websites/pve_proxmox_pve-docs`, `/proxmox/pve-docs`) - confirm each option you are about to write + +On a live host, Phase A discovery is read-only ([safe-change-protocol.md](references/safe-change-protocol.md)). After file changes in a repo, run **fuse-ai-pilot:sniper**. + +--- + +## Overview + +| Area | What matters on 2026-11-10 | +|------|----------------------------| +| **Release** | PVE 9.2 (2026-05-21, Debian 13.5, kernel 7.0, QEMU 11.0); PVE 8 is EOL since 2026-08; PBS 4.2 | +| **Cluster/HA** | QDevice for even node counts only; HA groups -> affinity rules; CRS `dynamic` + auto-rebalance (HA guests only); `disarm-ha`/`arm-ha` | +| **Storage** | GlusterFS dropped in 9.0; snapshots as volume chains are tech preview; Ceph Tentacle is the default for new installs | +| **Network/Firewall** | Core SDN supported, IPAM/DHCP and FRR tech preview; pin NIC names; nftables `proxmox-firewall` tech preview | +| **Guests** | OCI-image containers tech preview; privileged CTs need `Sys.Modify`; cgroup v1 gone | + +--- + +## Critical Rules + +1. **Read-only first** - `pveversion`, `pvecm status`, `ha-manager status`, `pvesm status`, `qm list`, `pct list` before any write; stop on lost quorum, fenced HA services or inactive storage. +2. **Never invent a command, option or API path** - every one used here is from the man pages, the admin guide or the API viewer (docs 9.2.13); anything else goes under "Not verified". +3. **Destructive and cluster-breaking commands are proposed, never run on your own** - `qm destroy`, `pct destroy`, `pvecm delnode`, `pvecm expected`, `pvesm remove`/`free`, `zfs destroy`, `pveceph osd destroy`, edits to `/etc/pve/corosync.conf`, `disarm-ha`: state the exact effect and quorum impact, then wait for the user's go naming the object. +4. **A verified backup precedes every irreversible step** - snapshot for reversible guest changes, `vzdump --protected 1` before destroy/upgrade; a backup counts once a restore was tested. +5. **One node at a time, always with a way back** - upgrades, reboots, network and firewall changes; console access or a timed revert; quorum and HA checked between nodes. +6. **Tech preview is said out loud** - nftables firewall, SDN IPAM/DHCP/FRR, OCI app containers, volume-chain snapshots, BTRFS. + +--- + +## Architecture + +``` +/etc/pve/ # pmxcfs, replicated cluster-wide, read-only without quorum +├── corosync.conf # membership + links (edit a copy, bump config_version) +├── storage.cfg # every storage definition +├── datacenter.cfg # crs:, migration, defaults +├── jobs.cfg # backup jobs (pvescheduler) +├── qemu-server/ lxc/ # .conf of this node's VMs / CTs +├── firewall/ nodes//host.fw # cluster.fw, .fw, host rules +├── sdn/ # zones, vnets, subnets, fabrics (+ .running-config) +└── priv/ # keys, storage passwords (root only) +/etc/network/interfaces(.new) # node networking (ifupdown2, ifreload -a) +/etc/apt/sources.list.d/*.sources # deb822 repositories +``` + +--- + +## Reference Guide + +### Concepts + +| Topic | Reference | When to Consult | +|-------|-----------|-----------------| +| **Install & repos** | [install-repositories.md](references/install-repositories.md) | Versions/EOL, enterprise vs no-subscription, deb822, apt 401 | +| **Cluster & quorum** | [cluster-quorum.md](references/cluster-quorum.md) | create/join, QDevice, delnode, expected votes, corosync.conf | +| **HA & CRS** | [ha-crs.md](references/ha-crs.md) | HA resources, affinity rules, fencing, rebalancing, maintenance | +| **Storage** | [storage-model.md](references/storage-model.md) | storage.cfg, type choice, pvesm, removal semantics | +| **Network & SDN** | [networking-sdn.md](references/networking-sdn.md) | bridges, VLANs, NIC pinning, zones/VNets | +| **Firewall** | [firewall.md](references/firewall.md) | enabling without lockout, guest rules, nftables preview | +| **VMs & CTs** | [guests-qm-pct.md](references/guests-qm-pct.md) | qm/pct lifecycle, cloud-init, destroy semantics | +| **API & access** | [api-access.md](references/api-access.md) | pvesh, pveum, roles, ACLs, API tokens | +| **Backups** | [backup-pbs.md](references/backup-pbs.md) | vzdump modes, jobs, retention, PBS storage, encryption | +| **Upgrade & import** | [upgrade-migration.md](references/upgrade-migration.md) | pve8to9, minor updates, ESXi/OVF/disk import | +| **Safe change** | [safe-change-protocol.md](references/safe-change-protocol.md) | any write on a live node; destructive command catalogue | + +### Templates + +| Template | When to Use | +|----------|-------------| +| [cloud-init-template.md](references/templates/cloud-init-template.md) | Build a cloud-init VM template and deploy clones | +| [vlan-aware-bridge.md](references/templates/vlan-aware-bridge.md) | Bond + VLAN-aware `vmbr0` + corosync NIC, with timed revert | +| [api-token-least-privilege.md](references/templates/api-token-least-privilege.md) | Automation user, role, pool, privsep token with expiry | +| [backup-job-pbs.md](references/templates/backup-job-pbs.md) | Attach PBS with encryption, scheduled job with retention | +| [pre-upgrade-checklist.md](references/templates/pre-upgrade-checklist.md) | 8.4 to 9.x upgrade, cluster and per-node evidence | + +--- + +## Quick Reference + +```bash +pveversion; pvecm status; ha-manager status; pvesm status; qm list; pct list # read-only state +pvesh get /cluster/resources --output-format json-pretty # whole cluster inventory +qm snapshot pre-change && qm config # reversible safety net +vzdump --storage --mode snapshot --protected 1 # irreversible-step safety net +pve8to9 --full; pve-firewall status # upgrade checklist; firewall compile check +``` + +--- + +## Best Practices + +### DO +- Enterprise repository in production, same subscription level on every node +- Corosync on its own NIC plus a second link on another physical network +- Groups, pools and privilege-separated, expiring API tokens for access +- `protection 1` on critical guests; pin NIC names before kernel or major upgrades + +### DON'T +- Power a deleted node back on with its old config; enable a `DROP` input policy before the `management` IP set holds your admin addresses +- Treat `pvesm remove` as data deletion, or `pvesm free`/`qm destroy` as reversible +- Run Docker directly on the PVE host - use a VM diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/api-access.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/api-access.md new file mode 100644 index 0000000..79cca6b --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/api-access.md @@ -0,0 +1,88 @@ +--- +name: api-access +description: Proxmox VE API automation and access control - pvesh, REST over 8006, pveum users/groups/roles/ACLs, privilege-separated API tokens, pools +when-to-use: Automating PVE (scripts, Terraform/Ansible credentials, monitoring), creating users or tokens, or auditing who can do what +keywords: pvesh, api2/json, PVEAPIToken, pveum, acl modify, role add, privsep, token, PVEAuditor, PVEVMAdmin, /vms, /pool, Permissions.Modify +priority: high +related: guests-qm-pct.md, backup-pbs.md +--- + +# API, `pvesh` and `pveum` + +**Load when:** something must call the API (automation, IaC provider, monitoring exporter), a user/token/role is created, or permissions are audited. The Terraform/OpenTofu provider side belongs to skill `devops-iac`; secret storage of the token to `devops-secrets`. + +Sources: https://pve.proxmox.com/pve-docs/pvesh.1.html , https://pve.proxmox.com/pve-docs/pveum.1.html , https://pve.proxmox.com/pve-docs/api-viewer/ (9.2.13). + +## Three ways in, one API + +All PVE CLI tools wrap the same REST API (JSON schema). `pvesh` calls it locally **without** the HTTPS server and only as root: + +| pvesh | HTTP | Example | +|-------|------|---------| +| `pvesh get ` | GET | `pvesh get /nodes --output-format json` | +| `pvesh create ` | POST | `pvesh create /cluster/backup ...` | +| `pvesh set ` | PUT | `pvesh set /cluster/sdn` | +| `pvesh delete ` | DELETE | | +| `pvesh usage -v` | schema | `pvesh usage cluster/options -v` | + +Output: `--output-format json|json-pretty|text|yaml`, `--noborder`, `--noheader`, `--quiet`. Remote clients use `https://:8006/api2/json/`. Console endpoints (VNC/terminal) refuse API tokens and need user privileges. + +## Least-privilege model + +| Concept | Rule | +|---------|------| +| User | `name@realm` (`pam`, `pve`, LDAP/AD, OpenID). `root@pam` is unconfined: never hand its password to automation | +| Group | Grant to groups, not users | +| Role | A set of privileges; built-ins start with `PVE` (prefix reserved). Examples: `PVEAuditor` (read-only), `PVEVMUser`, `PVEVMAdmin`, `PVEDatastoreUser`, `PVESDNUser`, `PVETemplateUser` | +| ACL path | `/`, `/vms`, `/vms/{vmid}`, `/storage/{storeid}`, `/pool/{poolname}`, `/nodes/{node}`, `/access/groups`, `/access/realms/{realmid}` | +| Inheritance | `--propagate 1` default; deeper paths replace inherited ones; user permissions replace group ones; `NoAccess` cancels all | +| Token, privsep=1 (default) | Effective rights = intersection of the user's and the token's own ACLs | +| Token, privsep=0 | Same rights as the user: avoid unless the integration truly needs them | + +Dangerous privileges to grant only deliberately: `Permissions.Modify`, `Sys.Modify`, `Sys.Console`, `VM.GuestAgent.Unrestricted`, `Datastore.Allocate`. + +## Create a scoped automation identity + +```bash +pveum user add automation@pve --comment "CI provisioning" +pveum role add CI_Provision --privs "VM.Allocate VM.Clone VM.Config.Disk VM.Config.CPU VM.Config.Memory VM.Config.Network VM.Config.Cloudinit VM.Config.Options VM.PowerMgmt VM.Audit Datastore.AllocateSpace Datastore.Audit" +pveum pool add ci-pool --comment "CI guests" +pveum acl modify /pool/ci-pool --users automation@pve --roles CI_Provision +pveum user token add automation@pve ci --privsep 1 --expire +pveum acl modify /pool/ci-pool --tokens 'automation@pve!ci' --roles CI_Provision +pveum user token permissions automation@pve ci # effective rights: verify, do not assume +``` + +The token secret is shown **once**. HTTP header: `Authorization: PVEAPIToken=automation@pve!ci=`. Rotation without losing ACLs (9.2): `pveum user token modify automation@pve ci --regenerate 1` - every consumer of the old secret loses access immediately. A complete version with storage/bridge ACLs is in [templates/api-token-least-privilege.md](templates/api-token-least-privilege.md). + +Which privileges an endpoint needs is listed per method in the API viewer (`["perm", , [privs]]`). + +## Destructive operations (propose, then the user's go) + +| Command | Effect | +|---------|--------| +| `pveum user token delete ` | Revokes the token; automation using it fails at once | +| `pveum user delete ` | Removes the user (its tokens stop working) | +| `pveum acl delete --roles --users/--groups/--tokens ...` | Removes the grant | +| `pveum acl modify / ... --roles Administrator` | Not destructive but cluster-wide full power: refuse without explicit owner request | + +## Verification + +```bash +pveum user permissions +pveum user token permissions --path /pool/ci-pool +curl -fsS -H "Authorization: PVEAPIToken=!=" https://:8006/api2/json/version +``` + +A call outside the granted path must return 403. + +## Pitfalls + +- Tokens without `--expire`: abandoned automation keeps working (the docs recommend setting an expiry). +- Granting on `/` because "it works": use a pool or `/vms/{vmid}` and `/storage/{id}`. +- 9.0 removed `VM.Monitor` and added `VM.Replicate`; 9.2 requires `VM.PowerMgmt` to start a VM after create/restore/rollback in custom roles (https://pve.proxmox.com/wiki/Roadmap). Re-check custom roles after upgrades. +- Printing the token secret in logs or CI output: treat it as a password. + +## Not verified + +- The exact privilege set each third-party tool needs (Terraform/OpenTofu providers, exporters): read that tool's documentation; the role above is a starting point, not a verified minimum. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/backup-pbs.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/backup-pbs.md new file mode 100644 index 0000000..342fecb --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/backup-pbs.md @@ -0,0 +1,88 @@ +--- +name: backup-pbs +description: Backups from the Proxmox VE side - vzdump modes, scheduled backup jobs (jobs.cfg, /cluster/backup), retention, protection, Proxmox Backup Server storage integration and client-side encryption +when-to-use: Defining or auditing backup jobs, attaching a PBS datastore to PVE, setting retention, or taking a pre-change backup before a risky operation +keywords: vzdump, jobs.cfg, /cluster/backup, schedule, prune-backups, keep-last, protected, fleecing, pbs storage, fingerprint, encryption-key, master-pubkey, qmrestore, pct restore +priority: high +related: storage-model.md, safe-change-protocol.md +--- + +# Backups and PBS Integration (PVE side) + +**Load when:** a backup job is created or audited, a PBS datastore is attached, retention is set, or a change needs a pre-change backup. PBS server internals (datastores, GC, verify, sync, S3 backend, tape) and **restore drills** belong to skill `devops-storage-backup`. + +Sources: https://pve.proxmox.com/pve-docs/vzdump.1.html , https://pve.proxmox.com/pve-docs/pve-admin-guide.html section 7.8 (pbs), https://pve.proxmox.com/pve-docs/api-viewer/ (`POST /cluster/backup`) - docs 9.2.13. PBS current release 4.2 (2026-04-29), PBS 4 EOL TBA (https://pbs.proxmox.com/docs/faq.html). + +## Modes + +| `--mode` | VM | Container | +|----------|----|-----------| +| `snapshot` (default) | Live backup, lowest downtime, small inconsistency risk; guest agent `fsfreeze` if `agent: 1` | Needs all backed-up volumes on snapshot-capable storage | +| `suspend` | Compatibility only; longer downtime, no consistency gain: use `snapshot` | rsync to `--tmpdir`, short suspend, second rsync; needs space | +| `stop` | Orderly shutdown, highest consistency, short downtime | Container stopped for the whole backup | + +QEMU guest agent option `freeze-fs` (9.2; aliases `freeze-fs-on-backup`, `guest-fsfreeze`) controls freeze/thaw. `--fleecing enabled=1,storage=` (VMs only) caches copy-before-write data in a local fleecing image so a slow backup target does not throttle or hang guest writes. CT extra mount points are only included when their Backup option is set; bind and device mounts are never backed up. + +## Attach a PBS datastore + +```bash +pvesm scan pbs @pbs --password --fingerprint +pvesm add pbs pbs-main --server --datastore \ + --username @pbs --fingerprint --password +pvesm status +``` + +- Use a dedicated PBS user or token with backup rights on that datastore only, not `root@pam`. The password is stored root-only in `/etc/pve/priv/storage/.pw`. +- `--fingerprint` is required for self-signed PBS certificates (`proxmox-backup-manager cert info` on the PBS host). +- Client-side encryption: `--encryption-key autogen` stores the key in `/etc/pve/priv/storage/.enc`. **Without the key, backups are unrecoverable**: export it at once (`proxmox-backup-client key paperkey ... --output-format text`) to a password manager and an offline copy. Optional `--master-pubkey` adds an RSA-wrapped copy of the key to every backup. +- Different encryption keys on one datastore defeat deduplication between them. + +## Scheduled jobs + +Jobs live in `/etc/pve/jobs.cfg`, run by `pvescheduler`, managed in Datacenter, Backup or via the API: + +```bash +pvesh create /cluster/backup --id nightly-prod --schedule "01:30" \ + --storage pbs-main --vmid 100,101,200 --mode snapshot --enabled 1 \ + --prune-backups keep-last=3,keep-daily=13,keep-weekly=8,keep-monthly=11 \ + --repeat-missed 1 --notes-template "{{guestname}} on {{node}}" +pvesh get /cluster/backup +``` + +- Schedules use the calendar-event syntax (admin guide appendix "Calendar Events"): `01:30` daily, `sat 02:00`, `mon..fri 22:00`. +- `--all 1` with `--exclude ` instead of `--vmid` to catch new guests automatically. +- `--repeat-missed 1` runs jobs missed while the node or scheduler was down. +- The legacy `starttime`/`dow` parameters are deprecated in favour of `schedule` (9.2). The `maxfiles` setting was dropped in 9.0: use `prune-backups`. +- Retention is usually set on the storage (`prune-backups` storage option); job-level values override it. Weeks follow ISO weeks. + +Full job template: [templates/backup-job-pbs.md](templates/backup-job-pbs.md). + +## One-off pre-change backup + +```bash +vzdump --storage pbs-main --mode snapshot --notes-template "pre-change {{vmid}}" --protected 1 +``` + +`--protected 1` excludes the backup from pruning (enforced by PVE/PBS, not by the filesystem). Remove protection once the change is accepted. + +## Restore (PVE side) + +`qmrestore ` for VMs, `pct restore ` for CTs. PBS supports live-restore of VMs: the VM starts while the restore is still ongoing (https://pve.proxmox.com/wiki/Migrate_to_Proxmox_VE). Restoring over an existing VMID replaces it: use a **new** VMID for tests. + +## Verification + +- `pvesh get /cluster/backup` shows the job, `enabled 1`, the right storage. +- Every production VMID appears in a job: compare `qm list` / `pct list` with `/etc/pve/jobs.cfg` (or the GUI's backup-job guest view). +- Task log of the last run: `OK` per guest. A backup is only proven by a restore to a test VMID - procedure in `devops-storage-backup`. + +## Pitfalls + +- Backups stored on the same node/pool as the guests: not a backup. +- Encryption key only on the PVE node that is being backed up: lost with the node. +- Destroying a guest without `--purge` leaves it in jobs (job warnings); with `--purge` it silently leaves the job. +- Veeam (third party) broke for VMs on QEMU machine version >= 10.0 after the 9.0 upgrade (https://pve.proxmox.com/wiki/Upgrade_from_8_to_9 known issues). + +## Not verified + +- An API endpoint listing guests not covered by any backup job was not checked; the grep comparison above is the verified-safe method. +- Behaviour of a live-restore that fails midway: not read at the source; test on a throwaway VMID before relying on live-restore. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/cluster-quorum.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/cluster-quorum.md new file mode 100644 index 0000000..4475371 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/cluster-quorum.md @@ -0,0 +1,99 @@ +--- +name: cluster-quorum +description: Proxmox VE clustering with corosync - create, join, quorum math, QDevice, node removal, corosync.conf edits +when-to-use: Building or changing a PVE cluster, a 2-node setup, losing quorum, removing a node, or touching corosync +keywords: pvecm, corosync, knet, quorum, expected votes, QDevice, corosync-qnetd, delnode, token_coefficient, pmxcfs +priority: high +related: ha-crs.md, safe-change-protocol.md +--- + +# Cluster, Corosync and Quorum + +**Load when:** creating/joining a cluster, designing the cluster network, running a 2-node cluster, a node is "not quorate", a node must leave, or `/etc/pve/corosync.conf` must change. + +Source for this file unless noted: https://pve.proxmox.com/pve-docs/pvecm.1.html (docs build 9.2.13). + +## Rules that decide the design + +| Rule | Consequence | +|------|-------------| +| Each node has 1 vote; state changes need a majority | Losing quorum makes `/etc/pve` read-only: no guest start, no config change | +| HA needs at least 3 votes | 2 nodes = add a QDevice as third vote | +| QDevice is supported for **even** node counts | For odd counts it gives N-1 votes and becomes almost a single point of failure: discouraged | +| Corosync needs low, stable latency, not bandwidth | Dedicated NIC (1 Gbit is enough), plus a second link on a different physical network; up to 8 links (`link0`..`link7`) | +| Ports | UDP 5405-5412 between nodes, TCP 22 (SSH tunnel), synchronized time | +| Joining node | Its `/etc/pve` is overwritten and it must hold **no guests** (VMID conflicts) | + +## Create and join + +```bash +# first node +pvecm create --link0 --link1 +# each joining node (authenticates against the API, verifies the SHA-256 fingerprint) +pvecm add --link0 --link1 +pvecm status # Quorate: Yes, Expected votes == Total votes +pvecm nodes +``` + +`--use_ssh` falls back to the legacy SSH join (needs root SSH on the existing node). Since 9.2, new clusters get `token_coefficient: 125` explicitly (default otherwise 650 ms). + +## QDevice (2-node or even clusters) + +```bash +# external host (any Debian-like box outside the cluster failure domain) +apt install corosync-qnetd +# every cluster node +apt install corosync-qdevice +# one cluster node, all nodes online, root key access to the external host +pvecm qdevice setup +pvecm status # Flags: Quorate Qdevice ; Qdevice column A,V,NMW +``` + +`NA` (not alive) = TCP 5403 to the qnetd host is blocked. Adding or removing a cluster node requires `pvecm qdevice remove` first, then re-setup once the count is even again. + +## Destructive operations (propose, then the user's go) + +| Command | Exact effect | Quorum / blast radius | +|---------|--------------|-----------------------| +| `pvecm delnode ` | "Removes a node from the cluster configuration" | Run from a remaining node **after** the node is powered off for good; if it boots again with its old config the cluster can break. Reinstall before any rejoin | +| `pvecm expected ` | "Tells corosync a new value of expected votes" | `pvecm expected 1` makes a lone node quorate: two partitions doing this = split brain. Only on a node that is provably the only one writing | +| editing `/etc/pve/corosync.conf` | Applied **as soon as the file changes** and propagated to all nodes | A bad save can drop membership cluster-wide | +| `pvecm qdevice remove` | Removes the external vote | A 2-node cluster loses its tie-breaker | + +### Node removal sequence (documented) + +1. Migrate all guests away; back up local data; remove the node from every replication job (otherwise the job becomes irremovable). +2. Remove the QDevice if configured. If the node runs Ceph: destroy its OSDs one by one and wait for `HEALTH_OK`, then MDS, monitor, manager, and `ceph osd crush remove ` (Ceph detail lives in skill `devops-storage-backup`). +3. Power the node off; it must not come back on this network with its current config. +4. From a remaining node: `pvecm delnode `; the message `could not kill node (error = cs_err_not_exist)` is harmless. +5. Verify with `pvecm nodes`; clean `/etc/pve/nodes/`, its key in `/etc/pve/priv/authorized_keys`, and HA rules that still name it. + +## Editing corosync.conf safely + +```bash +cp /etc/pve/corosync.conf /etc/pve/corosync.conf.new # edit the copy, never the live file +# edit .new and INCREMENT config_version +cp /etc/pve/corosync.conf /etc/pve/corosync.conf.bak +mv /etc/pve/corosync.conf.new /etc/pve/corosync.conf +systemctl status corosync ; journalctl -b -u corosync +``` + +Membership changes go through `pvecm`, never by hand. If corosync cannot start any more, fix `/etc/corosync/corosync.conf` (local copy) identically on all nodes. + +## Token timeout check (HA clusters) + +```bash +corosync-cmapctl | grep -Ew 'runtime.config.totem.token|runtime.config.totem.consensus' +``` + +token + consensus should stay below 30 s, and must stay below 45 s with HA (watchdog fences at 60 s). Formula: `token = 3000 + (nodes - 2) * token_coefficient`, `consensus = 1.2 * token`. Lower `token_coefficient` in the `totem` section (with a `config_version` bump) when the sum is too high. + +## Verification + +`pvecm status` shows `Quorate: Yes`, the expected link count per node, and (if used) `Qdevice` in Flags. `journalctl -u corosync` shows no repeated link down/up flapping. + +## Pitfalls + +- A single corosync link over a bond: documented as problematic in some failure scenarios; prefer two independent links. +- Storage or backup traffic on the corosync link: latency spikes cause fencing with HA. +- Separating a node without reinstall (`pmxcfs -l`, deleting corosync files) is documented as "not recommended"; shared storage must be split first. diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/firewall.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/firewall.md new file mode 100644 index 0000000..15f9134 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/firewall.md @@ -0,0 +1,101 @@ +--- +name: firewall +description: The Proxmox VE firewall - cluster/host/guest/VNet zones, config files, enabling without lockout, management IP set, ports, iptables vs nftables (tech preview) +when-to-use: Enabling or changing the PVE firewall, filtering guest traffic, opening access to the GUI/API, or evaluating proxmox-firewall (nftables) +keywords: pve-firewall, proxmox-firewall, cluster.fw, host.fw, VMID.fw, management, ipfilter, local_network, security group, nftables, 8006 +priority: high +related: networking-sdn.md, safe-change-protocol.md +--- + +# PVE Firewall + +**Load when:** enabling the firewall, writing rules for hosts or guests, locking down port 8006, or someone proposes the nftables backend. Host-level nftables rulesets outside PVE belong to skill `devops-networking`. + +Source unless noted: https://pve.proxmox.com/pve-docs/chapter-pve-firewall.html (9.2.13). + +## Model + +| Level | File | Directions | +|-------|------|-----------| +| Datacenter (cluster-wide) | `/etc/pve/firewall/cluster.fw` | in, out, forward (forward: nftables only) | +| Host | `/etc/pve/nodes//host.fw` | in, out, forward (nftables only); host rules override datacenter rules | +| VM / CT | `/etc/pve/firewall/.fw` | in, out | +| VNet (SDN) | per VNet | forward only, **nftables backend only** | + +Files are replicated by pmxcfs; the service on each node recompiles rules on change. Sections: `[OPTIONS]`, `[RULES]`, `[IPSET ]`, `[GROUP ]`, `[ALIASES]`. + +The firewall is **disabled by default**. Enabling it at datacenter level (`enable: 1`) blocks traffic to all hosts except the defaults below; a guest is filtered only if its vNIC has `firewall=1` and its own `.fw` enables it. + +## Enable without locking yourself out + +1. Keep an SSH session open to a node (documented advice) and have console/IPMI access. +2. Put admin source addresses into the `management` IP set first: + +```text +# /etc/pve/firewall/cluster.fw +[IPSET management] +192.168.2.10 +192.168.2.0/24 + +[OPTIONS] +enable: 1 +policy_in: DROP +``` + +3. Check the auto-detected cluster network: `pve-firewall localnet` (alias `local_network`; a single host on a public network should set it explicitly in `[ALIASES]`). +4. Compile and check before relying on it: `pve-firewall compile` and `pve-firewall status` (status reads and compiles all rules and prints configuration errors). + +## Defaults that stay open (policy DROP/REJECT) + +Loopback, established connections, IGMP; from `management` hosts: TCP 8006 (GUI/API), 5900-5999 (VNC), 3128 (SPICE), 22 (SSH), 60000-60050 (migration); UDP 5405-5412 corosync and multicast inside the cluster network. + +Ports used by PVE: 8006/tcp, 5900-5999/tcp, 3128/tcp, 22/tcp, 111/udp (rpcbind), 25/tcp outgoing, 5405-5412/udp, 60000-60050/tcp. + +## Guest rules + +```text +# /etc/pve/firewall/100.fw +[OPTIONS] +enable: 1 +policy_in: DROP + +[RULES] +IN SSH(ACCEPT) -source +management +IN HTTPS(ACCEPT) + +[IPSET ipfilter-net0] +192.168.2.10 +``` + +`ipfilter-net*` IP sets drop outgoing traffic whose source IP does not match the interface (anti-spoofing). The `blacklist` IP set is dropped by every host and guest. Macros (`SSH`, `HTTPS`, ...) are listed in the admin guide appendix "Firewall Macro Definitions". SDN generates IP sets `vnet-all`, `vnet-gateway`, `vnet-no-gateway`, `vnet-dhcp` per VNet (https://pve.proxmox.com/pve-docs/chapter-pvesdn.html). + +## iptables (`pve-firewall`) vs nftables (`proxmox-firewall`) + +`proxmox-firewall` is a **technology preview, "currently not suited for production use"**. The Roadmap aims to make it production-ready and eventually replace the iptables firewall (https://pve.proxmox.com/wiki/Roadmap). Do not present it as GA. + +Lab only: + +```bash +apt install proxmox-firewall +# per node: /etc/pve/nodes//host.fw -> [OPTIONS] nftables: 1 +systemctl status proxmox-firewall +/usr/libexec/proxmox/proxmox-firewall compile > firewall.json # debug: generated commands +``` + +After switching backends, **all running VMs and CTs must be restarted**. It manages tables `proxmox-firewall` and `proxmox-firewall-guests`; custom rules go in your own table. + +## Verification + +```bash +pve-firewall status # running, no compile errors +iptables-save | head # iptables backend: rules present +``` + +From an address outside `management`, port 8006 must be refused; from inside, reachable. + +## Pitfalls + +- `policy_in: DROP` at datacenter level with an empty `management` set and a remote admin: lockout. +- Forward and VNet rules on the iptables backend are **silently ignored**. +- Interface names in `cluster.fw` are not rewritten by `pve-network-interface-pinning`. +- `ebtables` is on by default cluster-wide (`ebtables: 1`). diff --git a/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/guests-qm-pct.md b/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/guests-qm-pct.md new file mode 100644 index 0000000..54dc148 --- /dev/null +++ b/.cursor-plugin/plugins/fuse-devops/skills/devops-proxmox/references/guests-qm-pct.md @@ -0,0 +1,92 @@ +--- +name: guests-qm-pct +description: VM (qm) and container (pct) lifecycle on Proxmox VE - create, snapshot, clone, migrate, templates and cloud-init, OCI images, protection and destroy semantics +when-to-use: Creating, changing, cloning, snapshotting, migrating or deleting VMs and containers, or building cloud-init templates +keywords: qm, pct, pveam, cloud-init, cicustom, ipconfig0, sshkeys, template, linked clone, snapshot, rollback, destroy, purge, protection, OCI, unprivileged +priority: high +related: storage-model.md, api-access.md, safe-change-protocol.md +--- + +# Guests: `qm` (VMs) and `pct` (containers) + +**Load when:** any VM/CT lifecycle task. Generic KVM/libvirt, Packer image builds and in-guest cloud-init authoring belong to skill `devops-virtualization`. + +Sources: https://pve.proxmox.com/pve-docs/qm.1.html , https://pve.proxmox.com/pve-docs/pct.1.html (9.2.13). + +## Read-only inventory first + +```bash +qm list ; pct list +qm config ; pct config +qm listsnapshot ; pct listsnapshot +ha-manager status | grep -E "vm:|ct:" +grep -n "" /etc/pve/jobs.cfg # backup jobs naming the guest +``` + +## VM vs container + +| Need | Choose | +|------|--------| +| Any OS, own kernel, live migration, PCI passthrough | VM (`qm`) | +| Linux system service, density, fast start | Unprivileged CT (`pct`, `--unprivileged` defaults to 1 on create) | +| Docker workloads | VM; running Docker directly on the PVE host is not recommended (https://pve.proxmox.com/wiki/FAQ) | +| Application from an OCI image | CT from OCI image: **technology preview** (since 9.1) | + +Creating **privileged** containers requires `Sys.Modify` since 9.0; cgroup v1 support was dropped in 9.0 and raw cgroup v1 entries are deprecated in 9.2 (https://pve.proxmox.com/wiki/Roadmap). Templates: `pveam update`, `pveam available --section system`, `pveam download local