diff --git a/internal/cmd/agents_golden_test.go b/internal/cmd/agents_golden_test.go new file mode 100644 index 0000000..dd30d01 --- /dev/null +++ b/internal/cmd/agents_golden_test.go @@ -0,0 +1,57 @@ +package cmd + +import ( + "net/http" + "testing" +) + +// The agent commands (init, mcp install, skill install) write config files and +// print what they did. Their base URL is an unreachable address so the MCP and +// docs URLs they write are fixed; the skill comes from a fake docs server. +const ( + gAgentBase = "http://127.0.0.1:1" + gAgentSkill = "---\nname: spicrawl-web-data-api\ndescription: golden test skill\n---\n\n# Spicrawl\n\nUse https://api.spicrawl.com for scraping.\n" +) + +func gSkillAPI() func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{"GET /skill.md": func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/markdown") + _, _ = w.Write([]byte(gAgentSkill)) + }}) +} + +func init() { + env := map[string]string{agentEnvSkillURL: "{server}/skill.md"} + // agent builds a case that runs in a fresh project directory. + agent := func(name string, human bool, files []string, setup func(t *testing.T, e *goldenEnv), args ...string) goldenCase { + return goldenCase{name: name, human: human, args: args, baseURL: gAgentBase, env: env, server: gSkillAPI(), files: files, setup: setup} + } + allMCP := []string{".mcp.json", ".cursor/mcp.json", ".vscode/mcp.json", ".codex/config.toml"} + existing := func(t *testing.T, e *goldenEnv) { + goldenWrite(t, e.Dir, ".mcp.json", `{"zeta": 1, "mcpServers": {"other": {"command": "x"}}}`) + } + + c := agent("agents/mcp_print_all", false, nil, nil, "mcp", "install", "--client", "all", "--dir", "{dir}", "--print") + c2 := agent("agents/mcp_install_claude", false, []string{".mcp.json"}, nil, "mcp", "install", "--client", "claude", "--dir", "{dir}") + c3 := agent("agents/mcp_install_all", false, allMCP, nil, "mcp", "install", "--client", "all", "--dir", "{dir}") + c4 := agent("agents/mcp_install_merges_existing", false, []string{".mcp.json"}, existing, "mcp", "install", "--client", "claude", "--dir", "{dir}") + c5 := agent("agents/mcp_err_no_client_detected", false, nil, nil, "mcp", "install", "--dir", "{dir}") + c5.stderr = true + c6 := agent("agents/skill_print", false, nil, nil, "skill", "install", "--print", "--dir", "{dir}") + c7 := agent("agents/skill_install_claude", false, []string{".claude/skills/spicrawl/SKILL.md"}, nil, "skill", "install", "--client", "claude", "--dir", "{dir}") + c8 := agent("agents/skill_install_codex_agents_md", false, []string{".agents/skills/spicrawl/SKILL.md", "AGENTS.md"}, nil, "skill", "install", "--client", "codex", "--agents-md", "--dir", "{dir}") + c9 := agent("agents/init_yes", false, []string{".mcp.json", ".claude/skills/spicrawl/SKILL.md", ".cursor/mcp.json"}, nil, "init", "--client", "claude,cursor", "--yes", "--dir", "{dir}") + c10 := agent("agents/init_needs_yes_without_terminal", false, nil, nil, "init", "--client", "claude", "--dir", "{dir}") + c10.stderr = true + c11 := agent("agents/init_err_no_client_detected", false, nil, nil, "init", "--yes", "--dir", "{dir}") + c11.stderr = true + + goldenAdd(c, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, + agent("agents_human/mcp_print_all", true, nil, nil, "mcp", "install", "--client", "all", "--dir", "{dir}", "--print"), + agent("agents_human/mcp_install_all", true, nil, nil, "mcp", "install", "--client", "all", "--dir", "{dir}"), + agent("agents_human/mcp_install_merges_existing", true, nil, existing, "mcp", "install", "--client", "claude", "--dir", "{dir}"), + agent("agents_human/skill_print", true, nil, nil, "skill", "install", "--print", "--dir", "{dir}"), + agent("agents_human/skill_install_claude", true, nil, nil, "skill", "install", "--client", "claude", "--dir", "{dir}"), + agent("agents_human/init_yes", true, nil, nil, "init", "--client", "claude,cursor", "--yes", "--dir", "{dir}"), + ) +} diff --git a/internal/cmd/batch_golden_test.go b/internal/cmd/batch_golden_test.go new file mode 100644 index 0000000..3ef11e9 --- /dev/null +++ b/internal/cmd/batch_golden_test.go @@ -0,0 +1,193 @@ +package cmd + +import ( + "fmt" + "net/http" + "sync/atomic" + "testing" +) + +const gBatchID = "01J9Z6V0Q8M4K2T7R3N5B1C9XA" + +// gJob renders a BatchJob the way the API does. +type gJob struct { + Status string + Done, Failed int + Total int + Open bool + Name string + Err string // error_code + Finished, Exp bool +} + +func (j gJob) JSON() string { + ok := j.Done - j.Failed + extra := "" + if j.Err != "" { + extra += fmt.Sprintf(`,"error_code":%q,"error_message":"failure threshold reached"`, j.Err) + } + if j.Status != "queued" { + extra += `,"started_at":"2026-09-22T06:00:05Z"` + } + if j.Finished { + extra += `,"finished_at":"2026-09-22T06:03:41Z"` + } + if j.Exp { + extra += `,"results_expire_at":"2026-09-25T06:00:00Z"` + } + pct := 0.0 + if j.Total > 0 { + pct = float64(j.Done) * 100 / float64(j.Total) + } + return fmt.Sprintf(`{"id":%q,"project_id":"proj_01J9Z","name":%q,"status":%q,"open":%t,`+ + `"status_url":"/v1/batch/%s","results_url":"/v1/batch/%s/results","params":{"js_render":true,"response_format":"markdown"},`+ + `"total_items":%d,"concurrency":10,"priority":100,"max_attempts":3,"estimated_credits":%d,"estimated_credits_micro":%d,`+ + `"progress":{"total":%d,"completed":%d,"succeeded":%d,"failed":%d,"cancelled":0,"skipped":0,"remaining":%d,"percent_complete":%.1f,"credits_charged":%d,"credits_charged_micro":%d,"bytes":%d},`+ + `"submitted_at":"2026-09-22T06:00:00Z"%s}`, + gBatchID, j.Name, j.Status, j.Open, gBatchID, gBatchID, j.Total, j.Total*3, j.Total*3000000, + j.Total, j.Done, ok, j.Failed, j.Total-j.Done, pct, j.Done*3, j.Done*3000000, j.Done*20480, extra) +} + +// gSeq answers each request with the next body (the last one repeats): a job +// that moves through queued, running and completed while the CLI polls it. +func gSeq(bodies ...string) http.HandlerFunc { + var n atomic.Int32 + return func(w http.ResponseWriter, r *http.Request) { + i := int(n.Add(1)) - 1 + gJSON(http.StatusOK, bodies[min(i, len(bodies)-1)])(w, r) + } +} + +const gBatchResults = `{"seq":0,"external_id":"sku-1","url":"https://example.com/a","status":"succeeded","http_status":200,"bytes":2048,"result":{"content":"\"

A

\\n\""}} +{"seq":1,"external_id":"","url":"https://example.com/b","status":"failed","http_status":null,"bytes":0,"error":{"code":"ERR::UPSTREAM::TIMEOUT","message":"the target did not answer"}} +{"seq":2,"external_id":"sku-3","url":"https://example.com/c?x=1&y=2","status":"succeeded","http_status":404,"bytes":512,"result":{"content":"\"not found\""}} +` + +const gBatchURLs = "# nightly targets\nhttps://example.com/a\n\nhttps://example.com/b\n" + +func gBatchSetupURLs(t *testing.T, e *goldenEnv) { goldenWrite(t, e.Dir, "urls.txt", gBatchURLs) } + +func init() { + queued := gJob{Status: "queued", Total: 2, Name: "nightly"} + running := gJob{Status: "running", Done: 1, Total: 2, Name: "nightly"} + completed := gJob{Status: "completed", Done: 3, Failed: 1, Total: 3, Name: "nightly", Finished: true, Exp: true} + failed := gJob{Status: "failed", Done: 2, Failed: 2, Total: 10, Name: "nightly", Finished: true, Err: "ERR::LIMIT::FAILURE_THRESHOLD"} + cancelled := gJob{Status: "cancelled", Done: 1, Total: 5, Name: "nightly", Finished: true} + open := gJob{Status: "running", Done: 4, Total: 9, Name: "crawl", Open: true} + + submit := goldenServe(map[string]http.HandlerFunc{"POST /v1/batch": gJSON(202, queued.JSON())}) + get := func(j gJob) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}": gJSON(200, j.JSON())}) + } + submitWait := func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{ + "POST /v1/batch": gJSON(202, queued.JSON()), + "GET /v1/batch/{id}": gSeq(queued.JSON(), running.JSON(), completed.JSON()), + })() + } + list := func(body string) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{"GET /v1/batch": gJSON(200, body)}) + } + two := `{"batches":[` + completed.JSON() + `,` + open.JSON() + `]}` + results := func(next string) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/results": func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/x-ndjson") + if next != "" && r.URL.Query().Get("cursor") == "" { + w.Header().Set("X-Next-Cursor", next) + } + _, _ = fmt.Fprint(w, gBatchResults) + }}) + } + job := func(method string, j gJob) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{method + " /v1/batch/{id}/{action}": gJSON(200, j.JSON())}) + } + wrapped := func(field string, n int) string { + return fmt.Sprintf(`{"job":%s,"%s":%d,"items_dispatched":%d}`, running.JSON(), field, n, n-1) + } + + goldenAdd( + // submit + goldenCase{name: "batch/submit_urls", args: []string{"batch", "submit", "{dir}/urls.txt", "--render", "--name", "nightly"}, setup: gBatchSetupURLs, server: submit}, + goldenCase{name: "batch/submit_items_jsonl", args: []string{"batch", "submit", "{dir}/items.jsonl", "--format", "markdown"}, + setup: func(t *testing.T, e *goldenEnv) { + goldenWrite(t, e.Dir, "items.jsonl", `{"url":"https://example.com/a","external_id":"sku-1"}`+"\n"+`{"url":"https://example.com/b","external_id":"sku-2","js_render":false}`+"\n") + }, server: submit}, + goldenCase{name: "batch/submit_stdin", args: []string{"batch", "submit", "-"}, stdin: gBatchURLs, server: submit}, + goldenCase{name: "batch/submit_wait", args: []string{"batch", "submit", "{dir}/urls.txt", "--wait", "--poll", "1ms"}, setup: gBatchSetupURLs, server: submitWait}, + goldenCase{name: "batch/submit_wait_open_is_usage_error", args: []string{"batch", "submit", "{dir}/urls.txt", "--wait", "--open"}, stderr: true, setup: gBatchSetupURLs, server: submit}, + goldenCase{name: "batch/submit_err_request", args: []string{"batch", "submit", "{dir}/urls.txt", "--format", "pdf"}, stderr: true, setup: gBatchSetupURLs, + server: goldenServe(map[string]http.HandlerFunc{"POST /v1/batch": gProblem(400, "ERR::REQUEST::INVALID_PARAMETER", "response_format pdf is not supported on batch")})}, + goldenCase{name: "batch_human/submit_urls", human: true, args: []string{"batch", "submit", "{dir}/urls.txt", "--name", "nightly"}, setup: gBatchSetupURLs, server: submit}, + goldenCase{name: "batch_human/submit_wait", human: true, args: []string{"batch", "submit", "{dir}/urls.txt", "--wait", "--poll", "1ms"}, setup: gBatchSetupURLs, server: submitWait}, + + // wait + goldenCase{name: "batch/wait_completed", args: []string{"batch", "wait", gBatchID, "--poll", "1ms"}, server: get(completed)}, + goldenCase{name: "batch/wait_sequence", args: []string{"batch", "wait", gBatchID, "--poll", "1ms"}, + server: func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}": gSeq(queued.JSON(), running.JSON(), completed.JSON())})() + }}, + goldenCase{name: "batch/wait_failed_exit4", args: []string{"batch", "wait", gBatchID, "--poll", "1ms"}, stderr: true, server: get(failed)}, + goldenCase{name: "batch/wait_cancelled_exit4", args: []string{"batch", "wait", gBatchID, "--poll", "1ms"}, stderr: true, server: get(cancelled)}, + goldenCase{name: "batch/wait_pending_exit11", args: []string{"batch", "wait", gBatchID, "--poll", "10ms", "--max-wait", "60ms"}, stderr: true, server: get(running)}, + goldenCase{name: "batch_human/wait_completed", human: true, args: []string{"batch", "wait", gBatchID, "--poll", "1ms"}, server: get(completed)}, + goldenCase{name: "batch_human/wait_failed_exit4", human: true, args: []string{"batch", "wait", gBatchID, "--poll", "1ms"}, server: get(failed)}, + + // get + goldenCase{name: "batch/get", args: []string{"batch", "get", gBatchID}, server: get(completed)}, + goldenCase{name: "batch/get_not_found", args: []string{"batch", "get", gBatchID}, stderr: true, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}": gProblem(404, "ERR::REQUEST::NOT_FOUND", "no such batch")})}, + goldenCase{name: "batch_human/get_completed", human: true, args: []string{"batch", "get", gBatchID}, server: get(completed)}, + goldenCase{name: "batch_human/get_open", human: true, args: []string{"batch", "get", gBatchID}, server: get(open)}, + goldenCase{name: "batch_human/get_failed", human: true, args: []string{"batch", "get", gBatchID}, server: get(failed)}, + + // list + goldenCase{name: "batch/list", args: []string{"batch", "list"}, server: list(two)}, + goldenCase{name: "batch/list_empty", args: []string{"batch", "list", "--status", "paused"}, server: list(`{"batches":[]}`)}, + goldenCase{name: "batch/list_next_cursor", args: []string{"batch", "list", "--limit", "2"}, server: list(`{"batches":[` + completed.JSON() + `],"next_cursor":"1790000000000000000.ABC"}`)}, + goldenCase{name: "batch/list_all_pages", args: []string{"batch", "list", "--all"}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch": func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("cursor") == "" { + gJSON(200, `{"batches":[`+completed.JSON()+`],"next_cursor":"1790000000000000000.ABC"}`)(w, r) + return + } + gJSON(200, `{"batches":[`+open.JSON()+`]}`)(w, r) + }})}, + goldenCase{name: "batch_human/list", human: true, args: []string{"batch", "list"}, server: list(two)}, + goldenCase{name: "batch_human/list_empty", human: true, args: []string{"batch", "list"}, server: list(`{"batches":[]}`)}, + goldenCase{name: "batch_human/list_next_cursor", human: true, args: []string{"batch", "list"}, server: list(`{"batches":[` + completed.JSON() + `],"next_cursor":"1790000000000000000.ABC"}`)}, + + // results and content + goldenCase{name: "batch/results", args: []string{"batch", "results", gBatchID}, server: results("")}, + goldenCase{name: "batch/results_more_pages", args: []string{"batch", "results", gBatchID, "--limit", "3"}, server: results("c2")}, + goldenCase{name: "batch/results_all_pages", args: []string{"batch", "results", gBatchID, "--all"}, server: results("c2")}, + goldenCase{name: "batch/results_output_file", args: []string{"batch", "results", gBatchID, "-o", "{dir}/results.jsonl"}, server: results(""), files: []string{"results.jsonl"}}, + goldenCase{name: "batch/results_empty", args: []string{"batch", "results", gBatchID}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/results": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(200) }})}, + goldenCase{name: "batch/results_expired", args: []string{"batch", "results", gBatchID}, stderr: true, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/results": gProblem(410, "ERR::REQUEST::BEYOND_RETENTION", "results expired")})}, + goldenCase{name: "batch_human/results", human: true, args: []string{"batch", "results", gBatchID}, server: results("")}, + goldenCase{name: "batch_human/results_empty", human: true, args: []string{"batch", "results", gBatchID}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/results": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(200) }})}, + goldenCase{name: "batch/content", args: []string{"batch", "content", gBatchID, "0"}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/tasks/{seq}/content": gJSON(200, `"

A

\n"`)})}, + goldenCase{name: "batch/content_output_file", args: []string{"batch", "content", gBatchID, "0", "-o", "{dir}/a.html"}, files: []string{"a.html"}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/tasks/{seq}/content": gJSON(200, `"

A

\n"`)})}, + goldenCase{name: "batch/content_not_finished", args: []string{"batch", "content", gBatchID, "7"}, stderr: true, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/tasks/{seq}/content": gProblem(409, "ERR::REQUEST::NOT_READY", "item 7 has not finished")})}, + goldenCase{name: "batch/content_bad_seq", args: []string{"batch", "content", gBatchID, "x"}, stderr: true}, + goldenCase{name: "batch_human/content", human: true, args: []string{"batch", "content", gBatchID, "0"}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/tasks/{seq}/content": gJSON(200, `"

A

\n"`)})}, + + // cancel, close, retry, append + goldenCase{name: "batch/cancel_yes", args: []string{"batch", "cancel", gBatchID, "--yes"}, server: job("POST", cancelled)}, + goldenCase{name: "batch/cancel_needs_yes_without_terminal", args: []string{"batch", "cancel", gBatchID}, stderr: true, server: job("POST", cancelled)}, + goldenCase{name: "batch_human/cancel_yes", human: true, args: []string{"batch", "cancel", gBatchID, "--yes"}, server: job("POST", cancelled)}, + goldenCase{name: "batch/close", args: []string{"batch", "close", gBatchID}, server: job("POST", running)}, + goldenCase{name: "batch/retry", args: []string{"batch", "retry", gBatchID}, server: goldenServe(map[string]http.HandlerFunc{"POST /v1/batch/{id}/retry": gJSON(200, wrapped("items_reset", 2))})}, + goldenCase{name: "batch_human/retry", human: true, args: []string{"batch", "retry", gBatchID}, server: goldenServe(map[string]http.HandlerFunc{"POST /v1/batch/{id}/retry": gJSON(200, wrapped("items_reset", 2))})}, + goldenCase{name: "batch/append", args: []string{"batch", "append", gBatchID, "{dir}/urls.txt"}, setup: gBatchSetupURLs, + server: goldenServe(map[string]http.HandlerFunc{"POST /v1/batch/{id}/items": gJSON(200, wrapped("items_added", 2))})}, + goldenCase{name: "batch_human/append", human: true, args: []string{"batch", "append", gBatchID, "{dir}/urls.txt"}, setup: gBatchSetupURLs, + server: goldenServe(map[string]http.HandlerFunc{"POST /v1/batch/{id}/items": gJSON(200, wrapped("items_added", 2))})}, + ) +} diff --git a/internal/cmd/config_golden_test.go b/internal/cmd/config_golden_test.go new file mode 100644 index 0000000..0f39c7d --- /dev/null +++ b/internal/cmd/config_golden_test.go @@ -0,0 +1,119 @@ +package cmd + +import ( + "net/http" + "testing" +) + +const gSavedKey = "spicrawl_live_0123456789abcdefghijklmnop" + +func gSetupConfig(t *testing.T, e *goldenEnv) { + goldenWrite(t, e.CfgDir, "config.json", `{"api_key":"`+gSavedKey+`","base_url":"https://api.example.test"}`) +} + +func gSetupKeyOnlyConfig(t *testing.T, e *goldenEnv) { + goldenWrite(t, e.CfgDir, "config.json", `{"api_key":"`+gSavedKey+`"}`) +} + +// gAuthAPI answers the one call login and auth status make: GET /v1/requests?limit=1. +func gAuthAPI(status int) func() http.HandlerFunc { + h := gJSON(200, `{"data":[],`+gLogPageEnd+`}`) + if status == 401 { + h = gProblem(401, "ERR::AUTH::INVALID_KEY", "the API key is not valid") + } + return goldenServe(map[string]http.HandlerFunc{"GET /v1/requests": h}) +} + +const gDocsHome = "# Spicrawl\n\n- [Quickstart](https://docs.spicrawl.com/quickstart.md)\n- [Anti-bot](https://docs.spicrawl.com/guides/anti-bot.md)\n" + +func gDocsAPI() func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{ + "GET /llms.txt": func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte(gDocsHome)) + }, + "GET /guides/anti-bot.md": func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/markdown") + _, _ = w.Write([]byte("# Anti-bot\n\nUse --stealth only when a page needs it.\n")) + }, + "GET /no-newline.md": func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("# No newline")) }, + }) +} + +func init() { + docsEnv := map[string]string{docsEnvURL: "{server}/"} + + goldenAdd( + // config + goldenCase{name: "config/get_all", args: []string{"config", "get"}, setup: gSetupConfig}, + goldenCase{name: "config/get_all_empty", args: []string{"config", "get"}}, + goldenCase{name: "config/get_base_url", args: []string{"config", "get", "base_url"}, setup: gSetupConfig}, + goldenCase{name: "config/get_api_key_masked", args: []string{"config", "get", "api_key"}, setup: gSetupConfig}, + goldenCase{name: "config/get_api_key_show_secret", args: []string{"config", "get", "api_key", "--show-secret"}, setup: gSetupConfig}, + goldenCase{name: "config/get_unset", args: []string{"config", "get", "base_url"}}, + goldenCase{name: "config/set_base_url", args: []string{"config", "set", "base_url", "http://localhost:8080/"}}, + goldenCase{name: "config/set_api_key", args: []string{"config", "set", "api_key", gSavedKey}}, + goldenCase{name: "config/unset_api_key", args: []string{"config", "unset", "api_key"}, setup: gSetupConfig}, + goldenCase{name: "config/path", args: []string{"config", "path"}}, + goldenCase{name: "config/err_unknown_key", args: []string{"config", "get", "nope"}, stderr: true}, + goldenCase{name: "config/err_bad_base_url", args: []string{"config", "set", "base_url", "ftp://x"}, stderr: true}, + goldenCase{name: "config_human/get_all", human: true, args: []string{"config", "get"}, setup: gSetupConfig}, + goldenCase{name: "config_human/get_all_empty", human: true, args: []string{"config", "get"}}, + goldenCase{name: "config_human/get_base_url", human: true, args: []string{"config", "get", "base_url"}, setup: gSetupConfig}, + goldenCase{name: "config_human/get_unset", human: true, args: []string{"config", "get", "base_url"}}, + goldenCase{name: "config_human/set_api_key", human: true, args: []string{"config", "set", "api_key", gSavedKey}}, + goldenCase{name: "config_human/unset_api_key", human: true, args: []string{"config", "unset", "api_key"}, setup: gSetupConfig}, + goldenCase{name: "config_human/path", human: true, args: []string{"config", "path"}}, + + // auth + goldenCase{name: "auth/status_ok", args: []string{"auth", "status"}, server: gAuthAPI(200)}, + goldenCase{name: "auth/status_offline", args: []string{"auth", "status", "--offline"}}, + goldenCase{name: "auth/status_key_from_file", args: []string{"auth", "status", "--offline"}, noKey: true, setup: gSetupKeyOnlyConfig}, + goldenCase{name: "auth/status_rejected_exit3", args: []string{"auth", "status"}, stderr: true, server: gAuthAPI(401)}, + goldenCase{name: "auth/status_no_key_exit3", args: []string{"auth", "status"}, noKey: true, stderr: true}, + goldenCase{name: "auth_human/status_ok", human: true, args: []string{"auth", "status"}, server: gAuthAPI(200)}, + goldenCase{name: "auth_human/status_rejected_exit3", human: true, args: []string{"auth", "status"}, server: gAuthAPI(401)}, + goldenCase{name: "auth_human/status_no_key_exit3", human: true, args: []string{"auth", "status"}, noKey: true}, + goldenCase{name: "auth/login_ok", args: []string{"login", "--api-key", gSavedKey}, server: gAuthAPI(200)}, + goldenCase{name: "auth/login_rejected_exit3", args: []string{"login", "--api-key", gSavedKey}, stderr: true, server: gAuthAPI(401)}, + goldenCase{name: "auth/login_needs_key_without_terminal", args: []string{"login"}, noKey: true, stderr: true, server: gAuthAPI(200)}, + goldenCase{name: "auth_human/login_ok", human: true, args: []string{"login", "--api-key", gSavedKey}, server: gAuthAPI(200)}, + goldenCase{name: "auth/logout_removed", args: []string{"logout"}, setup: gSetupConfig}, + goldenCase{name: "auth/logout_nothing_saved", args: []string{"logout"}}, + goldenCase{name: "auth_human/logout_removed", human: true, args: []string{"logout"}, setup: gSetupConfig}, + goldenCase{name: "auth_human/logout_nothing_saved", human: true, args: []string{"logout"}}, + + // version, exit codes, root errors + goldenCase{name: "meta/version", args: []string{"version"}}, + goldenCase{name: "meta/version_human", human: true, args: []string{"version"}}, + goldenCase{name: "meta/exit_codes", args: []string{"exit-codes"}}, + goldenCase{name: "meta/exit_codes_human", human: true, args: []string{"exit-codes"}}, + goldenCase{name: "meta/err_unknown_command", args: []string{"zzzzzz"}, stderr: true}, + goldenCase{name: "meta/err_unknown_subcommand_suggests", args: []string{"batch", "lst"}, stderr: true}, + goldenCase{name: "meta/err_unknown_flag", args: []string{"scrape", "https://example.com", "--nope"}, stderr: true}, + + // schema: printed from the embedded files (no network) + goldenCase{name: "schema/list", args: []string{"schema"}}, + goldenCase{name: "schema/list_flag", args: []string{"schema", "--list"}}, + goldenCase{name: "schema/session", args: []string{"schema", "session"}, digest: true}, + goldenCase{name: "schema/scrape", args: []string{"schema", "scrape"}, digest: true}, + goldenCase{name: "schema/batch", args: []string{"schema", "batch"}, digest: true}, + goldenCase{name: "schema/batch_item", args: []string{"schema", "batch-item"}, digest: true}, + goldenCase{name: "schema/err_unknown", args: []string{"schema", "nope"}, stderr: true}, + goldenCase{name: "schema_human/list", human: true, args: []string{"schema"}}, + goldenCase{name: "schema_human/session", human: true, args: []string{"schema", "session"}, digest: true}, + goldenCase{name: "schema_human/scrape", human: true, args: []string{"schema", "scrape"}, digest: true}, + goldenCase{name: "schema_human/batch", human: true, args: []string{"schema", "batch"}, digest: true}, + goldenCase{name: "schema_human/batch_item", human: true, args: []string{"schema", "batch-item"}, digest: true}, + + // docs: the Markdown is the data, so it is printed as is whether or not stdout is a terminal + goldenCase{name: "docs/topic_raw_markdown", args: []string{"docs", "guides/anti-bot"}, env: docsEnv, server: gDocsAPI()}, + goldenCase{name: "docs/list_raw_markdown", args: []string{"docs", "--list"}, env: docsEnv, server: gDocsAPI()}, + goldenCase{name: "docs/no_topic_is_the_index", args: []string{"docs"}, env: docsEnv, server: gDocsAPI()}, + goldenCase{name: "docs/adds_missing_newline", args: []string{"docs", "no-newline"}, env: docsEnv, server: gDocsAPI()}, + goldenCase{name: "docs/json_flag_wraps", args: []string{"docs", "guides/anti-bot"}, explicitJSON: true, env: docsEnv, server: gDocsAPI()}, + goldenCase{name: "docs/err_unknown_topic", args: []string{"docs", "missing"}, stderr: true, env: docsEnv, server: gDocsAPI()}, + goldenCase{name: "docs/err_bad_topic", args: []string{"docs", "../etc/passwd"}, stderr: true, env: docsEnv, server: gDocsAPI()}, + goldenCase{name: "docs_human/topic_raw_markdown", human: true, args: []string{"docs", "guides/anti-bot"}, env: docsEnv, server: gDocsAPI()}, + ) +} diff --git a/internal/cmd/harness_golden_test.go b/internal/cmd/harness_golden_test.go new file mode 100644 index 0000000..de5858f --- /dev/null +++ b/internal/cmd/harness_golden_test.go @@ -0,0 +1,464 @@ +package cmd + +// Golden tests: they lock what the CLI writes to stdout (byte for byte) and the +// exit code it returns, for every command and its main modes, so that terminal +// work (spinners, colour, TUI) cannot change what scripts and AI agents parse. +// DESIGN.md "Contract agents rely on" is the rule they enforce: stdout is +// data only, JSON when stdout is not a terminal, stable exit codes, and the +// problem document on stderr in JSON mode. +// +// Every case runs three times against its own fake API server, with the +// environment a non-interactive caller can have: nothing set, NO_COLOR=1 and +// CI=true. All three must produce the same stdout, the same exit code and a +// stderr (and stdout) with no ANSI escape, so a feature that colours or +// animates must stay off when the output is not a terminal. +// +// Files (testdata/golden/.): +// +// .stdout stdout, normalized (see goldenNormalize) +// .stdout.sha256 instead of .stdout for outputs too big to review (digest case flag) +// .exit the exit code +// .stderr stderr, only for cases where stderr is itself machine output +// (JSON mode: the problem document); other stderr is prose +// .file. a file the command was asked to write (-o, --screenshot-dir) +// +// Update them after an intended change, then review the diff: +// +// go test ./internal/cmd -run TestGolden -update (or GOLDEN_UPDATE=1 go test ./...) +// +// (-update is a flag of this package's test binary only, so use GOLDEN_UPDATE=1 +// when running ./... .) -update also deletes golden files no case produces. +// +// The default mode of a case is what a script sees: stdout not a terminal, so +// JSON unless the command says otherwise (docs, raw documents). A case with +// human set runs the code path a person at a terminal gets (humanOutput), which +// is the only way to reach it without a pty. + +import ( + "bytes" + "crypto/sha256" + "flag" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "testing" + + "github.com/OfficialSpicrawl/cli/internal/api" +) + +var goldenUpdate = flag.Bool("update", false, "rewrite the files under testdata/golden (or set GOLDEN_UPDATE=1)") + +func goldenWantUpdate() bool { return *goldenUpdate || os.Getenv("GOLDEN_UPDATE") == "1" } + +// goldenKey is the fake API key every case runs with (unless noKey). +const goldenKey = "spicrawl_test_0123456789abcdefghij" + +// goldenCase is one invocation of the CLI. +type goldenCase struct { + name string // "group/case": the golden file stem; unique + args []string // after the global flags; {server}, {dir} and {home} are replaced + human bool // run the human (terminal) code path instead of the script one + + explicitJSON bool // pass --json (the default for a script is the same) + noKey bool // no API key in the environment + baseURL string // --base-url, default: the fake server + server func() http.HandlerFunc // the fake API; built fresh for every run + env map[string]string // extra environment; tokens as in args + stdin string // what os.Stdin holds (never a terminal) + setup func(t *testing.T, e *goldenEnv) + + stderr bool // golden stderr too: it is machine-readable in this case + digest bool // golden a sha256 of stdout instead of the bytes (big outputs) + files []string // paths under {dir} the command writes; their bytes are golden too +} + +// goldenEnv is the sandbox of one run. +type goldenEnv struct { + Dir string // working/project directory + Home string // $HOME + CfgDir string // holds config.json ($SPICRAWL_CONFIG) + Server string // fake API URL, "" when the case has none +} + +var goldenCases []goldenCase + +// goldenAdd registers cases; each *_golden_test.go calls it from init(). +func goldenAdd(cs ...goldenCase) { + seen := map[string]bool{} + for _, c := range goldenCases { + seen[c.name] = true + } + for _, c := range cs { + if seen[c.name] { + panic("duplicate golden case " + c.name) + } + seen[c.name] = true + goldenCases = append(goldenCases, c) + } +} + +// goldenVariants are the environments every case must be insensitive to. +var goldenVariants = []struct { + name string + env map[string]string +}{ + {"plain", nil}, + {"NO_COLOR", map[string]string{"NO_COLOR": "1"}}, + {"CI", map[string]string{"CI": "true"}}, +} + +func TestGolden(t *testing.T) { + for _, c := range goldenCases { + t.Run(strings.ReplaceAll(c.name, "/", "."), func(t *testing.T) { + for _, v := range goldenVariants { + t.Run(v.name, func(t *testing.T) { + goldenRun(t, c, goldenWantUpdate() && v.name == "plain", v.env) + }) + } + }) + } +} + +func goldenRun(t *testing.T, c goldenCase, update bool, variantEnv map[string]string) { + t.Helper() + r, e := goldenExec(t, c, variantEnv) + norm := func(s string) string { return goldenNormalize(s, e) } + + // Not a terminal, so never colour or animation, whatever NO_COLOR/CI say. + for name, s := range map[string]string{"stdout": r.stdout, "stderr": r.stderr} { + if strings.Contains(s, "\x1b") { + t.Errorf("%s has an ANSI escape on a non-terminal: %q", name, s) + } + } + if strings.Contains(r.stderr, "\r") { + t.Errorf("stderr has a carriage return (an in-place spinner or progress line) on a non-terminal: %q", r.stderr) + } + if r.stray != "" { + t.Errorf("the command wrote to os.Stdout directly instead of the stdout writer: %q", r.stray) + } + + out := norm(r.stdout) + if c.digest { + sum := sha256.Sum256([]byte(out)) + goldenCompare(t, update, c.name, ".stdout.sha256", []byte(fmt.Sprintf("sha256 %x\nbytes %d\n", sum, len(out)))) + } else { + goldenCompare(t, update, c.name, ".stdout", []byte(out)) + } + goldenCompare(t, update, c.name, ".exit", []byte(fmt.Sprintf("%d\n", r.code))) + if c.stderr { + goldenCompare(t, update, c.name, ".stderr", []byte(norm(r.stderr))) + } + for _, rel := range c.files { + b, err := os.ReadFile(filepath.Join(e.Dir, rel)) + if err != nil { + t.Errorf("expected the command to write %s: %v", rel, err) + continue + } + goldenCompare(t, update, c.name, ".file."+strings.ReplaceAll(rel, "/", "__"), []byte(norm(string(b)))) + } +} + +type goldenResult struct { + stdout, stderr, stray string + code int +} + +// goldenExec runs c in a fresh sandbox with the variant's extra environment. +func goldenExec(t *testing.T, c goldenCase, variantEnv map[string]string) (goldenResult, *goldenEnv) { + t.Helper() + e := &goldenEnv{Dir: t.TempDir(), Home: t.TempDir(), CfgDir: t.TempDir()} + + // A clean, deterministic environment, whatever the developer's shell has. + for _, k := range []string{ + "NO_COLOR", "CI", "FORCE_COLOR", "CLICOLOR", "CLICOLOR_FORCE", "SPICRAWL_API_KEY", "SPICRAWL_BASE_URL", + agentEnvMCPURL, agentEnvSkillURL, docsEnvURL, docsEnvHost, + } { + goldenUnsetenv(t, k) + } + t.Setenv("HOME", e.Home) + t.Setenv("XDG_CONFIG_HOME", "") + t.Setenv("CODEX_HOME", "") + t.Setenv("TERM", "xterm-256color") // a colour-capable TERM: only the non-terminal stdout may keep colour off + t.Setenv("COLORTERM", "truecolor") + t.Setenv("SPICRAWL_CONFIG", filepath.Join(e.CfgDir, "config.json")) + if !c.noKey { + t.Setenv("SPICRAWL_API_KEY", goldenKey) + } + + if c.server != nil { + srv := httptest.NewServer(c.server()) + t.Cleanup(srv.Close) + e.Server = srv.URL + } + tokens := strings.NewReplacer("{server}", e.Server, "{dir}", e.Dir, "{home}", e.Home) + for k, v := range c.env { + t.Setenv(k, tokens.Replace(v)) + } + for k, v := range variantEnv { + t.Setenv(k, v) + } + if c.setup != nil { + c.setup(t, e) + } + + base := c.baseURL + if base == "" { + base = e.Server + } + args := []string{"--base-url", base} + if c.explicitJSON { + args = append(args, "--json") + } + for _, a := range c.args { + args = append(args, tokens.Replace(a)) + } + + // Never a terminal: not even when `go test` itself runs in one. + tmp := func(name string) *os.File { + f, err := os.CreateTemp(t.TempDir(), name) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { f.Close() }) + return f + } + in, so, se := tmp("stdin"), tmp("stdout"), tmp("stderr") + _, _ = in.WriteString(c.stdin) + _, _ = in.Seek(0, 0) + + var ( + ob, eb bytes.Buffer + code int + ) + func() { + prevIn, prevOut, prevErr := os.Stdin, os.Stdout, os.Stderr + prevW, prevEW, prevHuman := stdout, stderr, humanOutput + os.Stdin, os.Stdout, os.Stderr = in, so, se + stdout, stderr, humanOutput = &ob, &eb, c.human + defer func() { + os.Stdin, os.Stdout, os.Stderr = prevIn, prevOut, prevErr + stdout, stderr, humanOutput = prevW, prevEW, prevHuman + resetFlags(rootCmd) + }() + rootCmd.SetArgs(args) + code = Execute() + }() + stray, _ := os.ReadFile(so.Name()) + return goldenResult{stdout: ob.String(), stderr: eb.String(), stray: string(stray), code: code}, e +} + +// goldenUnsetenv removes k for the test (t.Setenv alone can only set it). +func goldenUnsetenv(t *testing.T, k string) { + t.Helper() + t.Setenv(k, "") + _ = os.Unsetenv(k) +} + +// goldenCompare checks got against testdata/golden/, or rewrites +// the file when update is set. +func goldenCompare(t *testing.T, update bool, name, ext string, got []byte) { + t.Helper() + path := goldenPath(name, ext) + if update { + if old, err := os.ReadFile(path); err == nil && bytes.Equal(old, got) { + return + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, got, 0o644); err != nil { + t.Fatal(err) + } + return + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("no golden file %s (run: go test ./internal/cmd -run TestGolden -update)", path) + } + if !bytes.Equal(want, got) { + t.Errorf("%s differs from the golden file (intended? run: go test ./internal/cmd -run TestGolden -update)\n%s", path, goldenDiff(want, got)) + } +} + +func goldenPath(name, ext string) string { + return filepath.Join("testdata", "golden", filepath.FromSlash(name)+ext) +} + +// goldenDiff shows the first line where want and got differ. +func goldenDiff(want, got []byte) string { + w, g := strings.Split(string(want), "\n"), strings.Split(string(got), "\n") + for i := 0; i < len(w) || i < len(g); i++ { + var wl, gl string + if i < len(w) { + wl = w[i] + } + if i < len(g) { + gl = g[i] + } + if wl != gl || i >= len(w) || i >= len(g) { + return fmt.Sprintf("first difference at line %d:\n want %q\n got %q\n(want %d bytes, got %d bytes)", i+1, wl, gl, len(want), len(got)) + } + } + return "(same lines, different bytes)" +} + +// TestGoldenNoOrphans fails when testdata/golden holds a file no registered +// case produces (a renamed or deleted case); -update removes them. +func TestGoldenNoOrphans(t *testing.T) { + want := map[string]bool{} + for _, c := range goldenCases { + stdoutExt := ".stdout" + if c.digest { + stdoutExt = ".stdout.sha256" + } + exts := []string{stdoutExt, ".exit"} + if c.stderr { + exts = append(exts, ".stderr") + } + for _, f := range c.files { + exts = append(exts, ".file."+strings.ReplaceAll(f, "/", "__")) + } + for _, ext := range exts { + want[filepath.ToSlash(goldenPath(c.name, ext))] = true + } + } + var orphans []string + _ = filepath.WalkDir(filepath.Join("testdata", "golden"), func(p string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() && d.Name() != ".gitattributes" && !want[filepath.ToSlash(p)] { + orphans = append(orphans, p) + } + return nil + }) + sort.Strings(orphans) + for _, p := range orphans { + if goldenWantUpdate() { + _ = os.Remove(p) + continue + } + t.Errorf("orphan golden file %s: no case produces it (run: go test ./internal/cmd -run TestGolden -update)", p) + } +} + +// ---- normalization ---------------------------------------------------------- + +var ( + goldenTimestampRE = regexp.MustCompile(`\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})`) + goldenReqIDJSONRE = regexp.MustCompile(`("request_id":\s*")[^"]+(")`) + goldenReqIDTextRE = regexp.MustCompile(`(request id: )\S+( \(spicrawl logs get )\S+(\))`) +) + +// goldenNormalize turns the parts of an output that differ between runs and +// machines into fixed placeholders, so a golden file changes only when the +// CLI's output does. It is applied to stdout, stderr and written files: +// +// - the fake server's address (random port) -> SERVER (127.0.0.1:41233 -> SERVER) +// - the sandbox directories (random temp paths) -> , , +// - build identity: CLI version, Go version, OS and architecture +// (api.Version is stamped by -ldflags; the others are the host's) +// -> , , , +// - RFC 3339 timestamps (the fixtures' and any the CLI might stamp) -> +// - non-empty request ids ("request_id" fields, "request id:" lines) -> +// +// Empty values are left alone, so a field that goes missing or empty still +// shows. The CLI prints no measured durations today (the durations in its output +// are flags echoed back, e.g. "after 30m0s"), so there is nothing to mask; if +// one is ever added, add its pattern here rather than letting goldens flake. +func goldenNormalize(s string, e *goldenEnv) string { + if e != nil { + for _, d := range []struct{ path, name string }{{e.Dir, ""}, {e.Home, ""}, {e.CfgDir, ""}} { + s = strings.ReplaceAll(s, d.path, d.name) + if real, err := filepath.EvalSymlinks(d.path); err == nil && real != d.path { // macOS: /var -> /private/var + s = strings.ReplaceAll(s, real, d.name) + } + } + if e.Server != "" { + s = strings.ReplaceAll(s, strings.TrimPrefix(e.Server, "http://"), "SERVER") + } + } + for _, r := range [][2]string{ + {`"version": "` + api.Version + `"`, `"version": ""`}, + {"spicrawl " + api.Version + " (", "spicrawl ("}, + {runtime.Version(), ""}, + {`"os": "` + runtime.GOOS + `"`, `"os": ""`}, + {`"arch": "` + runtime.GOARCH + `"`, `"arch": ""`}, + {runtime.GOOS + "/" + runtime.GOARCH + ")", "/)"}, + } { + s = strings.ReplaceAll(s, r[0], r[1]) + } + s = goldenTimestampRE.ReplaceAllString(s, "") + s = goldenReqIDJSONRE.ReplaceAllString(s, "${1}${2}") + s = goldenReqIDTextRE.ReplaceAllString(s, "${1}${2}${3}") + return s +} + +func TestGoldenNormalize(t *testing.T) { + e := &goldenEnv{Dir: "/tmp/x/001", Home: "/tmp/x/002", CfgDir: "/tmp/x/003", Server: "http://127.0.0.1:41233"} + in := "ws://127.0.0.1:41233/v1 /tmp/x/001/a /tmp/x/003/config.json 2026-09-22T06:00:00.5Z 2026-09-22T06:00:00+02:00 " + + `"request_id": "01J9ZQ5N" "request_id": "" request id: 01ABC (spicrawl logs get 01ABC) ` + + `"version": "` + api.Version + `" ` + runtime.Version() + " 2026-09-22" + want := "ws://SERVER/v1 /a /config.json " + + `"request_id": "" "request_id": "" request id: (spicrawl logs get ) ` + + `"version": "" 2026-09-22` + if got := goldenNormalize(in, e); got != want { + t.Errorf("got\n%s\nwant\n%s", got, want) + } +} + +// ---- fake API building blocks ------------------------------------------------- + +// goldenServe builds a fake API from "METHOD /path" patterns (net/http ServeMux +// syntax, so "GET /v1/batch/{id}" works). Anything else is a 404 problem. +func goldenServe(routes map[string]http.HandlerFunc) func() http.HandlerFunc { + return func() http.HandlerFunc { + mux := http.NewServeMux() + for pattern, h := range routes { + mux.HandleFunc(pattern, h) + } + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + writeProblem(w, http.StatusNotFound, "ERR::REQUEST::NOT_FOUND", false) + }) + return mux.ServeHTTP + } +} + +// gJSON answers with a JSON body. +func gJSON(status int, body string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = io.WriteString(w, body) + } +} + +// gProblem answers with an RFC 7807 problem document (a full one: detail, +// request id and a hint, so the human error renderer has every line to show). +func gProblem(status int, code, detail string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/problem+json") + w.Header().Set("X-Request-Id", "01J9ZQ9ERR0000000000000000") + w.WriteHeader(status) + fmt.Fprintf(w, `{"type":"https://docs.spicrawl.com/errors","title":%q,"status":%d,"code":%q,"detail":%q,"retryable":false,"doc_url":"https://docs.spicrawl.com/errors","request_id":"01J9ZQ9ERR0000000000000000","target_status":null,"diagnostics":{"hint":"check the request and try again"}}`, + http.StatusText(status), status, code, detail) + } +} + +// goldenWrite creates path (relative to dir) with content, making parent +// directories, for a case's setup. +func goldenWrite(t *testing.T, dir, rel, content string) string { + t.Helper() + p := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return p +} diff --git a/internal/cmd/ops_golden_test.go b/internal/cmd/ops_golden_test.go new file mode 100644 index 0000000..bf3c944 --- /dev/null +++ b/internal/cmd/ops_golden_test.go @@ -0,0 +1,174 @@ +package cmd + +import ( + "net/http" +) + +// ---- logs ---- + +const ( + gLogOK = `{"id":"01M0FK1EP2E7FBMWAR7W10SMXW","created_at":"2026-09-22T10:00:00Z","project_id":"proj_01J9Z","status":"success","http_status":200,"engine":"fetch",` + + `"credits_micro":1500000,"url":"https://example.com/a","attempts":2,"bytes_in":2048,"bytes_out":512,"proxy_bytes":4096,` + + `"spans":{"admission_ms":1,"dispatch_ms":2,"worker_ms":3,"queue_ms":0,"engine_ms":80,"upstream_ms":120,"extract_ms":null,"total_ms":210},` + + `"proxy":{"source":"pool","reason":"default","provider":"dc-1","requested_tier":"datacenter","requested_country":"de","country":"de","endpoint_id":"ep-1","endpoints":["ep-1","ep-2"],"sticky":true}}` + + gLogBlocked = `{"id":"01M0FK1EP2E7FBMWAR7W10SMXX","created_at":"2026-09-22T10:00:01Z","project_id":"proj_01J9Z","status":"blocked","http_status":403,"engine":"",` + + `"credits_micro":0,"url":"https://example.com/b?x=1&y=2","error_code":"ERR::UPSTREAM::BLOCKED","error_detail":"bot challenge",` + + `"spans":{"admission_ms":1,"dispatch_ms":null,"worker_ms":null,"queue_ms":0,"engine_ms":null,"upstream_ms":120,"extract_ms":null,"total_ms":130},` + + `"proxy":{"source":"pool"},"blocked":{"vendor":"cloudflare","rule":"r-17","signal":"cf-mitigated"}}` + + gLogPageEnd = `"page":{"has_more":false,"retention_hours":24,"retained_since":"2026-09-21T10:00:00Z"}` + gLogPageMore = `"page":{"has_more":true,"next_before":"2026-09-22T10:00:00.5Z","next_before_id":"01M0FK1EP2E7FBMWAR7W10SMXW","retention_hours":24,"retained_since":"2026-09-21T10:00:00Z"}` +) + +// ---- status ---- + +func gStatusAPI(ready bool, keyStatus int, workers string) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{ + "GET /readyz": func(w http.ResponseWriter, r *http.Request) { + if ready { + gJSON(200, `{"status":"ready","checks":{"postgres":"ok","redis":"ok"},"cdp":"enabled"}`)(w, r) + return + } + gJSON(503, `{"status":"not_ready","checks":{"postgres":"ok","redis":"unreachable"},"cdp":"disabled"}`)(w, r) + }, + "GET /v1/requests": func(w http.ResponseWriter, r *http.Request) { + switch keyStatus { + case 401: + gProblem(401, "ERR::AUTH::INVALID_KEY", "the API key is not valid")(w, r) + case 403: + gProblem(403, "ERR::AUTH::INSUFFICIENT_SCOPE", "the key lacks the read scope")(w, r) + default: + gJSON(200, `{"data":[],`+gLogPageEnd+`}`)(w, r) + } + }, + "GET /v1/workers": gJSON(200, workers), + }) +} + +// gStatusAPIWorkersDown is a healthy API whose fleet endpoint fails. +func gStatusAPIWorkersDown() func() http.HandlerFunc { + ok := gStatusAPI(true, 200, "")() + return func() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/workers" { + gProblem(503, "ERR::INTERNAL::UNAVAILABLE", "fleet registry is down")(w, r) + return + } + ok(w, r) + } + } +} + +const gWorkers = `{"kinds":[{"kind":"browser","present":3,"ready":2,"workers":[]},{"kind":"egress","present":1,"ready":1,"workers":[]},{"kind":"batch","present":2,"ready":0,"workers":[]}]}` + +// ---- usage ---- + +const ( + gUsageDay = `{"period":{"start":"2026-08-01","end":"2026-08-04","days":3,"bounds":"[start, end)"},"group_by":"day","groups":[` + + `{"key":"2026-08-01","label":"","metrics":{"requests":10,"credits":12500000,"engine_ms":4200}},` + + `{"key":"2026-08-02","label":"","metrics":{"requests":15,"credits":18500000,"engine_ms":4800}}],` + + `"totals":{"requests":25,"credits":31000000,"engine_ms":9000},"unattributed":{"requests":0,"credits":0,"engine_ms":0},` + + `"usage_as_of":"2026-09-22T06:00:00Z","stale_seconds":120,"warnings":["rollup is partial for 2026-08-03"]}` + + gUsageEngine = `{"period":{"start":"2026-08-01","end":"2026-09-01","days":31,"bounds":"[start, end)"},"group_by":"engine","groups":[` + + `{"key":"fetch","label":"Fetch","metrics":{"requests":40,"credits":40000000}},{"key":"chromium","label":"","metrics":{"requests":5,"credits":25000000}}],` + + `"totals":{"requests":47,"credits":66000000},"unattributed":{"requests":2,"credits":1000000},"usage_as_of":null,"stale_seconds":7200}` + + gUsageKey = `{"period":{"start":"2026-08-01","end":"2026-09-01","days":31,"bounds":"[start, end)"},"group_by":"key","keys":[` + + `{"key_id":"K1","name":"ci-key","prefix":"spicrawl_live_ab12","project_id":"proj_01J9Z","revoked":false,"days":[],"totals":{"requests":7,"client_cli":3}},` + + `{"key_id":"K2","name":"old","prefix":"spicrawl_live_cd34","project_id":"proj_01J9Z","revoked":true,"days":[],"totals":{"requests":1}}]}` + + gUsageSummary = `{"period":{"start":"2026-09-01","end":"2026-10-01","days":30,"bounds":"[start, end)"},"period_elapsed_days":22,` + + `"plan":{"code":"starter","name":"Starter","subscription_status":"active","period_source":"subscription"},` + + `"credits":{"included_micro":50000000000,"used_micro":12345670000,"remaining_micro":37654330000,"overage_micro":0,"used_basis_points":2469},` + + `"allowance":{"limit_micro":50000000000,"used_micro":12400000000,"remaining_micro":37600000000,"unlimited":false,"resets_at":"2026-10-01T00:00:00Z"},` + + `"metrics":{"requests":1234,"credits":12345670000,"extractions":55},"warnings":["allowance counter is 40s behind"]}` + + gUsageRecon = `{"day":"2026-09-21","interpretation":"The request log holds no rows for the day; drift is not evidence of overbilling.",` + + `"drift":[{"metric":"requests","billed":10,"derived":0}]}` +) + +func init() { + get := func(path, body string) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{"GET " + path: gJSON(200, body)}) + } + logs := func(body string) func() http.HandlerFunc { return get("/v1/requests", body) } + both := `{"data":[` + gLogOK + `,` + gLogBlocked + `],` + gLogPageEnd + `}` + logsPaged := goldenServe(map[string]http.HandlerFunc{"GET /v1/requests": func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("before") == "" { + gJSON(200, `{"data":[`+gLogOK+`],`+gLogPageMore+`}`)(w, r) + return + } + gJSON(200, `{"data":[`+gLogBlocked+`],`+gLogPageEnd+`}`)(w, r) + }}) + logGet := func(body string) func() http.HandlerFunc { return get("/v1/requests/{id}", body) } + + goldenAdd( + goldenCase{name: "logs/list", args: []string{"logs"}, server: logs(both)}, + goldenCase{name: "logs/errors", args: []string{"logs", "--errors", "--limit", "20"}, server: logs(`{"data":[` + gLogBlocked + `],` + gLogPageEnd + `}`)}, + goldenCase{name: "logs/jsonl", args: []string{"logs", "--jsonl"}, server: logs(both)}, + goldenCase{name: "logs/all_pages", args: []string{"logs", "--all"}, server: logsPaged}, + goldenCase{name: "logs/all_pages_jsonl", args: []string{"logs", "--all", "--jsonl"}, server: logsPaged}, + goldenCase{name: "logs/has_more", args: []string{"logs"}, server: logs(`{"data":[` + gLogOK + `],` + gLogPageMore + `}`)}, + goldenCase{name: "logs/empty", args: []string{"logs"}, server: logs(`{"data":[],` + gLogPageEnd + `}`)}, + goldenCase{name: "logs/get", args: []string{"logs", "get", "01M0FK1EP2E7FBMWAR7W10SMXW"}, server: logGet(gLogOK)}, + goldenCase{name: "logs/get_not_found", args: []string{"logs", "get", "nope"}, stderr: true, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/requests/{id}": gProblem(404, "ERR::REQUEST::NOT_FOUND", "no such request")})}, + goldenCase{name: "logs/err_usage_status", args: []string{"logs", "--status", "weird"}, stderr: true}, + goldenCase{name: "logs/err_usage_errors_and_status", args: []string{"logs", "--errors", "--status", "failed"}, stderr: true}, + goldenCase{name: "logs_human/list", human: true, args: []string{"logs"}, server: logs(both)}, + goldenCase{name: "logs_human/all_projects", human: true, args: []string{"logs", "--all-projects", "--errors"}, server: logs(both)}, + goldenCase{name: "logs_human/has_more", human: true, args: []string{"logs"}, server: logs(`{"data":[` + gLogOK + `],` + gLogPageMore + `}`)}, + goldenCase{name: "logs_human/empty", human: true, args: []string{"logs"}, server: logs(`{"data":[],` + gLogPageEnd + `}`)}, + goldenCase{name: "logs_human/get_full", human: true, args: []string{"logs", "get", "x"}, server: logGet(gLogOK)}, + goldenCase{name: "logs_human/get_blocked", human: true, args: []string{"logs", "get", "x"}, server: logGet(gLogBlocked)}, + + // status: the report is on stdout in every outcome, exit 0 / 3 / 10 + goldenCase{name: "status/ok", args: []string{"status"}, server: gStatusAPI(true, 200, gWorkers)}, + goldenCase{name: "status/not_ready", args: []string{"status"}, server: gStatusAPI(false, 200, gWorkers)}, + goldenCase{name: "status/key_invalid_exit3", args: []string{"status"}, stderr: true, server: gStatusAPI(true, 401, gWorkers)}, + goldenCase{name: "status/key_without_read_scope", args: []string{"status"}, server: gStatusAPI(true, 403, gWorkers)}, + goldenCase{name: "status/no_key_exit3", args: []string{"status"}, noKey: true, stderr: true, server: gStatusAPI(true, 200, gWorkers)}, + goldenCase{name: "status/workers_error", args: []string{"status"}, server: gStatusAPIWorkersDown()}, + goldenCase{name: "status/unreachable_exit10", args: []string{"status"}, baseURL: "http://127.0.0.1:1"}, + goldenCase{name: "status_human/ok", human: true, args: []string{"status"}, server: gStatusAPI(true, 200, gWorkers)}, + goldenCase{name: "status_human/not_ready", human: true, args: []string{"status"}, server: gStatusAPI(false, 200, gWorkers)}, + goldenCase{name: "status_human/key_invalid_exit3", human: true, args: []string{"status"}, server: gStatusAPI(true, 401, gWorkers)}, + goldenCase{name: "status_human/no_key_exit3", human: true, args: []string{"status"}, noKey: true, server: gStatusAPI(true, 200, gWorkers)}, + goldenCase{name: "status_human/unreachable_exit10", human: true, args: []string{"status"}, baseURL: "http://127.0.0.1:1"}, + + // usage + goldenCase{name: "usage/by_day", args: []string{"usage", "--from", "2026-08-01", "--to", "2026-08-04"}, server: get("/v1/usage", gUsageDay)}, + goldenCase{name: "usage/by_key", args: []string{"usage", "--group-by", "key"}, server: get("/v1/usage", gUsageKey)}, + goldenCase{name: "usage/summary", args: []string{"usage", "summary"}, server: get("/v1/usage/summary", gUsageSummary)}, + goldenCase{name: "usage/reconciliation", args: []string{"usage", "reconciliation", "--day", "2026-09-21"}, server: get("/v1/usage/reconciliation", gUsageRecon)}, + goldenCase{name: "usage/err_scope", args: []string{"usage"}, stderr: true, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/usage": gProblem(403, "ERR::AUTH::INSUFFICIENT_SCOPE", "the key lacks the read scope")})}, + goldenCase{name: "usage/err_usage_to_before_from", args: []string{"usage", "--from", "2026-09-01", "--to", "2026-09-01"}, stderr: true}, + goldenCase{name: "usage_human/by_day", human: true, args: []string{"usage"}, server: get("/v1/usage", gUsageDay)}, + goldenCase{name: "usage_human/by_engine_unattributed", human: true, args: []string{"usage", "--group-by", "engine"}, server: get("/v1/usage", gUsageEngine)}, + goldenCase{name: "usage_human/by_key", human: true, args: []string{"usage", "--group-by", "key"}, server: get("/v1/usage", gUsageKey)}, + goldenCase{name: "usage_human/summary", human: true, args: []string{"usage", "summary"}, server: get("/v1/usage/summary", gUsageSummary)}, + goldenCase{name: "usage_human/reconciliation", human: true, args: []string{"usage", "reconciliation"}, server: get("/v1/usage/reconciliation", gUsageRecon)}, + goldenCase{name: "usage_human/err_scope", human: true, args: []string{"usage"}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/usage": gProblem(403, "ERR::AUTH::INSUFFICIENT_SCOPE", "the key lacks the read scope")})}, + + // browser url + goldenCase{name: "browser/url", args: []string{"browser", "url", "--country", "de"}, server: gBrowser(200)}, + goldenCase{name: "browser/url_quiet_prints_bare_url", args: []string{"-q", "browser", "url"}, server: gBrowser(200)}, + goldenCase{name: "browser/url_quiet_json", args: []string{"-q", "--json", "browser", "url"}, server: gBrowser(200)}, + goldenCase{name: "browser/not_enabled_exit8", args: []string{"browser", "url"}, stderr: true, server: gBrowser(501)}, + goldenCase{name: "browser/err_usage_ttl", args: []string{"browser", "url", "--ttl", "5"}, stderr: true}, + goldenCase{name: "browser/err_no_key", args: []string{"browser", "url"}, noKey: true, stderr: true, server: gBrowser(200)}, + goldenCase{name: "browser_human/url", human: true, args: []string{"browser", "url"}, server: gBrowser(200)}, + ) +} + +func gBrowser(status int) func() http.HandlerFunc { + h := gJSON(200, `{"path":"/v1/browser?token=wbt_testtoken&engine=chromium&proxy_country=de","expires_at":"2026-09-22T06:01:00Z"}`) + if status == 501 { + h = gProblem(501, "ERR::ENGINE::UNAVAILABLE", "cloud browser is not enabled") + } + return goldenServe(map[string]http.HandlerFunc{"POST /v1/browser/token": h}) +} diff --git a/internal/cmd/scrape_golden_test.go b/internal/cmd/scrape_golden_test.go new file mode 100644 index 0000000..1a78c87 --- /dev/null +++ b/internal/cmd/scrape_golden_test.go @@ -0,0 +1,136 @@ +package cmd + +import ( + "encoding/json" + "io" + "net/http" + "strconv" + "strings" +) + +// gScrapeRaw answers POST /v1/scrape with the raw document and the metadata +// headers the CLI wraps into its JSON object. +func gScrapeRaw(contentType, body string, target int, warnings ...string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + h := w.Header() + h.Set("Content-Type", contentType) + h.Set("X-Request-Id", "01J9ZQ5N8S4T9WY3L6P7Q1C2DE") + h.Set("X-Credits-Charged", "2") + h.Set("X-Request-Cost", "2") + h.Set("X-Engine", "fetch") + h.Set("X-Target-Status", strconv.Itoa(target)) + h.Set("X-Proxy-Source", "pool") + h.Set("X-Final-Url", "https://example.com/final") + h.Set("Cache-State", "miss") + for _, wn := range warnings { + h.Add("X-Warning", wn) + } + _, _ = io.WriteString(w, body) + } +} + +// gScrapeEnv answers with the API's JSON envelope (response_format=json, or +// any of extract, ai, autoparse, links, network capture, screenshot). +func gScrapeEnv(extra string) http.HandlerFunc { + body := `{"url":"https://example.com/product/1","final_url":"https://example.com/product/1","status":200,` + + `"content":"

Anvil

$19.99 & free shipping

","headers":{"content-type":"text/html"},` + + `"truncated":false,"credits":3,"engine":"fetch","proxy_source":"pool","warnings":[]` + extra + `}` + return gJSON(http.StatusOK, body) +} + +func gScrape(h http.HandlerFunc) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{"POST /v1/scrape": h}) +} + +const gHTMLDoc = "\nExample Domain\n

Example Domain

Prices from $19.99 & up.

\n" + +func init() { + const u = "https://example.com/product/1" + md := gScrapeRaw("text/markdown; charset=utf-8", "# Example Domain\n\nPrices from $19.99 & up.\n", 200, "FORMAT_COERCED: markdown requested, html served") + pdf := gScrapeRaw("application/pdf", "%PDF-1.4\n1 0 obj\n<<>>\nendobj\n%%EOF\n", 200) + shot := "UE5HREFUQQ==" // base64 "PNGDATA" + shots := gScrapeEnv(`,"screenshots":[{"label":"page","format":"png","width":1280,"height":720,"data":"` + shot + `"},{"label":"page","format":"png","width":1280,"height":720,"data":"` + shot + `"}]`) + + goldenAdd( + // ---- what a script gets: stdout is not a terminal, so one JSON object ---- + goldenCase{name: "scrape/html", args: []string{"scrape", u, "--format", "html"}, server: gScrape(gScrapeRaw("text/html; charset=utf-8", gHTMLDoc, 200))}, + goldenCase{name: "scrape/markdown", args: []string{"scrape", u, "--format", "markdown"}, server: gScrape(md)}, + goldenCase{name: "scrape/markdown_json_flag", args: []string{"scrape", u, "--format", "markdown"}, explicitJSON: true, server: gScrape(md)}, + goldenCase{name: "scrape/markdown_jsonl", args: []string{"scrape", u, "--format", "markdown", "--jsonl"}, server: gScrape(md)}, + goldenCase{name: "scrape/text", args: []string{"scrape", u, "--format", "text"}, server: gScrape(gScrapeRaw("text/plain; charset=utf-8", "Example Domain\nPrices from $19.99 & up.", 200))}, + goldenCase{name: "scrape/json_envelope", args: []string{"scrape", u, "--format", "json"}, server: gScrape(gScrapeEnv(`,"links":null`))}, + goldenCase{name: "scrape/json_envelope_jsonl", args: []string{"scrape", u, "--format", "json", "--jsonl"}, server: gScrape(gScrapeEnv(``))}, + goldenCase{name: "scrape/pdf_base64", args: []string{"scrape", u, "--format", "pdf"}, server: gScrape(pdf)}, + goldenCase{name: "scrape/extract", args: []string{"scrape", u, "--extract", `{"title":"h1","price":".price"}`}, server: gScrape(gScrapeEnv(`,"data":{"title":"Anvil","price":"$19.99 & free shipping"}`))}, + goldenCase{name: "scrape/ai", args: []string{"scrape", u, "--ai", "the product name and price"}, server: gScrape(gScrapeEnv(`,"data":{"name":"Anvil","price":19.99}`))}, + goldenCase{name: "scrape/autoparse", args: []string{"scrape", u, "--autoparse"}, server: gScrape(gScrapeEnv(`,"data":{"json_ld":[{"@type":"Product","name":"Anvil"}],"open_graph":{"og:title":"Anvil"}}`))}, + goldenCase{name: "scrape/links", args: []string{"scrape", u, "--links"}, server: gScrape(gScrapeEnv(`,"links":["https://example.com/a","https://example.com/b?x=1&y=2"]`))}, + goldenCase{name: "scrape/network_capture", args: []string{"scrape", u, "--render", "--network-capture", `{"url_patterns":["*/api/*"]}`}, + server: gScrape(gScrapeEnv(`,"network":[{"url":"https://example.com/api/price","status":200,"resource_type":"xhr","content_type":"application/json","body":"{\"price\":19.99}"}]`))}, + goldenCase{name: "scrape/screenshot_dir", args: []string{"scrape", u, "--render", "--screenshot", "--screenshot-dir", "{dir}/shots"}, server: gScrape(shots), + files: []string{"shots/page.png", "shots/page-2.png"}}, + goldenCase{name: "scrape/output_file", args: []string{"scrape", u, "--format", "markdown", "-o", "{dir}/page.md"}, server: gScrape(md), files: []string{"page.md"}}, + goldenCase{name: "scrape/output_stdout_dash", args: []string{"scrape", u, "--format", "markdown", "-o", "-"}, server: gScrape(md)}, + + // target answered non-2xx: the document is still printed, exit 6, target_error + goldenCase{name: "scrape/target_503", args: []string{"scrape", u}, stderr: true, server: gScrape(gScrapeRaw("text/html", "

Service Unavailable

\n", 503))}, + goldenCase{name: "scrape/target_404_allowed", args: []string{"scrape", u, "--allowed-status", "404"}, server: gScrape(gScrapeRaw("text/html", "

Not Found

\n", 404))}, + goldenCase{name: "scrape/target_503_output_file_not_written", args: []string{"scrape", u, "-o", "{dir}/page.html"}, stderr: true, server: gScrape(gScrapeRaw("text/html", "

Down

\n", 503))}, + goldenCase{name: "scrape/target_403_envelope", args: []string{"scrape", u, "--extract", `{"t":"h1"}`}, stderr: true, + server: gScrape(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Target-Status", "403") + gScrapeEnv(`,"data":null`)(w, r) + })}, + + // many URLs on stdin: one compact JSON line each (completion order; --concurrency 1 makes it input order) + goldenCase{name: "scrape/stdin_many", args: []string{"scrape", "-", "--format", "markdown", "--concurrency", "1"}, stderr: true, + stdin: "# targets\nhttps://example.com/ok\n\nhttps://example.com/down\nhttps://example.com/denied\n", + server: gScrape(gScrapeStdinMany)}, + goldenCase{name: "scrape/stdin_many_all_ok", args: []string{"scrape", "-", "--format", "markdown", "--concurrency", "1"}, stdin: "https://example.com/ok\nhttps://example.com/ok2\n", server: gScrape(gScrapeStdinMany)}, + goldenCase{name: "scrape/stdin_many_empty", args: []string{"scrape", "-"}, stderr: true, stdin: "\n# nothing\n", server: gScrape(gScrapeStdinMany)}, + + // failures: the problem document goes to stderr, stdout stays empty + goldenCase{name: "scrape/err_auth", args: []string{"scrape", u}, stderr: true, server: gScrape(gProblem(401, "ERR::AUTH::INVALID_KEY", "the API key is not valid"))}, + goldenCase{name: "scrape/err_limit", args: []string{"scrape", u}, stderr: true, server: gScrape(gProblem(429, "ERR::LIMIT::RATE", "slow down"))}, + goldenCase{name: "scrape/err_request", args: []string{"scrape", u, "--render", "--engine", "nope"}, stderr: true, server: gScrape(gProblem(400, "ERR::REQUEST::INVALID_PARAMETER", "engine: unknown value"))}, + goldenCase{name: "scrape/err_upstream", args: []string{"scrape", u}, stderr: true, server: gScrape(gProblem(502, "ERR::UPSTREAM::BLOCKED", "the target served a bot challenge"))}, + goldenCase{name: "scrape/err_engine", args: []string{"scrape", u, "--ai", "x"}, stderr: true, server: gScrape(gProblem(501, "ERR::ENGINE::UNAVAILABLE", "ai_extract is not available"))}, + goldenCase{name: "scrape/err_usage_ai_exclusive", args: []string{"scrape", u, "--ai", "x", "--ai-schema", `{"type":"object"}`}, stderr: true, server: gScrape(md)}, + goldenCase{name: "scrape/err_usage_bad_extract", args: []string{"scrape", u, "--extract", "{nope"}, server: gScrape(md)}, + goldenCase{name: "scrape/err_usage_no_url", args: []string{"scrape"}, stderr: true, server: gScrape(md)}, + goldenCase{name: "scrape/err_no_key", args: []string{"scrape", u}, noKey: true, stderr: true, server: gScrape(md)}, + goldenCase{name: "scrape/err_unreachable", args: []string{"scrape", u}, baseURL: "http://127.0.0.1:1"}, // stderr names the OS's dial error + + // ---- what a person gets (human mode): the document itself ---- + goldenCase{name: "scrape_human/html", human: true, args: []string{"scrape", u, "--format", "html"}, server: gScrape(gScrapeRaw("text/html", gHTMLDoc, 200))}, + goldenCase{name: "scrape_human/markdown", human: true, args: []string{"scrape", u, "--format", "markdown"}, server: gScrape(md)}, + goldenCase{name: "scrape_human/markdown_meta", human: true, args: []string{"scrape", u, "--format", "markdown", "--meta"}, server: gScrape(md)}, + goldenCase{name: "scrape_human/text_no_trailing_newline", human: true, args: []string{"scrape", u, "--format", "text"}, server: gScrape(gScrapeRaw("text/plain", "Example Domain", 200))}, + goldenCase{name: "scrape_human/extract_data", human: true, args: []string{"scrape", u, "--extract", `{"title":"h1"}`}, server: gScrape(gScrapeEnv(`,"data":{"title":"Anvil","price":"$19.99 & free shipping"}`))}, + goldenCase{name: "scrape_human/links", human: true, args: []string{"scrape", u, "--links", "--format", "markdown"}, server: gScrape(gScrapeEnv(`,"links":["https://example.com/a","https://example.com/b?x=1&y=2"]`))}, + goldenCase{name: "scrape_human/network", human: true, args: []string{"scrape", u, "--render", "--network-capture", `{}`}, + server: gScrape(gScrapeEnv(`,"network":[{"url":"https://example.com/api/price","status":200,"resource_type":"xhr","content_type":"application/json"},{"url":"https://example.com/x.png","resource_type":"image"}]`))}, + goldenCase{name: "scrape_human/target_503", human: true, args: []string{"scrape", u}, server: gScrape(gScrapeRaw("text/html", "

Service Unavailable

\n", 503))}, + goldenCase{name: "scrape_human/output_file", human: true, args: []string{"scrape", u, "--format", "markdown", "-o", "{dir}/page.md"}, server: gScrape(md), files: []string{"page.md"}}, + goldenCase{name: "scrape_human/pdf_needs_file", human: true, args: []string{"scrape", u, "--format", "pdf"}, server: gScrape(pdf)}, + goldenCase{name: "scrape_human/pdf_output_file", human: true, args: []string{"scrape", u, "--format", "pdf", "-o", "{dir}/page.pdf"}, server: gScrape(pdf), files: []string{"page.pdf"}}, + goldenCase{name: "scrape_human/err_auth", human: true, args: []string{"scrape", u}, server: gScrape(gProblem(401, "ERR::AUTH::INVALID_KEY", "the API key is not valid"))}, + ) +} + +// gScrapeStdinMany answers per URL: /ok* is a page, /down a target 503, /denied an API problem. +func gScrapeStdinMany(w http.ResponseWriter, r *http.Request) { + var req struct { + URL string `json:"url"` + } + b, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(b, &req) + switch { + case strings.HasSuffix(req.URL, "/down"): + gScrapeRaw("text/markdown", "# Down\n", 503)(w, r) + case strings.HasSuffix(req.URL, "/denied"): + gProblem(403, "ERR::UPSTREAM::BLOCKED", "blocked by the target")(w, r) + default: + gScrapeRaw("text/markdown", "# "+req.URL+"\n", 200)(w, r) + } +} diff --git a/internal/cmd/sessions_golden_test.go b/internal/cmd/sessions_golden_test.go new file mode 100644 index 0000000..6decc8b --- /dev/null +++ b/internal/cmd/sessions_golden_test.go @@ -0,0 +1,69 @@ +package cmd + +import ( + "net/http" + "testing" +) + +const ( + gSessionID = "01J9ZQ4M7R3T8VX2K5N6P0B1CD" + + gSession = `{"id":"` + gSessionID + `","project_id":"proj_01J9Z","engine":"chromium","status":"active","sticky_key":"login-1",` + + `"proxy":{"tier":"residential","country":"de","region_pool":"eu"},"created_at":"2026-09-22T00:00:00Z","last_used_at":"2026-09-22T00:30:00Z",` + + `"expires_at":"2026-09-22T02:00:00Z","hard_expires_at":"2026-09-23T00:00:00Z","usage_count":7,"context":null}` + + gSessionNew = `{"id":"` + gSessionID + `","project_id":"proj_01J9Z","engine":"obscura","status":"active","sticky_key":null,` + + `"proxy":{"tier":"datacenter"},"created_at":"2026-09-22T00:00:00Z","last_used_at":null,"expires_at":"2026-09-22T00:30:00Z",` + + `"usage_count":0,"warnings":["engine obscura cannot persist IndexedDB"]}` + + gSessionReleased = `{"id":"` + gSessionID + `","project_id":"proj_01J9Z","engine":"chromium","status":"released","sticky_key":"login-1",` + + `"proxy":{"tier":"residential","country":"de"},"created_at":"2026-09-22T00:00:00Z","last_used_at":"2026-09-22T00:30:00Z",` + + `"expires_at":"2026-09-22T02:00:00Z","released_at":"2026-09-22T00:45:00Z","retired":"released by the caller","usage_count":7}` + + gSessionContext = `{"session_id":"` + gSessionID + `","session_context":{"cookies":[{"name":"sid","value":"abc123","domain":"example.com","path":"/"}],"local_storage":{"theme":"dark"}}}` +) + +func init() { + one := func(method, path, body string, status int) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{method + " " + path: gJSON(status, body)}) + } + list := func(body string) func() http.HandlerFunc { return one("GET", "/v1/sessions", body, 200) } + page := `{"sessions":[` + gSession + `,` + gSessionReleased + `]}` + more := `{"sessions":[` + gSession + `],"next_cursor":"1790000000000000000.ABC"}` + del := goldenServe(map[string]http.HandlerFunc{"DELETE /v1/sessions/{id}": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }}) + + goldenAdd( + goldenCase{name: "sessions/create", args: []string{"sessions", "create", "--engine", "obscura"}, server: one("POST", "/v1/sessions", gSessionNew, 201)}, + goldenCase{name: "sessions/create_session_context_file", args: []string{"sessions", "create", "--session-context", "{dir}/ctx.json"}, server: one("POST", "/v1/sessions", gSession, 201), + setup: func(t *testing.T, e *goldenEnv) { goldenWrite(t, e.Dir, "ctx.json", gSessionContext) }}, + goldenCase{name: "sessions/create_usage_error", args: []string{"sessions", "create", "--rotate-ip", "--sticky-key", "k1"}, stderr: true}, + goldenCase{name: "sessions/list", args: []string{"sessions", "list"}, server: list(page)}, + goldenCase{name: "sessions/list_empty", args: []string{"sessions", "list", "--status", "expired"}, server: list(`{"sessions":[]}`)}, + goldenCase{name: "sessions/list_next_cursor", args: []string{"sessions", "list", "--limit", "1"}, server: list(more)}, + goldenCase{name: "sessions/list_all_pages", args: []string{"sessions", "list", "--all"}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/sessions": func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("cursor") == "" { + gJSON(200, more)(w, r) + return + } + gJSON(200, `{"sessions":[`+gSessionReleased+`]}`)(w, r) + }})}, + goldenCase{name: "sessions/get", args: []string{"sessions", "get", gSessionID}, server: one("GET", "/v1/sessions/{id}", gSession, 200)}, + goldenCase{name: "sessions/get_not_found", args: []string{"sessions", "get", gSessionID}, stderr: true, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/sessions/{id}": gProblem(404, "ERR::SESSION::NOT_FOUND", "no such session")})}, + goldenCase{name: "sessions/context", args: []string{"sessions", "context", gSessionID}, server: one("GET", "/v1/sessions/{id}/context", gSessionContext, 200)}, + goldenCase{name: "sessions/release", args: []string{"sessions", "release", gSessionID}, server: one("POST", "/v1/sessions/{id}/release", gSessionReleased, 200)}, + goldenCase{name: "sessions/delete_yes", args: []string{"sessions", "delete", gSessionID, "--yes"}, server: del}, + goldenCase{name: "sessions/delete_needs_yes_without_terminal", args: []string{"sessions", "delete", gSessionID}, stderr: true, server: del}, + + goldenCase{name: "sessions_human/create", human: true, args: []string{"sessions", "create"}, server: one("POST", "/v1/sessions", gSessionNew, 201)}, + goldenCase{name: "sessions_human/list", human: true, args: []string{"sessions", "list"}, server: list(page)}, + goldenCase{name: "sessions_human/list_empty", human: true, args: []string{"sessions", "list"}, server: list(`{"sessions":[]}`)}, + goldenCase{name: "sessions_human/list_next_cursor", human: true, args: []string{"sessions", "list"}, server: list(more)}, + goldenCase{name: "sessions_human/get", human: true, args: []string{"sessions", "get", gSessionID}, server: one("GET", "/v1/sessions/{id}", gSession, 200)}, + goldenCase{name: "sessions_human/get_released", human: true, args: []string{"sessions", "get", gSessionID}, server: one("GET", "/v1/sessions/{id}", gSessionReleased, 200)}, + goldenCase{name: "sessions_human/context", human: true, args: []string{"sessions", "context", gSessionID}, server: one("GET", "/v1/sessions/{id}/context", gSessionContext, 200)}, + goldenCase{name: "sessions_human/release", human: true, args: []string{"sessions", "release", gSessionID}, server: one("POST", "/v1/sessions/{id}/release", gSessionReleased, 200)}, + goldenCase{name: "sessions_human/delete_yes", human: true, args: []string{"sessions", "delete", gSessionID, "--yes"}, server: del}, + ) +} diff --git a/internal/cmd/testdata/golden/.gitattributes b/internal/cmd/testdata/golden/.gitattributes new file mode 100644 index 0000000..fa1385d --- /dev/null +++ b/internal/cmd/testdata/golden/.gitattributes @@ -0,0 +1 @@ +* -text diff --git a/internal/cmd/testdata/golden/agents/init_err_no_client_detected.exit b/internal/cmd/testdata/golden/agents/init_err_no_client_detected.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_err_no_client_detected.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/agents/init_err_no_client_detected.stderr b/internal/cmd/testdata/golden/agents/init_err_no_client_detected.stderr new file mode 100644 index 0000000..28738e9 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_err_no_client_detected.stderr @@ -0,0 +1,4 @@ +{ + "error": "no agent client detected in or your home directory; pass --client claude|cursor|vscode|codex or all", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/agents/init_err_no_client_detected.stdout b/internal/cmd/testdata/golden/agents/init_err_no_client_detected.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.exit b/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.stderr b/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.stderr new file mode 100644 index 0000000..2f9e2aa --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.stderr @@ -0,0 +1,9 @@ +clients: Claude Code +MCP server: http://127.0.0.1:1/mcp +plan: + create claude mcp .mcp.json + create claude skill .claude/skills/spicrawl/SKILL.md +{ + "error": "init changes files and stdin is not a terminal: re-run with --yes to apply the plan above", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.stdout b/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/agents/init_yes.exit b/internal/cmd/testdata/golden/agents/init_yes.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_yes.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/init_yes.file..claude__skills__spicrawl__SKILL.md b/internal/cmd/testdata/golden/agents/init_yes.file..claude__skills__spicrawl__SKILL.md new file mode 100644 index 0000000..9a9132e --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_yes.file..claude__skills__spicrawl__SKILL.md @@ -0,0 +1,8 @@ +--- +name: spicrawl +description: golden test skill +--- + +# Spicrawl + +Use http://127.0.0.1:1 for scraping. diff --git a/internal/cmd/testdata/golden/agents/init_yes.file..cursor__mcp.json b/internal/cmd/testdata/golden/agents/init_yes.file..cursor__mcp.json new file mode 100644 index 0000000..4277328 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_yes.file..cursor__mcp.json @@ -0,0 +1,10 @@ +{ + "mcpServers": { + "spicrawl": { + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${env:SPICRAWL_API_KEY}" + } + } + } +} diff --git a/internal/cmd/testdata/golden/agents/init_yes.file..mcp.json b/internal/cmd/testdata/golden/agents/init_yes.file..mcp.json new file mode 100644 index 0000000..698f8f0 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_yes.file..mcp.json @@ -0,0 +1,11 @@ +{ + "mcpServers": { + "spicrawl": { + "type": "http", + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${SPICRAWL_API_KEY}" + } + } + } +} diff --git a/internal/cmd/testdata/golden/agents/init_yes.stdout b/internal/cmd/testdata/golden/agents/init_yes.stdout new file mode 100644 index 0000000..82dcdd7 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_yes.stdout @@ -0,0 +1,27 @@ +[ + { + "client": "claude", + "kind": "mcp", + "file": "/.mcp.json", + "action": "created" + }, + { + "client": "cursor", + "kind": "mcp", + "file": "/.cursor/mcp.json", + "action": "created" + }, + { + "client": "claude", + "kind": "skill", + "file": "/.claude/skills/spicrawl/SKILL.md", + "action": "created" + }, + { + "client": "cursor", + "kind": "skill", + "file": "/.claude/skills/spicrawl/SKILL.md", + "action": "skipped", + "reason": "already reads this skill" + } +] diff --git a/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.exit b/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.stderr b/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.stderr new file mode 100644 index 0000000..28738e9 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.stderr @@ -0,0 +1,4 @@ +{ + "error": "no agent client detected in or your home directory; pass --client claude|cursor|vscode|codex or all", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.stdout b/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/agents/mcp_install_all.exit b/internal/cmd/testdata/golden/agents/mcp_install_all.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_all.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/mcp_install_all.file..codex__config.toml b/internal/cmd/testdata/golden/agents/mcp_install_all.file..codex__config.toml new file mode 100644 index 0000000..feae81b --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_all.file..codex__config.toml @@ -0,0 +1,3 @@ +[mcp_servers.spicrawl] +url = "http://127.0.0.1:1/mcp" +bearer_token_env_var = "SPICRAWL_API_KEY" diff --git a/internal/cmd/testdata/golden/agents/mcp_install_all.file..cursor__mcp.json b/internal/cmd/testdata/golden/agents/mcp_install_all.file..cursor__mcp.json new file mode 100644 index 0000000..4277328 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_all.file..cursor__mcp.json @@ -0,0 +1,10 @@ +{ + "mcpServers": { + "spicrawl": { + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${env:SPICRAWL_API_KEY}" + } + } + } +} diff --git a/internal/cmd/testdata/golden/agents/mcp_install_all.file..mcp.json b/internal/cmd/testdata/golden/agents/mcp_install_all.file..mcp.json new file mode 100644 index 0000000..698f8f0 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_all.file..mcp.json @@ -0,0 +1,11 @@ +{ + "mcpServers": { + "spicrawl": { + "type": "http", + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${SPICRAWL_API_KEY}" + } + } + } +} diff --git a/internal/cmd/testdata/golden/agents/mcp_install_all.file..vscode__mcp.json b/internal/cmd/testdata/golden/agents/mcp_install_all.file..vscode__mcp.json new file mode 100644 index 0000000..8c81ec7 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_all.file..vscode__mcp.json @@ -0,0 +1,19 @@ +{ + "servers": { + "spicrawl": { + "type": "http", + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${input:spicrawl-api-key}" + } + } + }, + "inputs": [ + { + "type": "promptString", + "id": "spicrawl-api-key", + "description": "Spicrawl API key", + "password": true + } + ] +} diff --git a/internal/cmd/testdata/golden/agents/mcp_install_all.stdout b/internal/cmd/testdata/golden/agents/mcp_install_all.stdout new file mode 100644 index 0000000..3ff9d2e --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_all.stdout @@ -0,0 +1,26 @@ +[ + { + "client": "claude", + "kind": "mcp", + "file": "/.mcp.json", + "action": "created" + }, + { + "client": "cursor", + "kind": "mcp", + "file": "/.cursor/mcp.json", + "action": "created" + }, + { + "client": "vscode", + "kind": "mcp", + "file": "/.vscode/mcp.json", + "action": "created" + }, + { + "client": "codex", + "kind": "mcp", + "file": "/.codex/config.toml", + "action": "created" + } +] diff --git a/internal/cmd/testdata/golden/agents/mcp_install_claude.exit b/internal/cmd/testdata/golden/agents/mcp_install_claude.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_claude.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/mcp_install_claude.file..mcp.json b/internal/cmd/testdata/golden/agents/mcp_install_claude.file..mcp.json new file mode 100644 index 0000000..698f8f0 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_claude.file..mcp.json @@ -0,0 +1,11 @@ +{ + "mcpServers": { + "spicrawl": { + "type": "http", + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${SPICRAWL_API_KEY}" + } + } + } +} diff --git a/internal/cmd/testdata/golden/agents/mcp_install_claude.stdout b/internal/cmd/testdata/golden/agents/mcp_install_claude.stdout new file mode 100644 index 0000000..d72903e --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_claude.stdout @@ -0,0 +1,8 @@ +[ + { + "client": "claude", + "kind": "mcp", + "file": "/.mcp.json", + "action": "created" + } +] diff --git a/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.exit b/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.file..mcp.json b/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.file..mcp.json new file mode 100644 index 0000000..4d19b70 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.file..mcp.json @@ -0,0 +1,15 @@ +{ + "zeta": 1, + "mcpServers": { + "other": { + "command": "x" + }, + "spicrawl": { + "type": "http", + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${SPICRAWL_API_KEY}" + } + } + } +} diff --git a/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.stdout b/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.stdout new file mode 100644 index 0000000..edcf64a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.stdout @@ -0,0 +1,9 @@ +[ + { + "client": "claude", + "kind": "mcp", + "file": "/.mcp.json", + "action": "updated", + "backup": "/.mcp.json.bak" + } +] diff --git a/internal/cmd/testdata/golden/agents/mcp_print_all.exit b/internal/cmd/testdata/golden/agents/mcp_print_all.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_print_all.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/mcp_print_all.stdout b/internal/cmd/testdata/golden/agents/mcp_print_all.stdout new file mode 100644 index 0000000..72f63cc --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_print_all.stdout @@ -0,0 +1,26 @@ +[ + { + "client": "claude", + "file": "/.mcp.json", + "format": "json", + "snippet": "{\n \"mcpServers\": {\n \"spicrawl\": {\n \"type\": \"http\",\n \"url\": \"http://127.0.0.1:1/mcp\",\n \"headers\": {\n \"Authorization\": \"Bearer ${SPICRAWL_API_KEY}\"\n }\n }\n }\n}\n" + }, + { + "client": "cursor", + "file": "/.cursor/mcp.json", + "format": "json", + "snippet": "{\n \"mcpServers\": {\n \"spicrawl\": {\n \"url\": \"http://127.0.0.1:1/mcp\",\n \"headers\": {\n \"Authorization\": \"Bearer ${env:SPICRAWL_API_KEY}\"\n }\n }\n }\n}\n" + }, + { + "client": "vscode", + "file": "/.vscode/mcp.json", + "format": "json", + "snippet": "{\n \"servers\": {\n \"spicrawl\": {\n \"type\": \"http\",\n \"url\": \"http://127.0.0.1:1/mcp\",\n \"headers\": {\n \"Authorization\": \"Bearer ${input:spicrawl-api-key}\"\n }\n }\n },\n \"inputs\": [\n {\n \"type\": \"promptString\",\n \"id\": \"spicrawl-api-key\",\n \"description\": \"Spicrawl API key\",\n \"password\": true\n }\n ]\n}\n" + }, + { + "client": "codex", + "file": "/.codex/config.toml", + "format": "toml", + "snippet": "[mcp_servers.spicrawl]\nurl = \"http://127.0.0.1:1/mcp\"\nbearer_token_env_var = \"SPICRAWL_API_KEY\"\n" + } +] diff --git a/internal/cmd/testdata/golden/agents/skill_install_claude.exit b/internal/cmd/testdata/golden/agents/skill_install_claude.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_install_claude.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/skill_install_claude.file..claude__skills__spicrawl__SKILL.md b/internal/cmd/testdata/golden/agents/skill_install_claude.file..claude__skills__spicrawl__SKILL.md new file mode 100644 index 0000000..9a9132e --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_install_claude.file..claude__skills__spicrawl__SKILL.md @@ -0,0 +1,8 @@ +--- +name: spicrawl +description: golden test skill +--- + +# Spicrawl + +Use http://127.0.0.1:1 for scraping. diff --git a/internal/cmd/testdata/golden/agents/skill_install_claude.stdout b/internal/cmd/testdata/golden/agents/skill_install_claude.stdout new file mode 100644 index 0000000..1f9c4d9 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_install_claude.stdout @@ -0,0 +1,8 @@ +[ + { + "client": "claude", + "kind": "skill", + "file": "/.claude/skills/spicrawl/SKILL.md", + "action": "created" + } +] diff --git a/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.exit b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.file..agents__skills__spicrawl__SKILL.md b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.file..agents__skills__spicrawl__SKILL.md new file mode 100644 index 0000000..9a9132e --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.file..agents__skills__spicrawl__SKILL.md @@ -0,0 +1,8 @@ +--- +name: spicrawl +description: golden test skill +--- + +# Spicrawl + +Use http://127.0.0.1:1 for scraping. diff --git a/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.file.AGENTS.md b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.file.AGENTS.md new file mode 100644 index 0000000..54caf39 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.file.AGENTS.md @@ -0,0 +1,10 @@ + +## Web data (Spicrawl) + +To fetch, render or extract data from web pages, use Spicrawl instead of ad-hoc +HTTP requests: the `spicrawl` MCP server (tools `spicrawl_*`) or the CLI +(`spicrawl scrape --format markdown`, `spicrawl batch submit urls.txt`). +The API key is read from `$SPICRAWL_API_KEY`; never print or commit it. +Start cheap (plain fetch), then add `--render`, `--premium-proxy`, `--stealth` +only when the page needs it. Full guide: the `spicrawl` skill, or http://SERVER/skill.md. + diff --git a/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.stdout b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.stdout new file mode 100644 index 0000000..2c047d7 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.stdout @@ -0,0 +1,14 @@ +[ + { + "client": "codex", + "kind": "skill", + "file": "/.agents/skills/spicrawl/SKILL.md", + "action": "created" + }, + { + "client": "codex", + "kind": "agents_md", + "file": "/AGENTS.md", + "action": "created" + } +] diff --git a/internal/cmd/testdata/golden/agents/skill_print.exit b/internal/cmd/testdata/golden/agents/skill_print.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_print.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/skill_print.stdout b/internal/cmd/testdata/golden/agents/skill_print.stdout new file mode 100644 index 0000000..ccd5736 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_print.stdout @@ -0,0 +1,4 @@ +{ + "content": "---\nname: spicrawl\ndescription: golden test skill\n---\n\n# Spicrawl\n\nUse http://127.0.0.1:1 for scraping.\n", + "source": "http://SERVER/skill.md" +} diff --git a/internal/cmd/testdata/golden/agents_human/init_yes.exit b/internal/cmd/testdata/golden/agents_human/init_yes.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/init_yes.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents_human/init_yes.stdout b/internal/cmd/testdata/golden/agents_human/init_yes.stdout new file mode 100644 index 0000000..3073b3e --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/init_yes.stdout @@ -0,0 +1,5 @@ +CLIENT KIND ACTION FILE NOTE +claude mcp created .mcp.json +cursor mcp created .cursor/mcp.json +claude skill created .claude/skills/spicrawl/SKILL.md +cursor skill skipped .claude/skills/spicrawl/SKILL.md already reads this skill diff --git a/internal/cmd/testdata/golden/agents_human/mcp_install_all.exit b/internal/cmd/testdata/golden/agents_human/mcp_install_all.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/mcp_install_all.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents_human/mcp_install_all.stdout b/internal/cmd/testdata/golden/agents_human/mcp_install_all.stdout new file mode 100644 index 0000000..3042314 --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/mcp_install_all.stdout @@ -0,0 +1,5 @@ +CLIENT KIND ACTION FILE NOTE +claude mcp created .mcp.json +cursor mcp created .cursor/mcp.json +vscode mcp created .vscode/mcp.json +codex mcp created .codex/config.toml diff --git a/internal/cmd/testdata/golden/agents_human/mcp_install_merges_existing.exit b/internal/cmd/testdata/golden/agents_human/mcp_install_merges_existing.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/mcp_install_merges_existing.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents_human/mcp_install_merges_existing.stdout b/internal/cmd/testdata/golden/agents_human/mcp_install_merges_existing.stdout new file mode 100644 index 0000000..fccf189 --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/mcp_install_merges_existing.stdout @@ -0,0 +1,2 @@ +CLIENT KIND ACTION FILE NOTE +claude mcp updated .mcp.json backup: .mcp.json.bak diff --git a/internal/cmd/testdata/golden/agents_human/mcp_print_all.exit b/internal/cmd/testdata/golden/agents_human/mcp_print_all.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/mcp_print_all.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents_human/mcp_print_all.stdout b/internal/cmd/testdata/golden/agents_human/mcp_print_all.stdout new file mode 100644 index 0000000..beb2ecf --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/mcp_print_all.stdout @@ -0,0 +1,50 @@ +# Claude Code: merge into .mcp.json +{ + "mcpServers": { + "spicrawl": { + "type": "http", + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${SPICRAWL_API_KEY}" + } + } + } +} + +# Cursor: merge into .cursor/mcp.json +{ + "mcpServers": { + "spicrawl": { + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${env:SPICRAWL_API_KEY}" + } + } + } +} + +# VS Code (Copilot): merge into .vscode/mcp.json +{ + "servers": { + "spicrawl": { + "type": "http", + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${input:spicrawl-api-key}" + } + } + }, + "inputs": [ + { + "type": "promptString", + "id": "spicrawl-api-key", + "description": "Spicrawl API key", + "password": true + } + ] +} + +# Codex: merge into .codex/config.toml +[mcp_servers.spicrawl] +url = "http://127.0.0.1:1/mcp" +bearer_token_env_var = "SPICRAWL_API_KEY" diff --git a/internal/cmd/testdata/golden/agents_human/skill_install_claude.exit b/internal/cmd/testdata/golden/agents_human/skill_install_claude.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/skill_install_claude.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents_human/skill_install_claude.stdout b/internal/cmd/testdata/golden/agents_human/skill_install_claude.stdout new file mode 100644 index 0000000..4fe0518 --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/skill_install_claude.stdout @@ -0,0 +1,2 @@ +CLIENT KIND ACTION FILE NOTE +claude skill created .claude/skills/spicrawl/SKILL.md diff --git a/internal/cmd/testdata/golden/agents_human/skill_print.exit b/internal/cmd/testdata/golden/agents_human/skill_print.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/skill_print.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents_human/skill_print.stdout b/internal/cmd/testdata/golden/agents_human/skill_print.stdout new file mode 100644 index 0000000..9a9132e --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/skill_print.stdout @@ -0,0 +1,8 @@ +--- +name: spicrawl +description: golden test skill +--- + +# Spicrawl + +Use http://127.0.0.1:1 for scraping. diff --git a/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.exit b/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.stderr b/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.stderr new file mode 100644 index 0000000..2472206 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.stderr @@ -0,0 +1,4 @@ +{ + "error": "no API key given and stdin is not a terminal: pass --api-key or set SPICRAWL_API_KEY (create a key at https://app.spicrawl.com/dashboard/keys)", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.stdout b/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/auth/login_ok.exit b/internal/cmd/testdata/golden/auth/login_ok.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth/login_ok.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth/login_ok.stdout b/internal/cmd/testdata/golden/auth/login_ok.stdout new file mode 100644 index 0000000..b58b7b9 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/login_ok.stdout @@ -0,0 +1,5 @@ +{ + "base_url": "http://SERVER", + "key": "spicrawl_live_0123…mnop", + "saved": "/config.json" +} diff --git a/internal/cmd/testdata/golden/auth/login_rejected_exit3.exit b/internal/cmd/testdata/golden/auth/login_rejected_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/auth/login_rejected_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/auth/login_rejected_exit3.stderr b/internal/cmd/testdata/golden/auth/login_rejected_exit3.stderr new file mode 100644 index 0000000..cbdc3b7 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/login_rejected_exit3.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Unauthorized", + "status": 401, + "code": "ERR::AUTH::INVALID_KEY", + "detail": "the API key is not valid", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/auth/login_rejected_exit3.stdout b/internal/cmd/testdata/golden/auth/login_rejected_exit3.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/auth/logout_nothing_saved.exit b/internal/cmd/testdata/golden/auth/logout_nothing_saved.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth/logout_nothing_saved.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth/logout_nothing_saved.stdout b/internal/cmd/testdata/golden/auth/logout_nothing_saved.stdout new file mode 100644 index 0000000..a00ad10 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/logout_nothing_saved.stdout @@ -0,0 +1,4 @@ +{ + "path": "/config.json", + "removed": false +} diff --git a/internal/cmd/testdata/golden/auth/logout_removed.exit b/internal/cmd/testdata/golden/auth/logout_removed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth/logout_removed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth/logout_removed.stdout b/internal/cmd/testdata/golden/auth/logout_removed.stdout new file mode 100644 index 0000000..8f7f696 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/logout_removed.stdout @@ -0,0 +1,4 @@ +{ + "path": "/config.json", + "removed": true +} diff --git a/internal/cmd/testdata/golden/auth/status_key_from_file.exit b/internal/cmd/testdata/golden/auth/status_key_from_file.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_key_from_file.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth/status_key_from_file.stdout b/internal/cmd/testdata/golden/auth/status_key_from_file.stdout new file mode 100644 index 0000000..4ae02a9 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_key_from_file.stdout @@ -0,0 +1,8 @@ +{ + "api_key": "spicrawl_live_0123…mnop", + "api_key_source": "file", + "base_url": "https://api.spicrawl.com", + "base_url_source": "default", + "config_path": "/config.json", + "key_valid": null +} diff --git a/internal/cmd/testdata/golden/auth/status_no_key_exit3.exit b/internal/cmd/testdata/golden/auth/status_no_key_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_no_key_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/auth/status_no_key_exit3.stderr b/internal/cmd/testdata/golden/auth/status_no_key_exit3.stderr new file mode 100644 index 0000000..6b478b2 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_no_key_exit3.stderr @@ -0,0 +1,4 @@ +{ + "error": "no API key: run \"spicrawl login\", set SPICRAWL_API_KEY, or pass --api-key", + "exit_code": 3 +} diff --git a/internal/cmd/testdata/golden/auth/status_no_key_exit3.stdout b/internal/cmd/testdata/golden/auth/status_no_key_exit3.stdout new file mode 100644 index 0000000..be37f14 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_no_key_exit3.stdout @@ -0,0 +1,8 @@ +{ + "api_key": null, + "api_key_source": null, + "base_url": "https://api.spicrawl.com", + "base_url_source": "default", + "config_path": "/config.json", + "key_valid": null +} diff --git a/internal/cmd/testdata/golden/auth/status_offline.exit b/internal/cmd/testdata/golden/auth/status_offline.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_offline.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth/status_offline.stdout b/internal/cmd/testdata/golden/auth/status_offline.stdout new file mode 100644 index 0000000..232df19 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_offline.stdout @@ -0,0 +1,8 @@ +{ + "api_key": "spicrawl_test_0123…ghij", + "api_key_source": "env", + "base_url": "https://api.spicrawl.com", + "base_url_source": "default", + "config_path": "/config.json", + "key_valid": null +} diff --git a/internal/cmd/testdata/golden/auth/status_ok.exit b/internal/cmd/testdata/golden/auth/status_ok.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_ok.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth/status_ok.stdout b/internal/cmd/testdata/golden/auth/status_ok.stdout new file mode 100644 index 0000000..3f3611c --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_ok.stdout @@ -0,0 +1,8 @@ +{ + "api_key": "spicrawl_test_0123…ghij", + "api_key_source": "env", + "base_url": "http://SERVER", + "base_url_source": "flag", + "config_path": "/config.json", + "key_valid": true +} diff --git a/internal/cmd/testdata/golden/auth/status_rejected_exit3.exit b/internal/cmd/testdata/golden/auth/status_rejected_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_rejected_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/auth/status_rejected_exit3.stderr b/internal/cmd/testdata/golden/auth/status_rejected_exit3.stderr new file mode 100644 index 0000000..47478ac --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_rejected_exit3.stderr @@ -0,0 +1,4 @@ +{ + "error": "the API rejected the key: ERR::AUTH::INVALID_KEY: the API key is not valid", + "exit_code": 3 +} diff --git a/internal/cmd/testdata/golden/auth/status_rejected_exit3.stdout b/internal/cmd/testdata/golden/auth/status_rejected_exit3.stdout new file mode 100644 index 0000000..a9c0aad --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_rejected_exit3.stdout @@ -0,0 +1,9 @@ +{ + "api_key": "spicrawl_test_0123…ghij", + "api_key_source": "env", + "base_url": "http://SERVER", + "base_url_source": "flag", + "check_error": "ERR::AUTH::INVALID_KEY: the API key is not valid", + "config_path": "/config.json", + "key_valid": false +} diff --git a/internal/cmd/testdata/golden/auth_human/login_ok.exit b/internal/cmd/testdata/golden/auth_human/login_ok.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/login_ok.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth_human/login_ok.stdout b/internal/cmd/testdata/golden/auth_human/login_ok.stdout new file mode 100644 index 0000000..69d5c68 --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/login_ok.stdout @@ -0,0 +1,2 @@ +Logged in. Key spicrawl_live_0123…mnop saved to /config.json +Base URL: http://SERVER diff --git a/internal/cmd/testdata/golden/auth_human/logout_nothing_saved.exit b/internal/cmd/testdata/golden/auth_human/logout_nothing_saved.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/logout_nothing_saved.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth_human/logout_nothing_saved.stdout b/internal/cmd/testdata/golden/auth_human/logout_nothing_saved.stdout new file mode 100644 index 0000000..fd5b5ae --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/logout_nothing_saved.stdout @@ -0,0 +1 @@ +No API key saved in /config.json diff --git a/internal/cmd/testdata/golden/auth_human/logout_removed.exit b/internal/cmd/testdata/golden/auth_human/logout_removed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/logout_removed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth_human/logout_removed.stdout b/internal/cmd/testdata/golden/auth_human/logout_removed.stdout new file mode 100644 index 0000000..9fe5b85 --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/logout_removed.stdout @@ -0,0 +1 @@ +Removed the API key from /config.json diff --git a/internal/cmd/testdata/golden/auth_human/status_no_key_exit3.exit b/internal/cmd/testdata/golden/auth_human/status_no_key_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/status_no_key_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/auth_human/status_no_key_exit3.stdout b/internal/cmd/testdata/golden/auth_human/status_no_key_exit3.stdout new file mode 100644 index 0000000..8281987 --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/status_no_key_exit3.stdout @@ -0,0 +1,5 @@ +FIELD VALUE +api key (none) +base url https://api.spicrawl.com (from default) +config /config.json +key works not checked diff --git a/internal/cmd/testdata/golden/auth_human/status_ok.exit b/internal/cmd/testdata/golden/auth_human/status_ok.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/status_ok.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth_human/status_ok.stdout b/internal/cmd/testdata/golden/auth_human/status_ok.stdout new file mode 100644 index 0000000..ff80741 --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/status_ok.stdout @@ -0,0 +1,5 @@ +FIELD VALUE +api key spicrawl_test_0123…ghij (from env) +base url http://SERVER (from flag) +config /config.json +key works yes diff --git a/internal/cmd/testdata/golden/auth_human/status_rejected_exit3.exit b/internal/cmd/testdata/golden/auth_human/status_rejected_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/status_rejected_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/auth_human/status_rejected_exit3.stdout b/internal/cmd/testdata/golden/auth_human/status_rejected_exit3.stdout new file mode 100644 index 0000000..7fc2897 --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/status_rejected_exit3.stdout @@ -0,0 +1,5 @@ +FIELD VALUE +api key spicrawl_test_0123…ghij (from env) +base url http://SERVER (from flag) +config /config.json +key works no diff --git a/internal/cmd/testdata/golden/batch/append.exit b/internal/cmd/testdata/golden/batch/append.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/append.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/append.stdout b/internal/cmd/testdata/golden/batch/append.stdout new file mode 100644 index 0000000..d89a0bf --- /dev/null +++ b/internal/cmd/testdata/golden/batch/append.stdout @@ -0,0 +1,38 @@ +{ + "job": { + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "running", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 2, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 6, + "estimated_credits_micro": 6000000, + "progress": { + "total": 2, + "completed": 1, + "succeeded": 1, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 1, + "percent_complete": 50.0, + "credits_charged": 3, + "credits_charged_micro": 3000000, + "bytes": 20480 + }, + "submitted_at": "", + "started_at": "" + }, + "items_added": 2, + "items_dispatched": 1 +} diff --git a/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.exit b/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.stderr b/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.stderr new file mode 100644 index 0000000..578268c --- /dev/null +++ b/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.stderr @@ -0,0 +1,4 @@ +{ + "error": "refusing to cancel batch 01J9Z6V0Q8M4K2T7R3N5B1C9XA without confirmation: cancelling cannot be undone; pass --yes", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.stdout b/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/cancel_yes.exit b/internal/cmd/testdata/golden/batch/cancel_yes.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/cancel_yes.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/cancel_yes.stdout b/internal/cmd/testdata/golden/batch/cancel_yes.stdout new file mode 100644 index 0000000..55bef45 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/cancel_yes.stdout @@ -0,0 +1,35 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "cancelled", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 5, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 15, + "estimated_credits_micro": 15000000, + "progress": { + "total": 5, + "completed": 1, + "succeeded": 1, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 4, + "percent_complete": 20.0, + "credits_charged": 3, + "credits_charged_micro": 3000000, + "bytes": 20480 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/close.exit b/internal/cmd/testdata/golden/batch/close.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/close.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/close.stdout b/internal/cmd/testdata/golden/batch/close.stdout new file mode 100644 index 0000000..211ab47 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/close.stdout @@ -0,0 +1,34 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "running", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 2, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 6, + "estimated_credits_micro": 6000000, + "progress": { + "total": 2, + "completed": 1, + "succeeded": 1, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 1, + "percent_complete": 50.0, + "credits_charged": 3, + "credits_charged_micro": 3000000, + "bytes": 20480 + }, + "submitted_at": "", + "started_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/content.exit b/internal/cmd/testdata/golden/batch/content.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/content.stdout b/internal/cmd/testdata/golden/batch/content.stdout new file mode 100644 index 0000000..619f212 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content.stdout @@ -0,0 +1 @@ +"

A

\n" diff --git a/internal/cmd/testdata/golden/batch/content_bad_seq.exit b/internal/cmd/testdata/golden/batch/content_bad_seq.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content_bad_seq.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/batch/content_bad_seq.stderr b/internal/cmd/testdata/golden/batch/content_bad_seq.stderr new file mode 100644 index 0000000..7a43a17 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content_bad_seq.stderr @@ -0,0 +1,4 @@ +{ + "error": "seq \"x\": want the item's 0-based position (\"seq\" on a result line)", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/batch/content_bad_seq.stdout b/internal/cmd/testdata/golden/batch/content_bad_seq.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/content_not_finished.exit b/internal/cmd/testdata/golden/batch/content_not_finished.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content_not_finished.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/batch/content_not_finished.stderr b/internal/cmd/testdata/golden/batch/content_not_finished.stderr new file mode 100644 index 0000000..2cb7ffa --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content_not_finished.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Conflict", + "status": 409, + "code": "ERR::REQUEST::NOT_READY", + "detail": "item 7 has not finished", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/batch/content_not_finished.stdout b/internal/cmd/testdata/golden/batch/content_not_finished.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/content_output_file.exit b/internal/cmd/testdata/golden/batch/content_output_file.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content_output_file.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/content_output_file.file.a.html b/internal/cmd/testdata/golden/batch/content_output_file.file.a.html new file mode 100644 index 0000000..cd15ba6 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content_output_file.file.a.html @@ -0,0 +1 @@ +

A

diff --git a/internal/cmd/testdata/golden/batch/content_output_file.stdout b/internal/cmd/testdata/golden/batch/content_output_file.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/get.exit b/internal/cmd/testdata/golden/batch/get.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/get.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/get.stdout b/internal/cmd/testdata/golden/batch/get.stdout new file mode 100644 index 0000000..f30f585 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/get.stdout @@ -0,0 +1,36 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "completed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 3, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 9, + "estimated_credits_micro": 9000000, + "progress": { + "total": 3, + "completed": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "skipped": 0, + "remaining": 0, + "percent_complete": 100.0, + "credits_charged": 9, + "credits_charged_micro": 9000000, + "bytes": 61440 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "", + "results_expire_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/get_not_found.exit b/internal/cmd/testdata/golden/batch/get_not_found.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/get_not_found.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/batch/get_not_found.stderr b/internal/cmd/testdata/golden/batch/get_not_found.stderr new file mode 100644 index 0000000..e52b1ba --- /dev/null +++ b/internal/cmd/testdata/golden/batch/get_not_found.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Not Found", + "status": 404, + "code": "ERR::REQUEST::NOT_FOUND", + "detail": "no such batch", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/batch/get_not_found.stdout b/internal/cmd/testdata/golden/batch/get_not_found.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/list.exit b/internal/cmd/testdata/golden/batch/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/list.stdout b/internal/cmd/testdata/golden/batch/list.stdout new file mode 100644 index 0000000..d93642d --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list.stdout @@ -0,0 +1,74 @@ +{ + "batches": [ + { + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "completed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 3, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 9, + "estimated_credits_micro": 9000000, + "progress": { + "total": 3, + "completed": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "skipped": 0, + "remaining": 0, + "percent_complete": 100.0, + "credits_charged": 9, + "credits_charged_micro": 9000000, + "bytes": 61440 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "", + "results_expire_at": "" + }, + { + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "crawl", + "status": "running", + "open": true, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 9, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 27, + "estimated_credits_micro": 27000000, + "progress": { + "total": 9, + "completed": 4, + "succeeded": 4, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 5, + "percent_complete": 44.4, + "credits_charged": 12, + "credits_charged_micro": 12000000, + "bytes": 81920 + }, + "submitted_at": "", + "started_at": "" + } + ] +} diff --git a/internal/cmd/testdata/golden/batch/list_all_pages.exit b/internal/cmd/testdata/golden/batch/list_all_pages.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list_all_pages.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/list_all_pages.stdout b/internal/cmd/testdata/golden/batch/list_all_pages.stdout new file mode 100644 index 0000000..d93642d --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list_all_pages.stdout @@ -0,0 +1,74 @@ +{ + "batches": [ + { + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "completed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 3, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 9, + "estimated_credits_micro": 9000000, + "progress": { + "total": 3, + "completed": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "skipped": 0, + "remaining": 0, + "percent_complete": 100.0, + "credits_charged": 9, + "credits_charged_micro": 9000000, + "bytes": 61440 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "", + "results_expire_at": "" + }, + { + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "crawl", + "status": "running", + "open": true, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 9, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 27, + "estimated_credits_micro": 27000000, + "progress": { + "total": 9, + "completed": 4, + "succeeded": 4, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 5, + "percent_complete": 44.4, + "credits_charged": 12, + "credits_charged_micro": 12000000, + "bytes": 81920 + }, + "submitted_at": "", + "started_at": "" + } + ] +} diff --git a/internal/cmd/testdata/golden/batch/list_empty.exit b/internal/cmd/testdata/golden/batch/list_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/list_empty.stdout b/internal/cmd/testdata/golden/batch/list_empty.stdout new file mode 100644 index 0000000..fef4d1b --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list_empty.stdout @@ -0,0 +1,3 @@ +{ + "batches": [] +} diff --git a/internal/cmd/testdata/golden/batch/list_next_cursor.exit b/internal/cmd/testdata/golden/batch/list_next_cursor.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list_next_cursor.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/list_next_cursor.stdout b/internal/cmd/testdata/golden/batch/list_next_cursor.stdout new file mode 100644 index 0000000..2518c02 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list_next_cursor.stdout @@ -0,0 +1,41 @@ +{ + "batches": [ + { + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "completed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 3, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 9, + "estimated_credits_micro": 9000000, + "progress": { + "total": 3, + "completed": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "skipped": 0, + "remaining": 0, + "percent_complete": 100.0, + "credits_charged": 9, + "credits_charged_micro": 9000000, + "bytes": 61440 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "", + "results_expire_at": "" + } + ], + "next_cursor": "1790000000000000000.ABC" +} diff --git a/internal/cmd/testdata/golden/batch/results.exit b/internal/cmd/testdata/golden/batch/results.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/results.stdout b/internal/cmd/testdata/golden/batch/results.stdout new file mode 100644 index 0000000..a3ce132 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results.stdout @@ -0,0 +1,3 @@ +{"seq":0,"external_id":"sku-1","url":"https://example.com/a","status":"succeeded","http_status":200,"bytes":2048,"result":{"content":"\"

A

\\n\""}} +{"seq":1,"external_id":"","url":"https://example.com/b","status":"failed","http_status":null,"bytes":0,"error":{"code":"ERR::UPSTREAM::TIMEOUT","message":"the target did not answer"}} +{"seq":2,"external_id":"sku-3","url":"https://example.com/c?x=1&y=2","status":"succeeded","http_status":404,"bytes":512,"result":{"content":"\"not found\""}} diff --git a/internal/cmd/testdata/golden/batch/results_all_pages.exit b/internal/cmd/testdata/golden/batch/results_all_pages.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_all_pages.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/results_all_pages.stdout b/internal/cmd/testdata/golden/batch/results_all_pages.stdout new file mode 100644 index 0000000..bcb5037 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_all_pages.stdout @@ -0,0 +1,6 @@ +{"seq":0,"external_id":"sku-1","url":"https://example.com/a","status":"succeeded","http_status":200,"bytes":2048,"result":{"content":"\"

A

\\n\""}} +{"seq":1,"external_id":"","url":"https://example.com/b","status":"failed","http_status":null,"bytes":0,"error":{"code":"ERR::UPSTREAM::TIMEOUT","message":"the target did not answer"}} +{"seq":2,"external_id":"sku-3","url":"https://example.com/c?x=1&y=2","status":"succeeded","http_status":404,"bytes":512,"result":{"content":"\"not found\""}} +{"seq":0,"external_id":"sku-1","url":"https://example.com/a","status":"succeeded","http_status":200,"bytes":2048,"result":{"content":"\"

A

\\n\""}} +{"seq":1,"external_id":"","url":"https://example.com/b","status":"failed","http_status":null,"bytes":0,"error":{"code":"ERR::UPSTREAM::TIMEOUT","message":"the target did not answer"}} +{"seq":2,"external_id":"sku-3","url":"https://example.com/c?x=1&y=2","status":"succeeded","http_status":404,"bytes":512,"result":{"content":"\"not found\""}} diff --git a/internal/cmd/testdata/golden/batch/results_empty.exit b/internal/cmd/testdata/golden/batch/results_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/results_empty.stdout b/internal/cmd/testdata/golden/batch/results_empty.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/results_expired.exit b/internal/cmd/testdata/golden/batch/results_expired.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_expired.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/batch/results_expired.stderr b/internal/cmd/testdata/golden/batch/results_expired.stderr new file mode 100644 index 0000000..b68510a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_expired.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Gone", + "status": 410, + "code": "ERR::REQUEST::BEYOND_RETENTION", + "detail": "results expired", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/batch/results_expired.stdout b/internal/cmd/testdata/golden/batch/results_expired.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/results_more_pages.exit b/internal/cmd/testdata/golden/batch/results_more_pages.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_more_pages.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/results_more_pages.stdout b/internal/cmd/testdata/golden/batch/results_more_pages.stdout new file mode 100644 index 0000000..a3ce132 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_more_pages.stdout @@ -0,0 +1,3 @@ +{"seq":0,"external_id":"sku-1","url":"https://example.com/a","status":"succeeded","http_status":200,"bytes":2048,"result":{"content":"\"

A

\\n\""}} +{"seq":1,"external_id":"","url":"https://example.com/b","status":"failed","http_status":null,"bytes":0,"error":{"code":"ERR::UPSTREAM::TIMEOUT","message":"the target did not answer"}} +{"seq":2,"external_id":"sku-3","url":"https://example.com/c?x=1&y=2","status":"succeeded","http_status":404,"bytes":512,"result":{"content":"\"not found\""}} diff --git a/internal/cmd/testdata/golden/batch/results_output_file.exit b/internal/cmd/testdata/golden/batch/results_output_file.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_output_file.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/results_output_file.file.results.jsonl b/internal/cmd/testdata/golden/batch/results_output_file.file.results.jsonl new file mode 100644 index 0000000..a3ce132 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_output_file.file.results.jsonl @@ -0,0 +1,3 @@ +{"seq":0,"external_id":"sku-1","url":"https://example.com/a","status":"succeeded","http_status":200,"bytes":2048,"result":{"content":"\"

A

\\n\""}} +{"seq":1,"external_id":"","url":"https://example.com/b","status":"failed","http_status":null,"bytes":0,"error":{"code":"ERR::UPSTREAM::TIMEOUT","message":"the target did not answer"}} +{"seq":2,"external_id":"sku-3","url":"https://example.com/c?x=1&y=2","status":"succeeded","http_status":404,"bytes":512,"result":{"content":"\"not found\""}} diff --git a/internal/cmd/testdata/golden/batch/results_output_file.stdout b/internal/cmd/testdata/golden/batch/results_output_file.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/retry.exit b/internal/cmd/testdata/golden/batch/retry.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/retry.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/retry.stdout b/internal/cmd/testdata/golden/batch/retry.stdout new file mode 100644 index 0000000..351570f --- /dev/null +++ b/internal/cmd/testdata/golden/batch/retry.stdout @@ -0,0 +1,38 @@ +{ + "job": { + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "running", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 2, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 6, + "estimated_credits_micro": 6000000, + "progress": { + "total": 2, + "completed": 1, + "succeeded": 1, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 1, + "percent_complete": 50.0, + "credits_charged": 3, + "credits_charged_micro": 3000000, + "bytes": 20480 + }, + "submitted_at": "", + "started_at": "" + }, + "items_reset": 2, + "items_dispatched": 1 +} diff --git a/internal/cmd/testdata/golden/batch/submit_err_request.exit b/internal/cmd/testdata/golden/batch/submit_err_request.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_err_request.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/batch/submit_err_request.stderr b/internal/cmd/testdata/golden/batch/submit_err_request.stderr new file mode 100644 index 0000000..69386d4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_err_request.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Bad Request", + "status": 400, + "code": "ERR::REQUEST::INVALID_PARAMETER", + "detail": "response_format pdf is not supported on batch", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/batch/submit_err_request.stdout b/internal/cmd/testdata/golden/batch/submit_err_request.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/submit_items_jsonl.exit b/internal/cmd/testdata/golden/batch/submit_items_jsonl.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_items_jsonl.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/submit_items_jsonl.stdout b/internal/cmd/testdata/golden/batch/submit_items_jsonl.stdout new file mode 100644 index 0000000..2f58ee9 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_items_jsonl.stdout @@ -0,0 +1,33 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "queued", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 2, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 6, + "estimated_credits_micro": 6000000, + "progress": { + "total": 2, + "completed": 0, + "succeeded": 0, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 2, + "percent_complete": 0.0, + "credits_charged": 0, + "credits_charged_micro": 0, + "bytes": 0 + }, + "submitted_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/submit_stdin.exit b/internal/cmd/testdata/golden/batch/submit_stdin.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_stdin.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/submit_stdin.stdout b/internal/cmd/testdata/golden/batch/submit_stdin.stdout new file mode 100644 index 0000000..2f58ee9 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_stdin.stdout @@ -0,0 +1,33 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "queued", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 2, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 6, + "estimated_credits_micro": 6000000, + "progress": { + "total": 2, + "completed": 0, + "succeeded": 0, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 2, + "percent_complete": 0.0, + "credits_charged": 0, + "credits_charged_micro": 0, + "bytes": 0 + }, + "submitted_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/submit_urls.exit b/internal/cmd/testdata/golden/batch/submit_urls.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_urls.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/submit_urls.stdout b/internal/cmd/testdata/golden/batch/submit_urls.stdout new file mode 100644 index 0000000..2f58ee9 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_urls.stdout @@ -0,0 +1,33 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "queued", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 2, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 6, + "estimated_credits_micro": 6000000, + "progress": { + "total": 2, + "completed": 0, + "succeeded": 0, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 2, + "percent_complete": 0.0, + "credits_charged": 0, + "credits_charged_micro": 0, + "bytes": 0 + }, + "submitted_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/submit_wait.exit b/internal/cmd/testdata/golden/batch/submit_wait.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_wait.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/submit_wait.stdout b/internal/cmd/testdata/golden/batch/submit_wait.stdout new file mode 100644 index 0000000..f30f585 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_wait.stdout @@ -0,0 +1,36 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "completed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 3, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 9, + "estimated_credits_micro": 9000000, + "progress": { + "total": 3, + "completed": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "skipped": 0, + "remaining": 0, + "percent_complete": 100.0, + "credits_charged": 9, + "credits_charged_micro": 9000000, + "bytes": 61440 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "", + "results_expire_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.exit b/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.stderr b/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.stderr new file mode 100644 index 0000000..8b9f9fa --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.stderr @@ -0,0 +1,4 @@ +{ + "error": "--wait with --open would never finish: an open job completes only after \"spicrawl batch close\"", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.stdout b/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.exit b/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.stderr b/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.stderr new file mode 100644 index 0000000..bd49396 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.stderr @@ -0,0 +1,4 @@ +{ + "error": "batch 01J9Z6V0Q8M4K2T7R3N5B1C9XA cancelled (1 of 5 items finished)", + "exit_code": 4 +} diff --git a/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.stdout b/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.stdout new file mode 100644 index 0000000..55bef45 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.stdout @@ -0,0 +1,35 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "cancelled", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 5, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 15, + "estimated_credits_micro": 15000000, + "progress": { + "total": 5, + "completed": 1, + "succeeded": 1, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 4, + "percent_complete": 20.0, + "credits_charged": 3, + "credits_charged_micro": 3000000, + "bytes": 20480 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/wait_completed.exit b/internal/cmd/testdata/golden/batch/wait_completed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_completed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/wait_completed.stdout b/internal/cmd/testdata/golden/batch/wait_completed.stdout new file mode 100644 index 0000000..f30f585 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_completed.stdout @@ -0,0 +1,36 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "completed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 3, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 9, + "estimated_credits_micro": 9000000, + "progress": { + "total": 3, + "completed": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "skipped": 0, + "remaining": 0, + "percent_complete": 100.0, + "credits_charged": 9, + "credits_charged_micro": 9000000, + "bytes": 61440 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "", + "results_expire_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/wait_failed_exit4.exit b/internal/cmd/testdata/golden/batch/wait_failed_exit4.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_failed_exit4.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/batch/wait_failed_exit4.stderr b/internal/cmd/testdata/golden/batch/wait_failed_exit4.stderr new file mode 100644 index 0000000..93c36d9 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_failed_exit4.stderr @@ -0,0 +1,4 @@ +{ + "error": "batch 01J9Z6V0Q8M4K2T7R3N5B1C9XA failed (2 of 10 items finished): ERR::LIMIT::FAILURE_THRESHOLD failure threshold reached", + "exit_code": 4 +} diff --git a/internal/cmd/testdata/golden/batch/wait_failed_exit4.stdout b/internal/cmd/testdata/golden/batch/wait_failed_exit4.stdout new file mode 100644 index 0000000..4f404fb --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_failed_exit4.stdout @@ -0,0 +1,37 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "failed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 10, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 30, + "estimated_credits_micro": 30000000, + "progress": { + "total": 10, + "completed": 2, + "succeeded": 0, + "failed": 2, + "cancelled": 0, + "skipped": 0, + "remaining": 8, + "percent_complete": 20.0, + "credits_charged": 6, + "credits_charged_micro": 6000000, + "bytes": 40960 + }, + "submitted_at": "", + "error_code": "ERR::LIMIT::FAILURE_THRESHOLD", + "error_message": "failure threshold reached", + "started_at": "", + "finished_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/wait_pending_exit11.exit b/internal/cmd/testdata/golden/batch/wait_pending_exit11.exit new file mode 100644 index 0000000..b4de394 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_pending_exit11.exit @@ -0,0 +1 @@ +11 diff --git a/internal/cmd/testdata/golden/batch/wait_pending_exit11.stderr b/internal/cmd/testdata/golden/batch/wait_pending_exit11.stderr new file mode 100644 index 0000000..e3b9954 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_pending_exit11.stderr @@ -0,0 +1,4 @@ +{ + "error": "batch 01J9Z6V0Q8M4K2T7R3N5B1C9XA is still running after 60ms; resume with: spicrawl batch wait 01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "exit_code": 11 +} diff --git a/internal/cmd/testdata/golden/batch/wait_pending_exit11.stdout b/internal/cmd/testdata/golden/batch/wait_pending_exit11.stdout new file mode 100644 index 0000000..211ab47 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_pending_exit11.stdout @@ -0,0 +1,34 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "running", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 2, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 6, + "estimated_credits_micro": 6000000, + "progress": { + "total": 2, + "completed": 1, + "succeeded": 1, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 1, + "percent_complete": 50.0, + "credits_charged": 3, + "credits_charged_micro": 3000000, + "bytes": 20480 + }, + "submitted_at": "", + "started_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/wait_sequence.exit b/internal/cmd/testdata/golden/batch/wait_sequence.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_sequence.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/wait_sequence.stdout b/internal/cmd/testdata/golden/batch/wait_sequence.stdout new file mode 100644 index 0000000..f30f585 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_sequence.stdout @@ -0,0 +1,36 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "completed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 3, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 9, + "estimated_credits_micro": 9000000, + "progress": { + "total": 3, + "completed": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "skipped": 0, + "remaining": 0, + "percent_complete": 100.0, + "credits_charged": 9, + "credits_charged_micro": 9000000, + "bytes": 61440 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "", + "results_expire_at": "" +} diff --git a/internal/cmd/testdata/golden/batch_human/append.exit b/internal/cmd/testdata/golden/batch_human/append.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/append.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/append.stdout b/internal/cmd/testdata/golden/batch_human/append.stdout new file mode 100644 index 0000000..5f03531 --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/append.stdout @@ -0,0 +1,10 @@ +added 2 items (1 dispatched now, the rest by the recovery sweep; do not re-append them) + +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status running +progress 1/2 (50.0%) succeeded 1 failed 0 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 3 charged, 6 reserved +submitted +started diff --git a/internal/cmd/testdata/golden/batch_human/cancel_yes.exit b/internal/cmd/testdata/golden/batch_human/cancel_yes.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/cancel_yes.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/cancel_yes.stdout b/internal/cmd/testdata/golden/batch_human/cancel_yes.stdout new file mode 100644 index 0000000..8d0f250 --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/cancel_yes.stdout @@ -0,0 +1,9 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status cancelled +progress 1/5 (20.0%) succeeded 1 failed 0 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 3 charged, 15 reserved +submitted +started +finished diff --git a/internal/cmd/testdata/golden/batch_human/content.exit b/internal/cmd/testdata/golden/batch_human/content.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/content.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/content.stdout b/internal/cmd/testdata/golden/batch_human/content.stdout new file mode 100644 index 0000000..cd15ba6 --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/content.stdout @@ -0,0 +1 @@ +

A

diff --git a/internal/cmd/testdata/golden/batch_human/get_completed.exit b/internal/cmd/testdata/golden/batch_human/get_completed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/get_completed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/get_completed.stdout b/internal/cmd/testdata/golden/batch_human/get_completed.stdout new file mode 100644 index 0000000..2f5584a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/get_completed.stdout @@ -0,0 +1,10 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status completed +progress 3/3 (100.0%) succeeded 2 failed 1 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 9 charged, 9 reserved +submitted +started +finished +results readable until diff --git a/internal/cmd/testdata/golden/batch_human/get_failed.exit b/internal/cmd/testdata/golden/batch_human/get_failed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/get_failed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/get_failed.stdout b/internal/cmd/testdata/golden/batch_human/get_failed.stdout new file mode 100644 index 0000000..a36a5fb --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/get_failed.stdout @@ -0,0 +1,10 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status failed +progress 2/10 (20.0%) succeeded 0 failed 2 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 6 charged, 30 reserved +submitted +started +finished +error ERR::LIMIT::FAILURE_THRESHOLD failure threshold reached diff --git a/internal/cmd/testdata/golden/batch_human/get_open.exit b/internal/cmd/testdata/golden/batch_human/get_open.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/get_open.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/get_open.stdout b/internal/cmd/testdata/golden/batch_human/get_open.stdout new file mode 100644 index 0000000..59e4918 --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/get_open.stdout @@ -0,0 +1,8 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name crawl +status running (open: accepting items until closed) +progress 4/9 (44.4%) succeeded 4 failed 0 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 12 charged, 27 reserved +submitted +started diff --git a/internal/cmd/testdata/golden/batch_human/list.exit b/internal/cmd/testdata/golden/batch_human/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/list.stdout b/internal/cmd/testdata/golden/batch_human/list.stdout new file mode 100644 index 0000000..8bcaf3b --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/list.stdout @@ -0,0 +1,3 @@ +ID STATUS NAME DONE SUBMITTED +01J9Z6V0Q8M4K2T7R3N5B1C9XA completed nightly 3/3 +01J9Z6V0Q8M4K2T7R3N5B1C9XA running crawl 4/9 diff --git a/internal/cmd/testdata/golden/batch_human/list_empty.exit b/internal/cmd/testdata/golden/batch_human/list_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/list_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/list_empty.stdout b/internal/cmd/testdata/golden/batch_human/list_empty.stdout new file mode 100644 index 0000000..ddc789b --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/list_empty.stdout @@ -0,0 +1 @@ +no batch jobs diff --git a/internal/cmd/testdata/golden/batch_human/list_next_cursor.exit b/internal/cmd/testdata/golden/batch_human/list_next_cursor.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/list_next_cursor.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/list_next_cursor.stdout b/internal/cmd/testdata/golden/batch_human/list_next_cursor.stdout new file mode 100644 index 0000000..d31059a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/list_next_cursor.stdout @@ -0,0 +1,2 @@ +ID STATUS NAME DONE SUBMITTED +01J9Z6V0Q8M4K2T7R3N5B1C9XA completed nightly 3/3 diff --git a/internal/cmd/testdata/golden/batch_human/results.exit b/internal/cmd/testdata/golden/batch_human/results.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/results.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/results.stdout b/internal/cmd/testdata/golden/batch_human/results.stdout new file mode 100644 index 0000000..279d254 --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/results.stdout @@ -0,0 +1,4 @@ +SEQ STATUS HTTP URL BYTES +0 succeeded 200 https://example.com/a [sku-1] 2048 +1 failed ERR::UPSTREAM::TIMEOUT - https://example.com/b 0 +2 succeeded 404 https://example.com/c?x=1&y=2 [sku-3] 512 diff --git a/internal/cmd/testdata/golden/batch_human/results_empty.exit b/internal/cmd/testdata/golden/batch_human/results_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/results_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/results_empty.stdout b/internal/cmd/testdata/golden/batch_human/results_empty.stdout new file mode 100644 index 0000000..22cb73b --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/results_empty.stdout @@ -0,0 +1 @@ +no finished items yet diff --git a/internal/cmd/testdata/golden/batch_human/retry.exit b/internal/cmd/testdata/golden/batch_human/retry.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/retry.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/retry.stdout b/internal/cmd/testdata/golden/batch_human/retry.stdout new file mode 100644 index 0000000..5e8da5c --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/retry.stdout @@ -0,0 +1,10 @@ +reset 2 failed items (1 dispatched now, the rest by the recovery sweep) + +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status running +progress 1/2 (50.0%) succeeded 1 failed 0 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 3 charged, 6 reserved +submitted +started diff --git a/internal/cmd/testdata/golden/batch_human/submit_urls.exit b/internal/cmd/testdata/golden/batch_human/submit_urls.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/submit_urls.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/submit_urls.stdout b/internal/cmd/testdata/golden/batch_human/submit_urls.stdout new file mode 100644 index 0000000..babaef9 --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/submit_urls.stdout @@ -0,0 +1,7 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status queued +progress 0/2 (0.0%) succeeded 0 failed 0 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 0 charged, 6 reserved +submitted diff --git a/internal/cmd/testdata/golden/batch_human/submit_wait.exit b/internal/cmd/testdata/golden/batch_human/submit_wait.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/submit_wait.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/submit_wait.stdout b/internal/cmd/testdata/golden/batch_human/submit_wait.stdout new file mode 100644 index 0000000..2f5584a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/submit_wait.stdout @@ -0,0 +1,10 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status completed +progress 3/3 (100.0%) succeeded 2 failed 1 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 9 charged, 9 reserved +submitted +started +finished +results readable until diff --git a/internal/cmd/testdata/golden/batch_human/wait_completed.exit b/internal/cmd/testdata/golden/batch_human/wait_completed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/wait_completed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/wait_completed.stdout b/internal/cmd/testdata/golden/batch_human/wait_completed.stdout new file mode 100644 index 0000000..2f5584a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/wait_completed.stdout @@ -0,0 +1,10 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status completed +progress 3/3 (100.0%) succeeded 2 failed 1 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 9 charged, 9 reserved +submitted +started +finished +results readable until diff --git a/internal/cmd/testdata/golden/batch_human/wait_failed_exit4.exit b/internal/cmd/testdata/golden/batch_human/wait_failed_exit4.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/wait_failed_exit4.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/batch_human/wait_failed_exit4.stdout b/internal/cmd/testdata/golden/batch_human/wait_failed_exit4.stdout new file mode 100644 index 0000000..a36a5fb --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/wait_failed_exit4.stdout @@ -0,0 +1,10 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status failed +progress 2/10 (20.0%) succeeded 0 failed 2 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 6 charged, 30 reserved +submitted +started +finished +error ERR::LIMIT::FAILURE_THRESHOLD failure threshold reached diff --git a/internal/cmd/testdata/golden/browser/err_no_key.exit b/internal/cmd/testdata/golden/browser/err_no_key.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/browser/err_no_key.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/browser/err_no_key.stderr b/internal/cmd/testdata/golden/browser/err_no_key.stderr new file mode 100644 index 0000000..6b478b2 --- /dev/null +++ b/internal/cmd/testdata/golden/browser/err_no_key.stderr @@ -0,0 +1,4 @@ +{ + "error": "no API key: run \"spicrawl login\", set SPICRAWL_API_KEY, or pass --api-key", + "exit_code": 3 +} diff --git a/internal/cmd/testdata/golden/browser/err_no_key.stdout b/internal/cmd/testdata/golden/browser/err_no_key.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/browser/err_usage_ttl.exit b/internal/cmd/testdata/golden/browser/err_usage_ttl.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/browser/err_usage_ttl.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/browser/err_usage_ttl.stderr b/internal/cmd/testdata/golden/browser/err_usage_ttl.stderr new file mode 100644 index 0000000..79518e5 --- /dev/null +++ b/internal/cmd/testdata/golden/browser/err_usage_ttl.stderr @@ -0,0 +1,4 @@ +{ + "error": "--ttl must be between 60 and 900 seconds", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/browser/err_usage_ttl.stdout b/internal/cmd/testdata/golden/browser/err_usage_ttl.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/browser/not_enabled_exit8.exit b/internal/cmd/testdata/golden/browser/not_enabled_exit8.exit new file mode 100644 index 0000000..45a4fb7 --- /dev/null +++ b/internal/cmd/testdata/golden/browser/not_enabled_exit8.exit @@ -0,0 +1 @@ +8 diff --git a/internal/cmd/testdata/golden/browser/not_enabled_exit8.stderr b/internal/cmd/testdata/golden/browser/not_enabled_exit8.stderr new file mode 100644 index 0000000..ec405ec --- /dev/null +++ b/internal/cmd/testdata/golden/browser/not_enabled_exit8.stderr @@ -0,0 +1,4 @@ +{ + "error": "cloud browser (CDP) is not enabled on this server; \"spicrawl status\" shows its cdp field", + "exit_code": 8 +} diff --git a/internal/cmd/testdata/golden/browser/not_enabled_exit8.stdout b/internal/cmd/testdata/golden/browser/not_enabled_exit8.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/browser/url.exit b/internal/cmd/testdata/golden/browser/url.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/browser/url.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/browser/url.stdout b/internal/cmd/testdata/golden/browser/url.stdout new file mode 100644 index 0000000..6b234ac --- /dev/null +++ b/internal/cmd/testdata/golden/browser/url.stdout @@ -0,0 +1,5 @@ +{ + "expires_at": "", + "single_use": true, + "url": "ws://SERVER/v1/browser?token=wbt_testtoken&engine=chromium&proxy_country=de" +} diff --git a/internal/cmd/testdata/golden/browser/url_quiet_json.exit b/internal/cmd/testdata/golden/browser/url_quiet_json.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/browser/url_quiet_json.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/browser/url_quiet_json.stdout b/internal/cmd/testdata/golden/browser/url_quiet_json.stdout new file mode 100644 index 0000000..6b234ac --- /dev/null +++ b/internal/cmd/testdata/golden/browser/url_quiet_json.stdout @@ -0,0 +1,5 @@ +{ + "expires_at": "", + "single_use": true, + "url": "ws://SERVER/v1/browser?token=wbt_testtoken&engine=chromium&proxy_country=de" +} diff --git a/internal/cmd/testdata/golden/browser/url_quiet_prints_bare_url.exit b/internal/cmd/testdata/golden/browser/url_quiet_prints_bare_url.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/browser/url_quiet_prints_bare_url.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/browser/url_quiet_prints_bare_url.stdout b/internal/cmd/testdata/golden/browser/url_quiet_prints_bare_url.stdout new file mode 100644 index 0000000..c1c624d --- /dev/null +++ b/internal/cmd/testdata/golden/browser/url_quiet_prints_bare_url.stdout @@ -0,0 +1 @@ +ws://SERVER/v1/browser?token=wbt_testtoken&engine=chromium&proxy_country=de diff --git a/internal/cmd/testdata/golden/browser_human/url.exit b/internal/cmd/testdata/golden/browser_human/url.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/browser_human/url.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/browser_human/url.stdout b/internal/cmd/testdata/golden/browser_human/url.stdout new file mode 100644 index 0000000..c1c624d --- /dev/null +++ b/internal/cmd/testdata/golden/browser_human/url.stdout @@ -0,0 +1 @@ +ws://SERVER/v1/browser?token=wbt_testtoken&engine=chromium&proxy_country=de diff --git a/internal/cmd/testdata/golden/config/err_bad_base_url.exit b/internal/cmd/testdata/golden/config/err_bad_base_url.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/config/err_bad_base_url.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/config/err_bad_base_url.stderr b/internal/cmd/testdata/golden/config/err_bad_base_url.stderr new file mode 100644 index 0000000..bfc7ab2 --- /dev/null +++ b/internal/cmd/testdata/golden/config/err_bad_base_url.stderr @@ -0,0 +1,4 @@ +{ + "error": "base_url must be an http(s) URL, got \"ftp://x\"", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/config/err_bad_base_url.stdout b/internal/cmd/testdata/golden/config/err_bad_base_url.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/config/err_unknown_key.exit b/internal/cmd/testdata/golden/config/err_unknown_key.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/config/err_unknown_key.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/config/err_unknown_key.stderr b/internal/cmd/testdata/golden/config/err_unknown_key.stderr new file mode 100644 index 0000000..4778ffc --- /dev/null +++ b/internal/cmd/testdata/golden/config/err_unknown_key.stderr @@ -0,0 +1,4 @@ +{ + "error": "unknown config key \"nope\" (keys: api_key, base_url)", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/config/err_unknown_key.stdout b/internal/cmd/testdata/golden/config/err_unknown_key.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/config/get_all.exit b/internal/cmd/testdata/golden/config/get_all.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_all.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/get_all.stdout b/internal/cmd/testdata/golden/config/get_all.stdout new file mode 100644 index 0000000..66518d7 --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_all.stdout @@ -0,0 +1,4 @@ +{ + "api_key": "spicrawl_live_0123…mnop", + "base_url": "https://api.example.test" +} diff --git a/internal/cmd/testdata/golden/config/get_all_empty.exit b/internal/cmd/testdata/golden/config/get_all_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_all_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/get_all_empty.stdout b/internal/cmd/testdata/golden/config/get_all_empty.stdout new file mode 100644 index 0000000..04e5250 --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_all_empty.stdout @@ -0,0 +1,4 @@ +{ + "api_key": null, + "base_url": null +} diff --git a/internal/cmd/testdata/golden/config/get_api_key_masked.exit b/internal/cmd/testdata/golden/config/get_api_key_masked.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_api_key_masked.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/get_api_key_masked.stdout b/internal/cmd/testdata/golden/config/get_api_key_masked.stdout new file mode 100644 index 0000000..2acb265 --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_api_key_masked.stdout @@ -0,0 +1,4 @@ +{ + "key": "api_key", + "value": "spicrawl_live_0123…mnop" +} diff --git a/internal/cmd/testdata/golden/config/get_api_key_show_secret.exit b/internal/cmd/testdata/golden/config/get_api_key_show_secret.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_api_key_show_secret.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/get_api_key_show_secret.stdout b/internal/cmd/testdata/golden/config/get_api_key_show_secret.stdout new file mode 100644 index 0000000..98a7f3a --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_api_key_show_secret.stdout @@ -0,0 +1,4 @@ +{ + "key": "api_key", + "value": "spicrawl_live_0123456789abcdefghijklmnop" +} diff --git a/internal/cmd/testdata/golden/config/get_base_url.exit b/internal/cmd/testdata/golden/config/get_base_url.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_base_url.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/get_base_url.stdout b/internal/cmd/testdata/golden/config/get_base_url.stdout new file mode 100644 index 0000000..b70c081 --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_base_url.stdout @@ -0,0 +1,4 @@ +{ + "key": "base_url", + "value": "https://api.example.test" +} diff --git a/internal/cmd/testdata/golden/config/get_unset.exit b/internal/cmd/testdata/golden/config/get_unset.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_unset.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/get_unset.stdout b/internal/cmd/testdata/golden/config/get_unset.stdout new file mode 100644 index 0000000..78c4aa4 --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_unset.stdout @@ -0,0 +1,4 @@ +{ + "key": "base_url", + "value": null +} diff --git a/internal/cmd/testdata/golden/config/path.exit b/internal/cmd/testdata/golden/config/path.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/path.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/path.stdout b/internal/cmd/testdata/golden/config/path.stdout new file mode 100644 index 0000000..026f21c --- /dev/null +++ b/internal/cmd/testdata/golden/config/path.stdout @@ -0,0 +1,3 @@ +{ + "path": "/config.json" +} diff --git a/internal/cmd/testdata/golden/config/set_api_key.exit b/internal/cmd/testdata/golden/config/set_api_key.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/set_api_key.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/set_api_key.stdout b/internal/cmd/testdata/golden/config/set_api_key.stdout new file mode 100644 index 0000000..7bb7b4c --- /dev/null +++ b/internal/cmd/testdata/golden/config/set_api_key.stdout @@ -0,0 +1,5 @@ +{ + "key": "api_key", + "path": "/config.json", + "value": "spicrawl_live_0123…mnop" +} diff --git a/internal/cmd/testdata/golden/config/set_base_url.exit b/internal/cmd/testdata/golden/config/set_base_url.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/set_base_url.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/set_base_url.stdout b/internal/cmd/testdata/golden/config/set_base_url.stdout new file mode 100644 index 0000000..a9c825a --- /dev/null +++ b/internal/cmd/testdata/golden/config/set_base_url.stdout @@ -0,0 +1,5 @@ +{ + "key": "base_url", + "path": "/config.json", + "value": "http://localhost:8080" +} diff --git a/internal/cmd/testdata/golden/config/unset_api_key.exit b/internal/cmd/testdata/golden/config/unset_api_key.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/unset_api_key.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/unset_api_key.stdout b/internal/cmd/testdata/golden/config/unset_api_key.stdout new file mode 100644 index 0000000..9c2295e --- /dev/null +++ b/internal/cmd/testdata/golden/config/unset_api_key.stdout @@ -0,0 +1,5 @@ +{ + "key": "api_key", + "path": "/config.json", + "value": null +} diff --git a/internal/cmd/testdata/golden/config_human/get_all.exit b/internal/cmd/testdata/golden/config_human/get_all.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_all.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config_human/get_all.stdout b/internal/cmd/testdata/golden/config_human/get_all.stdout new file mode 100644 index 0000000..25d4d5a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_all.stdout @@ -0,0 +1,3 @@ +KEY VALUE +api_key spicrawl_live_0123…mnop +base_url https://api.example.test diff --git a/internal/cmd/testdata/golden/config_human/get_all_empty.exit b/internal/cmd/testdata/golden/config_human/get_all_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_all_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config_human/get_all_empty.stdout b/internal/cmd/testdata/golden/config_human/get_all_empty.stdout new file mode 100644 index 0000000..cf47445 --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_all_empty.stdout @@ -0,0 +1,3 @@ +KEY VALUE +api_key +base_url diff --git a/internal/cmd/testdata/golden/config_human/get_base_url.exit b/internal/cmd/testdata/golden/config_human/get_base_url.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_base_url.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config_human/get_base_url.stdout b/internal/cmd/testdata/golden/config_human/get_base_url.stdout new file mode 100644 index 0000000..d07c895 --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_base_url.stdout @@ -0,0 +1 @@ +https://api.example.test diff --git a/internal/cmd/testdata/golden/config_human/get_unset.exit b/internal/cmd/testdata/golden/config_human/get_unset.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_unset.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config_human/get_unset.stdout b/internal/cmd/testdata/golden/config_human/get_unset.stdout new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_unset.stdout @@ -0,0 +1 @@ + diff --git a/internal/cmd/testdata/golden/config_human/path.exit b/internal/cmd/testdata/golden/config_human/path.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/path.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config_human/path.stdout b/internal/cmd/testdata/golden/config_human/path.stdout new file mode 100644 index 0000000..ac2f8ba --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/path.stdout @@ -0,0 +1 @@ +/config.json diff --git a/internal/cmd/testdata/golden/config_human/set_api_key.exit b/internal/cmd/testdata/golden/config_human/set_api_key.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/set_api_key.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config_human/set_api_key.stdout b/internal/cmd/testdata/golden/config_human/set_api_key.stdout new file mode 100644 index 0000000..7862456 --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/set_api_key.stdout @@ -0,0 +1 @@ +Set api_key = spicrawl_live_0123…mnop in /config.json diff --git a/internal/cmd/testdata/golden/config_human/unset_api_key.exit b/internal/cmd/testdata/golden/config_human/unset_api_key.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/unset_api_key.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config_human/unset_api_key.stdout b/internal/cmd/testdata/golden/config_human/unset_api_key.stdout new file mode 100644 index 0000000..e3caede --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/unset_api_key.stdout @@ -0,0 +1 @@ +Removed api_key from /config.json diff --git a/internal/cmd/testdata/golden/docs/adds_missing_newline.exit b/internal/cmd/testdata/golden/docs/adds_missing_newline.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/docs/adds_missing_newline.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/docs/adds_missing_newline.stdout b/internal/cmd/testdata/golden/docs/adds_missing_newline.stdout new file mode 100644 index 0000000..2876d08 --- /dev/null +++ b/internal/cmd/testdata/golden/docs/adds_missing_newline.stdout @@ -0,0 +1 @@ +# No newline diff --git a/internal/cmd/testdata/golden/docs/err_bad_topic.exit b/internal/cmd/testdata/golden/docs/err_bad_topic.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/docs/err_bad_topic.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/docs/err_bad_topic.stderr b/internal/cmd/testdata/golden/docs/err_bad_topic.stderr new file mode 100644 index 0000000..5ddc308 --- /dev/null +++ b/internal/cmd/testdata/golden/docs/err_bad_topic.stderr @@ -0,0 +1,4 @@ +{ + "error": "invalid docs topic \"../etc/passwd\": use a page path like guides/anti-bot (see spicrawl docs --list)", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/docs/err_bad_topic.stdout b/internal/cmd/testdata/golden/docs/err_bad_topic.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/docs/err_unknown_topic.exit b/internal/cmd/testdata/golden/docs/err_unknown_topic.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/docs/err_unknown_topic.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/docs/err_unknown_topic.stderr b/internal/cmd/testdata/golden/docs/err_unknown_topic.stderr new file mode 100644 index 0000000..09c70ed --- /dev/null +++ b/internal/cmd/testdata/golden/docs/err_unknown_topic.stderr @@ -0,0 +1,4 @@ +{ + "error": "no docs page at http://SERVER/missing.md (see spicrawl docs --list)", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/docs/err_unknown_topic.stdout b/internal/cmd/testdata/golden/docs/err_unknown_topic.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/docs/json_flag_wraps.exit b/internal/cmd/testdata/golden/docs/json_flag_wraps.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/docs/json_flag_wraps.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/docs/json_flag_wraps.stdout b/internal/cmd/testdata/golden/docs/json_flag_wraps.stdout new file mode 100644 index 0000000..254440e --- /dev/null +++ b/internal/cmd/testdata/golden/docs/json_flag_wraps.stdout @@ -0,0 +1,5 @@ +{ + "markdown": "# Anti-bot\n\nUse --stealth only when a page needs it.\n", + "topic": "guides/anti-bot", + "url": "http://SERVER/guides/anti-bot.md" +} diff --git a/internal/cmd/testdata/golden/docs/list_raw_markdown.exit b/internal/cmd/testdata/golden/docs/list_raw_markdown.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/docs/list_raw_markdown.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/docs/list_raw_markdown.stdout b/internal/cmd/testdata/golden/docs/list_raw_markdown.stdout new file mode 100644 index 0000000..8229e13 --- /dev/null +++ b/internal/cmd/testdata/golden/docs/list_raw_markdown.stdout @@ -0,0 +1,4 @@ +# Spicrawl + +- [Quickstart](https://docs.spicrawl.com/quickstart.md) +- [Anti-bot](https://docs.spicrawl.com/guides/anti-bot.md) diff --git a/internal/cmd/testdata/golden/docs/no_topic_is_the_index.exit b/internal/cmd/testdata/golden/docs/no_topic_is_the_index.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/docs/no_topic_is_the_index.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/docs/no_topic_is_the_index.stdout b/internal/cmd/testdata/golden/docs/no_topic_is_the_index.stdout new file mode 100644 index 0000000..8229e13 --- /dev/null +++ b/internal/cmd/testdata/golden/docs/no_topic_is_the_index.stdout @@ -0,0 +1,4 @@ +# Spicrawl + +- [Quickstart](https://docs.spicrawl.com/quickstart.md) +- [Anti-bot](https://docs.spicrawl.com/guides/anti-bot.md) diff --git a/internal/cmd/testdata/golden/docs/topic_raw_markdown.exit b/internal/cmd/testdata/golden/docs/topic_raw_markdown.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/docs/topic_raw_markdown.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/docs/topic_raw_markdown.stdout b/internal/cmd/testdata/golden/docs/topic_raw_markdown.stdout new file mode 100644 index 0000000..298b286 --- /dev/null +++ b/internal/cmd/testdata/golden/docs/topic_raw_markdown.stdout @@ -0,0 +1,3 @@ +# Anti-bot + +Use --stealth only when a page needs it. diff --git a/internal/cmd/testdata/golden/docs_human/topic_raw_markdown.exit b/internal/cmd/testdata/golden/docs_human/topic_raw_markdown.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/docs_human/topic_raw_markdown.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/docs_human/topic_raw_markdown.stdout b/internal/cmd/testdata/golden/docs_human/topic_raw_markdown.stdout new file mode 100644 index 0000000..298b286 --- /dev/null +++ b/internal/cmd/testdata/golden/docs_human/topic_raw_markdown.stdout @@ -0,0 +1,3 @@ +# Anti-bot + +Use --stealth only when a page needs it. diff --git a/internal/cmd/testdata/golden/logs/all_pages.exit b/internal/cmd/testdata/golden/logs/all_pages.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/all_pages.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/all_pages.stdout b/internal/cmd/testdata/golden/logs/all_pages.stdout new file mode 100644 index 0000000..066b2c2 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/all_pages.stdout @@ -0,0 +1,77 @@ +{ + "data": [ + { + "id": "01M0FK1EP2E7FBMWAR7W10SMXW", + "created_at": "", + "project_id": "proj_01J9Z", + "status": "success", + "http_status": 200, + "engine": "fetch", + "credits_micro": 1500000, + "url": "https://example.com/a", + "attempts": 2, + "bytes_in": 2048, + "bytes_out": 512, + "proxy_bytes": 4096, + "spans": { + "admission_ms": 1, + "dispatch_ms": 2, + "worker_ms": 3, + "queue_ms": 0, + "engine_ms": 80, + "upstream_ms": 120, + "extract_ms": null, + "total_ms": 210 + }, + "proxy": { + "source": "pool", + "reason": "default", + "provider": "dc-1", + "requested_tier": "datacenter", + "requested_country": "de", + "country": "de", + "endpoint_id": "ep-1", + "endpoints": [ + "ep-1", + "ep-2" + ], + "sticky": true + } + }, + { + "id": "01M0FK1EP2E7FBMWAR7W10SMXX", + "created_at": "", + "project_id": "proj_01J9Z", + "status": "blocked", + "http_status": 403, + "engine": "", + "credits_micro": 0, + "url": "https://example.com/b?x=1&y=2", + "error_code": "ERR::UPSTREAM::BLOCKED", + "error_detail": "bot challenge", + "spans": { + "admission_ms": 1, + "dispatch_ms": null, + "worker_ms": null, + "queue_ms": 0, + "engine_ms": null, + "upstream_ms": 120, + "extract_ms": null, + "total_ms": 130 + }, + "proxy": { + "source": "pool" + }, + "blocked": { + "vendor": "cloudflare", + "rule": "r-17", + "signal": "cf-mitigated" + } + } + ], + "page": { + "has_more": false, + "retained_since": "", + "retention_hours": 24 + } +} diff --git a/internal/cmd/testdata/golden/logs/all_pages_jsonl.exit b/internal/cmd/testdata/golden/logs/all_pages_jsonl.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/all_pages_jsonl.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/all_pages_jsonl.stdout b/internal/cmd/testdata/golden/logs/all_pages_jsonl.stdout new file mode 100644 index 0000000..22f025f --- /dev/null +++ b/internal/cmd/testdata/golden/logs/all_pages_jsonl.stdout @@ -0,0 +1,2 @@ +{"id":"01M0FK1EP2E7FBMWAR7W10SMXW","created_at":"","project_id":"proj_01J9Z","status":"success","http_status":200,"engine":"fetch","credits_micro":1500000,"url":"https://example.com/a","attempts":2,"bytes_in":2048,"bytes_out":512,"proxy_bytes":4096,"spans":{"admission_ms":1,"dispatch_ms":2,"worker_ms":3,"queue_ms":0,"engine_ms":80,"upstream_ms":120,"extract_ms":null,"total_ms":210},"proxy":{"source":"pool","reason":"default","provider":"dc-1","requested_tier":"datacenter","requested_country":"de","country":"de","endpoint_id":"ep-1","endpoints":["ep-1","ep-2"],"sticky":true}} +{"id":"01M0FK1EP2E7FBMWAR7W10SMXX","created_at":"","project_id":"proj_01J9Z","status":"blocked","http_status":403,"engine":"","credits_micro":0,"url":"https://example.com/b?x=1&y=2","error_code":"ERR::UPSTREAM::BLOCKED","error_detail":"bot challenge","spans":{"admission_ms":1,"dispatch_ms":null,"worker_ms":null,"queue_ms":0,"engine_ms":null,"upstream_ms":120,"extract_ms":null,"total_ms":130},"proxy":{"source":"pool"},"blocked":{"vendor":"cloudflare","rule":"r-17","signal":"cf-mitigated"}} diff --git a/internal/cmd/testdata/golden/logs/empty.exit b/internal/cmd/testdata/golden/logs/empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/empty.stdout b/internal/cmd/testdata/golden/logs/empty.stdout new file mode 100644 index 0000000..1ac3e6d --- /dev/null +++ b/internal/cmd/testdata/golden/logs/empty.stdout @@ -0,0 +1,8 @@ +{ + "data": [], + "page": { + "has_more": false, + "retained_since": "", + "retention_hours": 24 + } +} diff --git a/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.exit b/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.stderr b/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.stderr new file mode 100644 index 0000000..02a65d0 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.stderr @@ -0,0 +1,4 @@ +{ + "error": "--errors and --status cannot be combined", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.stdout b/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/logs/err_usage_status.exit b/internal/cmd/testdata/golden/logs/err_usage_status.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/err_usage_status.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/logs/err_usage_status.stderr b/internal/cmd/testdata/golden/logs/err_usage_status.stderr new file mode 100644 index 0000000..e4d9101 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/err_usage_status.stderr @@ -0,0 +1,4 @@ +{ + "error": "--status must be one of success, failed, timeout, blocked, rejected, cancelled, not \"weird\"", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/logs/err_usage_status.stdout b/internal/cmd/testdata/golden/logs/err_usage_status.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/logs/errors.exit b/internal/cmd/testdata/golden/logs/errors.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/errors.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/errors.stdout b/internal/cmd/testdata/golden/logs/errors.stdout new file mode 100644 index 0000000..3cfb105 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/errors.stdout @@ -0,0 +1,39 @@ +{ + "data": [ + { + "id": "01M0FK1EP2E7FBMWAR7W10SMXX", + "created_at": "", + "project_id": "proj_01J9Z", + "status": "blocked", + "http_status": 403, + "engine": "", + "credits_micro": 0, + "url": "https://example.com/b?x=1&y=2", + "error_code": "ERR::UPSTREAM::BLOCKED", + "error_detail": "bot challenge", + "spans": { + "admission_ms": 1, + "dispatch_ms": null, + "worker_ms": null, + "queue_ms": 0, + "engine_ms": null, + "upstream_ms": 120, + "extract_ms": null, + "total_ms": 130 + }, + "proxy": { + "source": "pool" + }, + "blocked": { + "vendor": "cloudflare", + "rule": "r-17", + "signal": "cf-mitigated" + } + } + ], + "page": { + "has_more": false, + "retained_since": "", + "retention_hours": 24 + } +} diff --git a/internal/cmd/testdata/golden/logs/get.exit b/internal/cmd/testdata/golden/logs/get.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/get.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/get.stdout b/internal/cmd/testdata/golden/logs/get.stdout new file mode 100644 index 0000000..82c6926 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/get.stdout @@ -0,0 +1,38 @@ +{ + "id": "01M0FK1EP2E7FBMWAR7W10SMXW", + "created_at": "", + "project_id": "proj_01J9Z", + "status": "success", + "http_status": 200, + "engine": "fetch", + "credits_micro": 1500000, + "url": "https://example.com/a", + "attempts": 2, + "bytes_in": 2048, + "bytes_out": 512, + "proxy_bytes": 4096, + "spans": { + "admission_ms": 1, + "dispatch_ms": 2, + "worker_ms": 3, + "queue_ms": 0, + "engine_ms": 80, + "upstream_ms": 120, + "extract_ms": null, + "total_ms": 210 + }, + "proxy": { + "source": "pool", + "reason": "default", + "provider": "dc-1", + "requested_tier": "datacenter", + "requested_country": "de", + "country": "de", + "endpoint_id": "ep-1", + "endpoints": [ + "ep-1", + "ep-2" + ], + "sticky": true + } +} diff --git a/internal/cmd/testdata/golden/logs/get_not_found.exit b/internal/cmd/testdata/golden/logs/get_not_found.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/get_not_found.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/logs/get_not_found.stderr b/internal/cmd/testdata/golden/logs/get_not_found.stderr new file mode 100644 index 0000000..856bebc --- /dev/null +++ b/internal/cmd/testdata/golden/logs/get_not_found.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Not Found", + "status": 404, + "code": "ERR::REQUEST::NOT_FOUND", + "detail": "no such request", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/logs/get_not_found.stdout b/internal/cmd/testdata/golden/logs/get_not_found.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/logs/has_more.exit b/internal/cmd/testdata/golden/logs/has_more.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/has_more.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/has_more.stdout b/internal/cmd/testdata/golden/logs/has_more.stdout new file mode 100644 index 0000000..63c2235 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/has_more.stdout @@ -0,0 +1,49 @@ +{ + "data": [ + { + "id": "01M0FK1EP2E7FBMWAR7W10SMXW", + "created_at": "", + "project_id": "proj_01J9Z", + "status": "success", + "http_status": 200, + "engine": "fetch", + "credits_micro": 1500000, + "url": "https://example.com/a", + "attempts": 2, + "bytes_in": 2048, + "bytes_out": 512, + "proxy_bytes": 4096, + "spans": { + "admission_ms": 1, + "dispatch_ms": 2, + "worker_ms": 3, + "queue_ms": 0, + "engine_ms": 80, + "upstream_ms": 120, + "extract_ms": null, + "total_ms": 210 + }, + "proxy": { + "source": "pool", + "reason": "default", + "provider": "dc-1", + "requested_tier": "datacenter", + "requested_country": "de", + "country": "de", + "endpoint_id": "ep-1", + "endpoints": [ + "ep-1", + "ep-2" + ], + "sticky": true + } + } + ], + "page": { + "has_more": true, + "next_before": "", + "next_before_id": "01M0FK1EP2E7FBMWAR7W10SMXW", + "retained_since": "", + "retention_hours": 24 + } +} diff --git a/internal/cmd/testdata/golden/logs/jsonl.exit b/internal/cmd/testdata/golden/logs/jsonl.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/jsonl.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/jsonl.stdout b/internal/cmd/testdata/golden/logs/jsonl.stdout new file mode 100644 index 0000000..22f025f --- /dev/null +++ b/internal/cmd/testdata/golden/logs/jsonl.stdout @@ -0,0 +1,2 @@ +{"id":"01M0FK1EP2E7FBMWAR7W10SMXW","created_at":"","project_id":"proj_01J9Z","status":"success","http_status":200,"engine":"fetch","credits_micro":1500000,"url":"https://example.com/a","attempts":2,"bytes_in":2048,"bytes_out":512,"proxy_bytes":4096,"spans":{"admission_ms":1,"dispatch_ms":2,"worker_ms":3,"queue_ms":0,"engine_ms":80,"upstream_ms":120,"extract_ms":null,"total_ms":210},"proxy":{"source":"pool","reason":"default","provider":"dc-1","requested_tier":"datacenter","requested_country":"de","country":"de","endpoint_id":"ep-1","endpoints":["ep-1","ep-2"],"sticky":true}} +{"id":"01M0FK1EP2E7FBMWAR7W10SMXX","created_at":"","project_id":"proj_01J9Z","status":"blocked","http_status":403,"engine":"","credits_micro":0,"url":"https://example.com/b?x=1&y=2","error_code":"ERR::UPSTREAM::BLOCKED","error_detail":"bot challenge","spans":{"admission_ms":1,"dispatch_ms":null,"worker_ms":null,"queue_ms":0,"engine_ms":null,"upstream_ms":120,"extract_ms":null,"total_ms":130},"proxy":{"source":"pool"},"blocked":{"vendor":"cloudflare","rule":"r-17","signal":"cf-mitigated"}} diff --git a/internal/cmd/testdata/golden/logs/list.exit b/internal/cmd/testdata/golden/logs/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/list.stdout b/internal/cmd/testdata/golden/logs/list.stdout new file mode 100644 index 0000000..066b2c2 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/list.stdout @@ -0,0 +1,77 @@ +{ + "data": [ + { + "id": "01M0FK1EP2E7FBMWAR7W10SMXW", + "created_at": "", + "project_id": "proj_01J9Z", + "status": "success", + "http_status": 200, + "engine": "fetch", + "credits_micro": 1500000, + "url": "https://example.com/a", + "attempts": 2, + "bytes_in": 2048, + "bytes_out": 512, + "proxy_bytes": 4096, + "spans": { + "admission_ms": 1, + "dispatch_ms": 2, + "worker_ms": 3, + "queue_ms": 0, + "engine_ms": 80, + "upstream_ms": 120, + "extract_ms": null, + "total_ms": 210 + }, + "proxy": { + "source": "pool", + "reason": "default", + "provider": "dc-1", + "requested_tier": "datacenter", + "requested_country": "de", + "country": "de", + "endpoint_id": "ep-1", + "endpoints": [ + "ep-1", + "ep-2" + ], + "sticky": true + } + }, + { + "id": "01M0FK1EP2E7FBMWAR7W10SMXX", + "created_at": "", + "project_id": "proj_01J9Z", + "status": "blocked", + "http_status": 403, + "engine": "", + "credits_micro": 0, + "url": "https://example.com/b?x=1&y=2", + "error_code": "ERR::UPSTREAM::BLOCKED", + "error_detail": "bot challenge", + "spans": { + "admission_ms": 1, + "dispatch_ms": null, + "worker_ms": null, + "queue_ms": 0, + "engine_ms": null, + "upstream_ms": 120, + "extract_ms": null, + "total_ms": 130 + }, + "proxy": { + "source": "pool" + }, + "blocked": { + "vendor": "cloudflare", + "rule": "r-17", + "signal": "cf-mitigated" + } + } + ], + "page": { + "has_more": false, + "retained_since": "", + "retention_hours": 24 + } +} diff --git a/internal/cmd/testdata/golden/logs_human/all_projects.exit b/internal/cmd/testdata/golden/logs_human/all_projects.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/all_projects.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs_human/all_projects.stdout b/internal/cmd/testdata/golden/logs_human/all_projects.stdout new file mode 100644 index 0000000..16b9d52 --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/all_projects.stdout @@ -0,0 +1,3 @@ +TIME PROJECT STATUS HTTP ENGINE CREDITS TOTAL_MS URL + proj_01J9Z success 200 fetch 1.5 210 https://example.com/a + proj_01J9Z blocked 403 — 0 130 https://example.com/b?x=1&y=2 diff --git a/internal/cmd/testdata/golden/logs_human/empty.exit b/internal/cmd/testdata/golden/logs_human/empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs_human/empty.stdout b/internal/cmd/testdata/golden/logs_human/empty.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/logs_human/get_blocked.exit b/internal/cmd/testdata/golden/logs_human/get_blocked.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/get_blocked.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs_human/get_blocked.stdout b/internal/cmd/testdata/golden/logs_human/get_blocked.stdout new file mode 100644 index 0000000..8024009 --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/get_blocked.stdout @@ -0,0 +1,28 @@ +id: 01M0FK1EP2E7FBMWAR7W10SMXX +time: +project: proj_01J9Z +status: blocked +url: https://example.com/b?x=1&y=2 +http status: 403 +engine: — +credits: 0 +error: ERR::UPSTREAM::BLOCKED +error detail: bot challenge + +spans (ms): + admission 1 + dispatch — + worker — + queue 0 + engine — + upstream 120 + extract — + total 130 + +proxy: + source: pool + +blocked: + vendor: cloudflare + rule: r-17 + signal: cf-mitigated diff --git a/internal/cmd/testdata/golden/logs_human/get_full.exit b/internal/cmd/testdata/golden/logs_human/get_full.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/get_full.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs_human/get_full.stdout b/internal/cmd/testdata/golden/logs_human/get_full.stdout new file mode 100644 index 0000000..1456d34 --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/get_full.stdout @@ -0,0 +1,30 @@ +id: 01M0FK1EP2E7FBMWAR7W10SMXW +time: +project: proj_01J9Z +status: success +url: https://example.com/a +http status: 200 +engine: fetch +credits: 1.5 +attempts: 2 +bytes: in 2048, out 512, via proxy 4096 + +spans (ms): + admission 1 + dispatch 2 + worker 3 + queue 0 + engine 80 + upstream 120 + extract — + total 210 + +proxy: + source: pool + reason: default + provider: dc-1 + requested: datacenter de + served country: de + endpoint: ep-1 + endpoints: ep-1 -> ep-2 + sticky: yes diff --git a/internal/cmd/testdata/golden/logs_human/has_more.exit b/internal/cmd/testdata/golden/logs_human/has_more.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/has_more.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs_human/has_more.stdout b/internal/cmd/testdata/golden/logs_human/has_more.stdout new file mode 100644 index 0000000..e7be0f5 --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/has_more.stdout @@ -0,0 +1,2 @@ +TIME STATUS HTTP ENGINE CREDITS TOTAL_MS URL + success 200 fetch 1.5 210 https://example.com/a diff --git a/internal/cmd/testdata/golden/logs_human/list.exit b/internal/cmd/testdata/golden/logs_human/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs_human/list.stdout b/internal/cmd/testdata/golden/logs_human/list.stdout new file mode 100644 index 0000000..4572b4b --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/list.stdout @@ -0,0 +1,3 @@ +TIME STATUS HTTP ENGINE CREDITS TOTAL_MS URL + success 200 fetch 1.5 210 https://example.com/a + blocked 403 — 0 130 https://example.com/b?x=1&y=2 diff --git a/internal/cmd/testdata/golden/meta/err_unknown_command.exit b/internal/cmd/testdata/golden/meta/err_unknown_command.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/err_unknown_command.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/meta/err_unknown_command.stderr b/internal/cmd/testdata/golden/meta/err_unknown_command.stderr new file mode 100644 index 0000000..52a5ca0 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/err_unknown_command.stderr @@ -0,0 +1,4 @@ +{ + "error": "unknown command \"zzzzzz\" for \"spicrawl\"", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/meta/err_unknown_command.stdout b/internal/cmd/testdata/golden/meta/err_unknown_command.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/meta/err_unknown_flag.exit b/internal/cmd/testdata/golden/meta/err_unknown_flag.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/err_unknown_flag.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/meta/err_unknown_flag.stderr b/internal/cmd/testdata/golden/meta/err_unknown_flag.stderr new file mode 100644 index 0000000..016d3db --- /dev/null +++ b/internal/cmd/testdata/golden/meta/err_unknown_flag.stderr @@ -0,0 +1,4 @@ +{ + "error": "unknown flag: --nope", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/meta/err_unknown_flag.stdout b/internal/cmd/testdata/golden/meta/err_unknown_flag.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.exit b/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.stderr b/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.stderr new file mode 100644 index 0000000..51e7fb5 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.stderr @@ -0,0 +1,4 @@ +{ + "error": "unknown command \"lst\" for \"spicrawl batch\"; did you mean get or list?", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.stdout b/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/meta/exit_codes.exit b/internal/cmd/testdata/golden/meta/exit_codes.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/meta/exit_codes.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/meta/exit_codes.stdout b/internal/cmd/testdata/golden/meta/exit_codes.stdout new file mode 100644 index 0000000..68909d7 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/exit_codes.stdout @@ -0,0 +1,72 @@ +[ + { + "code": 0, + "name": "ok", + "meaning": "success" + }, + { + "code": 1, + "name": "internal", + "meaning": "CLI bug, or a transient ERR::INTERNAL::* (retryable)" + }, + { + "code": 2, + "name": "usage", + "meaning": "bad flags or arguments, or an unreadable/malformed config file; nothing was sent" + }, + { + "code": 3, + "name": "auth", + "meaning": "ERR::AUTH::*, or no API key configured" + }, + { + "code": 4, + "name": "request", + "meaning": "ERR::REQUEST::*, ERR::SECURITY::*, ERR::EXTRACT::INVALID_RULES: fix the request" + }, + { + "code": 5, + "name": "limit", + "meaning": "ERR::LIMIT::*: rate, concurrency, quota or max_cost" + }, + { + "code": 6, + "name": "upstream", + "meaning": "ERR::UPSTREAM::*: the target site failed or served a bot challenge" + }, + { + "code": 7, + "name": "proxy", + "meaning": "ERR::PROXY::*" + }, + { + "code": 8, + "name": "engine", + "meaning": "ERR::ENGINE::*, ERR::EXTRACT::FAILED, or ERR::INTERNAL::UNAVAILABLE with retryable=false: capability not available on this deployment" + }, + { + "code": 9, + "name": "session", + "meaning": "ERR::SESSION::*" + }, + { + "code": 10, + "name": "network", + "meaning": "the Spicrawl API could not be reached (DNS, refused, TLS); nothing was sent" + }, + { + "code": 11, + "name": "pending", + "meaning": "a --wait gave up before the job finished" + }, + { + "code": 12, + "name": "timeout", + "meaning": "timed out waiting for the API, or the connection dropped after the request was sent; the request may have run and been billed: check `spicrawl logs` before retrying" + }, + { + "code": 130, + "name": "interrupted", + "meaning": "stopped with Ctrl-C" + } +] diff --git a/internal/cmd/testdata/golden/meta/exit_codes_human.exit b/internal/cmd/testdata/golden/meta/exit_codes_human.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/meta/exit_codes_human.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/meta/exit_codes_human.stdout b/internal/cmd/testdata/golden/meta/exit_codes_human.stdout new file mode 100644 index 0000000..9f53803 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/exit_codes_human.stdout @@ -0,0 +1,18 @@ +Exit codes (stable; scripts and agents may branch on them): + + 0 ok success + 1 internal CLI bug, or a transient ERR::INTERNAL::* (retryable) + 2 usage bad flags or arguments, or an unreadable/malformed config file; nothing was sent + 3 auth ERR::AUTH::*, or no API key configured + 4 request ERR::REQUEST::*, ERR::SECURITY::*, ERR::EXTRACT::INVALID_RULES: fix the request + 5 limit ERR::LIMIT::*: rate, concurrency, quota or max_cost + 6 upstream ERR::UPSTREAM::*: the target site failed or served a bot challenge + 7 proxy ERR::PROXY::* + 8 engine ERR::ENGINE::*, ERR::EXTRACT::FAILED, or ERR::INTERNAL::UNAVAILABLE with retryable=false: capability not available on this deployment + 9 session ERR::SESSION::* + 10 network the Spicrawl API could not be reached (DNS, refused, TLS); nothing was sent + 11 pending a --wait gave up before the job finished + 12 timeout timed out waiting for the API, or the connection dropped after the request was sent; the request may have run and been billed: check `spicrawl logs` before retrying + 130 interrupted stopped with Ctrl-C + +In JSON mode a failed command writes the API's problem document to stderr. diff --git a/internal/cmd/testdata/golden/meta/version.exit b/internal/cmd/testdata/golden/meta/version.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/meta/version.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/meta/version.stdout b/internal/cmd/testdata/golden/meta/version.stdout new file mode 100644 index 0000000..ba21f57 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/version.stdout @@ -0,0 +1,6 @@ +{ + "arch": "", + "go": "", + "os": "", + "version": "" +} diff --git a/internal/cmd/testdata/golden/meta/version_human.exit b/internal/cmd/testdata/golden/meta/version_human.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/meta/version_human.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/meta/version_human.stdout b/internal/cmd/testdata/golden/meta/version_human.stdout new file mode 100644 index 0000000..1e73a9b --- /dev/null +++ b/internal/cmd/testdata/golden/meta/version_human.stdout @@ -0,0 +1 @@ +spicrawl (, /) diff --git a/internal/cmd/testdata/golden/schema/batch.exit b/internal/cmd/testdata/golden/schema/batch.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema/batch.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema/batch.stdout.sha256 b/internal/cmd/testdata/golden/schema/batch.stdout.sha256 new file mode 100644 index 0000000..e3a3e98 --- /dev/null +++ b/internal/cmd/testdata/golden/schema/batch.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 266998850ee3992e4bd07755021c5f11ec6077c02834d03c83da88898731eb2a +bytes 102977 diff --git a/internal/cmd/testdata/golden/schema/batch_item.exit b/internal/cmd/testdata/golden/schema/batch_item.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema/batch_item.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema/batch_item.stdout.sha256 b/internal/cmd/testdata/golden/schema/batch_item.stdout.sha256 new file mode 100644 index 0000000..ae9bbaf --- /dev/null +++ b/internal/cmd/testdata/golden/schema/batch_item.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 9039dfe91d5728ae8bf229ddd0560d651b27fbd3c73fb64a14915f5f36a09f9a +bytes 48426 diff --git a/internal/cmd/testdata/golden/schema/err_unknown.exit b/internal/cmd/testdata/golden/schema/err_unknown.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/schema/err_unknown.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/schema/err_unknown.stderr b/internal/cmd/testdata/golden/schema/err_unknown.stderr new file mode 100644 index 0000000..c6f1050 --- /dev/null +++ b/internal/cmd/testdata/golden/schema/err_unknown.stderr @@ -0,0 +1,4 @@ +{ + "error": "unknown schema \"nope\" (available: batch, batch-item, scrape, session)", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/schema/err_unknown.stdout b/internal/cmd/testdata/golden/schema/err_unknown.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/schema/list.exit b/internal/cmd/testdata/golden/schema/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema/list.stdout b/internal/cmd/testdata/golden/schema/list.stdout new file mode 100644 index 0000000..34aaae7 --- /dev/null +++ b/internal/cmd/testdata/golden/schema/list.stdout @@ -0,0 +1,18 @@ +[ + { + "description": "POST /v1/batch and POST /v1/batch/{id}/items body", + "name": "batch" + }, + { + "description": "one element of a batch body's items[] (one line of a JSONL batch file)", + "name": "batch-item" + }, + { + "description": "POST /v1/scrape body (spicrawl scrape --body)", + "name": "scrape" + }, + { + "description": "POST /v1/sessions body", + "name": "session" + } +] diff --git a/internal/cmd/testdata/golden/schema/list_flag.exit b/internal/cmd/testdata/golden/schema/list_flag.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema/list_flag.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema/list_flag.stdout b/internal/cmd/testdata/golden/schema/list_flag.stdout new file mode 100644 index 0000000..34aaae7 --- /dev/null +++ b/internal/cmd/testdata/golden/schema/list_flag.stdout @@ -0,0 +1,18 @@ +[ + { + "description": "POST /v1/batch and POST /v1/batch/{id}/items body", + "name": "batch" + }, + { + "description": "one element of a batch body's items[] (one line of a JSONL batch file)", + "name": "batch-item" + }, + { + "description": "POST /v1/scrape body (spicrawl scrape --body)", + "name": "scrape" + }, + { + "description": "POST /v1/sessions body", + "name": "session" + } +] diff --git a/internal/cmd/testdata/golden/schema/scrape.exit b/internal/cmd/testdata/golden/schema/scrape.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema/scrape.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema/scrape.stdout.sha256 b/internal/cmd/testdata/golden/schema/scrape.stdout.sha256 new file mode 100644 index 0000000..5b7dc24 --- /dev/null +++ b/internal/cmd/testdata/golden/schema/scrape.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 676f4b8f19ba42a3665e03dc792447ae0d0f7994212b5aa6a38eeaaaf6491de8 +bytes 47911 diff --git a/internal/cmd/testdata/golden/schema/session.exit b/internal/cmd/testdata/golden/schema/session.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema/session.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema/session.stdout.sha256 b/internal/cmd/testdata/golden/schema/session.stdout.sha256 new file mode 100644 index 0000000..c53e2ef --- /dev/null +++ b/internal/cmd/testdata/golden/schema/session.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 ec7f95d65b603dca6468abb8aec21ca1acc14905ab47d6843c2633d9b6c7e16e +bytes 8827 diff --git a/internal/cmd/testdata/golden/schema_human/batch.exit b/internal/cmd/testdata/golden/schema_human/batch.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/batch.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema_human/batch.stdout.sha256 b/internal/cmd/testdata/golden/schema_human/batch.stdout.sha256 new file mode 100644 index 0000000..e3a3e98 --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/batch.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 266998850ee3992e4bd07755021c5f11ec6077c02834d03c83da88898731eb2a +bytes 102977 diff --git a/internal/cmd/testdata/golden/schema_human/batch_item.exit b/internal/cmd/testdata/golden/schema_human/batch_item.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/batch_item.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema_human/batch_item.stdout.sha256 b/internal/cmd/testdata/golden/schema_human/batch_item.stdout.sha256 new file mode 100644 index 0000000..ae9bbaf --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/batch_item.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 9039dfe91d5728ae8bf229ddd0560d651b27fbd3c73fb64a14915f5f36a09f9a +bytes 48426 diff --git a/internal/cmd/testdata/golden/schema_human/list.exit b/internal/cmd/testdata/golden/schema_human/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema_human/list.stdout b/internal/cmd/testdata/golden/schema_human/list.stdout new file mode 100644 index 0000000..c50f241 --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/list.stdout @@ -0,0 +1,5 @@ +NAME DESCRIBES +batch POST /v1/batch and POST /v1/batch/{id}/items body +batch-item one element of a batch body's items[] (one line of a JSONL batch file) +scrape POST /v1/scrape body (spicrawl scrape --body) +session POST /v1/sessions body diff --git a/internal/cmd/testdata/golden/schema_human/scrape.exit b/internal/cmd/testdata/golden/schema_human/scrape.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/scrape.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema_human/scrape.stdout.sha256 b/internal/cmd/testdata/golden/schema_human/scrape.stdout.sha256 new file mode 100644 index 0000000..5b7dc24 --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/scrape.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 676f4b8f19ba42a3665e03dc792447ae0d0f7994212b5aa6a38eeaaaf6491de8 +bytes 47911 diff --git a/internal/cmd/testdata/golden/schema_human/session.exit b/internal/cmd/testdata/golden/schema_human/session.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/session.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema_human/session.stdout.sha256 b/internal/cmd/testdata/golden/schema_human/session.stdout.sha256 new file mode 100644 index 0000000..c53e2ef --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/session.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 ec7f95d65b603dca6468abb8aec21ca1acc14905ab47d6843c2633d9b6c7e16e +bytes 8827 diff --git a/internal/cmd/testdata/golden/scrape/ai.exit b/internal/cmd/testdata/golden/scrape/ai.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/ai.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/ai.stdout b/internal/cmd/testdata/golden/scrape/ai.stdout new file mode 100644 index 0000000..0e82125 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/ai.stdout @@ -0,0 +1,18 @@ +{ + "url": "https://example.com/product/1", + "final_url": "https://example.com/product/1", + "status": 200, + "content": "

Anvil

$19.99 & free shipping

", + "headers": { + "content-type": "text/html" + }, + "truncated": false, + "credits": 3, + "engine": "fetch", + "proxy_source": "pool", + "warnings": [], + "data": { + "name": "Anvil", + "price": 19.99 + } +} diff --git a/internal/cmd/testdata/golden/scrape/autoparse.exit b/internal/cmd/testdata/golden/scrape/autoparse.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/autoparse.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/autoparse.stdout b/internal/cmd/testdata/golden/scrape/autoparse.stdout new file mode 100644 index 0000000..ae8c0ab --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/autoparse.stdout @@ -0,0 +1,25 @@ +{ + "url": "https://example.com/product/1", + "final_url": "https://example.com/product/1", + "status": 200, + "content": "

Anvil

$19.99 & free shipping

", + "headers": { + "content-type": "text/html" + }, + "truncated": false, + "credits": 3, + "engine": "fetch", + "proxy_source": "pool", + "warnings": [], + "data": { + "json_ld": [ + { + "@type": "Product", + "name": "Anvil" + } + ], + "open_graph": { + "og:title": "Anvil" + } + } +} diff --git a/internal/cmd/testdata/golden/scrape/err_auth.exit b/internal/cmd/testdata/golden/scrape/err_auth.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_auth.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/scrape/err_auth.stderr b/internal/cmd/testdata/golden/scrape/err_auth.stderr new file mode 100644 index 0000000..cbdc3b7 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_auth.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Unauthorized", + "status": 401, + "code": "ERR::AUTH::INVALID_KEY", + "detail": "the API key is not valid", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/scrape/err_auth.stdout b/internal/cmd/testdata/golden/scrape/err_auth.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_engine.exit b/internal/cmd/testdata/golden/scrape/err_engine.exit new file mode 100644 index 0000000..45a4fb7 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_engine.exit @@ -0,0 +1 @@ +8 diff --git a/internal/cmd/testdata/golden/scrape/err_engine.stderr b/internal/cmd/testdata/golden/scrape/err_engine.stderr new file mode 100644 index 0000000..fbf1f2f --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_engine.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Not Implemented", + "status": 501, + "code": "ERR::ENGINE::UNAVAILABLE", + "detail": "ai_extract is not available", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/scrape/err_engine.stdout b/internal/cmd/testdata/golden/scrape/err_engine.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_limit.exit b/internal/cmd/testdata/golden/scrape/err_limit.exit new file mode 100644 index 0000000..7ed6ff8 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_limit.exit @@ -0,0 +1 @@ +5 diff --git a/internal/cmd/testdata/golden/scrape/err_limit.stderr b/internal/cmd/testdata/golden/scrape/err_limit.stderr new file mode 100644 index 0000000..eb1a50a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_limit.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Too Many Requests", + "status": 429, + "code": "ERR::LIMIT::RATE", + "detail": "slow down", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/scrape/err_limit.stdout b/internal/cmd/testdata/golden/scrape/err_limit.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_no_key.exit b/internal/cmd/testdata/golden/scrape/err_no_key.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_no_key.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/scrape/err_no_key.stderr b/internal/cmd/testdata/golden/scrape/err_no_key.stderr new file mode 100644 index 0000000..6b478b2 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_no_key.stderr @@ -0,0 +1,4 @@ +{ + "error": "no API key: run \"spicrawl login\", set SPICRAWL_API_KEY, or pass --api-key", + "exit_code": 3 +} diff --git a/internal/cmd/testdata/golden/scrape/err_no_key.stdout b/internal/cmd/testdata/golden/scrape/err_no_key.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_request.exit b/internal/cmd/testdata/golden/scrape/err_request.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_request.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/scrape/err_request.stderr b/internal/cmd/testdata/golden/scrape/err_request.stderr new file mode 100644 index 0000000..2174ecc --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_request.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Bad Request", + "status": 400, + "code": "ERR::REQUEST::INVALID_PARAMETER", + "detail": "engine: unknown value", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/scrape/err_request.stdout b/internal/cmd/testdata/golden/scrape/err_request.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_unreachable.exit b/internal/cmd/testdata/golden/scrape/err_unreachable.exit new file mode 100644 index 0000000..f599e28 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_unreachable.exit @@ -0,0 +1 @@ +10 diff --git a/internal/cmd/testdata/golden/scrape/err_unreachable.stdout b/internal/cmd/testdata/golden/scrape/err_unreachable.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_upstream.exit b/internal/cmd/testdata/golden/scrape/err_upstream.exit new file mode 100644 index 0000000..1e8b314 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_upstream.exit @@ -0,0 +1 @@ +6 diff --git a/internal/cmd/testdata/golden/scrape/err_upstream.stderr b/internal/cmd/testdata/golden/scrape/err_upstream.stderr new file mode 100644 index 0000000..657d4ce --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_upstream.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Bad Gateway", + "status": 502, + "code": "ERR::UPSTREAM::BLOCKED", + "detail": "the target served a bot challenge", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/scrape/err_upstream.stdout b/internal/cmd/testdata/golden/scrape/err_upstream.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.exit b/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.stderr b/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.stderr new file mode 100644 index 0000000..0b659a1 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.stderr @@ -0,0 +1,4 @@ +{ + "error": "--ai and --ai-schema are mutually exclusive: the API takes a prompt or a schema, not both", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.stdout b/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_usage_bad_extract.exit b/internal/cmd/testdata/golden/scrape/err_usage_bad_extract.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_usage_bad_extract.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/scrape/err_usage_bad_extract.stdout b/internal/cmd/testdata/golden/scrape/err_usage_bad_extract.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_usage_no_url.exit b/internal/cmd/testdata/golden/scrape/err_usage_no_url.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_usage_no_url.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/scrape/err_usage_no_url.stderr b/internal/cmd/testdata/golden/scrape/err_usage_no_url.stderr new file mode 100644 index 0000000..9a97161 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_usage_no_url.stderr @@ -0,0 +1,4 @@ +{ + "error": "accepts 1 arg(s), received 0", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/scrape/err_usage_no_url.stdout b/internal/cmd/testdata/golden/scrape/err_usage_no_url.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/extract.exit b/internal/cmd/testdata/golden/scrape/extract.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/extract.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/extract.stdout b/internal/cmd/testdata/golden/scrape/extract.stdout new file mode 100644 index 0000000..559a5e8 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/extract.stdout @@ -0,0 +1,18 @@ +{ + "url": "https://example.com/product/1", + "final_url": "https://example.com/product/1", + "status": 200, + "content": "

Anvil

$19.99 & free shipping

", + "headers": { + "content-type": "text/html" + }, + "truncated": false, + "credits": 3, + "engine": "fetch", + "proxy_source": "pool", + "warnings": [], + "data": { + "title": "Anvil", + "price": "$19.99 & free shipping" + } +} diff --git a/internal/cmd/testdata/golden/scrape/html.exit b/internal/cmd/testdata/golden/scrape/html.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/html.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/html.stdout b/internal/cmd/testdata/golden/scrape/html.stdout new file mode 100644 index 0000000..d3ad295 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/html.stdout @@ -0,0 +1,15 @@ +{ + "cache_state": "miss", + "content": "\nExample Domain\n

Example Domain

Prices from $19.99 & up.

\n", + "content_type": "text/html; charset=utf-8", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 200, + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape/json_envelope.exit b/internal/cmd/testdata/golden/scrape/json_envelope.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/json_envelope.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/json_envelope.stdout b/internal/cmd/testdata/golden/scrape/json_envelope.stdout new file mode 100644 index 0000000..81d4384 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/json_envelope.stdout @@ -0,0 +1,15 @@ +{ + "url": "https://example.com/product/1", + "final_url": "https://example.com/product/1", + "status": 200, + "content": "

Anvil

$19.99 & free shipping

", + "headers": { + "content-type": "text/html" + }, + "truncated": false, + "credits": 3, + "engine": "fetch", + "proxy_source": "pool", + "warnings": [], + "links": null +} diff --git a/internal/cmd/testdata/golden/scrape/json_envelope_jsonl.exit b/internal/cmd/testdata/golden/scrape/json_envelope_jsonl.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/json_envelope_jsonl.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/json_envelope_jsonl.stdout b/internal/cmd/testdata/golden/scrape/json_envelope_jsonl.stdout new file mode 100644 index 0000000..dd48a48 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/json_envelope_jsonl.stdout @@ -0,0 +1 @@ +{"url":"https://example.com/product/1","final_url":"https://example.com/product/1","status":200,"content":"

Anvil

$19.99 & free shipping

","headers":{"content-type":"text/html"},"truncated":false,"credits":3,"engine":"fetch","proxy_source":"pool","warnings":[]} diff --git a/internal/cmd/testdata/golden/scrape/links.exit b/internal/cmd/testdata/golden/scrape/links.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/links.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/links.stdout b/internal/cmd/testdata/golden/scrape/links.stdout new file mode 100644 index 0000000..af3f6c0 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/links.stdout @@ -0,0 +1,18 @@ +{ + "url": "https://example.com/product/1", + "final_url": "https://example.com/product/1", + "status": 200, + "content": "

Anvil

$19.99 & free shipping

", + "headers": { + "content-type": "text/html" + }, + "truncated": false, + "credits": 3, + "engine": "fetch", + "proxy_source": "pool", + "warnings": [], + "links": [ + "https://example.com/a", + "https://example.com/b?x=1&y=2" + ] +} diff --git a/internal/cmd/testdata/golden/scrape/markdown.exit b/internal/cmd/testdata/golden/scrape/markdown.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/markdown.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/markdown.stdout b/internal/cmd/testdata/golden/scrape/markdown.stdout new file mode 100644 index 0000000..982714c --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/markdown.stdout @@ -0,0 +1,17 @@ +{ + "cache_state": "miss", + "content": "# Example Domain\n\nPrices from $19.99 & up.\n", + "content_type": "text/markdown; charset=utf-8", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 200, + "url": "https://example.com/product/1", + "warnings": [ + "FORMAT_COERCED: markdown requested, html served" + ] +} diff --git a/internal/cmd/testdata/golden/scrape/markdown_json_flag.exit b/internal/cmd/testdata/golden/scrape/markdown_json_flag.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/markdown_json_flag.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/markdown_json_flag.stdout b/internal/cmd/testdata/golden/scrape/markdown_json_flag.stdout new file mode 100644 index 0000000..982714c --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/markdown_json_flag.stdout @@ -0,0 +1,17 @@ +{ + "cache_state": "miss", + "content": "# Example Domain\n\nPrices from $19.99 & up.\n", + "content_type": "text/markdown; charset=utf-8", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 200, + "url": "https://example.com/product/1", + "warnings": [ + "FORMAT_COERCED: markdown requested, html served" + ] +} diff --git a/internal/cmd/testdata/golden/scrape/markdown_jsonl.exit b/internal/cmd/testdata/golden/scrape/markdown_jsonl.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/markdown_jsonl.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/markdown_jsonl.stdout b/internal/cmd/testdata/golden/scrape/markdown_jsonl.stdout new file mode 100644 index 0000000..e7ae612 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/markdown_jsonl.stdout @@ -0,0 +1 @@ +{"cache_state":"miss","content":"# Example Domain\n\nPrices from $19.99 & up.\n","content_type":"text/markdown; charset=utf-8","credits":2,"credits_charged":2,"engine":"fetch","final_url":"https://example.com/final","proxy_source":"pool","request_cost":2,"request_id":"","status":200,"url":"https://example.com/product/1","warnings":["FORMAT_COERCED: markdown requested, html served"]} diff --git a/internal/cmd/testdata/golden/scrape/network_capture.exit b/internal/cmd/testdata/golden/scrape/network_capture.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/network_capture.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/network_capture.stdout b/internal/cmd/testdata/golden/scrape/network_capture.stdout new file mode 100644 index 0000000..76c989a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/network_capture.stdout @@ -0,0 +1,23 @@ +{ + "url": "https://example.com/product/1", + "final_url": "https://example.com/product/1", + "status": 200, + "content": "

Anvil

$19.99 & free shipping

", + "headers": { + "content-type": "text/html" + }, + "truncated": false, + "credits": 3, + "engine": "fetch", + "proxy_source": "pool", + "warnings": [], + "network": [ + { + "url": "https://example.com/api/price", + "status": 200, + "resource_type": "xhr", + "content_type": "application/json", + "body": "{\"price\":19.99}" + } + ] +} diff --git a/internal/cmd/testdata/golden/scrape/output_file.exit b/internal/cmd/testdata/golden/scrape/output_file.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/output_file.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/output_file.file.page.md b/internal/cmd/testdata/golden/scrape/output_file.file.page.md new file mode 100644 index 0000000..f3485b7 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/output_file.file.page.md @@ -0,0 +1,3 @@ +# Example Domain + +Prices from $19.99 & up. diff --git a/internal/cmd/testdata/golden/scrape/output_file.stdout b/internal/cmd/testdata/golden/scrape/output_file.stdout new file mode 100644 index 0000000..2f8fbcb --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/output_file.stdout @@ -0,0 +1,17 @@ +{ + "cache_state": "miss", + "content_type": "text/markdown; charset=utf-8", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "output": "/page.md", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 200, + "url": "https://example.com/product/1", + "warnings": [ + "FORMAT_COERCED: markdown requested, html served" + ] +} diff --git a/internal/cmd/testdata/golden/scrape/output_stdout_dash.exit b/internal/cmd/testdata/golden/scrape/output_stdout_dash.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/output_stdout_dash.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/output_stdout_dash.stdout b/internal/cmd/testdata/golden/scrape/output_stdout_dash.stdout new file mode 100644 index 0000000..982714c --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/output_stdout_dash.stdout @@ -0,0 +1,17 @@ +{ + "cache_state": "miss", + "content": "# Example Domain\n\nPrices from $19.99 & up.\n", + "content_type": "text/markdown; charset=utf-8", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 200, + "url": "https://example.com/product/1", + "warnings": [ + "FORMAT_COERCED: markdown requested, html served" + ] +} diff --git a/internal/cmd/testdata/golden/scrape/pdf_base64.exit b/internal/cmd/testdata/golden/scrape/pdf_base64.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/pdf_base64.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/pdf_base64.stdout b/internal/cmd/testdata/golden/scrape/pdf_base64.stdout new file mode 100644 index 0000000..c513fb9 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/pdf_base64.stdout @@ -0,0 +1,15 @@ +{ + "cache_state": "miss", + "content_base64": "JVBERi0xLjQKMSAwIG9iago8PD4+CmVuZG9iagolJUVPRgo=", + "content_type": "application/pdf", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 200, + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape/screenshot_dir.exit b/internal/cmd/testdata/golden/scrape/screenshot_dir.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/screenshot_dir.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/screenshot_dir.file.shots__page-2.png b/internal/cmd/testdata/golden/scrape/screenshot_dir.file.shots__page-2.png new file mode 100644 index 0000000..0cadfa2 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/screenshot_dir.file.shots__page-2.png @@ -0,0 +1 @@ +PNGDATA \ No newline at end of file diff --git a/internal/cmd/testdata/golden/scrape/screenshot_dir.file.shots__page.png b/internal/cmd/testdata/golden/scrape/screenshot_dir.file.shots__page.png new file mode 100644 index 0000000..0cadfa2 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/screenshot_dir.file.shots__page.png @@ -0,0 +1 @@ +PNGDATA \ No newline at end of file diff --git a/internal/cmd/testdata/golden/scrape/screenshot_dir.stdout b/internal/cmd/testdata/golden/scrape/screenshot_dir.stdout new file mode 100644 index 0000000..678fb28 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/screenshot_dir.stdout @@ -0,0 +1,30 @@ +{ + "content": "

Anvil

$19.99 & free shipping

", + "credits": 3, + "engine": "fetch", + "final_url": "https://example.com/product/1", + "headers": { + "content-type": "text/html" + }, + "proxy_source": "pool", + "screenshots": [ + { + "file": "/shots/page.png", + "format": "png", + "height": 720, + "label": "page", + "width": 1280 + }, + { + "file": "/shots/page-2.png", + "format": "png", + "height": 720, + "label": "page", + "width": 1280 + } + ], + "status": 200, + "truncated": false, + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape/stdin_many.exit b/internal/cmd/testdata/golden/scrape/stdin_many.exit new file mode 100644 index 0000000..1e8b314 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/stdin_many.exit @@ -0,0 +1 @@ +6 diff --git a/internal/cmd/testdata/golden/scrape/stdin_many.stderr b/internal/cmd/testdata/golden/scrape/stdin_many.stderr new file mode 100644 index 0000000..01f0dcb --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/stdin_many.stderr @@ -0,0 +1,4 @@ +{ + "error": "2 of 3 URLs failed", + "exit_code": 6 +} diff --git a/internal/cmd/testdata/golden/scrape/stdin_many.stdout b/internal/cmd/testdata/golden/scrape/stdin_many.stdout new file mode 100644 index 0000000..57b2c98 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/stdin_many.stdout @@ -0,0 +1,3 @@ +{"cache_state":"miss","content":"# https://example.com/ok\n","content_type":"text/markdown","credits":2,"credits_charged":2,"engine":"fetch","final_url":"https://example.com/final","index":1,"proxy_source":"pool","request_cost":2,"request_id":"","status":200,"url":"https://example.com/ok","warnings":[]} +{"cache_state":"miss","content":"# Down\n","content_type":"text/markdown","credits":2,"credits_charged":2,"engine":"fetch","final_url":"https://example.com/final","index":2,"proxy_source":"pool","request_cost":2,"request_id":"","status":503,"target_error":"target returned 503","url":"https://example.com/down","warnings":[]} +{"error":{"type":"https://docs.spicrawl.com/errors","title":"Forbidden","status":403,"code":"ERR::UPSTREAM::BLOCKED","detail":"blocked by the target","retryable":false,"doc_url":"https://docs.spicrawl.com/errors","request_id":"","target_status":null,"diagnostics":{"hint":"check the request and try again"}},"index":3,"url":"https://example.com/denied"} diff --git a/internal/cmd/testdata/golden/scrape/stdin_many_all_ok.exit b/internal/cmd/testdata/golden/scrape/stdin_many_all_ok.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/stdin_many_all_ok.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/stdin_many_all_ok.stdout b/internal/cmd/testdata/golden/scrape/stdin_many_all_ok.stdout new file mode 100644 index 0000000..11a25ce --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/stdin_many_all_ok.stdout @@ -0,0 +1,2 @@ +{"cache_state":"miss","content":"# https://example.com/ok\n","content_type":"text/markdown","credits":2,"credits_charged":2,"engine":"fetch","final_url":"https://example.com/final","index":1,"proxy_source":"pool","request_cost":2,"request_id":"","status":200,"url":"https://example.com/ok","warnings":[]} +{"cache_state":"miss","content":"# https://example.com/ok2\n","content_type":"text/markdown","credits":2,"credits_charged":2,"engine":"fetch","final_url":"https://example.com/final","index":2,"proxy_source":"pool","request_cost":2,"request_id":"","status":200,"url":"https://example.com/ok2","warnings":[]} diff --git a/internal/cmd/testdata/golden/scrape/stdin_many_empty.exit b/internal/cmd/testdata/golden/scrape/stdin_many_empty.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/stdin_many_empty.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/scrape/stdin_many_empty.stderr b/internal/cmd/testdata/golden/scrape/stdin_many_empty.stderr new file mode 100644 index 0000000..e09eb43 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/stdin_many_empty.stderr @@ -0,0 +1,4 @@ +{ + "error": "no URLs on stdin", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/scrape/stdin_many_empty.stdout b/internal/cmd/testdata/golden/scrape/stdin_many_empty.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/target_403_envelope.exit b/internal/cmd/testdata/golden/scrape/target_403_envelope.exit new file mode 100644 index 0000000..1e8b314 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_403_envelope.exit @@ -0,0 +1 @@ +6 diff --git a/internal/cmd/testdata/golden/scrape/target_403_envelope.stderr b/internal/cmd/testdata/golden/scrape/target_403_envelope.stderr new file mode 100644 index 0000000..0ba524a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_403_envelope.stderr @@ -0,0 +1,4 @@ +{ + "error": "target returned 403 (pass --allowed-status 403 to accept it)", + "exit_code": 6 +} diff --git a/internal/cmd/testdata/golden/scrape/target_403_envelope.stdout b/internal/cmd/testdata/golden/scrape/target_403_envelope.stdout new file mode 100644 index 0000000..59e30fa --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_403_envelope.stdout @@ -0,0 +1,16 @@ +{ + "content": "

Anvil

$19.99 & free shipping

", + "credits": 3, + "data": null, + "engine": "fetch", + "final_url": "https://example.com/product/1", + "headers": { + "content-type": "text/html" + }, + "proxy_source": "pool", + "status": 200, + "target_error": "target returned 403", + "truncated": false, + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape/target_404_allowed.exit b/internal/cmd/testdata/golden/scrape/target_404_allowed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_404_allowed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/target_404_allowed.stdout b/internal/cmd/testdata/golden/scrape/target_404_allowed.stdout new file mode 100644 index 0000000..9f62169 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_404_allowed.stdout @@ -0,0 +1,15 @@ +{ + "cache_state": "miss", + "content": "

Not Found

\n", + "content_type": "text/html", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 404, + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape/target_503.exit b/internal/cmd/testdata/golden/scrape/target_503.exit new file mode 100644 index 0000000..1e8b314 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_503.exit @@ -0,0 +1 @@ +6 diff --git a/internal/cmd/testdata/golden/scrape/target_503.stderr b/internal/cmd/testdata/golden/scrape/target_503.stderr new file mode 100644 index 0000000..af601fb --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_503.stderr @@ -0,0 +1,4 @@ +{ + "error": "target returned 503 (pass --allowed-status 503 to accept it)", + "exit_code": 6 +} diff --git a/internal/cmd/testdata/golden/scrape/target_503.stdout b/internal/cmd/testdata/golden/scrape/target_503.stdout new file mode 100644 index 0000000..161f5b7 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_503.stdout @@ -0,0 +1,16 @@ +{ + "cache_state": "miss", + "content": "

Service Unavailable

\n", + "content_type": "text/html", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 503, + "target_error": "target returned 503", + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.exit b/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.exit new file mode 100644 index 0000000..1e8b314 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.exit @@ -0,0 +1 @@ +6 diff --git a/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.stderr b/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.stderr new file mode 100644 index 0000000..85c31dc --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.stderr @@ -0,0 +1,5 @@ +warning: not writing /page.html: the target failed (--allowed-status accepts it) +{ + "error": "target returned 503 (pass --allowed-status 503 to accept it)", + "exit_code": 6 +} diff --git a/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.stdout b/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.stdout new file mode 100644 index 0000000..3fee6c4 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.stdout @@ -0,0 +1,16 @@ +{ + "cache_state": "miss", + "content": "

Down

\n", + "content_type": "text/html", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 503, + "target_error": "target returned 503", + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape/text.exit b/internal/cmd/testdata/golden/scrape/text.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/text.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/text.stdout b/internal/cmd/testdata/golden/scrape/text.stdout new file mode 100644 index 0000000..071e7e5 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/text.stdout @@ -0,0 +1,15 @@ +{ + "cache_state": "miss", + "content": "Example Domain\nPrices from $19.99 & up.", + "content_type": "text/plain; charset=utf-8", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 200, + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape_human/err_auth.exit b/internal/cmd/testdata/golden/scrape_human/err_auth.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/err_auth.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/scrape_human/err_auth.stdout b/internal/cmd/testdata/golden/scrape_human/err_auth.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape_human/extract_data.exit b/internal/cmd/testdata/golden/scrape_human/extract_data.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/extract_data.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/extract_data.stdout b/internal/cmd/testdata/golden/scrape_human/extract_data.stdout new file mode 100644 index 0000000..a2ead9e --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/extract_data.stdout @@ -0,0 +1,4 @@ +{ + "price": "$19.99 & free shipping", + "title": "Anvil" +} diff --git a/internal/cmd/testdata/golden/scrape_human/html.exit b/internal/cmd/testdata/golden/scrape_human/html.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/html.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/html.stdout b/internal/cmd/testdata/golden/scrape_human/html.stdout new file mode 100644 index 0000000..5af3329 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/html.stdout @@ -0,0 +1,3 @@ + +Example Domain +

Example Domain

Prices from $19.99 & up.

diff --git a/internal/cmd/testdata/golden/scrape_human/links.exit b/internal/cmd/testdata/golden/scrape_human/links.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/links.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/links.stdout b/internal/cmd/testdata/golden/scrape_human/links.stdout new file mode 100644 index 0000000..e1e2411 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/links.stdout @@ -0,0 +1,4 @@ +

Anvil

$19.99 & free shipping

+--- links (2) --- +https://example.com/a +https://example.com/b?x=1&y=2 diff --git a/internal/cmd/testdata/golden/scrape_human/markdown.exit b/internal/cmd/testdata/golden/scrape_human/markdown.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/markdown.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/markdown.stdout b/internal/cmd/testdata/golden/scrape_human/markdown.stdout new file mode 100644 index 0000000..f3485b7 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/markdown.stdout @@ -0,0 +1,3 @@ +# Example Domain + +Prices from $19.99 & up. diff --git a/internal/cmd/testdata/golden/scrape_human/markdown_meta.exit b/internal/cmd/testdata/golden/scrape_human/markdown_meta.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/markdown_meta.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/markdown_meta.stdout b/internal/cmd/testdata/golden/scrape_human/markdown_meta.stdout new file mode 100644 index 0000000..f3485b7 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/markdown_meta.stdout @@ -0,0 +1,3 @@ +# Example Domain + +Prices from $19.99 & up. diff --git a/internal/cmd/testdata/golden/scrape_human/network.exit b/internal/cmd/testdata/golden/scrape_human/network.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/network.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/network.stdout b/internal/cmd/testdata/golden/scrape_human/network.stdout new file mode 100644 index 0000000..b21ace6 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/network.stdout @@ -0,0 +1,4 @@ +

Anvil

$19.99 & free shipping

+--- network (2) --- +200 xhr application/json https://example.com/api/price +- image - https://example.com/x.png diff --git a/internal/cmd/testdata/golden/scrape_human/output_file.exit b/internal/cmd/testdata/golden/scrape_human/output_file.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/output_file.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/output_file.file.page.md b/internal/cmd/testdata/golden/scrape_human/output_file.file.page.md new file mode 100644 index 0000000..f3485b7 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/output_file.file.page.md @@ -0,0 +1,3 @@ +# Example Domain + +Prices from $19.99 & up. diff --git a/internal/cmd/testdata/golden/scrape_human/output_file.stdout b/internal/cmd/testdata/golden/scrape_human/output_file.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape_human/pdf_needs_file.exit b/internal/cmd/testdata/golden/scrape_human/pdf_needs_file.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/pdf_needs_file.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/scrape_human/pdf_needs_file.stdout b/internal/cmd/testdata/golden/scrape_human/pdf_needs_file.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape_human/pdf_output_file.exit b/internal/cmd/testdata/golden/scrape_human/pdf_output_file.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/pdf_output_file.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/pdf_output_file.file.page.pdf b/internal/cmd/testdata/golden/scrape_human/pdf_output_file.file.page.pdf new file mode 100644 index 0000000..e8cf043 Binary files /dev/null and b/internal/cmd/testdata/golden/scrape_human/pdf_output_file.file.page.pdf differ diff --git a/internal/cmd/testdata/golden/scrape_human/pdf_output_file.stdout b/internal/cmd/testdata/golden/scrape_human/pdf_output_file.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape_human/target_503.exit b/internal/cmd/testdata/golden/scrape_human/target_503.exit new file mode 100644 index 0000000..1e8b314 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/target_503.exit @@ -0,0 +1 @@ +6 diff --git a/internal/cmd/testdata/golden/scrape_human/target_503.stdout b/internal/cmd/testdata/golden/scrape_human/target_503.stdout new file mode 100644 index 0000000..312cd2a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/target_503.stdout @@ -0,0 +1 @@ +

Service Unavailable

diff --git a/internal/cmd/testdata/golden/scrape_human/text_no_trailing_newline.exit b/internal/cmd/testdata/golden/scrape_human/text_no_trailing_newline.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/text_no_trailing_newline.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/text_no_trailing_newline.stdout b/internal/cmd/testdata/golden/scrape_human/text_no_trailing_newline.stdout new file mode 100644 index 0000000..85fedb4 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/text_no_trailing_newline.stdout @@ -0,0 +1 @@ +Example Domain diff --git a/internal/cmd/testdata/golden/sessions/context.exit b/internal/cmd/testdata/golden/sessions/context.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/context.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/context.stdout b/internal/cmd/testdata/golden/sessions/context.stdout new file mode 100644 index 0000000..a0f43bc --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/context.stdout @@ -0,0 +1,16 @@ +{ + "session_id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "session_context": { + "cookies": [ + { + "name": "sid", + "value": "abc123", + "domain": "example.com", + "path": "/" + } + ], + "local_storage": { + "theme": "dark" + } + } +} diff --git a/internal/cmd/testdata/golden/sessions/create.exit b/internal/cmd/testdata/golden/sessions/create.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/create.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/create.stdout b/internal/cmd/testdata/golden/sessions/create.stdout new file mode 100644 index 0000000..562dd8b --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/create.stdout @@ -0,0 +1,17 @@ +{ + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "obscura", + "status": "active", + "sticky_key": null, + "proxy": { + "tier": "datacenter" + }, + "created_at": "", + "last_used_at": null, + "expires_at": "", + "usage_count": 0, + "warnings": [ + "engine obscura cannot persist IndexedDB" + ] +} diff --git a/internal/cmd/testdata/golden/sessions/create_session_context_file.exit b/internal/cmd/testdata/golden/sessions/create_session_context_file.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/create_session_context_file.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/create_session_context_file.stdout b/internal/cmd/testdata/golden/sessions/create_session_context_file.stdout new file mode 100644 index 0000000..c5b29c0 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/create_session_context_file.stdout @@ -0,0 +1,18 @@ +{ + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "active", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de", + "region_pool": "eu" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "hard_expires_at": "", + "usage_count": 7, + "context": null +} diff --git a/internal/cmd/testdata/golden/sessions/create_usage_error.exit b/internal/cmd/testdata/golden/sessions/create_usage_error.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/create_usage_error.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/sessions/create_usage_error.stderr b/internal/cmd/testdata/golden/sessions/create_usage_error.stderr new file mode 100644 index 0000000..8135a56 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/create_usage_error.stderr @@ -0,0 +1,4 @@ +{ + "error": "--rotate-ip and --sticky-key cannot be combined: one lets the exit change, the other pins it", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/sessions/create_usage_error.stdout b/internal/cmd/testdata/golden/sessions/create_usage_error.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.exit b/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.stderr b/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.stderr new file mode 100644 index 0000000..c2f38d6 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.stderr @@ -0,0 +1,4 @@ +{ + "error": "refusing to delete session 01J9ZQ4M7R3T8VX2K5N6P0B1CD without confirmation: pass --yes", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.stdout b/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/sessions/delete_yes.exit b/internal/cmd/testdata/golden/sessions/delete_yes.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/delete_yes.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/delete_yes.stdout b/internal/cmd/testdata/golden/sessions/delete_yes.stdout new file mode 100644 index 0000000..b8dc394 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/delete_yes.stdout @@ -0,0 +1,4 @@ +{ + "deleted": true, + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD" +} diff --git a/internal/cmd/testdata/golden/sessions/get.exit b/internal/cmd/testdata/golden/sessions/get.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/get.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/get.stdout b/internal/cmd/testdata/golden/sessions/get.stdout new file mode 100644 index 0000000..c5b29c0 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/get.stdout @@ -0,0 +1,18 @@ +{ + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "active", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de", + "region_pool": "eu" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "hard_expires_at": "", + "usage_count": 7, + "context": null +} diff --git a/internal/cmd/testdata/golden/sessions/get_not_found.exit b/internal/cmd/testdata/golden/sessions/get_not_found.exit new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/get_not_found.exit @@ -0,0 +1 @@ +9 diff --git a/internal/cmd/testdata/golden/sessions/get_not_found.stderr b/internal/cmd/testdata/golden/sessions/get_not_found.stderr new file mode 100644 index 0000000..d1b1da5 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/get_not_found.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Not Found", + "status": 404, + "code": "ERR::SESSION::NOT_FOUND", + "detail": "no such session", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/sessions/get_not_found.stdout b/internal/cmd/testdata/golden/sessions/get_not_found.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/sessions/list.exit b/internal/cmd/testdata/golden/sessions/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/list.stdout b/internal/cmd/testdata/golden/sessions/list.stdout new file mode 100644 index 0000000..931ded3 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list.stdout @@ -0,0 +1,39 @@ +{ + "sessions": [ + { + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "active", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de", + "region_pool": "eu" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "hard_expires_at": "", + "usage_count": 7, + "context": null + }, + { + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "released", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "released_at": "", + "retired": "released by the caller", + "usage_count": 7 + } + ] +} diff --git a/internal/cmd/testdata/golden/sessions/list_all_pages.exit b/internal/cmd/testdata/golden/sessions/list_all_pages.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list_all_pages.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/list_all_pages.stdout b/internal/cmd/testdata/golden/sessions/list_all_pages.stdout new file mode 100644 index 0000000..931ded3 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list_all_pages.stdout @@ -0,0 +1,39 @@ +{ + "sessions": [ + { + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "active", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de", + "region_pool": "eu" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "hard_expires_at": "", + "usage_count": 7, + "context": null + }, + { + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "released", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "released_at": "", + "retired": "released by the caller", + "usage_count": 7 + } + ] +} diff --git a/internal/cmd/testdata/golden/sessions/list_empty.exit b/internal/cmd/testdata/golden/sessions/list_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/list_empty.stdout b/internal/cmd/testdata/golden/sessions/list_empty.stdout new file mode 100644 index 0000000..f56218a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list_empty.stdout @@ -0,0 +1,3 @@ +{ + "sessions": [] +} diff --git a/internal/cmd/testdata/golden/sessions/list_next_cursor.exit b/internal/cmd/testdata/golden/sessions/list_next_cursor.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list_next_cursor.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/list_next_cursor.stdout b/internal/cmd/testdata/golden/sessions/list_next_cursor.stdout new file mode 100644 index 0000000..e1a1240 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list_next_cursor.stdout @@ -0,0 +1,23 @@ +{ + "next_cursor": "1790000000000000000.ABC", + "sessions": [ + { + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "active", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de", + "region_pool": "eu" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "hard_expires_at": "", + "usage_count": 7, + "context": null + } + ] +} diff --git a/internal/cmd/testdata/golden/sessions/release.exit b/internal/cmd/testdata/golden/sessions/release.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/release.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/release.stdout b/internal/cmd/testdata/golden/sessions/release.stdout new file mode 100644 index 0000000..aaea824 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/release.stdout @@ -0,0 +1,17 @@ +{ + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "released", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "released_at": "", + "retired": "released by the caller", + "usage_count": 7 +} diff --git a/internal/cmd/testdata/golden/sessions_human/context.exit b/internal/cmd/testdata/golden/sessions_human/context.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/context.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/context.stdout b/internal/cmd/testdata/golden/sessions_human/context.stdout new file mode 100644 index 0000000..a0f43bc --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/context.stdout @@ -0,0 +1,16 @@ +{ + "session_id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "session_context": { + "cookies": [ + { + "name": "sid", + "value": "abc123", + "domain": "example.com", + "path": "/" + } + ], + "local_storage": { + "theme": "dark" + } + } +} diff --git a/internal/cmd/testdata/golden/sessions_human/create.exit b/internal/cmd/testdata/golden/sessions_human/create.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/create.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/create.stdout b/internal/cmd/testdata/golden/sessions_human/create.stdout new file mode 100644 index 0000000..69f419b --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/create.stdout @@ -0,0 +1,9 @@ +id: 01J9ZQ4M7R3T8VX2K5N6P0B1CD +status: active +engine: obscura +exit: datacenter +sticky key: (rotating) +uses: 0 +created: +last used: never +expires: diff --git a/internal/cmd/testdata/golden/sessions_human/delete_yes.exit b/internal/cmd/testdata/golden/sessions_human/delete_yes.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/delete_yes.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/delete_yes.stdout b/internal/cmd/testdata/golden/sessions_human/delete_yes.stdout new file mode 100644 index 0000000..a452206 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/delete_yes.stdout @@ -0,0 +1 @@ +deleted session 01J9ZQ4M7R3T8VX2K5N6P0B1CD diff --git a/internal/cmd/testdata/golden/sessions_human/get.exit b/internal/cmd/testdata/golden/sessions_human/get.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/get.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/get.stdout b/internal/cmd/testdata/golden/sessions_human/get.stdout new file mode 100644 index 0000000..20dce43 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/get.stdout @@ -0,0 +1,10 @@ +id: 01J9ZQ4M7R3T8VX2K5N6P0B1CD +status: active +engine: chromium +exit: residential (de) pool eu +sticky key: login-1 +uses: 7 +created: +last used: +expires: +hard expiry: diff --git a/internal/cmd/testdata/golden/sessions_human/get_released.exit b/internal/cmd/testdata/golden/sessions_human/get_released.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/get_released.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/get_released.stdout b/internal/cmd/testdata/golden/sessions_human/get_released.stdout new file mode 100644 index 0000000..a3d6efc --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/get_released.stdout @@ -0,0 +1,11 @@ +id: 01J9ZQ4M7R3T8VX2K5N6P0B1CD +status: released +engine: chromium +exit: residential (de) +sticky key: login-1 +uses: 7 +created: +last used: +expires: +released: +retired: released by the caller diff --git a/internal/cmd/testdata/golden/sessions_human/list.exit b/internal/cmd/testdata/golden/sessions_human/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/list.stdout b/internal/cmd/testdata/golden/sessions_human/list.stdout new file mode 100644 index 0000000..aa37fc0 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/list.stdout @@ -0,0 +1,3 @@ +ID ENGINE STATUS EXIT USES EXPIRES +01J9ZQ4M7R3T8VX2K5N6P0B1CD chromium active residential/de 7 +01J9ZQ4M7R3T8VX2K5N6P0B1CD chromium released residential/de 7 diff --git a/internal/cmd/testdata/golden/sessions_human/list_empty.exit b/internal/cmd/testdata/golden/sessions_human/list_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/list_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/list_empty.stdout b/internal/cmd/testdata/golden/sessions_human/list_empty.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/sessions_human/list_next_cursor.exit b/internal/cmd/testdata/golden/sessions_human/list_next_cursor.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/list_next_cursor.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/list_next_cursor.stdout b/internal/cmd/testdata/golden/sessions_human/list_next_cursor.stdout new file mode 100644 index 0000000..05429ad --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/list_next_cursor.stdout @@ -0,0 +1,2 @@ +ID ENGINE STATUS EXIT USES EXPIRES +01J9ZQ4M7R3T8VX2K5N6P0B1CD chromium active residential/de 7 diff --git a/internal/cmd/testdata/golden/sessions_human/release.exit b/internal/cmd/testdata/golden/sessions_human/release.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/release.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/release.stdout b/internal/cmd/testdata/golden/sessions_human/release.stdout new file mode 100644 index 0000000..a3d6efc --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/release.stdout @@ -0,0 +1,11 @@ +id: 01J9ZQ4M7R3T8VX2K5N6P0B1CD +status: released +engine: chromium +exit: residential (de) +sticky key: login-1 +uses: 7 +created: +last used: +expires: +released: +retired: released by the caller diff --git a/internal/cmd/testdata/golden/status/key_invalid_exit3.exit b/internal/cmd/testdata/golden/status/key_invalid_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/status/key_invalid_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/status/key_invalid_exit3.stderr b/internal/cmd/testdata/golden/status/key_invalid_exit3.stderr new file mode 100644 index 0000000..47478ac --- /dev/null +++ b/internal/cmd/testdata/golden/status/key_invalid_exit3.stderr @@ -0,0 +1,4 @@ +{ + "error": "the API rejected the key: ERR::AUTH::INVALID_KEY: the API key is not valid", + "exit_code": 3 +} diff --git a/internal/cmd/testdata/golden/status/key_invalid_exit3.stdout b/internal/cmd/testdata/golden/status/key_invalid_exit3.stdout new file mode 100644 index 0000000..d846600 --- /dev/null +++ b/internal/cmd/testdata/golden/status/key_invalid_exit3.stdout @@ -0,0 +1,14 @@ +{ + "api_reachable": true, + "ready": true, + "checks": { + "postgres": "ok", + "redis": "ok" + }, + "cdp": "enabled", + "key_valid": false, + "key_error": "ERR::AUTH::INVALID_KEY: the API key is not valid", + "key_source": "env", + "base_url": "http://SERVER", + "workers": null +} diff --git a/internal/cmd/testdata/golden/status/key_without_read_scope.exit b/internal/cmd/testdata/golden/status/key_without_read_scope.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/status/key_without_read_scope.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/status/key_without_read_scope.stdout b/internal/cmd/testdata/golden/status/key_without_read_scope.stdout new file mode 100644 index 0000000..dca8245 --- /dev/null +++ b/internal/cmd/testdata/golden/status/key_without_read_scope.stdout @@ -0,0 +1,27 @@ +{ + "api_reachable": true, + "ready": true, + "checks": { + "postgres": "ok", + "redis": "ok" + }, + "cdp": "enabled", + "key_valid": true, + "key_error": "key is valid but lacks the read scope", + "key_source": "env", + "base_url": "http://SERVER", + "workers": { + "batch": { + "present": 2, + "ready": 0 + }, + "browser": { + "present": 3, + "ready": 2 + }, + "egress": { + "present": 1, + "ready": 1 + } + } +} diff --git a/internal/cmd/testdata/golden/status/no_key_exit3.exit b/internal/cmd/testdata/golden/status/no_key_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/status/no_key_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/status/no_key_exit3.stderr b/internal/cmd/testdata/golden/status/no_key_exit3.stderr new file mode 100644 index 0000000..6b478b2 --- /dev/null +++ b/internal/cmd/testdata/golden/status/no_key_exit3.stderr @@ -0,0 +1,4 @@ +{ + "error": "no API key: run \"spicrawl login\", set SPICRAWL_API_KEY, or pass --api-key", + "exit_code": 3 +} diff --git a/internal/cmd/testdata/golden/status/no_key_exit3.stdout b/internal/cmd/testdata/golden/status/no_key_exit3.stdout new file mode 100644 index 0000000..5c64b55 --- /dev/null +++ b/internal/cmd/testdata/golden/status/no_key_exit3.stdout @@ -0,0 +1,14 @@ +{ + "api_reachable": true, + "ready": true, + "checks": { + "postgres": "ok", + "redis": "ok" + }, + "cdp": "enabled", + "key_valid": false, + "key_error": "no API key configured", + "key_source": "", + "base_url": "http://SERVER", + "workers": null +} diff --git a/internal/cmd/testdata/golden/status/not_ready.exit b/internal/cmd/testdata/golden/status/not_ready.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/status/not_ready.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/status/not_ready.stdout b/internal/cmd/testdata/golden/status/not_ready.stdout new file mode 100644 index 0000000..9610bd8 --- /dev/null +++ b/internal/cmd/testdata/golden/status/not_ready.stdout @@ -0,0 +1,26 @@ +{ + "api_reachable": true, + "ready": false, + "checks": { + "postgres": "ok", + "redis": "unreachable" + }, + "cdp": "disabled", + "key_valid": true, + "key_source": "env", + "base_url": "http://SERVER", + "workers": { + "batch": { + "present": 2, + "ready": 0 + }, + "browser": { + "present": 3, + "ready": 2 + }, + "egress": { + "present": 1, + "ready": 1 + } + } +} diff --git a/internal/cmd/testdata/golden/status/ok.exit b/internal/cmd/testdata/golden/status/ok.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/status/ok.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/status/ok.stdout b/internal/cmd/testdata/golden/status/ok.stdout new file mode 100644 index 0000000..92d7385 --- /dev/null +++ b/internal/cmd/testdata/golden/status/ok.stdout @@ -0,0 +1,26 @@ +{ + "api_reachable": true, + "ready": true, + "checks": { + "postgres": "ok", + "redis": "ok" + }, + "cdp": "enabled", + "key_valid": true, + "key_source": "env", + "base_url": "http://SERVER", + "workers": { + "batch": { + "present": 2, + "ready": 0 + }, + "browser": { + "present": 3, + "ready": 2 + }, + "egress": { + "present": 1, + "ready": 1 + } + } +} diff --git a/internal/cmd/testdata/golden/status/unreachable_exit10.exit b/internal/cmd/testdata/golden/status/unreachable_exit10.exit new file mode 100644 index 0000000..f599e28 --- /dev/null +++ b/internal/cmd/testdata/golden/status/unreachable_exit10.exit @@ -0,0 +1 @@ +10 diff --git a/internal/cmd/testdata/golden/status/unreachable_exit10.stdout b/internal/cmd/testdata/golden/status/unreachable_exit10.stdout new file mode 100644 index 0000000..63919ae --- /dev/null +++ b/internal/cmd/testdata/golden/status/unreachable_exit10.stdout @@ -0,0 +1,9 @@ +{ + "api_reachable": false, + "ready": false, + "cdp": "", + "key_valid": null, + "key_source": "env", + "base_url": "http://127.0.0.1:1", + "workers": null +} diff --git a/internal/cmd/testdata/golden/status/workers_error.exit b/internal/cmd/testdata/golden/status/workers_error.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/status/workers_error.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/status/workers_error.stdout b/internal/cmd/testdata/golden/status/workers_error.stdout new file mode 100644 index 0000000..5266310 --- /dev/null +++ b/internal/cmd/testdata/golden/status/workers_error.stdout @@ -0,0 +1,14 @@ +{ + "api_reachable": true, + "ready": true, + "checks": { + "postgres": "ok", + "redis": "ok" + }, + "cdp": "enabled", + "key_valid": true, + "key_source": "env", + "base_url": "http://SERVER", + "workers": null, + "workers_error": "ERR::INTERNAL::UNAVAILABLE: fleet registry is down" +} diff --git a/internal/cmd/testdata/golden/status_human/key_invalid_exit3.exit b/internal/cmd/testdata/golden/status_human/key_invalid_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/key_invalid_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/status_human/key_invalid_exit3.stdout b/internal/cmd/testdata/golden/status_human/key_invalid_exit3.stdout new file mode 100644 index 0000000..1d516c5 --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/key_invalid_exit3.stdout @@ -0,0 +1,5 @@ +base url: http://SERVER +reachable: yes +ready: yes +cdp: enabled +key: invalid (from env): ERR::AUTH::INVALID_KEY: the API key is not valid diff --git a/internal/cmd/testdata/golden/status_human/no_key_exit3.exit b/internal/cmd/testdata/golden/status_human/no_key_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/no_key_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/status_human/no_key_exit3.stdout b/internal/cmd/testdata/golden/status_human/no_key_exit3.stdout new file mode 100644 index 0000000..33c2360 --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/no_key_exit3.stdout @@ -0,0 +1,5 @@ +base url: http://SERVER +reachable: yes +ready: yes +cdp: enabled +key: invalid: no API key configured diff --git a/internal/cmd/testdata/golden/status_human/not_ready.exit b/internal/cmd/testdata/golden/status_human/not_ready.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/not_ready.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/status_human/not_ready.stdout b/internal/cmd/testdata/golden/status_human/not_ready.stdout new file mode 100644 index 0000000..c283eaf --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/not_ready.stdout @@ -0,0 +1,9 @@ +base url: http://SERVER +reachable: yes +ready: no (redis=unreachable) +cdp: disabled (cloud browser /v1/browser is not available on this deployment) +key: valid (from env) +workers: + browser 2 ready of 3 + egress 1 ready of 1 + batch 0 ready of 2 diff --git a/internal/cmd/testdata/golden/status_human/ok.exit b/internal/cmd/testdata/golden/status_human/ok.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/ok.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/status_human/ok.stdout b/internal/cmd/testdata/golden/status_human/ok.stdout new file mode 100644 index 0000000..6604db2 --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/ok.stdout @@ -0,0 +1,9 @@ +base url: http://SERVER +reachable: yes +ready: yes +cdp: enabled +key: valid (from env) +workers: + browser 2 ready of 3 + egress 1 ready of 1 + batch 0 ready of 2 diff --git a/internal/cmd/testdata/golden/status_human/unreachable_exit10.exit b/internal/cmd/testdata/golden/status_human/unreachable_exit10.exit new file mode 100644 index 0000000..f599e28 --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/unreachable_exit10.exit @@ -0,0 +1 @@ +10 diff --git a/internal/cmd/testdata/golden/status_human/unreachable_exit10.stdout b/internal/cmd/testdata/golden/status_human/unreachable_exit10.stdout new file mode 100644 index 0000000..df2dcbf --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/unreachable_exit10.stdout @@ -0,0 +1,2 @@ +base url: http://127.0.0.1:1 +reachable: no diff --git a/internal/cmd/testdata/golden/usage/by_day.exit b/internal/cmd/testdata/golden/usage/by_day.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage/by_day.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage/by_day.stdout b/internal/cmd/testdata/golden/usage/by_day.stdout new file mode 100644 index 0000000..837f85f --- /dev/null +++ b/internal/cmd/testdata/golden/usage/by_day.stdout @@ -0,0 +1,44 @@ +{ + "period": { + "start": "2026-08-01", + "end": "2026-08-04", + "days": 3, + "bounds": "[start, end)" + }, + "group_by": "day", + "groups": [ + { + "key": "2026-08-01", + "label": "", + "metrics": { + "requests": 10, + "credits": 12500000, + "engine_ms": 4200 + } + }, + { + "key": "2026-08-02", + "label": "", + "metrics": { + "requests": 15, + "credits": 18500000, + "engine_ms": 4800 + } + } + ], + "totals": { + "requests": 25, + "credits": 31000000, + "engine_ms": 9000 + }, + "unattributed": { + "requests": 0, + "credits": 0, + "engine_ms": 0 + }, + "usage_as_of": "", + "stale_seconds": 120, + "warnings": [ + "rollup is partial for 2026-08-03" + ] +} diff --git a/internal/cmd/testdata/golden/usage/by_key.exit b/internal/cmd/testdata/golden/usage/by_key.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage/by_key.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage/by_key.stdout b/internal/cmd/testdata/golden/usage/by_key.stdout new file mode 100644 index 0000000..925df59 --- /dev/null +++ b/internal/cmd/testdata/golden/usage/by_key.stdout @@ -0,0 +1,34 @@ +{ + "period": { + "start": "2026-08-01", + "end": "2026-09-01", + "days": 31, + "bounds": "[start, end)" + }, + "group_by": "key", + "keys": [ + { + "key_id": "K1", + "name": "ci-key", + "prefix": "spicrawl_live_ab12", + "project_id": "proj_01J9Z", + "revoked": false, + "days": [], + "totals": { + "requests": 7, + "client_cli": 3 + } + }, + { + "key_id": "K2", + "name": "old", + "prefix": "spicrawl_live_cd34", + "project_id": "proj_01J9Z", + "revoked": true, + "days": [], + "totals": { + "requests": 1 + } + } + ] +} diff --git a/internal/cmd/testdata/golden/usage/err_scope.exit b/internal/cmd/testdata/golden/usage/err_scope.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/usage/err_scope.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/usage/err_scope.stderr b/internal/cmd/testdata/golden/usage/err_scope.stderr new file mode 100644 index 0000000..28eb06b --- /dev/null +++ b/internal/cmd/testdata/golden/usage/err_scope.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Forbidden", + "status": 403, + "code": "ERR::AUTH::INSUFFICIENT_SCOPE", + "detail": "the key lacks the read scope", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/usage/err_scope.stdout b/internal/cmd/testdata/golden/usage/err_scope.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/usage/err_usage_to_before_from.exit b/internal/cmd/testdata/golden/usage/err_usage_to_before_from.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/usage/err_usage_to_before_from.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/usage/err_usage_to_before_from.stderr b/internal/cmd/testdata/golden/usage/err_usage_to_before_from.stderr new file mode 100644 index 0000000..a0cc871 --- /dev/null +++ b/internal/cmd/testdata/golden/usage/err_usage_to_before_from.stderr @@ -0,0 +1,4 @@ +{ + "error": "--to is exclusive and must be after --from", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/usage/err_usage_to_before_from.stdout b/internal/cmd/testdata/golden/usage/err_usage_to_before_from.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/usage/reconciliation.exit b/internal/cmd/testdata/golden/usage/reconciliation.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage/reconciliation.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage/reconciliation.stdout b/internal/cmd/testdata/golden/usage/reconciliation.stdout new file mode 100644 index 0000000..aea1bd6 --- /dev/null +++ b/internal/cmd/testdata/golden/usage/reconciliation.stdout @@ -0,0 +1,11 @@ +{ + "day": "2026-09-21", + "interpretation": "The request log holds no rows for the day; drift is not evidence of overbilling.", + "drift": [ + { + "metric": "requests", + "billed": 10, + "derived": 0 + } + ] +} diff --git a/internal/cmd/testdata/golden/usage/summary.exit b/internal/cmd/testdata/golden/usage/summary.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage/summary.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage/summary.stdout b/internal/cmd/testdata/golden/usage/summary.stdout new file mode 100644 index 0000000..2999943 --- /dev/null +++ b/internal/cmd/testdata/golden/usage/summary.stdout @@ -0,0 +1,37 @@ +{ + "period": { + "start": "2026-09-01", + "end": "2026-10-01", + "days": 30, + "bounds": "[start, end)" + }, + "period_elapsed_days": 22, + "plan": { + "code": "starter", + "name": "Starter", + "subscription_status": "active", + "period_source": "subscription" + }, + "credits": { + "included_micro": 50000000000, + "used_micro": 12345670000, + "remaining_micro": 37654330000, + "overage_micro": 0, + "used_basis_points": 2469 + }, + "allowance": { + "limit_micro": 50000000000, + "used_micro": 12400000000, + "remaining_micro": 37600000000, + "unlimited": false, + "resets_at": "" + }, + "metrics": { + "requests": 1234, + "credits": 12345670000, + "extractions": 55 + }, + "warnings": [ + "allowance counter is 40s behind" + ] +} diff --git a/internal/cmd/testdata/golden/usage_human/by_day.exit b/internal/cmd/testdata/golden/usage_human/by_day.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/by_day.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage_human/by_day.stdout b/internal/cmd/testdata/golden/usage_human/by_day.stdout new file mode 100644 index 0000000..3cb469e --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/by_day.stdout @@ -0,0 +1,6 @@ +period [2026-08-01, 2026-08-04) (3 days), by day + +DAY REQUESTS CREDITS ENGINE_MS +2026-08-01 10 12.5 4200 +2026-08-02 15 18.5 4800 +TOTAL 25 31 9000 diff --git a/internal/cmd/testdata/golden/usage_human/by_engine_unattributed.exit b/internal/cmd/testdata/golden/usage_human/by_engine_unattributed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/by_engine_unattributed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage_human/by_engine_unattributed.stdout b/internal/cmd/testdata/golden/usage_human/by_engine_unattributed.stdout new file mode 100644 index 0000000..c18b6a0 --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/by_engine_unattributed.stdout @@ -0,0 +1,7 @@ +period [2026-08-01, 2026-09-01) (31 days), by engine + +ENGINE REQUESTS CREDITS +Fetch 40 40 +chromium 5 25 +(unattributed) 2 1 +TOTAL 47 66 diff --git a/internal/cmd/testdata/golden/usage_human/by_key.exit b/internal/cmd/testdata/golden/usage_human/by_key.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/by_key.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage_human/by_key.stdout b/internal/cmd/testdata/golden/usage_human/by_key.stdout new file mode 100644 index 0000000..34fd23d --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/by_key.stdout @@ -0,0 +1,5 @@ +period [2026-08-01, 2026-09-01) (31 days), by key + +KEY NAME PREFIX REQUESTS CLIENT_CLI +K1 ci-key spicrawl_live_ab12 7 3 +K2 old spicrawl_live_cd34 1 0 diff --git a/internal/cmd/testdata/golden/usage_human/err_scope.exit b/internal/cmd/testdata/golden/usage_human/err_scope.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/err_scope.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/usage_human/err_scope.stdout b/internal/cmd/testdata/golden/usage_human/err_scope.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/usage_human/reconciliation.exit b/internal/cmd/testdata/golden/usage_human/reconciliation.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/reconciliation.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage_human/reconciliation.stdout b/internal/cmd/testdata/golden/usage_human/reconciliation.stdout new file mode 100644 index 0000000..aea1bd6 --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/reconciliation.stdout @@ -0,0 +1,11 @@ +{ + "day": "2026-09-21", + "interpretation": "The request log holds no rows for the day; drift is not evidence of overbilling.", + "drift": [ + { + "metric": "requests", + "billed": 10, + "derived": 0 + } + ] +} diff --git a/internal/cmd/testdata/golden/usage_human/summary.exit b/internal/cmd/testdata/golden/usage_human/summary.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/summary.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage_human/summary.stdout b/internal/cmd/testdata/golden/usage_human/summary.stdout new file mode 100644 index 0000000..96740c8 --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/summary.stdout @@ -0,0 +1,9 @@ +plan: Starter (active) +period: [2026-09-01, 2026-10-01), day 22 of 30 (subscription) +credits: 12345.67 of 50000 used (24.7%), 37654.33 remaining +allowance: 12400 of 50000 used (incl. credits held for in-flight work), 37600 remaining, resets + +METRIC VALUE +requests 1234 +credits 12345.67 +extractions 55