From 802c2ab076c90c5fabb7ab3ec66505acc4fd1b79 Mon Sep 17 00:00:00 2001 From: Julian Blacher Date: Sun, 1 Nov 2026 00:45:28 -0300 Subject: [PATCH] feat: Bitbucket Pipelines logs for --logs and get_failed_logs The Bitbucket client now implements github.RunLogReader: for a failed build status linking to a Pipelines result, it lists the pipeline's steps by build number, downloads the log of the first failed step, and keeps the output of the last script command ("+ ") up to the teardown. When there is no log (no pipeline scope, or a timeout), the step's error message is shown instead. Other CI systems are listed with their link, as before. Also: github.Client gets GetText (raw GET, tail-limited, counted), and GetJobLog now counts its request too. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + README.md | 7 +- TASKS.md | 1 + cmd/prw/checks.go | 6 +- cmd/prw/logs.go | 3 +- cmd/prw/mcp.go | 2 +- cmd/prw/wait.go | 7 +- internal/bitbucket/client.go | 2 +- internal/bitbucket/pipelines.go | 173 +++++++++++++++++++++++++++ internal/bitbucket/pipelines_test.go | 148 +++++++++++++++++++++++ internal/github/client.go | 39 +++++- internal/github/logs.go | 34 +++++- 12 files changed, 403 insertions(+), 20 deletions(-) create mode 100644 internal/bitbucket/pipelines.go create mode 100644 internal/bitbucket/pipelines_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 240e5a1..df57ec3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Bitbucket Pipelines logs: `prw checks --logs`, `prw wait --logs` and the MCP tool `get_failed_logs` show, for each failed Pipelines build, the output of the script command that failed in its first failed step (from the last `+ command` line up to the teardown), or the reason Bitbucket gives when there's no log (e.g. a timeout). Needs a token with the pipeline read scope. - Auto-watch on Bitbucket Cloud: `prw watch --mine` / `--review-requested` and `prw run` also find your Bitbucket PRs (when a Bitbucket token or `bitbucket_user` is set): your open PRs in every workspace (or `bitbucket_workspaces`), and open PRs listing you as a reviewer that you haven't approved or requested changes on, in repositories active in the last 30 days (rescanned every 15 minutes). Without a GitHub token, auto-watch works on Bitbucket alone. ### Changed diff --git a/README.md b/README.md index 81fd6ed..abea0ac 100644 --- a/README.md +++ b/README.md @@ -328,7 +328,7 @@ Runs superseded by a newer run of the same check (e.g. cancelled by `concurrency ### Why did it fail? `--logs` -`prw checks --logs` (and `prw wait --logs`) also print, for each failed GitHub Actions job, the output of the step that failed: its last lines up to the error, with timestamps, colors and runner markup stripped — what you'd scroll to in the browser. +`prw checks --logs` (and `prw wait --logs`) also print, for each failed GitHub Actions job or Bitbucket Pipelines build, the output of the step that failed: its last lines up to the error, with timestamps, colors and runner markup stripped — what you'd scroll to in the browser. ``` $ prw checks https://github.com/owner/repo/pull/123 --logs @@ -383,7 +383,7 @@ Tools: | Tool | What it does | |------|--------------| | `get_ci_status` | Current state of a PR (`success`, `failure`, `error`, `pending`, `merged`, `closed`, `no_ci`), check counts, failed checks with links. Returns immediately. | -| `get_failed_logs` | For each failed GitHub Actions job, the output of the step that failed (last `max_lines` lines up to the error, default 40). What an agent needs to fix the failure without a browser. | +| `get_failed_logs` | For each failed GitHub Actions job or Bitbucket Pipelines build, the output of the step that failed (last `max_lines` lines up to the error, default 40). What an agent needs to fix the failure without a browser. | | `get_flaky_checks` | A repository's jobs that failed and then passed on re-run of the same commit, with counts and links — so an agent can tell a known-flaky failure from one its change caused. | | `wait_for_ci` | Blocks until CI finishes (`timeout_seconds` default 600, max 3600). Fails fast on the first failed check, follows new pushes, sends progress notifications. On timeout it returns `pending` with `timed_out: true` so the agent can call it again. | | `list_watched` | PRs on the local watch list with their last known state. | @@ -406,7 +406,8 @@ prw checks https://bitbucket.org/acme/api/pull-requests/42 - `prw run` notifies about Bitbucket PRs just like GitHub ones, `prw list` shows them as `bitbucket:workspace/repo`, and the MCP tools accept their URLs. - **Auto-watch**: `prw watch --mine` follows your open PRs in all your workspaces (or those in `bitbucket_workspaces`). `--review-requested` follows open PRs that list you as a reviewer and that you haven't approved or requested changes on yet; Bitbucket can only search those per repository, so prw scans the repositories active in the last 30 days (up to 60 per workspace) every 15 minutes. - Bitbucket Cloud allows 1,000 API requests per hour, so `prw run` spaces out the checks of Bitbucket PRs whatever the poll interval: each is checked at most once a minute, and less often as the list grows (every 18 seconds × the number of Bitbucket PRs, e.g. every 9 minutes with 30), so checks use about 600 requests an hour and leave room for auto-watch and other commands. A rate limit on one service doesn't pause the other. -- Not yet for Bitbucket: `--logs` and `prw flaky`. +- **`--logs`** shows, for each failed Pipelines build, the output of the script command that failed in its first failed step (reading Pipelines logs needs a token with the pipeline read scope; without it, prw shows the reason Bitbucket gives, e.g. a timeout). Builds from other CI systems (Jenkins, ...) are listed with their link. +- Not yet for Bitbucket: `prw flaky`. **Authentication.** Public repositories work without credentials. For private ones, create an [Atlassian API token](https://id.atlassian.com/manage-profile/security/api-tokens) with Bitbucket read scopes (repositories and pull requests) and give prw your Atlassian email with it: diff --git a/TASKS.md b/TASKS.md index 821023b..c4515bd 100644 --- a/TASKS.md +++ b/TASKS.md @@ -100,6 +100,7 @@ Legend — Impact: 1–5 · Effort: S (≤½ day), M (1–2 days), L (3+ days) - 🆕 Bitbucket Cloud support (requested by the owner, who uses Bitbucket at work). Phase 1 done: PR identity carries the host; a Bitbucket client maps PRs, build statuses (Pipelines, Jenkins), participants and merge checks onto the existing types, so run / wait / checks / list / broadcast / MCP work unchanged; per-host rate-limit pauses; ≥60s between checks of a Bitbucket PR (1,000 req/h limit). Verified live on public atlassian/atlassian-connect-express PRs. Next: auto-watch (`--mine` / `--review-requested`) on Bitbucket, Pipelines step logs for `--logs`, then GitHub Enterprise (#2) on the same host abstraction. - ✅ `prw dashboard` (owner's use case: ~25 repos × dev/rc/master cherry-picks at work) — live table grouped by task across GitHub + Bitbucket, smart CI re-reads. Bitbucket discovery endpoints verified live anonymously (workspace user PRs, merged since, recent repos, reviewer filter); `/user` and `/user/workspaces` still need a check with the owner's token. Verified live on the owner's GitHub PRs. - 🚀 v0.5.0 released with Bitbucket Cloud and `prw dashboard` (PRs #24–#25). +- ✅ Bitbucket Pipelines logs for `--logs` / MCP `get_failed_logs` (RunLogReader on the Bitbucket client: steps by build number, log by pipeline + step UUID). Verified live anonymously up to the log download (needs the pipeline scope; the timeout reason is shown instead); the log parser is tested on a constructed log, to check against a real one with the owner's token. - ✅ Bitbucket auto-watch (`--mine` / `--review-requested`): per-host finders, review scan every 15 min skipping PRs already reviewed, check interval scaled to the number of Bitbucket PRs (600 req/h budget). Verified live with a public Bitbucket user + the owner's GitHub account in one watch list. - Next: #17 packaging (Homebrew/Scoop) + keychain token storage, #8 `gh` extension publishing — both need owner decisions diff --git a/cmd/prw/checks.go b/cmd/prw/checks.go index b1b27b7..51e6083 100644 --- a/cmd/prw/checks.go +++ b/cmd/prw/checks.go @@ -41,7 +41,7 @@ var ( func init() { rootCmd.AddCommand(checksCmd) checksCmd.Flags().BoolVar(&checksJSON, "json", false, "print the checks as JSON") - checksCmd.Flags().BoolVar(&checksLogs, "logs", false, "show the end of the log of each failed GitHub Actions job") + checksCmd.Flags().BoolVar(&checksLogs, "logs", false, "show the end of the log of each failed GitHub Actions job or Bitbucket Pipelines build") checksCmd.Flags().IntVar(&checksLogLines, "log-lines", github.DefaultLogLines, "lines of each failed job's log to show with --logs") } @@ -53,8 +53,8 @@ commit statuses) of a pull request's head commit: failures first, then running ones, then the rest. The PR does not need to be on the watch list. With --logs, it also prints the output of the step that failed in each failed -GitHub Actions job (the last --log-lines lines up to the error), so you can see -why without opening the browser. +GitHub Actions job or Bitbucket Pipelines build (the last --log-lines lines up to +the error), so you can see why without opening the browser. Exit codes: 0 all checks passed diff --git a/cmd/prw/logs.go b/cmd/prw/logs.go index 43c4fe8..c524ea3 100644 --- a/cmd/prw/logs.go +++ b/cmd/prw/logs.go @@ -12,7 +12,8 @@ import ( // maxMCPLogLines caps the log lines per failed job returned to agents. const maxMCPLogLines = 200 -// fetchFailedLogs gets log excerpts of the failed GitHub Actions jobs of a commit. +// fetchFailedLogs gets log excerpts of the failed GitHub Actions jobs (or Bitbucket +// Pipelines builds) of a commit. // Failed commit statuses (external CI) are listed too, with their link, since they // have no log in the API. func fetchFailedLogs(ctx context.Context, client ciClient, owner, repo, sha string, maxLines int) ([]github.FailedJobLog, error) { diff --git a/cmd/prw/mcp.go b/cmd/prw/mcp.go index 92078b8..a5f465f 100644 --- a/cmd/prw/mcp.go +++ b/cmd/prw/mcp.go @@ -469,7 +469,7 @@ var mcpTools = []map[string]interface{}{ { "name": "get_failed_logs", "title": "Get failed CI job logs", - "description": "For each failed GitHub Actions job on a pull request's head commit, return the output of the step that failed: its last lines up to the error, with timestamps and colors stripped. Use it after get_ci_status or wait_for_ci report a failure, to see why without opening the browser. Checks from other CI systems have no log here; their links are returned instead. At most 5 job logs are downloaded per call.", + "description": "For each failed GitHub Actions job or Bitbucket Pipelines build on a pull request's head commit, return the output of the step that failed: its last lines up to the error, with timestamps and colors stripped. Use it after get_ci_status or wait_for_ci report a failure, to see why without opening the browser. Checks from other CI systems have no log here; their links are returned instead. At most 5 job logs are downloaded per call.", "inputSchema": map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ diff --git a/cmd/prw/wait.go b/cmd/prw/wait.go index 3c7de8f..f1b27db 100644 --- a/cmd/prw/wait.go +++ b/cmd/prw/wait.go @@ -69,7 +69,7 @@ func init() { waitCmd.Flags().DurationVar(&waitTimeout, "timeout", 30*time.Minute, "give up after this long (0 = wait forever)") waitCmd.Flags().DurationVar(&waitInterval, "interval", 15*time.Second, "how often to poll GitHub") waitCmd.Flags().BoolVar(&waitJSON, "json", false, "print the result as JSON on stdout") - waitCmd.Flags().BoolVar(&waitLogs, "logs", false, "on failure, also print the end of the log of each failed GitHub Actions job") + waitCmd.Flags().BoolVar(&waitLogs, "logs", false, "on failure, also print the end of the log of each failed GitHub Actions job or Bitbucket Pipelines build") waitCmd.Flags().IntVar(&waitLogLines, "log-lines", github.DefaultLogLines, "lines of each failed job's log to show with --logs") } @@ -94,7 +94,7 @@ Exit codes: Progress goes to stderr; the result goes to stdout (as JSON with --json). With --logs, a failure also prints the output of the step that failed in each failed -GitHub Actions job (failed_logs in --json).`, +GitHub Actions job or Bitbucket Pipelines build (failed_logs in --json).`, Example: ` prw wait https://github.com/owner/repo/pull/123 prw wait https://github.com/owner/repo/pull/123 --timeout 10m --json prw wait https://github.com/owner/repo/pull/123 --logs @@ -179,7 +179,8 @@ type waitResult struct { SHA string `json:"sha,omitempty"` State string `json:"state"` // success, failure, error, pending, merged, closed FailedChecks []github.FailedCheck `json:"failed_checks,omitempty"` - // FailedLogs has the log excerpt of each failed GitHub Actions job (--logs). + // FailedLogs has the log excerpt of each failed GitHub Actions job or Pipelines + // build (--logs). FailedLogs []github.FailedJobLog `json:"failed_logs,omitempty"` TotalChecks int `json:"total_checks"` PendingChecks int `json:"pending_checks"` diff --git a/internal/bitbucket/client.go b/internal/bitbucket/client.go index a9e7e0b..c502e51 100644 --- a/internal/bitbucket/client.go +++ b/internal/bitbucket/client.go @@ -310,7 +310,7 @@ func (c *Client) GetReviews(ctx context.Context, workspace, repo string, number return reviews, nil } -// GetJobLog is not available for Bitbucket yet. +// GetJobLog is GitHub Actions only; Pipelines logs are read with RunLog. func (c *Client) GetJobLog(ctx context.Context, workspace, repo string, jobID int64) (string, error) { return "", ErrUnsupported } diff --git a/internal/bitbucket/pipelines.go b/internal/bitbucket/pipelines.go new file mode 100644 index 0000000..e0881c4 --- /dev/null +++ b/internal/bitbucket/pipelines.go @@ -0,0 +1,173 @@ +package bitbucket + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + + "github.com/devblac/prw/internal/github" +) + +// maxStepLogBytes is how much of the end of a step log is read: the failure is there. +const maxStepLogBytes = 4 << 20 + +var ( + // pipelineResultURL matches the link of a Pipelines build status, e.g. + // https://bitbucket.org/ws/repo/addon/pipelines/home#!/results/2435 or + // https://bitbucket.org/ws/repo/pipelines/results/2435. + pipelineResultURL = regexp.MustCompile(`/pipelines/(?:home#!/)?results/(\d+)`) + ansiEscape = regexp.MustCompile(`\x1b\[[0-9;?]*[A-Za-z]`) + // teardownLine starts the output Pipelines adds after the script. + teardownLine = regexp.MustCompile(`^(Searching for test report files|Finished scanning for test reports|Merged test suites|Searching for files matching artifact pattern)`) +) + +func pipelineBuildNumber(run github.CheckRun) (int, bool) { + m := pipelineResultURL.FindStringSubmatch(run.HTMLURL) + if m == nil { + return 0, false + } + n, err := strconv.Atoi(m[1]) + return n, err == nil +} + +// CanReadLog reports whether run is a Bitbucket Pipelines build, whose log prw reads. +func (c *Client) CanReadLog(run github.CheckRun) bool { + _, ok := pipelineBuildNumber(run) + return ok +} + +// NoLogReason explains why other builds (Jenkins, ...) have no excerpt. +func (c *Client) NoLogReason() string { + return "not a Bitbucket Pipelines build; its log isn't available through the Bitbucket API" +} + +type pipelineStep struct { + UUID string `json:"uuid"` + Name string `json:"name"` + State struct { + Result struct { + Name string `json:"name"` // SUCCESSFUL, FAILED, ERROR, STOPPED, NOT_RUN + Error struct { + Message string `json:"message"` + } `json:"error"` + } `json:"result"` + } `json:"state"` + Pipeline struct { + UUID string `json:"uuid"` + } `json:"pipeline"` + ScriptCommands []struct { + Name string `json:"name"` + } `json:"script_commands"` +} + +// RunLog returns the output of the command that failed in the first failed step of a +// Pipelines build. Reading logs needs a token with the pipeline read scope. +func (c *Client) RunLog(ctx context.Context, workspace, repo string, run github.CheckRun, maxLines int) (github.LogExcerpt, error) { + number, ok := pipelineBuildNumber(run) + if !ok { + return github.LogExcerpt{}, errors.New(c.NoLogReason()) + } + var resp struct { + Values []pipelineStep `json:"values"` + } + path := fmt.Sprintf("%s/pipelines/%d/steps/?pagelen=100", repoPath(workspace, repo), number) + if err := c.http.GetJSON(ctx, path, &resp); err != nil { + return github.LogExcerpt{}, fmt.Errorf("failed to list the steps of pipeline #%d: %w", number, err) + } + var step *pipelineStep + for i := range resp.Values { + switch strings.ToUpper(resp.Values[i].State.Result.Name) { + case "FAILED", "ERROR": + step = &resp.Values[i] + } + if step != nil { + break + } + } + if step == nil { + return github.LogExcerpt{}, fmt.Errorf("pipeline #%d has no failed step", number) + } + reason := step.State.Result.Error.Message // e.g. "Exceeded build time limit of 120 minutes." + + logPath := fmt.Sprintf("%s/pipelines/%s/steps/%s/log", repoPath(workspace, repo), url.PathEscape(step.Pipeline.UUID), url.PathEscape(step.UUID)) + log, err := c.http.GetText(ctx, logPath, maxStepLogBytes) + if err != nil { + var apiErr *github.APIError + if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound { + err = errors.New("log not available: reading Pipelines logs needs a Bitbucket token with the pipeline read scope, or the log expired") + } + if reason != "" { + return github.LogExcerpt{Step: step.Name, Lines: []string{reason}}, nil + } + return github.LogExcerpt{}, err + } + var commands []string + for _, cmd := range step.ScriptCommands { + commands = append(commands, cmd.Name) + } + excerpt := ExtractStepFailure(log, commands, maxLines) + if reason != "" { + excerpt.Lines = append(excerpt.Lines, reason) + } + return excerpt, nil +} + +// ExtractStepFailure returns the output of the last script command a Pipelines step +// ran — the one that failed, since a step stops at the first failing command — up to +// the teardown, at most maxLines lines. Pipelines echoes each command as "+ ". +// Without such a line it returns the end of the log. +func ExtractStepFailure(log string, commands []string, maxLines int) github.LogExcerpt { + if maxLines <= 0 { + maxLines = github.DefaultLogLines + } + raw := strings.Split(strings.ReplaceAll(log, "\r\n", "\n"), "\n") + lines := make([]string, len(raw)) + for i, l := range raw { + lines[i] = strings.TrimRight(ansiEscape.ReplaceAllString(l, ""), "\r") + } + + known := map[string]bool{} + for _, c := range commands { + // A multi-line command is echoed with its first line. + first, _, _ := strings.Cut(c, "\n") + known[strings.TrimSpace(first)] = true + } + start, step := -1, "" + for i := len(lines) - 1; i >= 0 && start < 0; i-- { + if cmd, ok := strings.CutPrefix(lines[i], "+ "); ok && known[strings.TrimSpace(cmd)] { + start, step = i+1, strings.TrimSpace(cmd) + } + } + if start < 0 { // commands unknown or not echoed: fall back to any "+ " line + for i := len(lines) - 1; i >= 0 && start < 0; i-- { + if cmd, ok := strings.CutPrefix(lines[i], "+ "); ok { + start, step = i+1, strings.TrimSpace(cmd) + } + } + } + if start < 0 { + start = 0 + } + + end := len(lines) + for i := start; i < len(lines); i++ { + if teardownLine.MatchString(lines[i]) { + end = i + break + } + } + for end > start && strings.TrimSpace(lines[end-1]) == "" { + end-- + } + from := end - maxLines + if from < start { + from = start + } + out := append([]string{}, lines[from:end]...) + return github.LogExcerpt{Step: step, Lines: out, Truncated: from > start} +} diff --git a/internal/bitbucket/pipelines_test.go b/internal/bitbucket/pipelines_test.go new file mode 100644 index 0000000..207a43c --- /dev/null +++ b/internal/bitbucket/pipelines_test.go @@ -0,0 +1,148 @@ +package bitbucket + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/devblac/prw/internal/github" +) + +// stepLog is shaped like a Pipelines step log: setup commands, then each script +// command echoed as "+ " with its output, then the teardown. +const stepLog = "+ umask 000\n" + + "+ retry 6 $CHECKOUT_SCRIPT\n" + + "Initialized empty Git repository\n" + + "+ npm install\n" + + "added 812 packages in 21s\n" + + "+ npm test\n" + + "\x1b[32m> atlassian-connect-express@12.0.0 test\x1b[0m\n" + + "> jest\r\n" + + "FAIL test/store.test.js\n" + + " ● store › saves a client\n" + + " expected 1, received 0\n" + + "\n" + + "Searching for test report files in directories named [test-results, failsafe-reports] recursively\n" + + "Finished scanning for test reports. Found 0 test report files.\n" + + "Merged test suites, total number tests is 0, with 0 failures and 0 errors.\n" + +func TestExtractStepFailure(t *testing.T) { + got := ExtractStepFailure(stepLog, []string{"npm install", "npm test", "npm run lint:all"}, 50) + if got.Step != "npm test" || got.Truncated { + t.Errorf("step = %q, truncated = %v", got.Step, got.Truncated) + } + want := "> atlassian-connect-express@12.0.0 test|> jest|FAIL test/store.test.js| ● store › saves a client| expected 1, received 0" + if s := strings.Join(got.Lines, "|"); s != want { + t.Errorf("lines =\n%s\nwant\n%s", s, want) + } + + short := ExtractStepFailure(stepLog, []string{"npm test"}, 2) + if !short.Truncated || len(short.Lines) != 2 || short.Lines[1] != " expected 1, received 0" { + t.Errorf("short = %+v", short) + } + + // Commands unknown: the last echoed command is used. + if got := ExtractStepFailure(stepLog, nil, 50); got.Step != "npm test" { + t.Errorf("without commands: step = %q", got.Step) + } + // Nothing echoed: the end of the log. + if got := ExtractStepFailure("one\ntwo\nthree\n", nil, 2); strings.Join(got.Lines, "|") != "two|three" || !got.Truncated { + t.Errorf("plain log = %+v", got) + } +} + +const stepsJSON = `{"values": [ + {"uuid": "{s1}", "name": "Build", "state": {"result": {"name": "SUCCESSFUL"}}, "pipeline": {"uuid": "{p1}"}}, + {"uuid": "{s2}", "name": "Test", "state": {"result": {"name": "FAILED"%s}}, "pipeline": {"uuid": "{p1}"}, + "script_commands": [{"name": "npm install"}, {"name": "npm test"}]} +]}` + +func newPipelinesFake(t *testing.T, stepError string, logStatus int) (*Client, *[]string) { + var paths []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.EscapedPath()) + switch r.URL.EscapedPath() { + case "/repositories/acme/api/pipelines/2435/steps/": + fmt.Fprintf(w, stepsJSON, stepError) + case "/repositories/acme/api/pipelines/%7Bp1%7D/steps/%7Bs2%7D/log": + if logStatus != http.StatusOK { + w.WriteHeader(logStatus) + fmt.Fprint(w, `{"error": {"message": "Not found"}}`) + return + } + fmt.Fprint(w, stepLog) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + c := NewClient("me@example.com", "token") + c.SetBaseURL(srv.URL) + return c, &paths +} + +func pipelineRun(name, url string) github.CheckRun { + return github.CheckRun{ID: 1, Name: name, HTMLURL: url, Status: "completed", Conclusion: "failure"} +} + +func TestRunLog(t *testing.T) { + c, paths := newPipelinesFake(t, "", http.StatusOK) + run := pipelineRun("Pipeline #2435 for main", "https://bitbucket.org/acme/api/addon/pipelines/home#!/results/2435") + if !c.CanReadLog(run) || c.CanReadLog(pipelineRun("Jenkins", "https://ci.example.com/job/42")) { + t.Fatal("CanReadLog must accept Pipelines links only") + } + if !c.CanReadLog(pipelineRun("new link", "https://bitbucket.org/acme/api/pipelines/results/2435")) { + t.Error("CanReadLog must accept the newer /pipelines/results/N links") + } + + got, err := c.RunLog(context.Background(), "acme", "api", run, 50) + if err != nil { + t.Fatalf("%v (requests: %v)", err, *paths) + } + if got.Step != "npm test" || got.Lines[len(got.Lines)-1] != " expected 1, received 0" { + t.Errorf("excerpt = %+v", got) + } + if c.RequestsInLastHour() != 2 { + t.Errorf("requests = %d, want 2 (steps + log)", c.RequestsInLastHour()) + } +} + +func TestRunLog_NoLog(t *testing.T) { + c, _ := newPipelinesFake(t, "", http.StatusNotFound) + run := pipelineRun("p", "https://bitbucket.org/acme/api/addon/pipelines/home#!/results/2435") + if _, err := c.RunLog(context.Background(), "acme", "api", run, 50); err == nil || !strings.Contains(err.Error(), "pipeline read scope") { + t.Errorf("err = %v, want a hint about the token scope", err) + } + + // A step that failed without a log (e.g. a timeout) still says why. + c, _ = newPipelinesFake(t, `, "error": {"message": "Exceeded build time limit of 120 minutes."}`, http.StatusNotFound) + got, err := c.RunLog(context.Background(), "acme", "api", run, 50) + if err != nil || strings.Join(got.Lines, "") != "Exceeded build time limit of 120 minutes." { + t.Errorf("got %+v, %v", got, err) + } +} + +func TestFailedJobLogs_UsesPipelinesLogs(t *testing.T) { + c, _ := newPipelinesFake(t, "", http.StatusOK) + runs := []github.CheckRun{ + pipelineRun("Pipeline #2435 for main", "https://bitbucket.org/acme/api/addon/pipelines/home#!/results/2435"), + pipelineRun("Jenkins", "https://ci.example.com/job/42"), + } + runs[1].ID = 2 + logs, err := github.FailedJobLogs(context.Background(), c, "acme", "api", runs, 50) + if err != nil { + t.Fatal(err) + } + if len(logs) != 2 { + t.Fatalf("logs = %+v", logs) + } + if logs[0].Name != "Jenkins" || !strings.Contains(logs[0].Error, "not a Bitbucket Pipelines build") { + t.Errorf("Jenkins entry = %+v", logs[0]) + } + if logs[1].Excerpt == nil || logs[1].Excerpt.Step != "npm test" { + t.Errorf("Pipelines entry = %+v", logs[1]) + } +} diff --git a/internal/github/client.go b/internal/github/client.go index b1b5f87..d2248fe 100644 --- a/internal/github/client.go +++ b/internal/github/client.go @@ -506,19 +506,18 @@ func hasCISignal(status *CombinedStatus, runs []CheckRun) bool { return legacyState != "" && !(legacyState == "pending" && status.TotalCount == 0) } -// getJSON performs an authenticated GET request and decodes the JSON response into v. // GetJSON GETs BaseURL+path and decodes the JSON response into v, with the client's // authentication, ETag cache and rate-limit handling. func (c *Client) GetJSON(ctx context.Context, path string, v interface{}) error { return c.getJSON(ctx, path, v) } -func (c *Client) getJSON(ctx context.Context, path string, v interface{}) error { +// newRequest builds an authenticated GET request for BaseURL+path. +func (c *Client) newRequest(ctx context.Context, path string) (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, "GET", c.BaseURL+path, nil) if err != nil { - return err + return nil, err } - // GitHub rejects an empty "Bearer " with 401 even for public repos, so only send the // header when there is a token. switch { @@ -532,6 +531,38 @@ func (c *Client) getJSON(ctx context.Context, path string, v interface{}) error accept = "application/vnd.github.v3+json" } req.Header.Set("Accept", accept) + return req, nil +} + +// GetText GETs BaseURL+path, following redirects, and returns at most the last limit +// bytes of the body (for logs). It isn't cached. +func (c *Client) GetText(ctx context.Context, path string, limit int) (string, error) { + req, err := c.newRequest(ctx, path) + if err != nil { + return "", err + } + req.Header.Set("Accept", "text/plain") + c.countRequest() + resp, err := c.HTTPClient.Do(req) + if err != nil { + return "", fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + c.recordRateLimit(resp.Header) + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) + apiErr := newAPIError(resp, body) + apiErr.Service = c.Service + return "", apiErr + } + return readTail(resp.Body, limit) +} + +func (c *Client) getJSON(ctx context.Context, path string, v interface{}) error { + req, err := c.newRequest(ctx, path) + if err != nil { + return err + } c.mu.Lock() cached, haveCached := c.cache[path] diff --git a/internal/github/logs.go b/internal/github/logs.go index 6017239..5832fe9 100644 --- a/internal/github/logs.go +++ b/internal/github/logs.go @@ -50,6 +50,7 @@ func (c *Client) GetJobLog(ctx context.Context, owner, repo string, jobID int64) } req.Header.Set("Accept", "application/vnd.github.v3+json") + c.countRequest() resp, err := c.HTTPClient.Do(req) if err != nil { return "", fmt.Errorf("request failed: %w", err) @@ -196,6 +197,17 @@ type LogFetcher interface { GetJobLog(ctx context.Context, owner, repo string, jobID int64) (string, error) } +// RunLogReader is implemented by clients of hosts whose CI isn't GitHub Actions +// (Bitbucket Pipelines): FailedJobLogs then asks it for the excerpt of each failed run +// it can read, instead of downloading GitHub Actions job logs. +type RunLogReader interface { + // CanReadLog reports whether run's log can be read, e.g. it's a Pipelines build. + CanReadLog(run CheckRun) bool + RunLog(ctx context.Context, owner, repo string, run CheckRun, maxLines int) (LogExcerpt, error) + // NoLogReason explains why the other runs have no excerpt. + NoLogReason() string +} + // FailedJobLog is the log excerpt of one failed check, or why there is none. type FailedJobLog struct { Name string `json:"name"` @@ -209,12 +221,13 @@ type FailedJobLog struct { const maxLogFetches = 5 // FailedJobLogs returns log excerpts for the failed (not cancelled) runs among runs, -// newest run per check only. Runs that aren't GitHub Actions jobs, and failures past -// the first maxLogFetches, get an Error explaining why there's no excerpt. If ctx is -// cancelled it returns ctx.Err(). +// newest run per check only. Runs that aren't GitHub Actions jobs (or, when c is a +// RunLogReader, runs it can't read), and failures past the first maxLogFetches, get an +// Error explaining why there's no excerpt. If ctx is cancelled it returns ctx.Err(). func FailedJobLogs(ctx context.Context, c LogFetcher, owner, repo string, runs []CheckRun, maxLines int) ([]FailedJobLog, error) { var out []FailedJobLog fetched := 0 + reader, hostLogs := c.(RunLogReader) latest := LatestCheckRuns(runs) sorted := make([]CheckRun, len(latest)) copy(sorted, latest) @@ -229,10 +242,23 @@ func FailedJobLogs(ctx context.Context, c LogFetcher, owner, repo string, runs [ } entry := FailedJobLog{Name: run.Name, URL: run.HTMLURL} switch { - case !IsActionsJob(run): + case hostLogs && !reader.CanReadLog(run): + entry.Error = reader.NoLogReason() + case !hostLogs && !IsActionsJob(run): entry.Error = "not a GitHub Actions job; prw only reads GitHub Actions logs so far" case fetched >= maxLogFetches: entry.Error = fmt.Sprintf("skipped: only the first %d failed jobs' logs are fetched", maxLogFetches) + case hostLogs: + fetched++ + excerpt, err := reader.RunLog(ctx, owner, repo, run, maxLines) + if ctx.Err() != nil { + return nil, ctx.Err() + } + if err != nil { + entry.Error = err.Error() + } else { + entry.Excerpt = &excerpt + } default: fetched++ log, err := c.GetJobLog(ctx, owner, repo, run.ID)