From 79faf11b374117b5dd68f8eaa22680e584258687 Mon Sep 17 00:00:00 2001 From: Fabio Bonelli Date: Tue, 10 Nov 2026 12:47:51 +0000 Subject: [PATCH] fix: bound Gitea scanner requests --- scanner/gitea.go | 38 +++++++++++++++++++++++------------ scanner/gitea_test.go | 3 +++ scanner/gitea_timeout_test.go | 37 ++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 scanner/gitea_timeout_test.go diff --git a/scanner/gitea.go b/scanner/gitea.go index f850e5a..a860183 100644 --- a/scanner/gitea.go +++ b/scanner/gitea.go @@ -9,6 +9,7 @@ import ( "net/url" "os" "strings" + "time" "github.com/italia/publiccode-crawler/v4/common" log "github.com/sirupsen/logrus" @@ -16,10 +17,21 @@ import ( var errNotFound = errors.New("not found") -type GiteaScanner struct{} +const giteaRequestTimeout = 30 * time.Second + +type GiteaScanner struct { + httpClient *http.Client +} func NewGiteaScanner() GiteaScanner { - return GiteaScanner{} + return GiteaScanner{ + httpClient: &http.Client{ + Transport: nil, + CheckRedirect: nil, + Jar: nil, + Timeout: giteaRequestTimeout, + }, + } } type giteaRepo struct { @@ -54,7 +66,7 @@ func (scanner GiteaScanner) List( } for page := 1; ; page++ { - repos, err := fetchPage(&groupURL, owner, limit, page) + repos, err := fetchPage(scanner.httpClient, &groupURL, owner, limit, page) if err != nil { return fmt.Errorf("GiteaScanner: %w", err) } @@ -89,7 +101,7 @@ func (scanner GiteaScanner) Scan( owner, repoName := parts[0], parts[1] apiURL := fmt.Sprintf("%s://%s/api/v1/repos/%s/%s", repoURL.Scheme, repoURL.Host, owner, repoName) - repo, err := giteaFetchRepo(apiURL) + repo, err := giteaFetchRepo(scanner.httpClient, apiURL) if err != nil { return fmt.Errorf("GiteaScanner: %w", err) } @@ -131,11 +143,11 @@ func addGiteaRepo( return nil } -func giteaOrgReposPage(base *url.URL, owner string, limit, page int) ([]giteaRepo, error) { +func giteaOrgReposPage(httpClient *http.Client, base *url.URL, owner string, limit, page int) ([]giteaRepo, error) { apiURL := fmt.Sprintf("%s://%s/api/v1/orgs/%s/repos?limit=%d&page=%d", base.Scheme, base.Host, url.PathEscape(owner), limit, page) - repos, err := giteaFetchRepoList(apiURL) + repos, err := giteaFetchRepoList(httpClient, apiURL) if !errors.Is(err, errNotFound) { return repos, err } @@ -144,10 +156,10 @@ func giteaOrgReposPage(base *url.URL, owner string, limit, page int) ([]giteaRep userURL := fmt.Sprintf("%s://%s/api/v1/users/%s/repos?limit=%d&page=%d", base.Scheme, base.Host, url.PathEscape(owner), limit, page) - return giteaFetchRepoList(userURL) + return giteaFetchRepoList(httpClient, userURL) } -func giteaInstanceReposPage(base *url.URL, _ string, limit, page int) ([]giteaRepo, error) { +func giteaInstanceReposPage(httpClient *http.Client, base *url.URL, _ string, limit, page int) ([]giteaRepo, error) { apiURL := fmt.Sprintf("%s://%s/api/v1/repos/search?limit=%d&page=%d", base.Scheme, base.Host, limit, page) @@ -156,7 +168,7 @@ func giteaInstanceReposPage(base *url.URL, _ string, limit, page int) ([]giteaRe return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := httpClient.Do(req) if err != nil { return nil, fmt.Errorf("GET %s: %w", apiURL, err) } @@ -174,13 +186,13 @@ func giteaInstanceReposPage(base *url.URL, _ string, limit, page int) ([]giteaRe return result.Data, nil } -func giteaFetchRepoList(apiURL string) ([]giteaRepo, error) { +func giteaFetchRepoList(httpClient *http.Client, apiURL string) ([]giteaRepo, error) { req, err := giteaNewRequest(apiURL) if err != nil { return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := httpClient.Do(req) if err != nil { return nil, fmt.Errorf("GET %s: %w", apiURL, err) } @@ -202,13 +214,13 @@ func giteaFetchRepoList(apiURL string) ([]giteaRepo, error) { return repos, nil } -func giteaFetchRepo(apiURL string) (giteaRepo, error) { +func giteaFetchRepo(httpClient *http.Client, apiURL string) (giteaRepo, error) { req, err := giteaNewRequest(apiURL) if err != nil { return giteaRepo{}, err } - resp, err := http.DefaultClient.Do(req) + resp, err := httpClient.Do(req) if err != nil { return giteaRepo{}, fmt.Errorf("GET %s: %w", apiURL, err) } diff --git a/scanner/gitea_test.go b/scanner/gitea_test.go index c90f909..a648031 100644 --- a/scanner/gitea_test.go +++ b/scanner/gitea_test.go @@ -37,10 +37,13 @@ func giteaPublisher() common.Publisher { } func TestGiteaScanner_ScanRepo_success(t *testing.T) { + t.Setenv("GITEA_TOKEN", "test-token") + repo := giteaRepoJSON("myrepo", "myorg/myrepo", "main", false, false, false) ts := newGiteaTestServer(t, func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "/api/v1/repos/myorg/myrepo", r.URL.Path) + assert.Equal(t, "token test-token", r.Header.Get("Authorization")) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(repo) diff --git a/scanner/gitea_timeout_test.go b/scanner/gitea_timeout_test.go new file mode 100644 index 0000000..877dd36 --- /dev/null +++ b/scanner/gitea_timeout_test.go @@ -0,0 +1,37 @@ +package scanner + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/italia/publiccode-crawler/v4/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewGiteaScannerSetsRequestTimeout(t *testing.T) { + scanner := NewGiteaScanner() + + assert.Equal(t, giteaRequestTimeout, scanner.httpClient.Timeout) +} + +func TestGiteaScannerCancelsStalledResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) { + <-request.Context().Done() + })) + t.Cleanup(server.Close) + + repoURL, err := url.Parse(server.URL + "/owner/repository") + require.NoError(t, err) + + scanner := NewGiteaScanner() + scanner.httpClient.Timeout = time.Second + + err = scanner.Scan(*repoURL, common.Publisher{}, make(chan common.Repository)) + + require.Error(t, err) + assert.ErrorContains(t, err, "context deadline exceeded") +}