Limit Gitea scanner request duration

scanner/gitea.go

@@ -9,17 +9,29 @@ import (
 	"net/url"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/italia/publiccode-crawler/v4/common"
 	log "github.com/sirupsen/logrus"
 )
 
 var errNotFound = errors.New("not found")
 
-type GiteaScanner struct{}
+const giteaRequestTimeout = 30 * time.Second
+
+type GiteaScanner struct {
+	httpClient *http.Client
+}
 
 func NewGiteaScanner() GiteaScanner {
-	return GiteaScanner{}
+	return GiteaScanner{
+		httpClient: &http.Client{
+			Transport:     nil,
+			CheckRedirect: nil,
+			Jar:           nil,
+			Timeout:       giteaRequestTimeout,
+		},
+	}
 }
 
 type giteaRepo struct {
@@ -54,7 +66,7 @@ func (scanner GiteaScanner) List(
 	}
 
 	for page := 1; ; page++ {
-		repos, err := fetchPage(&groupURL, owner, limit, page)
+		repos, err := fetchPage(scanner.httpClient, &groupURL, owner, limit, page)
 		if err != nil {
 			return fmt.Errorf("GiteaScanner: %w", err)
 		}
@@ -89,7 +101,7 @@ func (scanner GiteaScanner) Scan(
 	owner, repoName := parts[0], parts[1]
 	apiURL := fmt.Sprintf("%s://%s/api/v1/repos/%s/%s", repoURL.Scheme, repoURL.Host, owner, repoName)
 
-	repo, err := giteaFetchRepo(apiURL)
+	repo, err := giteaFetchRepo(scanner.httpClient, apiURL)
 	if err != nil {
 		return fmt.Errorf("GiteaScanner: %w", err)
 	}
@@ -131,11 +143,11 @@ func addGiteaRepo(
 	return nil
 }
 
-func giteaOrgReposPage(base *url.URL, owner string, limit, page int) ([]giteaRepo, error) {
+func giteaOrgReposPage(httpClient *http.Client, base *url.URL, owner string, limit, page int) ([]giteaRepo, error) {
 	apiURL := fmt.Sprintf("%s://%s/api/v1/orgs/%s/repos?limit=%d&page=%d",
 		base.Scheme, base.Host, url.PathEscape(owner), limit, page)
 
-	repos, err := giteaFetchRepoList(apiURL)
+	repos, err := giteaFetchRepoList(httpClient, apiURL)
 	if !errors.Is(err, errNotFound) {
 		return repos, err
 	}
@@ -144,10 +156,10 @@ func giteaOrgReposPage(base *url.URL, owner string, limit, page int) ([]giteaRep
 	userURL := fmt.Sprintf("%s://%s/api/v1/users/%s/repos?limit=%d&page=%d",
 		base.Scheme, base.Host, url.PathEscape(owner), limit, page)
 
-	return giteaFetchRepoList(userURL)
+	return giteaFetchRepoList(httpClient, userURL)
 }
 
-func giteaInstanceReposPage(base *url.URL, _ string, limit, page int) ([]giteaRepo, error) {
+func giteaInstanceReposPage(httpClient *http.Client, base *url.URL, _ string, limit, page int) ([]giteaRepo, error) {
 	apiURL := fmt.Sprintf("%s://%s/api/v1/repos/search?limit=%d&page=%d",
 		base.Scheme, base.Host, limit, page)
 
@@ -156,7 +168,7 @@ func giteaInstanceReposPage(base *url.URL, _ string, limit, page int) ([]giteaRe
 		return nil, err
 	}
 
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := httpClient.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("GET %s: %w", apiURL, err)
 	}
@@ -174,13 +186,13 @@ func giteaInstanceReposPage(base *url.URL, _ string, limit, page int) ([]giteaRe
 	return result.Data, nil
 }
 
-func giteaFetchRepoList(apiURL string) ([]giteaRepo, error) {
+func giteaFetchRepoList(httpClient *http.Client, apiURL string) ([]giteaRepo, error) {
 	req, err := giteaNewRequest(apiURL)
 	if err != nil {
 		return nil, err
 	}
 
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := httpClient.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("GET %s: %w", apiURL, err)
 	}
@@ -202,13 +214,13 @@ func giteaFetchRepoList(apiURL string) ([]giteaRepo, error) {
 	return repos, nil
 }
 
-func giteaFetchRepo(apiURL string) (giteaRepo, error) {
+func giteaFetchRepo(httpClient *http.Client, apiURL string) (giteaRepo, error) {
 	req, err := giteaNewRequest(apiURL)
 	if err != nil {
 		return giteaRepo{}, err
 	}
 
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := httpClient.Do(req)
 	if err != nil {
 		return giteaRepo{}, fmt.Errorf("GET %s: %w", apiURL, err)
 	}

scanner/gitea_test.go

@@ -37,10 +37,13 @@ func giteaPublisher() common.Publisher {
 }
 
 func TestGiteaScanner_ScanRepo_success(t *testing.T) {
+	t.Setenv("GITEA_TOKEN", "test-token")
+
 	repo := giteaRepoJSON("myrepo", "myorg/myrepo", "main", false, false, false)
 
 	ts := newGiteaTestServer(t, func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, "/api/v1/repos/myorg/myrepo", r.URL.Path)
+		assert.Equal(t, "token test-token", r.Header.Get("Authorization"))
 
 		w.Header().Set("Content-Type", "application/json")
 		_ = json.NewEncoder(w).Encode(repo)

scanner/gitea_timeout_test.go

@@ -0,0 +1,37 @@
+package scanner
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/italia/publiccode-crawler/v4/common"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewGiteaScannerSetsRequestTimeout(t *testing.T) {
+	scanner := NewGiteaScanner()
+
+	assert.Equal(t, giteaRequestTimeout, scanner.httpClient.Timeout)
+}
+
+func TestGiteaScannerCancelsStalledResponse(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) {
+		<-request.Context().Done()
+	}))
+	t.Cleanup(server.Close)
+
+	repoURL, err := url.Parse(server.URL + "/owner/repository")
+	require.NoError(t, err)
+
+	scanner := NewGiteaScanner()
+	scanner.httpClient.Timeout = time.Second
+
+	err = scanner.Scan(*repoURL, common.Publisher{}, make(chan common.Repository))
+
+	require.Error(t, err)
+	assert.ErrorContains(t, err, "context deadline exceeded")
+}