From 645a26c7a31fef74fd373ce52d15258406a79aa3 Mon Sep 17 00:00:00 2001 From: AK Date: Sat, 8 Aug 2026 05:04:04 -0700 Subject: [PATCH] fix: resolve Go packages and suggest fuzzy matches --- docs/configuration.md | 6 ++ docs/plugins.md | 13 ++- docs/security.md | 3 +- plugins/audit.go | 29 +++++- plugins/audit_test.go | 12 +++ plugins/pkg.go | 201 +++++++++++++++++++++++++++++++++++++++++- plugins/pkg_test.go | 60 +++++++++++++ 7 files changed, 316 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index a9cc657..4fece10 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -181,6 +181,12 @@ URL content. Package, audit, and Docker lookups use the configured catalog from `data/ports.txt` and has a fixed bounded response; it has no `max_length` setting. +Package metadata and OSV audit commands first use exact package/module names. +When an exact lookup fails, they perform a bounded best-effort search through +the relevant public package index and return suggestions; fuzzy candidates are +not automatically audited. Go module metadata uses the Go proxy's canonical +version field, including for `gopkg.in/...` module paths. + `daily` provides `!daily` in channels. Each authenticated account can claim once per UTC calendar day, regardless of channel or network; users without an account tag are limited by network and nickname. Different users can each diff --git a/docs/plugins.md b/docs/plugins.md index 124fbdf..3855c89 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -112,7 +112,9 @@ requests. Use `!pkg go`, `!pkg npm`, or `!pkg pip` for current registry metadata, with an optional version for a specific release. `!package` is an alias. The plugin uses the public Go module proxy, npm registry, and PyPI endpoints, requires no -API keys, and bounds both request time and response length. +API keys, and bounds both request time and response length. The Go module proxy +response is handled using its canonical `Version` field, so paths such as +`gopkg.in/irc.v3` work correctly. Examples: @@ -120,10 +122,13 @@ Examples: !pkg go github.com/variablenix/GoBot !pkg npm lodash !pkg pip requests 2.32.3 +!pkg go irc ~~~ Responses include the registry version, a sanitized description when present, -and the canonical package page. `!package` is the only alias. +and the canonical package page. If an exact name is not found, GoBot performs a +bounded best-effort search and returns possible Go, npm, or PyPI matches with +links; use the suggested full name for metadata. `!package` is the only alias. ## Ports @@ -144,7 +149,9 @@ key is required. With no version, the request omits the OSV `version` field, then fetches the latest registry version and evaluates OSV affected ranges. With a version, it performs an exact OSV query. Severity comes from OSV's database-specific or severity fields, and fixed versions are shown when OSV -provides them. +provides them. If the package cannot be resolved exactly, GoBot returns a +bounded list of possible Go, npm, or PyPI package names; it does not audit all +fuzzy matches automatically. ## Docker Hub diff --git a/docs/security.md b/docs/security.md index ba466a6..3d0c29d 100644 --- a/docs/security.md +++ b/docs/security.md @@ -33,7 +33,8 @@ dependencies, and deployment configuration maintained. - The URL title plugin rejects loopback, private, link-local, multicast, and local host targets to reduce SSRF risk. - External HTTP lookups use timeouts and bound response sizes. Package, audit, - and Docker requests use fixed public provider hosts. + and Docker requests use fixed public provider hosts; package suggestions use + only the fixed public Go, npm, and PyPI index hosts. - The paste plugin's URL mode is different: it fetches a user-supplied HTTP or HTTPS URL from the bot host and follows redirects. Treat it as an outbound network capability. Only enable it where users are trusted and host/network diff --git a/plugins/audit.go b/plugins/audit.go index b976ae1..ee5ea0c 100644 --- a/plugins/audit.go +++ b/plugins/audit.go @@ -53,7 +53,7 @@ type auditVulnerability struct { func (p *Audit) Name() string { return "audit" } func (p *Audit) Commands() []string { return []string{"audit", "vuln", "osv"} } func (p *Audit) Help() string { - return "!audit [version] — discover known OSV vulnerabilities (aliases: !vuln, !osv)" + return "!audit [version] — discover known OSV vulnerabilities; failed exact names get fuzzy suggestions (aliases: !vuln, !osv)" } func (p *Audit) Init(c bot.PluginConfig, _ *storage.DB) error { p.cfg = c; return nil } @@ -92,11 +92,21 @@ func (p *Audit) Handle(b *bot.Bot, m bot.Message) bool { maxLength = configured } if version != "" { + if len(response.Vulns) == 0 { + if _, metadataErr := lookupPackageMetadata(ctx, ecosystem, parts[1], version); metadataErr == errPackageNotFound { + b.Send(m.ReplyTarget(), truncateRunes(formatAuditSuggestions(ctx, ecosystem, parts[1]), maxLength)) + return true + } + } b.Send(m.ReplyTarget(), truncateRunes(formatAuditExact(parts[1], version, response.Vulns, maxShown), maxLength)) return true } latest, err := lookupPackageMetadata(ctx, ecosystem, parts[1], "") if err != nil { + if err == errPackageNotFound { + b.Send(m.ReplyTarget(), truncateRunes(formatAuditSuggestions(ctx, ecosystem, parts[1]), maxLength)) + return true + } b.Send(m.ReplyTarget(), "[audit] latest package version could not be determined") return true } @@ -118,6 +128,23 @@ func (p *Audit) Handle(b *bot.Bot, m bot.Message) bool { return true } +func formatAuditSuggestions(ctx context.Context, ecosystem, query string) string { + query = cleanExternalText(query) + candidates, err := searchPackageCandidates(ctx, ecosystem, query) + if err != nil || len(candidates) == 0 { + return fmt.Sprintf("[audit] %s not found; use the full package/module name", query) + } + items := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + name := cleanExternalText(candidate.Name) + if candidate.Version != "" { + name += " " + cleanExternalText(candidate.Version) + } + items = append(items, name) + } + return fmt.Sprintf("[audit] no exact match for %s; possible packages: %s", query, strings.Join(items, "; ")) +} + func queryOSV(ctx context.Context, ecosystem, name, version string) (osvResponse, error) { request := map[string]interface{}{"package": map[string]string{"name": name, "ecosystem": ecosystem}} if version != "" { diff --git a/plugins/audit_test.go b/plugins/audit_test.go index 6dc0900..321e11d 100644 --- a/plugins/audit_test.go +++ b/plugins/audit_test.go @@ -41,3 +41,15 @@ func TestAuditFormatsCVESeverityFixedAndMore(t *testing.T) { t.Fatal("OSV range matching is incorrect") } } + +func TestFormatAuditSuggestionsUsesSearchResults(t *testing.T) { + old := apiHTTPClient + t.Cleanup(func() { apiHTTPClient = old }) + apiHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { + return newPluginResponse(http.StatusOK, `{"objects":[{"package":{"name":"libirc-client","version":"0.0.3"}}]}`), nil + })} + got := formatAuditSuggestions(t.Context(), "npm", "libirc") + if !strings.Contains(got, "possible packages: libirc-client 0.0.3") { + t.Fatalf("audit suggestion output = %q", got) + } +} diff --git a/plugins/pkg.go b/plugins/pkg.go index c770241..20aabdb 100644 --- a/plugins/pkg.go +++ b/plugins/pkg.go @@ -12,6 +12,7 @@ import ( "github.com/variablenix/GoBot/bot" "github.com/variablenix/GoBot/storage" + "golang.org/x/net/html" ) type Pkg struct{ cfg bot.PluginConfig } @@ -23,10 +24,17 @@ type packageMetadata struct { Description string } +type packageCandidate struct { + Name string + Version string + Description string + URL string +} + func (p *Pkg) Name() string { return "pkg" } func (p *Pkg) Commands() []string { return []string{"pkg", "package"} } func (p *Pkg) Help() string { - return "!pkg [version] — show package metadata (alias: !package)" + return "!pkg [version] — show package metadata; failed exact names get fuzzy suggestions (alias: !package)" } func (p *Pkg) Init(c bot.PluginConfig, _ *storage.DB) error { p.cfg = c; return nil } @@ -55,7 +63,7 @@ func (p *Pkg) Handle(b *bot.Bot, m bot.Message) bool { metadata, err := lookupPackageMetadata(ctx, ecosystem, parts[1], version) if err != nil { if err == errPackageNotFound { - b.Send(m.ReplyTarget(), fmt.Sprintf("[%s] %s not found", ecosystem, cleanExternalText(parts[1]))) + b.Send(m.ReplyTarget(), truncateRunes(formatPackageSuggestions(ctx, ecosystem, parts[1]), packageMaxLength(p.cfg))) } else { b.Send(m.ReplyTarget(), fmt.Sprintf("[%s] package lookup is temporarily unavailable", ecosystem)) } @@ -141,7 +149,7 @@ func lookupPackageMetadata(ctx context.Context, ecosystem, name, version string) metadata.Version, _ = info["version"].(string) metadata.Description, _ = info["summary"].(string) } else { - metadata.Version, _ = payload["version"].(string) + metadata.Version = packagePayloadString(payload, "version", "Version") metadata.Description, _ = payload["description"].(string) } if metadata.Version == "" { @@ -150,6 +158,193 @@ func lookupPackageMetadata(ctx context.Context, ecosystem, name, version string) return metadata, nil } +func packagePayloadString(payload map[string]interface{}, keys ...string) string { + for _, key := range keys { + if value, ok := payload[key].(string); ok && strings.TrimSpace(value) != "" { + return value + } + } + return "" +} + +func formatPackageSuggestions(ctx context.Context, ecosystem, query string) string { + query = cleanExternalText(query) + candidates, err := searchPackageCandidates(ctx, ecosystem, query) + if err != nil || len(candidates) == 0 { + return fmt.Sprintf("[%s] %s not found; use the full package/module name", ecosystem, query) + } + items := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + name := cleanExternalText(candidate.Name) + if candidate.Version != "" { + name += " " + cleanExternalText(candidate.Version) + } + if candidate.URL != "" { + name += " (" + cleanExternalText(candidate.URL) + ")" + } + items = append(items, name) + } + return fmt.Sprintf("[%s] no exact match for %s; possible matches: %s", ecosystem, query, strings.Join(items, "; ")) +} + +func searchPackageCandidates(ctx context.Context, ecosystem, query string) ([]packageCandidate, error) { + if query == "" || len([]rune(query)) > 240 { + return nil, errPackageNotFound + } + switch ecosystem { + case "Go": + return searchGoPackages(ctx, query) + case "npm": + return searchNPMPackages(ctx, query) + case "PyPI": + return searchPyPIPackages(ctx, query) + default: + return nil, errPackageNotFound + } +} + +func searchGoPackages(ctx context.Context, query string) ([]packageCandidate, error) { + endpoint := "https://pkg.go.dev/search?m=package&limit=5&q=" + url.QueryEscape(query) + body, err := getPackageResponse(ctx, endpoint, 2*1024*1024) + if err != nil { + return nil, err + } + doc, err := html.Parse(strings.NewReader(string(body))) + if err != nil { + return nil, err + } + candidates := make([]packageCandidate, 0, 5) + walkHTML(doc, func(node *html.Node) { + if len(candidates) >= 5 || node.Type != html.ElementNode || node.Data != "a" || !hasHTMLAttribute(node, "data-test-id", "snippet-title") { + return + } + href := htmlAttribute(node, "href") + if !strings.HasPrefix(href, "/") || strings.ContainsAny(href, "?#\r\n") { + return + } + module := strings.TrimPrefix(href, "/") + if module == "" { + return + } + candidates = append(candidates, packageCandidate{Name: module, URL: "https://pkg.go.dev/" + module}) + }) + return candidates, nil +} + +func searchNPMPackages(ctx context.Context, query string) ([]packageCandidate, error) { + endpoint := "https://registry.npmjs.org/-/v1/search?text=" + url.QueryEscape(query) + "&size=5" + body, err := getPackageResponse(ctx, endpoint, 2*1024*1024) + if err != nil { + return nil, err + } + var response struct { + Objects []struct { + Package struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Links struct { + NPM string `json:"npm"` + } `json:"links"` + } `json:"package"` + } `json:"objects"` + } + if err := json.Unmarshal(body, &response); err != nil { + return nil, err + } + candidates := make([]packageCandidate, 0, len(response.Objects)) + for _, object := range response.Objects { + if object.Package.Name == "" { + continue + } + link := object.Package.Links.NPM + if link == "" { + link = "https://www.npmjs.com/package/" + url.PathEscape(object.Package.Name) + } + candidates = append(candidates, packageCandidate{Name: object.Package.Name, Version: object.Package.Version, Description: object.Package.Description, URL: link}) + } + return candidates, nil +} + +func searchPyPIPackages(ctx context.Context, query string) ([]packageCandidate, error) { + endpoint := "https://pypi.org/search/?q=" + url.QueryEscape(query) + body, err := getPackageResponse(ctx, endpoint, 2*1024*1024) + if err != nil { + return nil, err + } + doc, err := html.Parse(strings.NewReader(string(body))) + if err != nil { + return nil, err + } + candidates := make([]packageCandidate, 0, 5) + walkHTML(doc, func(node *html.Node) { + if len(candidates) >= 5 || node.Type != html.ElementNode || node.Data != "a" || !hasHTMLClass(node, "package-snippet") { + return + } + href := htmlAttribute(node, "href") + prefix := "/project/" + if !strings.HasPrefix(href, prefix) { + return + } + name := strings.Trim(strings.TrimPrefix(href, prefix), "/") + if name == "" || strings.ContainsAny(name, "?#\r\n") { + return + } + candidates = append(candidates, packageCandidate{Name: name, URL: "https://pypi.org/project/" + name + "/"}) + }) + return candidates, nil +} + +func getPackageResponse(ctx context.Context, endpoint string, limit int64) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/json, text/html") + req.Header.Set("User-Agent", "GoBot/1.0 (IRC bot; package search)") + res, err := apiHTTPClient.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode < 200 || res.StatusCode >= 300 { + return nil, fmt.Errorf("package search returned HTTP %d", res.StatusCode) + } + return io.ReadAll(io.LimitReader(res.Body, limit)) +} + +func walkHTML(node *html.Node, visit func(*html.Node)) { + if node == nil { + return + } + visit(node) + for child := node.FirstChild; child != nil; child = child.NextSibling { + walkHTML(child, visit) + } +} + +func htmlAttribute(node *html.Node, key string) string { + for _, attribute := range node.Attr { + if attribute.Key == key { + return strings.TrimSpace(attribute.Val) + } + } + return "" +} + +func hasHTMLAttribute(node *html.Node, key, value string) bool { + return htmlAttribute(node, key) == value +} + +func hasHTMLClass(node *html.Node, class string) bool { + for _, value := range strings.Fields(htmlAttribute(node, "class")) { + if value == class { + return true + } + } + return false +} + func packagePath(value string) string { parts := strings.Split(strings.Trim(value, "/"), "/") for i := range parts { diff --git a/plugins/pkg_test.go b/plugins/pkg_test.go index b7a905a..c57b01d 100644 --- a/plugins/pkg_test.go +++ b/plugins/pkg_test.go @@ -26,3 +26,63 @@ func TestLookupPackageMetadataNPM(t *testing.T) { } } } + +func TestLookupPackageMetadataGoProxyVersion(t *testing.T) { + old := apiHTTPClient + t.Cleanup(func() { apiHTTPClient = old }) + apiHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { + if r.URL.String() != "https://proxy.golang.org/gopkg.in/irc.v3/@latest" { + t.Fatalf("unexpected Go module endpoint: %s", r.URL) + } + return newPluginResponse(http.StatusOK, `{"Version":"v3.1.4","Time":"2021-01-19T17:45:41Z"}`), nil + })} + metadata, err := lookupPackageMetadata(t.Context(), "Go", "gopkg.in/irc.v3", "") + if err != nil { + t.Fatal(err) + } + if metadata.Version != "v3.1.4" { + t.Fatalf("Go module version = %q, want v3.1.4", metadata.Version) + } +} + +func TestSearchNPMPackageCandidates(t *testing.T) { + old := apiHTTPClient + t.Cleanup(func() { apiHTTPClient = old }) + apiHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { + if r.URL.String() != "https://registry.npmjs.org/-/v1/search?text=libirc&size=5" { + t.Fatalf("unexpected npm search endpoint: %s", r.URL) + } + return newPluginResponse(http.StatusOK, `{"objects":[{"package":{"name":"libirc-client","version":"0.0.3","links":{"npm":"https://www.npmjs.com/package/libirc-client"}}}]}`), nil + })} + candidates, err := searchPackageCandidates(t.Context(), "npm", "libirc") + if err != nil || len(candidates) != 1 || candidates[0].Name != "libirc-client" { + t.Fatalf("npm candidates = %+v, %v", candidates, err) + } +} + +func TestSearchGoPackageCandidates(t *testing.T) { + old := apiHTTPClient + t.Cleanup(func() { apiHTTPClient = old }) + apiHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { + if r.URL.String() != "https://pkg.go.dev/search?m=package&limit=5&q=irc" { + t.Fatalf("unexpected Go search endpoint: %s", r.URL) + } + return newPluginResponse(http.StatusOK, `irc`), nil + })} + candidates, err := searchPackageCandidates(t.Context(), "Go", "irc") + if err != nil || len(candidates) != 1 || candidates[0].Name != "github.com/go-irc/irc" { + t.Fatalf("Go candidates = %+v, %v", candidates, err) + } +} + +func TestFormatPackageSuggestionsUsesSearchResults(t *testing.T) { + old := apiHTTPClient + t.Cleanup(func() { apiHTTPClient = old }) + apiHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) { + return newPluginResponse(http.StatusOK, `{"objects":[{"package":{"name":"libirc-client","version":"0.0.3"}}]}`), nil + })} + got := formatPackageSuggestions(t.Context(), "npm", "libirc") + if !strings.Contains(got, "possible matches: libirc-client 0.0.3") { + t.Fatalf("suggestion output = %q", got) + } +}