LCORE-5177: Vulnerability script refactoring, enhancement, and fixes

pyproject.toml

@@ -164,6 +164,7 @@ dev = [
     "pygal>=3.1.3",
     "pygaljs>=1.0.2",
     "pdm>=2.29.2",
+    "pandas>=2.2.3",
 ]
 ogxlibdev = [
     # To check OGX API provider dependecies:

scripts/vulnerability_report.py

@@ -22,6 +22,7 @@
                         GitHub repository.
   -r, --retrieve-issues
                         Retrieve issues
+  -a, --all-actions     Retrieve issues for all LCore repositories, generates graphs and reports
   -g, --generate-graphs
                         Generate graphs with vulnerabilities info
   -svg, --svg-output    Generate graphs in SVG format
@@ -43,12 +44,21 @@
 from typing import Any
 
 import matplotlib.pyplot as plt
+import pandas as pd
 import requests
 from dateutil import parser
 
 type DependabotAlert = dict[str, Any]
 type DependabotAlerts = list[DependabotAlert]
 
+DEFAULT_ORGANIZATION = "lightspeed-core"
+
+LCORE_REPOSITORIES = [
+    "lightspeed-stack",
+    "lightspeed-providers",
+    "rag-content",
+]
+
 
 def create_argument_parser() -> ArgumentParser:
     """Create and configure argument parser object.
@@ -75,14 +85,14 @@ def create_argument_parser() -> ArgumentParser:
         "--organization",
         default="",
         help="GitHub organization.",
-        required=True,
+        required=False,
     )
 
     cli_parser.add_argument(
         "--repository",
         default="",
         help="GitHub repository.",
-        required=True,
+        required=False,
     )
 
     cli_parser.add_argument(
@@ -93,6 +103,14 @@ def create_argument_parser() -> ArgumentParser:
         help="Retrieve issues",
     )
 
+    cli_parser.add_argument(
+        "-a",
+        "--all-actions",
+        action="store_true",
+        default=False,
+        help="Retrieve issues for all LCore repositories, generates graphs and reports",
+    )
+
     cli_parser.add_argument(
         "-g",
         "--generate-graphs",
@@ -150,11 +168,18 @@ def check_args(args: Namespace) -> None:
         ValueError: If graph generation is requested but neither SVG nor PNG output is selected.
 
     """
-    if args.generate_graphs:
+    if args.all_actions or args.generate_graphs:
         # when graphs should be generated, output format need to be specified too
         if not args.svg_output and not args.png_output:
             msg = "Graphs generation was requested: you need to select the PNG and/or SVG output."
             raise ValueError(msg)
+    if not args.all_actions:
+        if args.organization == "":
+            msg = "Organization name needs to be specified"
+            raise ValueError(msg)
+        if args.repository == "":
+            msg = "Repository name needs to be specified"
+            raise ValueError(msg)
 
 
 def dependabot_file_name(args: Namespace) -> str:
@@ -540,7 +565,7 @@ def generate_vuln_for_days_graph(
     _, ax = plt.subplots()
     data = stat["days"]["days"]
     ax.hist(data, bins=30, edgecolor="black")
-    ax.set_ylim(top=100)
+    ax.set_ylim(top=150)
     ax.set_xlabel("Days")
     ax.set_title("Fixed in day(s)")
     save_graph(prefix, "days", svg_output, png_output)
@@ -585,11 +610,39 @@ def generate_new_cve_dates_graph(
         png_output (bool): Whether to save the graph as PNG.
 
     """
-    _, ax = plt.subplots()
+    _, ax = plt.subplots(figsize=(14, 7))
+
     data = stat["dates"]
+    if not data:
+        save_graph(prefix, "timeline", svg_output, png_output)
+        return
+
     dates, counts = zip(*sorted(data.items(), key=lambda x: x[0]))
-    ax.plot(dates, counts)
+    ax.plot(dates, counts, label="CVEs per day")
     ax.set_title("New CVEs timeline")
+    cve_series = pd.Series(counts, index=pd.to_datetime(list(dates)))
+    moving_average = cve_series.rolling(window=7, min_periods=1, center=True).mean()
+    ax.plot(
+        moving_average.index,
+        moving_average.values,
+        label="7-day moving average",
+        linewidth=2,
+    )
+
+    # horizontal line with "total" average
+    avg = cve_series.mean()
+    ax.axhline(
+        avg,
+        color="red",
+        linestyle="--",
+        linewidth=1.5,
+        label=f"Day average ({avg:.2f})",
+    )
+
+    ax.legend()
+
+    plt.setp(ax.get_xticklabels(), rotation=45, ha="right")
+    plt.subplots_adjust(bottom=0.20)
     save_graph(prefix, "timeline", svg_output, png_output)
 
 
@@ -733,6 +786,11 @@ def retrieve_issues(organization: str, repository: str, dependabot_file: str) ->
         print("GITHUB_TOKEN environment variable must be set!")
         return 1
 
+    print("Retrieving issues:")
+    print(f"    organization: {organization}")
+    print(f"    repository:   {repository}")
+    print(f"    output file:  {dependabot_file}")
+
     url = f"https://api.github.com/repos/{organization}/{repository}/dependabot/alerts"
 
     headers = {
@@ -772,12 +830,59 @@ def retrieve_issues(organization: str, repository: str, dependabot_file: str) ->
             f.write("]\n")
             raise
 
-    print("Saved all alerts to dependabot_alerts.json")
+    print(f"Saved all alerts to {dependabot_file}")
 
     # everything's ok
     return 0
 
 
+def process_issues_for_repository(
+    organization: str,
+    repository: str,
+    dependabot_file: str,
+    svg_output: bool,
+    png_output: bool,
+    retrieve_issues_operation: bool = True,
+    generate_graphs_operation: bool = True,
+    generate_page_operation: bool = True,
+) -> int:
+    """Retrieve alerts or process the local file and generate selected outputs.
+
+    Parameters:
+        organization (str): name of GitHub organization
+        repository (str): name of GitHub repository
+        dependabot_file (str): file to store Dependabot issues
+        svg_output (bool): if set to true, CVE statistic is exported into SVG graphs
+        png_output (bool): if set to true, CVE statistic is exported into PNG graphs
+        retrieve_issues_operation (bool): enable retrieving Dependabot issues
+        generate_graphs_operation (bool): enable generating graphs in SVG or PNG format
+        generate_page_operation (bool): enable generating page with overall status
+
+    Retrieval takes precedence over report generation and returns its status.
+    Otherwise return 0 after processing; file, data, and plotting errors propagate.
+    """
+    prefix = repository
+    result = 0
+    if retrieve_issues_operation:
+        # retrieve issues
+        result += retrieve_issues(organization, repository, dependabot_file)
+
+    if result > 0:
+        print("Error retrieving issues")
+        print("Skipping graphs and pages generation")
+        return 1
+
+    stat = process_dependabot_file(dependabot_file)
+
+    if generate_graphs_operation:
+        generate_graphs(stat, prefix, svg_output, png_output)
+
+    if generate_page_operation:
+        generate_page(stat, prefix, organization, repository)
+
+    return 0
+
+
 def main() -> int:
     """
     CLI entry point that retrieves Dependabot issues and produces Vulnerability report.
@@ -809,22 +914,34 @@ def main() -> int:
     cli_parser = create_argument_parser()
     args = cli_parser.parse_args()
     check_args(args)
-    dependabot_file = dependabot_file_name(args)
-    prefix = args.repository
-
-    if args.retrieve_issues:
-        # just retrieve issues, no other operation will be performed
-        return retrieve_issues(args.organization, args.repository, dependabot_file)
-
-    stat = process_dependabot_file(dependabot_file)
-
-    if args.generate_graphs:
-        generate_graphs(stat, prefix, args.svg_output, args.png_output)
 
-    if args.generate_page:
-        generate_page(stat, prefix, args.organization, args.repository)
+    if args.all_actions:
+        errors = 0
+        for repository in LCORE_REPOSITORIES:
+            organization = DEFAULT_ORGANIZATION
+            dependabot_file = f"{organization}__{repository}.json"
+            errors += process_issues_for_repository(
+                organization,
+                repository,
+                dependabot_file,
+                args.svg_output,
+                args.png_output,
+            )
+        return 1 if errors > 0 else 0
+
+    # processing for one selected repository
+    dependabot_file = dependabot_file_name(args)
 
-    return 0
+    return process_issues_for_repository(
+        args.organization,
+        args.repository,
+        dependabot_file,
+        args.svg_output,
+        args.png_output,
+        args.retrieve_issues,
+        args.generate_graphs,
+        args.generate_page,
+    )
 
 
 if __name__ == "__main__":