diff --git a/pyproject.toml b/pyproject.toml index 01a53ea07..efc9ca9bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -164,6 +164,7 @@ dev = [ "pygal>=3.1.3", "pygaljs>=1.0.2", "pdm>=2.29.2", + "pandas>=2.2.3", ] ogxlibdev = [ # To check OGX API provider dependecies: diff --git a/scripts/vulnerability_report.py b/scripts/vulnerability_report.py index 1026a7ae1..4cc0d35db 100644 --- a/scripts/vulnerability_report.py +++ b/scripts/vulnerability_report.py @@ -22,6 +22,7 @@ GitHub repository. -r, --retrieve-issues Retrieve issues + -a, --all-actions Retrieve issues for all LCore repositories, generates graphs and reports -g, --generate-graphs Generate graphs with vulnerabilities info -svg, --svg-output Generate graphs in SVG format @@ -43,12 +44,21 @@ from typing import Any import matplotlib.pyplot as plt +import pandas as pd import requests from dateutil import parser type DependabotAlert = dict[str, Any] type DependabotAlerts = list[DependabotAlert] +DEFAULT_ORGANIZATION = "lightspeed-core" + +LCORE_REPOSITORIES = [ + "lightspeed-stack", + "lightspeed-providers", + "rag-content", +] + def create_argument_parser() -> ArgumentParser: """Create and configure argument parser object. @@ -75,14 +85,14 @@ def create_argument_parser() -> ArgumentParser: "--organization", default="", help="GitHub organization.", - required=True, + required=False, ) cli_parser.add_argument( "--repository", default="", help="GitHub repository.", - required=True, + required=False, ) cli_parser.add_argument( @@ -93,6 +103,14 @@ def create_argument_parser() -> ArgumentParser: help="Retrieve issues", ) + cli_parser.add_argument( + "-a", + "--all-actions", + action="store_true", + default=False, + help="Retrieve issues for all LCore repositories, generates graphs and reports", + ) + cli_parser.add_argument( "-g", "--generate-graphs", @@ -150,11 +168,18 @@ def check_args(args: Namespace) -> None: ValueError: If graph generation is requested but neither SVG nor PNG output is selected. """ - if args.generate_graphs: + if args.all_actions or args.generate_graphs: # when graphs should be generated, output format need to be specified too if not args.svg_output and not args.png_output: msg = "Graphs generation was requested: you need to select the PNG and/or SVG output." raise ValueError(msg) + if not args.all_actions: + if args.organization == "": + msg = "Organization name needs to be specified" + raise ValueError(msg) + if args.repository == "": + msg = "Repository name needs to be specified" + raise ValueError(msg) def dependabot_file_name(args: Namespace) -> str: @@ -540,7 +565,7 @@ def generate_vuln_for_days_graph( _, ax = plt.subplots() data = stat["days"]["days"] ax.hist(data, bins=30, edgecolor="black") - ax.set_ylim(top=100) + ax.set_ylim(top=150) ax.set_xlabel("Days") ax.set_title("Fixed in day(s)") save_graph(prefix, "days", svg_output, png_output) @@ -585,11 +610,39 @@ def generate_new_cve_dates_graph( png_output (bool): Whether to save the graph as PNG. """ - _, ax = plt.subplots() + _, ax = plt.subplots(figsize=(14, 7)) + data = stat["dates"] + if not data: + save_graph(prefix, "timeline", svg_output, png_output) + return + dates, counts = zip(*sorted(data.items(), key=lambda x: x[0])) - ax.plot(dates, counts) + ax.plot(dates, counts, label="CVEs per day") ax.set_title("New CVEs timeline") + cve_series = pd.Series(counts, index=pd.to_datetime(list(dates))) + moving_average = cve_series.rolling(window=7, min_periods=1, center=True).mean() + ax.plot( + moving_average.index, + moving_average.values, + label="7-day moving average", + linewidth=2, + ) + + # horizontal line with "total" average + avg = cve_series.mean() + ax.axhline( + avg, + color="red", + linestyle="--", + linewidth=1.5, + label=f"Day average ({avg:.2f})", + ) + + ax.legend() + + plt.setp(ax.get_xticklabels(), rotation=45, ha="right") + plt.subplots_adjust(bottom=0.20) save_graph(prefix, "timeline", svg_output, png_output) @@ -733,6 +786,11 @@ def retrieve_issues(organization: str, repository: str, dependabot_file: str) -> print("GITHUB_TOKEN environment variable must be set!") return 1 + print("Retrieving issues:") + print(f" organization: {organization}") + print(f" repository: {repository}") + print(f" output file: {dependabot_file}") + url = f"https://api.github.com/repos/{organization}/{repository}/dependabot/alerts" headers = { @@ -772,12 +830,59 @@ def retrieve_issues(organization: str, repository: str, dependabot_file: str) -> f.write("]\n") raise - print("Saved all alerts to dependabot_alerts.json") + print(f"Saved all alerts to {dependabot_file}") # everything's ok return 0 +def process_issues_for_repository( + organization: str, + repository: str, + dependabot_file: str, + svg_output: bool, + png_output: bool, + retrieve_issues_operation: bool = True, + generate_graphs_operation: bool = True, + generate_page_operation: bool = True, +) -> int: + """Retrieve alerts or process the local file and generate selected outputs. + + Parameters: + organization (str): name of GitHub organization + repository (str): name of GitHub repository + dependabot_file (str): file to store Dependabot issues + svg_output (bool): if set to true, CVE statistic is exported into SVG graphs + png_output (bool): if set to true, CVE statistic is exported into PNG graphs + retrieve_issues_operation (bool): enable retrieving Dependabot issues + generate_graphs_operation (bool): enable generating graphs in SVG or PNG format + generate_page_operation (bool): enable generating page with overall status + + Retrieval takes precedence over report generation and returns its status. + Otherwise return 0 after processing; file, data, and plotting errors propagate. + """ + prefix = repository + result = 0 + if retrieve_issues_operation: + # retrieve issues + result += retrieve_issues(organization, repository, dependabot_file) + + if result > 0: + print("Error retrieving issues") + print("Skipping graphs and pages generation") + return 1 + + stat = process_dependabot_file(dependabot_file) + + if generate_graphs_operation: + generate_graphs(stat, prefix, svg_output, png_output) + + if generate_page_operation: + generate_page(stat, prefix, organization, repository) + + return 0 + + def main() -> int: """ CLI entry point that retrieves Dependabot issues and produces Vulnerability report. @@ -809,22 +914,34 @@ def main() -> int: cli_parser = create_argument_parser() args = cli_parser.parse_args() check_args(args) - dependabot_file = dependabot_file_name(args) - prefix = args.repository - - if args.retrieve_issues: - # just retrieve issues, no other operation will be performed - return retrieve_issues(args.organization, args.repository, dependabot_file) - - stat = process_dependabot_file(dependabot_file) - - if args.generate_graphs: - generate_graphs(stat, prefix, args.svg_output, args.png_output) - if args.generate_page: - generate_page(stat, prefix, args.organization, args.repository) + if args.all_actions: + errors = 0 + for repository in LCORE_REPOSITORIES: + organization = DEFAULT_ORGANIZATION + dependabot_file = f"{organization}__{repository}.json" + errors += process_issues_for_repository( + organization, + repository, + dependabot_file, + args.svg_output, + args.png_output, + ) + return 1 if errors > 0 else 0 + + # processing for one selected repository + dependabot_file = dependabot_file_name(args) - return 0 + return process_issues_for_repository( + args.organization, + args.repository, + dependabot_file, + args.svg_output, + args.png_output, + args.retrieve_issues, + args.generate_graphs, + args.generate_page, + ) if __name__ == "__main__": diff --git a/uv.lock b/uv.lock index 4573cff11..e9594cb8b 100644 --- a/uv.lock +++ b/uv.lock @@ -2465,6 +2465,7 @@ dev = [ { name = "memray" }, { name = "mypy" }, { name = "openapi-to-md" }, + { name = "pandas" }, { name = "pdm" }, { name = "pip" }, { name = "pybuild-deps" }, @@ -2588,6 +2589,7 @@ dev = [ { name = "memray", specifier = ">=1.13.0" }, { name = "mypy", specifier = ">=2.0.0" }, { name = "openapi-to-md", specifier = ">=0.1.0b2" }, + { name = "pandas", specifier = ">=2.2.3" }, { name = "pdm", specifier = ">=2.29.2" }, { name = "pip", specifier = "==26.1" }, { name = "pybuild-deps", specifier = ">=0.5.0" },