Improve logging configuration and failure diagnosability

core/engine.py

@@ -126,8 +126,9 @@ def test_permissions(
                 logs = "Both Reader and Cost Management Reader roles validation failed."
 
         except ClientAuthenticationError as e:
+            # Expected outcome of a permission test; surfaced to the user via `logs`.
             logs = f"Azure credentials validation failed: {str(e)}"
-            logger.error(logs)
+            logger.warning(logs)
         except Exception as e:
             logs = f"Azure permission test failed: {str(e)}"
             logger.error(logs)
@@ -231,8 +232,9 @@ def test_permissions(
                         logs += f" Details: {details}"
 
         except NoCredentialsError as e:
+            # Expected outcome of a permission test; surfaced to the user via `logs`.
             logs = f"AWS credentials validation failed: {str(e)}"
-            logger.error(logs)
+            logger.warning(logs)
         except Exception as e:
             logs = f"AWS permission test failed: {str(e)}"
             logger.error(logs)
@@ -321,7 +323,12 @@ def sync_assessment(
     )
 
     if not result.get("success"):
-        raise RuntimeError(f"Assessment sync failed: {result.get('logs')}")
+        return {
+            "success": False,
+            "online": True,
+            "payload": None,
+            "logs": f"Assessment sync failed: {result.get('logs')}",
+        }
 
     logger.debug(result)
 
@@ -353,7 +360,12 @@ def sync_assessment(
 
     except Exception as e:
         logger.error("Error saving server risks to local DB: %s", str(e), exc_info=True)
-        raise RuntimeError(f"Failed to store server risks: {str(e)}")
+        return {
+            "success": False,
+            "online": True,
+            "payload": None,
+            "logs": f"Failed to store server risks: {str(e)}",
+        }
 
     try:
         scoring = payload.get("scoring_data")
@@ -378,7 +390,12 @@ def sync_assessment(
 
     except Exception as e:
         logger.error("Error saving scoring data to local DB: %s", str(e), exc_info=True)
-        raise RuntimeError(f"Failed to store scoring data: {str(e)}")
+        return {
+            "success": False,
+            "online": True,
+            "payload": None,
+            "logs": f"Failed to store scoring data: {str(e)}",
+        }
 
     return result
 

core/utils_aws.py

@@ -126,8 +126,19 @@ def build_aws_resource_inventory(
                     }
                 )
 
-            except Exception:
-                # logger.error(f"Error while processing {service_name}", exc_info=True)
+            except Exception as exc:
+                # Expected for services the caller can't access or that aren't
+                # available in a region. Keep at DEBUG (run.log only) so it never
+                # floods the console, while still distinguishing a failed service
+                # from an empty one. The message alone is the useful signal; the
+                # botocore stack is noise, so no exc_info here.
+                logger.debug(
+                    "Error processing %s.%s in %s: %s",
+                    service_name,
+                    operation_name,
+                    region,
+                    exc,
+                )
                 continue
 
         # Save raw data to a JSON file

core/utils_azure.py

@@ -35,7 +35,9 @@ def is_resource_inventory_empty(
             # logger.info("Resources found in the resource group.")
             return False
     except AzureError as e:
-        logger.error(
+        # Re-raised to build_azure_resource_inventory, which logs at ERROR with
+        # the traceback; keep this at DEBUG to avoid double-logging.
+        logger.debug(
             f"Error checking Azure resource inventory: {str(e)}", exc_info=True
         )
         raise

core/utils_db.py

@@ -2,9 +2,8 @@
 import sqlite3
 import logging
 
-# Configure logger for database operations
+# Configure logger for database operations (level left to the root handlers)
 logger = logging.getLogger("core.engine.db")
-logger.setLevel(logging.INFO)
 
 # Default master database
 MASTER_DATABASE = "datasets/data.db"
@@ -28,7 +27,7 @@ def connect(db_path=MASTER_DATABASE):
         conn = sqlite3.connect(db_path)
         return conn
     except sqlite3.Error as e:
-        logger.error(f"Error connecting to database: {e}")
+        logger.debug(f"Error connecting to database: {e}")
         raise
 
 
@@ -44,7 +43,7 @@ def load_data(table_name, db_path=MASTER_DATABASE):
         conn.close()
         return [dict(zip(columns, row)) for row in rows]
     except sqlite3.Error as e:
-        logger.error(f"Error loading data from table '{table_name}': {e}")
+        logger.debug(f"Error loading data from table '{table_name}': {e}")
         raise
 
 
@@ -58,7 +57,7 @@ def execute_query(query, params=None, db_path=MASTER_DATABASE):
         conn.close()
         return rowcount
     except sqlite3.Error as e:
-        logger.error(f"Error executing query: {e}")
+        logger.debug(f"Error executing query: {e}")
         raise
 
 
@@ -72,7 +71,7 @@ def fetch_one(query, params=None, db_path=MASTER_DATABASE):
         conn.close()
         return dict(zip(columns, row)) if row else None
     except sqlite3.Error as e:
-        logger.error(f"Error fetching data: {e}")
+        logger.debug(f"Error fetching data: {e}")
         raise
 
 
@@ -86,5 +85,5 @@ def fetch_all(query, params=None, db_path=MASTER_DATABASE):
         conn.close()
         return [dict(zip(columns, row)) for row in rows]
     except sqlite3.Error as e:
-        logger.error(f"Error fetching data: {e}")
+        logger.debug(f"Error fetching data: {e}")
         raise

core/utils_report.py

@@ -47,7 +47,6 @@
 
 # Configure logger
 logger = logging.getLogger("core.engine.report")
-logger.setLevel(logging.INFO)
 
 
 def anonymize_string(s: str, num_visible: int = 4) -> str:

core/utils_report_egress.py

@@ -36,7 +36,6 @@
 PDF_HEADER_TITLE = "EscapeCloud Community Edition - Data & Egress"
 
 logger = logging.getLogger("core.engine.report_egress")
-logger.setLevel(logging.INFO)
 
 DEFAULT_PRICING_ZONE = "zone1"
 UNIT_DIVISORS = {"GB": 10**9, "GiB": 2**30}

core/utils_report_html.py

@@ -10,7 +10,6 @@
 
 # Configure logger
 logger = logging.getLogger("core.engine.report_html")
-logger.setLevel(logging.INFO)
 
 
 def transform_cost_inventory_for_html(

core/utils_report_json.py

@@ -11,7 +11,6 @@
 
 # Configure logger
 logger = logging.getLogger("core.engine.report_json")
-logger.setLevel(logging.INFO)
 
 
 def transform_resource_inventory_for_json(

core/utils_report_pdf.py

@@ -26,7 +26,6 @@
 
 # Configure logger
 logger = logging.getLogger("core.engine.report_pdf")
-logger.setLevel(logging.INFO)
 
 
 def transform_resource_inventory_for_pdf(

core/utils_sync.py

@@ -14,7 +14,6 @@
 
 # Configure logger
 logger = logging.getLogger("core.engine.sync")
-logger.setLevel(logging.INFO)
 
 _ASSESS_PATH = "/api/v1/assessments/"
 

main.py

@@ -6,7 +6,9 @@
 import sys
 import os
 import getpass
+import traceback
 from rich.console import Console
+from rich.logging import RichHandler
 from datetime import datetime
 from botocore.exceptions import NoCredentialsError, ProfileNotFound
 from azure.identity import DefaultAzureCredential, ClientSecretCredential
@@ -53,20 +55,64 @@
 from utils import codes
 from utils.version import __version__
 
-# Configure the root logger to ensure logs propagate from all modules
-logging.basicConfig(
-    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-)
-logging.getLogger("botocore").setLevel(logging.WARNING)
-logging.getLogger("boto3").setLevel(logging.WARNING)
-
-# Configure the logger
+# Configure the logger (level is left to the handlers, see configure_logging)
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 # Initialize the console object
 console = Console()
 
+# Third-party loggers kept quiet unless the user opts into deep verbosity (-vv)
+_THIRD_PARTY_NOISY = ("botocore", "boto3", "azure")
+
+# Loggers with no diagnostic value ever — pinned to WARNING regardless of verbosity,
+#  so they never flood the -vv console or run.log.
+_ALWAYS_QUIET = ("PIL",)
+
+
+def configure_logging(verbose: int = 0) -> None:
+    """
+    verbose == 0 -> WARNING (default; the Rich step UI is the primary output)
+    verbose == 1 -> INFO    (--verbose: show our interaction narrative)
+    verbose >= 2 -> DEBUG   (-vv: also un-mute third-party libraries)
+    """
+    root = logging.getLogger()
+    root.setLevel(logging.DEBUG)
+    root.handlers.clear()
+
+    if verbose == 0:
+        console_level = logging.WARNING
+    elif verbose == 1:
+        console_level = logging.INFO
+    else:
+        console_level = logging.DEBUG
+
+    console_handler = RichHandler(
+        console=console, show_path=False, rich_tracebacks=True
+    )
+    console_handler.setLevel(console_level)
+    root.addHandler(console_handler)
+
+    third_party_level = logging.DEBUG if verbose >= 2 else logging.WARNING
+    for name in _THIRD_PARTY_NOISY:
+        logging.getLogger(name).setLevel(third_party_level)
+
+    for name in _ALWAYS_QUIET:
+        logging.getLogger(name).setLevel(logging.WARNING)
+
+
+def add_run_log_handler(report_path: str) -> None:
+    # A logging-setup failure must never abort the assessment, so degrade
+    # gracefully if run.log cannot be created (e.g. dir missing or read-only).
+    try:
+        file_handler = logging.FileHandler(os.path.join(report_path, "run.log"))
+        file_handler.setLevel(logging.DEBUG)
+        file_handler.setFormatter(
+            logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+        )
+        logging.getLogger().addHandler(file_handler)
+    except OSError as exc:
+        logger.warning("Could not create run.log in %s: %s", report_path, exc)
+
 
 class ConfigError(Exception):
     pass
@@ -419,6 +465,10 @@ def run_assessment(
     # Record the assessment start time to propagate across stages
     started_at = int(time.time())
 
+    # Bound up front so the error handler can reference it even if the crash
+    # happens before the report directory is created.
+    report_path = None
+
     try:
         # Preliminary Stage: Validate configuration & create directory
         console.print("-------------------------------------------")
@@ -448,6 +498,7 @@ def run_assessment(
         # Create directories
         try:
             report_path, raw_data_path = create_directory()
+            add_run_log_handler(report_path)
             print_step("Directory successfully created.", status="ok")
         except RuntimeError as e:
             print_step("Directory creation failed.", status="error", logs=str(e))
@@ -756,6 +807,20 @@ def run_assessment(
 
     except Exception as e:
         console.print(f"[red]Unexpected error: {e}[/red]")
+        # Persist the full traceback so a one-line error is actionable. Falls
+        # back to the cwd when the crash happened before the report dir existed.
+        target_dir = report_path or os.getcwd()
+        try:
+            log_file = os.path.join(target_dir, f"error-{int(time.time())}.log")
+            with open(log_file, "w", encoding="utf-8") as fh:
+                fh.write(traceback.format_exc())
+            console.print(f"[yellow]Full traceback written to: {log_file}[/yellow]")
+        except OSError:
+            console.print(
+                "[yellow]Could not write a traceback file; see run.log.[/yellow]"
+            )
+        # Also funnel to run.log at DEBUG (kept off the default console).
+        logger.debug("Unexpected error", exc_info=True)
         sys.exit(codes.UNEXPECTED)
 
 
@@ -791,8 +856,20 @@ def parse_arguments():
         dest="cloud_provider", help="Specify the cloud provider (aws or azure)."
     )
 
+    # Shared options available on every subcommand (e.g. `aws --verbose`)
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="Increase log verbosity (-v for INFO, -vv for DEBUG + third-party).",
+    )
+
     # Subparser for AWS
-    aws_parser = subparsers.add_parser("aws", help="Perform an AWS assessment.")
+    aws_parser = subparsers.add_parser(
+        "aws", parents=[common], help="Perform an AWS assessment."
+    )
     aws_group = aws_parser.add_mutually_exclusive_group(required=False)
     aws_group.add_argument(
         "--config", type=str, help="Path to the configuration file (JSON format)."
@@ -827,7 +904,9 @@ def parse_arguments():
     )
 
     # Subparser for Azure
-    azure_parser = subparsers.add_parser("azure", help="Perform an Azure assessment.")
+    azure_parser = subparsers.add_parser(
+        "azure", parents=[common], help="Perform an Azure assessment."
+    )
     azure_group = azure_parser.add_mutually_exclusive_group(required=False)
     azure_group.add_argument(
         "--config", type=str, help="Path to the configuration file (JSON format)."
@@ -870,6 +949,9 @@ def main():
     # (ASCII art, dataset download).
     args = parse_arguments()
 
+    # Configure logging before any side effects so verbosity applies everywhere.
+    configure_logging(getattr(args, "verbose", 0))
+
     # Print ASCII art
     console.print(ascii_art, style="bold cyan")