Improve error reporting

Cargo.toml

@@ -14,7 +14,7 @@ hakari-package = "workspace-hack"
 
 [workspace.package]
 edition = "2024"
-version = "0.36.0"
+version = "0.37.0"
 
 [workspace.dependencies]
 anyhow = { version = "1.0.102" }

libs/opsqueue_python/src/consumer.rs

@@ -1,3 +1,4 @@
+use opsqueue::tracing::as_dyn_error;
 use std::future::IntoFuture;
 use std::sync::Arc;
 use std::time::Duration;
@@ -199,13 +200,16 @@ impl ConsumerClient {
                     }
                     // In essence we 'catch `Exception` (but _not_ `BaseException` here)
                     Err(CError(L(e))) => {
-                        tracing::info!("Opsqueue consumer closing because of exception: {e:?}");
+                        tracing::info!(
+                            error = as_dyn_error(&e),
+                            "Opsqueue consumer closing because of exception"
+                        );
                         return CError(L(e));
                     }
                     Err(CError(R(err))) => {
                         tracing::warn!(
-                            "Opsqueue consumer encountered a Rust error, but will continue: {}",
-                            err
+                            error = as_dyn_error(&err),
+                            "Opsqueue consumer encountered a Rust error, but will continue",
                         );
                     }
                 }

libs/opsqueue_python/src/errors.rs

@@ -70,7 +70,7 @@ pub type CPyResult<T, E> = Result<T, CError<E>>;
 /// allowing things like `KeyboardInterrupt`, `SystemExit` or `MemoryError`,
 /// to trigger cleanup-and-exit.
 #[derive(thiserror::Error, Debug)]
-#[error("Fatal Python exception: {0}")]
+#[error("Fatal Python exception")]
 pub struct FatalPythonException(#[from] pub PyErr);
 
 impl From<CError<FatalPythonException>> for PyErr {

opsqueue/app/main.rs

@@ -4,9 +4,9 @@ use opentelemetry_otlp::SpanExporter;
 use opentelemetry_resource_detectors::HostResourceDetector;
 use opentelemetry_resource_detectors::{OsResourceDetector, ProcessResourceDetector};
 use opentelemetry_sdk::trace::{RandomIdGenerator, Sampler, SdkTracerProvider};
+use opsqueue::tracing::as_dyn_error;
 use opsqueue::{common::submission::db::periodically_cleanup_old, config::Config, prometheus};
 use std::{
-    error::Error,
     sync::{Arc, atomic::AtomicBool},
     time::Duration,
 };
@@ -104,7 +104,7 @@ pub async fn async_main() {
             res = tokio::signal::ctrl_c() => {
                 match res {
                     Ok(()) => tracing::warn!("Received Ctrl-C signal"),
-                    Err(ref err) => tracing::error!(error = err as &dyn Error, "Error while waiting for Ctrl-C signal"),
+                    Err(ref err) => tracing::error!(error = as_dyn_error(err), "Error while waiting for Ctrl-C signal"),
                 }
             },
         };
@@ -138,6 +138,7 @@ fn init_sentry() -> sentry::ClientInitGuard {
         traces_sample_rate: 0.0,
         send_default_pii: true,
         release: sentry::release_name!(),
+        in_app_include: vec!["opsqueue::"],
         ..Default::default()
     };
     sentry::init(options)

opsqueue/src/common/errors.rs

@@ -16,11 +16,8 @@ use super::{
     submission::{SubmissionCancelled, SubmissionCompleted, SubmissionFailed, SubmissionId},
 };
 
-// #[derive(Error, Debug, Clone, Serialize, Deserialize)]
-// #[error("Low-level database error: {0:?}")]
-// pub struct DatabaseError(#[from] pub serde_error::Error);
 #[cfg_attr(feature = "server-logic", derive(Error, Debug))]
-#[cfg_attr(feature = "server-logic", error("Low-level database error: {0:?}"))]
+#[cfg_attr(feature = "server-logic", error("Low-level database error"))]
 #[cfg(feature = "server-logic")]
 pub struct DatabaseError(#[from] pub sqlx::Error);
 
@@ -136,7 +133,7 @@ impl<L, R1, R2> From<E<R1, R2>> for E<L, E<R1, R2>> {
 }
 
 #[derive(Error, Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
-#[error("You are using Opsqueue incorrectly. Details: {0}")]
+#[error("You are using Opsqueue incorrectly")]
 pub struct IncorrectUsage<E>(#[from] pub E);
 
 #[derive(Error, Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]

opsqueue/src/common/mod.rs

@@ -51,18 +51,30 @@ impl std::fmt::Display for MaxSubmissions {
     }
 }
 
+impl std::str::FromStr for MaxSubmissions {
+    type Err = ParseMaxSubmissionsError;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let value: NonZero<u64> = s.parse()?;
+        Ok(MaxSubmissions::new(value)?)
+    }
+}
+
 #[derive(Debug, thiserror::Error)]
 pub enum ParseMaxSubmissionsError {
     #[error(transparent)]
-    NotANumber(#[from] std::num::ParseIntError),
+    NotANumber(std::num::ParseIntError),
     #[error(transparent)]
-    TooLarge(#[from] MaxSubmissionsTooLarge),
+    TooLarge(MaxSubmissionsTooLarge),
 }
 
-impl std::str::FromStr for MaxSubmissions {
-    type Err = ParseMaxSubmissionsError;
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let value: NonZero<u64> = s.parse()?;
-        Ok(MaxSubmissions::new(value)?)
+impl From<std::num::ParseIntError> for ParseMaxSubmissionsError {
+    fn from(value: std::num::ParseIntError) -> Self {
+        Self::NotANumber(value)
+    }
+}
+
+impl From<MaxSubmissionsTooLarge> for ParseMaxSubmissionsError {
+    fn from(value: MaxSubmissionsTooLarge) -> Self {
+        Self::TooLarge(value)
     }
 }

opsqueue/src/common/submission.rs

@@ -276,6 +276,7 @@ impl Submission {
 
 #[cfg(feature = "server-logic")]
 pub mod db {
+    use crate::tracing::as_dyn_error;
     use crate::{
         common::{
             MaxSubmissions, StrategicMetadataMap,
@@ -1179,7 +1180,7 @@ pub mod db {
             }
             .await;
             if let Err(e) = res {
-                tracing::error!("Error during periodic cleanup: {}", e);
+                tracing::error!(error = as_dyn_error(&e), "Error during periodic cleanup");
             }
             tokio::time::sleep(PERIODIC_CLEANUP_INTERVAL).await;
         }

opsqueue/src/consumer/client.rs

@@ -1,6 +1,6 @@
+use crate::tracing::{anyhow_as_dyn_error, as_dyn_error};
 use std::{
     collections::HashMap,
-    error::Error,
     str::FromStr,
     sync::{
         Arc,
@@ -160,10 +160,15 @@ impl OuterClient {
     async fn initialize(&self) -> Client {
         tracing::info!("Initializing (or re-initializing) consumer client connection...");
         (|| Client::new(&self.1))
-        .retry(retry_policy())
-        .notify(|err, duration| { tracing::debug!("Error establishing consumer client WS connection. (Will retry in {duration:?}). Details: {err:?}") })
-        .await
-        .expect("Infinite retries should never return Err")
+            .retry(retry_policy())
+            .notify(|err, duration| {
+                tracing::debug!(
+                    error = anyhow_as_dyn_error(err),
+                    "Error establishing consumer client WS connection. (Will retry in {duration:?})"
+                );
+            })
+            .await
+            .expect("Infinite retries should never return Err")
     }
 
     /// When `false` is returned, the next call to the client will attempt to restore the connection.
@@ -356,7 +361,7 @@ impl Client {
                             break;
                         }
                         Some(Err(e)) => {
-                            tracing::error!("Opsqueue consumer client background task closing, reason: {e}");
+                            tracing::error!(error = as_dyn_error(&e), "Opsqueue consumer client background task closing, reason");
                             break;
                         },
                         Some(Ok(msg)) => {
@@ -389,7 +394,7 @@ impl Client {
                                                             failure: "Requester disappeared".into(),
                                                         },
                                                     ).await else { continue; };
-                                                    tracing::error!(error = &internal_error as &dyn Error, chunk_id = ?chunk_id, "Failed to return unhandled reservation to server");
+                                                    tracing::error!(error = as_dyn_error(&internal_error), chunk_id = ?chunk_id, "Failed to return unhandled reservation to server");
                                                 }
                                             },
                                         }
@@ -416,7 +421,7 @@ impl Client {
                                                         failure: "Reservation expired".into(),
                                                     },
                                                 ).await else { continue; };
-                                                tracing::error!(error = &internal_error as &dyn Error, chunk_id = ?chunk_id, "Failed to return expired reservation to server");
+                                                tracing::error!(error = as_dyn_error(&internal_error), chunk_id = ?chunk_id, "Failed to return expired reservation to server");
                                             },
                                         }
                                     }
@@ -519,10 +524,10 @@ impl Client {
 
 #[derive(thiserror::Error, Debug)]
 pub enum InternalConsumerClientError {
-    #[error("Low-level error in the websocket connection: {0}")]
+    #[error("Low-level error in the websocket connection")]
     LowLevelWebsocketError(#[from] tokio_tungstenite::tungstenite::Error),
     #[error(
-        "The oneshot channel to receive a sync response to an earlier request was dropped before a response was received: {0}"
+        "The oneshot channel to receive a sync response to an earlier request was dropped before a response was received"
     )]
     OneshotSenderDropped(#[from] RecvError),
     #[error("Expected the sync response of kind {expected} but received {actual:?}")]

opsqueue/src/consumer/server/conn.rs

@@ -1,3 +1,4 @@
+use crate::tracing::as_dyn_error;
 use std::{
     sync::{
         Arc,
@@ -216,7 +217,7 @@ impl ConsumerConn {
                     .await;
                 match chunks_or_err {
                     Err(e) => {
-                        tracing::error!("Failed to reserve chunks: {}", e);
+                        tracing::error!(error = as_dyn_error(&e), "Failed to reserve chunks");
                         match e {
                             // In the unlikely event of a DatabaseError,
                             // this is not the client's fault but _our_ fault.

opsqueue/src/consumer/server/mod.rs

@@ -1,3 +1,4 @@
+use crate::tracing::{anyhow_as_dyn_error, as_dyn_error};
 use std::{
     collections::HashSet,
     sync::{Arc, Mutex},
@@ -124,12 +125,14 @@ async fn ws_accept_handler(
             Ok(()) => {}
             Err(e) if e.is_internal_error() => {
                 tracing::error!(
-                    "Closed websocket connection because of internal error, details: {e:?}"
+                    error = as_dyn_error(&e),
+                    "Closed websocket connection because of internal error"
                 );
             }
             Err(e) => {
                 tracing::warn!(
-                    "Closed websocket connection because of client error, details: {e:?}"
+                    error = as_dyn_error(&e),
+                    "Closed websocket connection because of client error"
                 );
             }
         }
@@ -298,7 +301,10 @@ impl Completer {
         .await;
         match res {
             Ok(()) => {}
-            Err(err) => tracing::error!("Error in chunk completer: {err}"),
+            Err(err) => tracing::error!(
+                error = anyhow_as_dyn_error(&err),
+                "Error in chunk completer"
+            ),
         }
     }
 }

opsqueue/src/db/mod.rs

@@ -53,6 +53,7 @@
 //! # Ok (()) }
 //! ```
 
+use crate::tracing::as_dyn_error;
 use std::{marker::PhantomData, num::NonZero, time::Duration};
 
 use futures::future::BoxFuture;
@@ -215,8 +216,11 @@ impl DBPools {
                 })
                 .await
                 .is_ok(),
-            Err(error) => {
-                tracing::error!("DB unhealthy; could not acquire DB connection: {error:?}");
+            Err(err) => {
+                tracing::error!(
+                    error = as_dyn_error(&err),
+                    "DB unhealthy; could not acquire DB connection"
+                );
                 false
             }
         }

opsqueue/src/object_store/mod.rs

@@ -50,9 +50,10 @@ impl std::fmt::Display for ChunkType {
 #[derive(thiserror::Error, Debug)]
 pub enum ChunkRetrievalError {
     #[error(
-        "Failed to retrieve chunk ({submission_prefix}, {chunk_index}, {chunk_type}) from object store: {source}"
+        "Failed to retrieve chunk ({submission_prefix}, {chunk_index}, {chunk_type}) from object store"
     )]
     ObjectStoreError {
+        #[source]
         source: object_store::Error,
         submission_prefix: Box<str>,
         chunk_index: chunk::ChunkIndex,
@@ -63,19 +64,21 @@ pub enum ChunkRetrievalError {
 #[derive(thiserror::Error, Debug)]
 pub enum ChunkStorageError {
     #[error(
-        "Failed to store chunk ({submission_prefix}, {chunk_index}, {chunk_type}) to object store: {source}"
+        "Failed to store chunk ({submission_prefix}, {chunk_index}, {chunk_type}) to object store"
     )]
     ObjectStoreError {
+        #[source]
         source: object_store::Error,
         submission_prefix: Box<str>,
         chunk_index: chunk::ChunkIndex,
         chunk_type: ChunkType,
     },
-    #[error("Failed to read chunk element from stream/iterator at index {chunk_index}: ")]
+    #[error("Failed to read chunk element from stream/iterator at index {chunk_index}")]
     ChunkContentsEvalError {
         submission_prefix: Box<str>,
         chunk_index: chunk::ChunkIndex,
         chunk_type: ChunkType,
+        #[source]
         source: anyhow::Error,
     },
 }
@@ -84,19 +87,20 @@ pub enum ChunkStorageError {
 pub enum ChunksStorageError {
     #[error(transparent)]
     ChunkStorageError(#[from] ChunkStorageError),
-    #[error("Failed to read chunk element from stream/iterator: {source}")]
+    #[error("Failed to read chunk element from stream/iterator")]
     ChunkContentsEvalError {
         submission_prefix: Box<str>,
         chunk_type: ChunkType,
+        #[source]
         source: anyhow::Error,
     },
 }
 
 #[derive(thiserror::Error, Debug)]
 pub enum NewObjectStoreClientError {
-    #[error("Failed to parse URL: {0}")]
+    #[error("Failed to parse URL")]
     UrlParseFailure(#[from] url::ParseError),
-    #[error("URL is not valid object store URL {0}")]
+    #[error("URL is not valid object store URL")]
     ObjectStoreUrlFailure(#[from] object_store::Error),
 }