diff --git a/src/handlers/http/ingest.rs b/src/handlers/http/ingest.rs index a98e8ab3a..c61b81bae 100644 --- a/src/handlers/http/ingest.rs +++ b/src/handlers/http/ingest.rs @@ -553,6 +553,8 @@ pub enum PostError { MissingQueryParameter, #[error(transparent)] MetastoreError(#[from] MetastoreError), + #[error("Stream {0} is being deleted, please retry after some time")] + StreamBeingDeleted(String), } impl actix_web::ResponseError for PostError { @@ -586,6 +588,8 @@ impl actix_web::ResponseError for PostError { StreamNotFound(_) => StatusCode::NOT_FOUND, + StreamBeingDeleted(_) => StatusCode::CONFLICT, + MetastoreError(e) => e.status_code(), } } diff --git a/src/handlers/http/logstream.rs b/src/handlers/http/logstream.rs index f1c6e8f5b..a48aee89a 100644 --- a/src/handlers/http/logstream.rs +++ b/src/handlers/http/logstream.rs @@ -28,7 +28,10 @@ use crate::rbac::Users; use crate::rbac::role::Action; use crate::stats::{Stats, event_labels_date, storage_size_labels_date}; use crate::storage::retention::Retention; -use crate::storage::{ObjectStoreFormat, StreamInfo, StreamType}; +use crate::storage::{ + ObjectStoreFormat, StreamInfo, StreamType, + object_storage::{spawn_stream_deletion, stream_json_path, to_bytes, tombstone_path}, +}; use crate::tenants::TenantNotFound; use crate::utils::actix::extract_session_key_from_req; use crate::utils::get_tenant_id_from_request; @@ -63,17 +66,56 @@ pub async fn delete( return Err(StreamNotFound(stream_name).into()); } + // Fetched once, up front: every step below this point is either + // infallible or best-effort, so nothing after this line can bail out + // with "stream not found" partway through an already-durably-started + // deletion. + let stream = PARSEABLE.get_stream(&stream_name, &tenant_id)?; + + // Flip the in-memory guard before any `.await` point: check_or_load_stream's + // resident-stream fast path doesn't itself consult is_tombstoned, so a + // concurrent request on this node could otherwise slip through in the + // window between the tombstone becoming durable and this flag being set. + stream.mark_deleting(); + let objectstore = PARSEABLE.storage.get_object_store(); - // Delete from storage - objectstore.delete_stream(&stream_name, &tenant_id).await?; + // Durable marker first: if the process crashes anywhere after this + // point, restart-recovery resumes the deletion instead of silently + // leaving the stream half-deleted with no record of it. + objectstore + .put_object( + &tombstone_path(&stream_name, &tenant_id), + to_bytes(&()), + &tenant_id, + ) + .await?; + + // Best-effort: makes the stream vanish from listings almost + // immediately. Not fatal if it fails -- is_deleting()/is_tombstoned() + // checks already block reads and writes regardless of whether this file + // is gone yet. + if let Err(e) = objectstore + .delete_object(&stream_json_path(&stream_name, &tenant_id), &tenant_id) + .await + { + warn!( + "failed to eagerly delete stream.json for {stream_name}, will be removed with the rest of the prefix: {e}" + ); + } + + // Scheduled immediately once the stream is durably tombstoned and + // flagged locally, before any of the remaining best-effort steps -- + // none of them are allowed to leave the deletion itself unscheduled if + // they fail. + spawn_stream_deletion(stream_name.clone(), tenant_id.clone()); + // Delete from staging - let stream_dir = PARSEABLE.get_or_create_stream(&stream_name, &tenant_id); - if let Err(err) = fs::remove_dir_all(&stream_dir.data_path) { + if let Err(err) = fs::remove_dir_all(&stream.data_path) { warn!( "failed to delete local data for stream {} with error {err}. Clean {} manually", stream_name, - stream_dir.data_path.to_string_lossy() + stream.data_path.to_string_lossy() ) } @@ -85,12 +127,10 @@ pub async fn delete( .await?; } - // Delete from memory - PARSEABLE.streams.delete(&stream_name, &tenant_id); - stats::delete_stats(&stream_name, "json", &tenant_id) - .unwrap_or_else(|e| warn!("failed to delete stats for stream {}: {:?}", stream_name, e)); - - Ok((format!("log stream {stream_name} deleted"), StatusCode::OK)) + Ok(( + format!("log stream {stream_name} deletion started"), + StatusCode::ACCEPTED, + )) } pub async fn list(req: HttpRequest) -> Result { @@ -186,6 +226,9 @@ pub async fn get_schema( } let stream = PARSEABLE.get_stream(&stream_name, &tenant_id)?; + if stream.is_deleting() { + return Err(StreamNotFound(stream_name.clone()).into()); + } match update_schema_when_distributed(&vec![stream_name.clone()], &tenant_id).await { Ok(_) => { let schema = stream.get_schema(); @@ -313,6 +356,12 @@ pub async fn get_stats( { return Err(StreamNotFound(stream_name.clone()).into()); } + if PARSEABLE + .get_stream(&stream_name, &tenant_id) + .is_ok_and(|stream| stream.is_deleting()) + { + return Err(StreamNotFound(stream_name.clone()).into()); + } let query_string = req.query_string(); if !query_string.is_empty() { @@ -378,6 +427,12 @@ pub async fn get_stream_info( { return Err(StreamNotFound(stream_name.clone()).into()); } + if PARSEABLE + .get_stream(&stream_name, &tenant_id) + .is_ok_and(|stream| stream.is_deleting()) + { + return Err(StreamNotFound(stream_name.clone()).into()); + } let storage = PARSEABLE.storage().get_object_store(); diff --git a/src/handlers/http/modal/ingest/ingestor_logstream.rs b/src/handlers/http/modal/ingest/ingestor_logstream.rs index 9f7414baa..02b281813 100644 --- a/src/handlers/http/modal/ingest/ingestor_logstream.rs +++ b/src/handlers/http/modal/ingest/ingestor_logstream.rs @@ -31,7 +31,6 @@ use crate::{ catalog::remove_manifest_from_snapshot, handlers::http::logstream::error::StreamError, parseable::{PARSEABLE, StreamNotFound}, - stats, utils::get_tenant_id_from_request, }; @@ -78,6 +77,7 @@ pub async fn delete( let tenant_id = get_tenant_id_from_request(&req); // Delete from staging let stream_dir = PARSEABLE.get_stream(&stream_name, &tenant_id)?; + stream_dir.mark_deleting(); // delete staging only for ingest server or standalone server // else skip @@ -91,12 +91,16 @@ pub async fn delete( ) } - // Delete from memory - PARSEABLE.streams.delete(&stream_name, &tenant_id); - stats::delete_stats(&stream_name, "json", &tenant_id) - .unwrap_or_else(|e| warn!("failed to delete stats for stream {}: {:?}", stream_name, e)); - - Ok((format!("log stream {stream_name} deleted"), StatusCode::OK)) + // Not removed from memory here: this node doesn't run the background + // deletion job, so it doesn't know when the underlying prefix is + // actually gone. The entry is reaped once `sync_all_streams` notices + // the tombstone has cleared (see its is_deleting()/is_tombstoned() + // self-heal check) -- until then, `is_deleting()` keeps rejecting + // ingestion for this stream with a clear "being deleted" error. + Ok(( + format!("log stream {stream_name} deletion started"), + StatusCode::OK, + )) } pub async fn put_stream( diff --git a/src/handlers/http/modal/query/querier_logstream.rs b/src/handlers/http/modal/query/querier_logstream.rs index 2bb170104..de6ae1f4b 100644 --- a/src/handlers/http/modal/query/querier_logstream.rs +++ b/src/handlers/http/modal/query/querier_logstream.rs @@ -45,12 +45,14 @@ use crate::{ utils::{IngestionStats, QueriedStats, StorageStats, merge_queried_stats}, }, logstream::error::StreamError, - modal::{NodeMetadata, NodeType}, }, }, parseable::{PARSEABLE, StreamNotFound}, stats, - storage::{ObjectStoreFormat, StreamType}, + storage::{ + ObjectStoreFormat, StreamType, + object_storage::{spawn_stream_deletion, stream_json_path, to_bytes, tombstone_path}, + }, utils::get_tenant_id_from_request, }; const STATS_DATE_QUERY_PARAM: &str = "date"; @@ -73,52 +75,82 @@ pub async fn delete( return Err(StreamNotFound(stream_name.clone()).into()); } + // Fetched once, up front: every step below this point is either + // infallible or best-effort, so nothing after this line can bail out + // with "stream not found" partway through an already-durably-started + // deletion. + let stream = PARSEABLE.get_stream(&stream_name, &tenant_id)?; + + // Flip the in-memory guard before any `.await` point: check_or_load_stream's + // resident-stream fast path doesn't itself consult is_tombstoned, so a + // concurrent request on this node could otherwise slip through in the + // window between the tombstone becoming durable and this flag being set. + stream.mark_deleting(); + let objectstore = PARSEABLE.storage.get_object_store(); - // Delete from storage - objectstore.delete_stream(&stream_name, &tenant_id).await?; - let stream_dir = PARSEABLE.get_or_create_stream(&stream_name, &tenant_id); - if let Err(err) = fs::remove_dir_all(&stream_dir.data_path) { - warn!( - "failed to delete local data for stream {} with error {err}. Clean {} manually", - stream_name, - stream_dir.data_path.to_string_lossy() + + // Durable marker first: if the process crashes anywhere after this + // point, restart-recovery resumes the deletion instead of silently + // leaving the stream half-deleted with no record of it. + objectstore + .put_object( + &tombstone_path(&stream_name, &tenant_id), + to_bytes(&()), + &tenant_id, ) - } + .await?; - if let Some(hot_tier_manager) = GLOBAL_HOTTIER.get() - && hot_tier_manager.check_stream_hot_tier_exists(&stream_name, &tenant_id) + // Best-effort: makes the stream vanish from listings almost + // immediately, without touching every listing endpoint individually. + // Not fatal if it fails -- is_deleting()/is_tombstoned() checks already + // block reads and writes regardless of whether this file is gone yet. + if let Err(e) = objectstore + .delete_object(&stream_json_path(&stream_name, &tenant_id), &tenant_id) + .await { - hot_tier_manager - .delete_hot_tier(&stream_name, &tenant_id) - .await?; + warn!( + "failed to eagerly delete stream.json for {stream_name}, will be removed with the rest of the prefix: {e}" + ); } - let ingestor_metadata: Vec = - cluster::get_node_info(NodeType::Ingestor, &tenant_id) - .await - .map_err(|err| { - error!("Fatal: failed to get ingestor info: {:?}", err); - err - })?; + // Scheduled immediately once the stream is durably tombstoned and + // flagged locally, before any of the remaining best-effort steps -- + // none of them are allowed to leave the deletion itself unscheduled if + // they fail. + spawn_stream_deletion(stream_name.clone(), tenant_id.clone()); - for ingestor in ingestor_metadata { + let fanout_stream_name = stream_name.clone(); + cluster::for_each_live_node(&tenant_id, move |node| { let url = format!( "{}{}/logstream/{}/sync", - ingestor.domain_name, + node.domain_name, base_path_without_preceding_slash(), - stream_name + fanout_stream_name ); + async move { cluster::send_stream_delete_request(&url, node).await } + }) + .await?; - // delete the stream - cluster::send_stream_delete_request(&url, ingestor.clone()).await?; + if let Err(err) = fs::remove_dir_all(&stream.data_path) { + warn!( + "failed to delete local data for stream {} with error {err}. Clean {} manually", + stream_name, + stream.data_path.to_string_lossy() + ) } - // Delete from memory - PARSEABLE.streams.delete(&stream_name, &tenant_id); - stats::delete_stats(&stream_name, "json", &tenant_id) - .unwrap_or_else(|e| warn!("failed to delete stats for stream {}: {:?}", stream_name, e)); + if let Some(hot_tier_manager) = GLOBAL_HOTTIER.get() + && hot_tier_manager.check_stream_hot_tier_exists(&stream_name, &tenant_id) + { + hot_tier_manager + .delete_hot_tier(&stream_name, &tenant_id) + .await?; + } - Ok((format!("log stream {stream_name} deleted"), StatusCode::OK)) + Ok(( + format!("log stream {stream_name} deletion started"), + StatusCode::ACCEPTED, + )) } pub async fn put_stream( diff --git a/src/handlers/http/modal/utils/ingest_utils.rs b/src/handlers/http/modal/utils/ingest_utils.rs index d3fcba7ac..07c85395b 100644 --- a/src/handlers/http/modal/utils/ingest_utils.rs +++ b/src/handlers/http/modal/utils/ingest_utils.rs @@ -509,6 +509,10 @@ pub fn validate_stream_for_ingestion( ) -> Result<(), PostError> { let stream = PARSEABLE.get_stream(stream_name, tenant_id)?; + if stream.is_deleting() { + return Err(PostError::StreamBeingDeleted(stream_name.to_string())); + } + // Validate that the stream's log source is compatible stream .get_log_source() diff --git a/src/handlers/http/query.rs b/src/handlers/http/query.rs index 34c8a7327..dd97bd08b 100644 --- a/src/handlers/http/query.rs +++ b/src/handlers/http/query.rs @@ -559,6 +559,22 @@ pub async fn create_streams_for_distributed( streams: Vec, tenant_id: &Option, ) -> Result<(), QueryError> { + // A stream that's already resident in memory but flagged `deleting` + // must reject the query outright. Checked unconditionally, ahead of + // the mode gate below, since this function backs every query-side + // call site (ad-hoc queries, alerts, saved query context, traces), + // not just the querier's own reload path. + for stream_name in &streams { + if PARSEABLE.streams.contains(stream_name, tenant_id) + && let Ok(stream) = PARSEABLE.get_stream(stream_name, tenant_id) + && stream.is_deleting() + { + return Err(QueryError::StreamNotFound(StreamNotFound( + stream_name.clone(), + ))); + } + } + if PARSEABLE.options.mode != Mode::Query && PARSEABLE.options.mode != Mode::Prism { return Ok(()); } diff --git a/src/metadata.rs b/src/metadata.rs index 983456447..3a4ec2e1c 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -99,6 +99,11 @@ pub struct LogStreamMetadata { pub dataset_tags: Vec, pub dataset_labels: Vec, pub infer_timestamp: bool, + /// Transient, in-memory only — never persisted to `ObjectStoreFormat`. + /// Set once a deletion has been initiated for this stream so that + /// readers/writers reached via an already-resident `Arc` reject + /// it instead of racing the background deletion. + pub deleting: bool, } impl Default for LogStreamMetadata { @@ -121,6 +126,7 @@ impl Default for LogStreamMetadata { dataset_tags: Vec::new(), dataset_labels: Vec::new(), infer_timestamp: true, + deleting: false, } } } diff --git a/src/metastore/metastores/object_store_metastore.rs b/src/metastore/metastores/object_store_metastore.rs index a5cda811e..b6b446b97 100644 --- a/src/metastore/metastores/object_store_metastore.rs +++ b/src/metastore/metastores/object_store_metastore.rs @@ -55,7 +55,7 @@ use crate::{ storage::{ ALERTS_ROOT_DIRECTORY, ObjectStorage, ObjectStorageError, PARSEABLE_ROOT_DIRECTORY, SETTINGS_ROOT_DIRECTORY, STREAM_METADATA_FILE_NAME, STREAM_ROOT_DIRECTORY, - TARGETS_ROOT_DIRECTORY, + TARGETS_ROOT_DIRECTORY, TOMBSTONE_ROOT_DIRECTORY, object_storage::{ alert_json_path, alert_state_json_path, filter_path, manifest_path, mttr_json_path, outbound_http_policy_json_path, parseable_json_path, schema_path, stream_json_path, @@ -1433,6 +1433,7 @@ impl Metastore for ObjectStoreMetastore { && name != USERS_ROOT_DIR && name != SETTINGS_ROOT_DIRECTORY && name != ALERTS_ROOT_DIRECTORY + && name != TOMBSTONE_ROOT_DIRECTORY }) .collect::>(); for stream in streams { diff --git a/src/migration/mod.rs b/src/migration/mod.rs index 989cdbbac..354c21782 100644 --- a/src/migration/mod.rs +++ b/src/migration/mod.rs @@ -35,7 +35,10 @@ use crate::{ metrics::fetch_stats_from_storage, option::Mode, parseable::{DEFAULT_TENANT, PARSEABLE, Parseable}, - storage::{ObjectStorage, ObjectStoreFormat, PARSEABLE_METADATA_FILE_NAME, StorageMetadata}, + storage::{ + ObjectStorage, ObjectStoreFormat, PARSEABLE_METADATA_FILE_NAME, StorageMetadata, + object_storage::{is_tombstoned, list_tombstoned_streams, spawn_stream_deletion}, + }, }; fn get_version(metadata: &serde_json::Value) -> Option<&str> { @@ -213,8 +216,12 @@ pub async fn run_migration(config: &Parseable) -> anyhow::Result<()> { let mut futures = Vec::new(); for tenant_id in tenants { - // Get all stream names - let stream_names = PARSEABLE.metastore.list_streams(&tenant_id).await?; + // Get all stream names, plus any stream whose `.stream.json` is + // already gone because it's mid-deletion -- `list_streams` alone + // would miss it, and `migration_stream` needs the chance to resume + // that deletion below if the process crashed before finishing it. + let mut stream_names = PARSEABLE.metastore.list_streams(&tenant_id).await?; + stream_names.extend(list_tombstoned_streams(storage.as_ref(), &tenant_id).await?); // Create futures for each stream migration let f = stream_names.into_iter().map(|stream_name| { @@ -267,6 +274,23 @@ async fn migration_stream( storage: &dyn ObjectStorage, tenant_id: &Option, ) -> anyhow::Result> { + if is_tombstoned(storage, stream, tenant_id).await? { + // Left mid-deletion by a node that crashed or restarted before the + // background job finished. Resume it here rather than treating the + // stream as a normal (possibly schema-less) migration candidate -- + // `create_schema_from_metastore` below can itself error out on a + // partially-swept schema file, which would otherwise turn "resume + // deletion" into "abort node startup". Only a query/standalone node + // ever owns this job (mirrors the same guard in + // `object_storage::sync_all_streams`'s self-heal check) -- an + // ingestor just skips the stream here and waits for the tombstone to + // clear. + if PARSEABLE.options.mode != Mode::Ingest { + spawn_stream_deletion(stream.to_string(), tenant_id.clone()); + } + return Ok(None); + } + let mut arrow_schema: Schema = Schema::empty(); let schema = storage @@ -513,6 +537,7 @@ pub async fn setup_logstream_metadata( dataset_tags, dataset_labels, infer_timestamp, + deleting: false, }; Ok(metadata) diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs index 70ba8208f..e55a5d863 100644 --- a/src/parseable/mod.rs +++ b/src/parseable/mod.rs @@ -77,7 +77,8 @@ use crate::{ static_schema::{StaticSchema, convert_static_schema_to_arrow_schema}, storage::{ ObjectStorage, ObjectStorageError, ObjectStorageProvider, ObjectStoreFormat, Owner, - Permisssion, StorageMetadata, StreamType, put_remote_metadata, + Permisssion, StorageMetadata, StreamType, object_storage::is_tombstoned, + put_remote_metadata, }, tenants::{Service, TENANT_METADATA}, validator, @@ -472,6 +473,11 @@ impl Parseable { ) -> Result { // Proceed to create log stream if it doesn't exist let storage = self.storage.get_object_store(); + // A deletion in progress (or left unfinished by a crashed node) must + // never be resurrected by a concurrent lazy reload. + if is_tombstoned(storage.as_ref(), stream_name, tenant_id).await? { + return Ok(false); + } let streams = PARSEABLE.metastore.list_streams(tenant_id).await?; if !streams.contains(stream_name) { return Ok(false); @@ -783,6 +789,21 @@ impl Parseable { let stream_in_memory_dont_update = self.streams.contains(stream_name, tenant_id) && !update_stream_flag; + // A stream still resident with is_deleting()=true is functionally + // gone (reads/writes are already rejected elsewhere), but its entry + // isn't removed from memory until the background deletion job + // finishes -- surface that distinctly rather than telling the + // caller it "already exists", which reads as if nothing were wrong. + if stream_in_memory_dont_update + && let Ok(stream) = self.get_stream(stream_name, tenant_id) + && stream.is_deleting() + { + return Err(StreamError::Custom { + msg: format!("Logstream {stream_name} is being deleted, please retry shortly"), + status: StatusCode::CONFLICT, + }); + } + // check if stream in storage only if not in memory // for Parseable OSS, create_update_stream is called only from query node // for Parseable Enterprise, create_update_stream is called from prism node diff --git a/src/parseable/streams.rs b/src/parseable/streams.rs index 2262f599c..a8592eccd 100644 --- a/src/parseable/streams.rs +++ b/src/parseable/streams.rs @@ -1217,8 +1217,12 @@ impl Stream { } /// Stores the provided stream metadata in memory mapping - pub async fn set_metadata(&self, updated_metadata: LogStreamMetadata) { - *self.metadata.write().expect(LOCK_EXPECT) = updated_metadata; + pub async fn set_metadata(&self, mut updated_metadata: LogStreamMetadata) { + let mut metadata = self.metadata.write().expect(LOCK_EXPECT); + // mark_deleting() is documented as monotonic -- a reload racing a + // delete must not silently clear it back to false. + updated_metadata.deleting |= metadata.deleting; + *metadata = updated_metadata; } pub fn get_first_event(&self) -> Option { @@ -1352,6 +1356,17 @@ impl Stream { self.metadata.read().expect(LOCK_EXPECT).hot_tier_enabled } + /// Marks this stream as being deleted. Once set, this flag is never + /// cleared for this in-memory entry — a deletion in progress runs to + /// completion (or is resumed on restart), it is never cancelled. + pub fn mark_deleting(&self) { + self.metadata.write().expect(LOCK_EXPECT).deleting = true; + } + + pub fn is_deleting(&self) -> bool { + self.metadata.read().expect(LOCK_EXPECT).deleting + } + pub fn get_stream_type(&self) -> StreamType { self.metadata.read().expect(LOCK_EXPECT).stream_type } @@ -1744,6 +1759,22 @@ mod tests { ); } + #[test] + fn test_mark_deleting_sets_is_deleting() { + let options = Arc::new(Options::default()); + let stream = Stream::new( + options, + "test_stream", + LogStreamMetadata::default(), + None, + &None, + ); + + assert!(!stream.is_deleting()); + stream.mark_deleting(); + assert!(stream.is_deleting()); + } + #[test] fn test_staging_with_special_characters() { let stream_name = "test_stream_!@#$%^&*()"; diff --git a/src/storage/localfs.rs b/src/storage/localfs.rs index bafd80404..54f5e2300 100644 --- a/src/storage/localfs.rs +++ b/src/storage/localfs.rs @@ -44,11 +44,13 @@ use crate::{ option::validation, parseable::{DEFAULT_TENANT, LogStream}, storage::SETTINGS_ROOT_DIRECTORY, + storage::object_storage::tombstone_path, }; use super::{ ALERTS_ROOT_DIRECTORY, ObjectStorage, ObjectStorageError, ObjectStorageProvider, PARSEABLE_ROOT_DIRECTORY, STREAM_METADATA_FILE_NAME, STREAM_ROOT_DIRECTORY, + TOMBSTONE_ROOT_DIRECTORY, }; #[derive(Debug, Clone, clap::Args)] @@ -533,6 +535,7 @@ impl ObjectStorage for LocalFS { USERS_ROOT_DIR, ALERTS_ROOT_DIRECTORY, SETTINGS_ROOT_DIRECTORY, + TOMBSTONE_ROOT_DIRECTORY, ]; let result = fs::read_dir(&self.root).await; @@ -553,7 +556,7 @@ impl ObjectStorage for LocalFS { let entries: Vec = directories.try_collect().await?; let entries = entries .into_iter() - .map(|entry| dir_with_stream(entry, ignore_dir)); + .map(|entry| dir_with_stream(entry, ignore_dir, &self.root)); let logstream_dirs: Vec> = FuturesUnordered::from_iter(entries).try_collect().await?; @@ -570,6 +573,7 @@ impl ObjectStorage for LocalFS { PARSEABLE_ROOT_DIRECTORY, ALERTS_ROOT_DIRECTORY, SETTINGS_ROOT_DIRECTORY, + TOMBSTONE_ROOT_DIRECTORY, ]; let result = fs::read_dir(&self.root).await; @@ -860,6 +864,7 @@ async fn dir_with_old_stream( async fn dir_with_stream( entry: DirEntry, ignore_dirs: &[&str], + root: &Path, ) -> Result, ObjectStorageError> { let dir_name = entry .path() @@ -883,6 +888,14 @@ async fn dir_with_stream( if stream_json_path.exists() { Ok(Some(dir_name)) + } else if tombstone_path(&dir_name, &None).to_path(root).exists() { + // Mid-async-deletion: `.stream.json` is deleted eagerly by the + // DELETE handler well before the background job finishes + // physically clearing the rest of the prefix, so a directory + // without it is expected here, not corrupt -- don't fail the + // whole listing over a stream that's in the middle of being + // deleted. + Ok(None) } else { let err: Box = format!("found {}", entry.path().display()).into(); @@ -913,3 +926,86 @@ impl From for ObjectStorageError { ObjectStorageError::UnhandledError(Box::new(e)) } } + +#[cfg(test)] +mod list_streams_tombstone_tests { + use temp_dir::TempDir; + + use super::{LocalFS, ObjectStorage}; + use crate::storage::object_storage::{to_bytes, tombstone_path}; + use crate::storage::{STREAM_METADATA_FILE_NAME, STREAM_ROOT_DIRECTORY}; + use relative_path::RelativePathBuf; + + // Deliberately not using `object_storage::stream_json_path` here: it + // reads the global PARSEABLE.options.mode, which isn't initialized under + // `cargo test` and crashes the whole test binary. This replicates its + // non-Ingest-mode path (tenant/stream/.stream/.stream.json) directly. + fn stream_json_path_for_test(stream_name: &str) -> RelativePathBuf { + RelativePathBuf::from_iter([ + "", + stream_name, + STREAM_ROOT_DIRECTORY, + STREAM_METADATA_FILE_NAME, + ]) + } + + #[tokio::test] + async fn normal_stream_is_still_listed() { + let dir = TempDir::new().unwrap(); + let storage = LocalFS::new(dir.path().to_path_buf()); + + storage + .put_object(&stream_json_path_for_test("mystream"), to_bytes(&()), &None) + .await + .unwrap(); + + let listed = storage.list_streams().await.unwrap(); + assert!(listed.contains("mystream")); + } + + #[tokio::test] + async fn stream_mid_deletion_is_skipped_not_a_listing_error() { + let dir = TempDir::new().unwrap(); + let storage = LocalFS::new(dir.path().to_path_buf()); + + // Set up as the DELETE handler leaves it: stream.json already gone, + // tombstone marker present, rest of the directory (and its other + // files) still there because the background delete hasn't finished. + storage + .put_object( + &stream_json_path_for_test("deleting-stream"), + to_bytes(&()), + &None, + ) + .await + .unwrap(); + storage + .delete_object(&stream_json_path_for_test("deleting-stream"), &None) + .await + .unwrap(); + storage + .put_object( + &tombstone_path("deleting-stream", &None), + to_bytes(&()), + &None, + ) + .await + .unwrap(); + + let listed = storage.list_streams().await.unwrap(); + assert!(!listed.contains("deleting-stream")); + } + + #[tokio::test] + async fn genuinely_corrupt_directory_still_errors() { + let dir = TempDir::new().unwrap(); + let storage = LocalFS::new(dir.path().to_path_buf()); + + // A real directory with neither a stream.json nor a tombstone is + // still treated as unexpected/corrupt, not silently skipped -- + // the fix narrows the exception to the tombstoned case specifically. + std::fs::create_dir_all(dir.path().join("not-a-stream")).unwrap(); + + assert!(storage.list_streams().await.is_err()); + } +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 5c6a2e36c..6271f14cf 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -310,6 +310,17 @@ pub const ALERTS_ROOT_DIRECTORY: &str = ".alerts"; pub const SETTINGS_ROOT_DIRECTORY: &str = ".settings"; pub const TARGETS_ROOT_DIRECTORY: &str = ".targets"; pub const MANIFEST_FILE: &str = "manifest.json"; +// top-level registry of streams currently being deleted; kept outside every +// stream's own prefix so a bulk prefix-delete can never sweep up a marker +// that's supposed to survive it (see is_tombstoned/tombstone_path) +pub const TOMBSTONE_ROOT_DIRECTORY: &str = ".tombstones"; +// the marker itself lives one level below `{tenant}/{stream_name}/`, not as +// a leaf key directly named after the stream: list_dirs_relative on every +// backend (S3/GCS/Azure via list-with-delimiter's common_prefixes, LocalFS +// via read_dir + is_dir) only surfaces child *directories*, never leaf +// objects, so a tombstone recorded as a bare `{stream_name}` key would be +// invisible to the restart-recovery scan that discovers tombstoned streams +pub const TOMBSTONE_MARKER_FILE_NAME: &str = ".tombstone"; // max concurrent request allowed for datafusion object store, overridable per // backend with P_MAX_OBJECT_STORE_REQUESTS. diff --git a/src/storage/object_storage.rs b/src/storage/object_storage.rs index 38b2809be..68e458a3f 100644 --- a/src/storage/object_storage.rs +++ b/src/storage/object_storage.rs @@ -30,6 +30,7 @@ use crate::metrics::{EVENTS_STORAGE_SIZE_DATE, LIFETIME_EVENTS_STORAGE_SIZE, STO use crate::option::Mode; use crate::parseable::DEFAULT_TENANT; use crate::parseable::{LogStream, PARSEABLE, Stream}; +use crate::stats; use crate::stats::FullStats; use crate::storage::SETTINGS_ROOT_DIRECTORY; use crate::storage::TARGETS_ROOT_DIRECTORY; @@ -41,13 +42,14 @@ use arrow_schema::Schema; use async_trait::async_trait; use bytes::Bytes; use chrono::{DateTime, Utc}; +use dashmap::DashMap; use dashmap::mapref::entry::Entry; use datafusion::{datasource::listing::ListingTableUrl, execution::runtime_env::RuntimeEnvBuilder}; use itertools::Itertools; use object_store::ListResult; use object_store::ObjectMeta; use object_store::buffered::BufReader; -use once_cell::sync::OnceCell; +use once_cell::sync::{Lazy, OnceCell}; use rayon::prelude::*; use relative_path::RelativePath; use relative_path::RelativePathBuf; @@ -68,7 +70,8 @@ use ulid::Ulid; use super::{ ALERTS_ROOT_DIRECTORY, MANIFEST_FILE, ObjectStorageError, ObjectStoreFormat, PARSEABLE_METADATA_FILE_NAME, PARSEABLE_ROOT_DIRECTORY, SCHEMA_FILE_NAME, - STREAM_METADATA_FILE_NAME, STREAM_ROOT_DIRECTORY, retention::Retention, + STREAM_METADATA_FILE_NAME, STREAM_ROOT_DIRECTORY, TOMBSTONE_MARKER_FILE_NAME, + TOMBSTONE_ROOT_DIRECTORY, retention::Retention, }; /// Context for upload operations containing stream information @@ -1334,6 +1337,70 @@ fn stream_relative_path( } } +/// Dedupes concurrent background deletion jobs for the same (tenant, stream) +/// so a repeat DELETE request, or a restart-triggered resume racing a job +/// already spawned before the restart, can't run `delete_stream` twice +/// concurrently against the same prefix. Mirrors `ACTIVE_OBJECT_STORE_SYNC_FILES` +/// in `sync.rs`. Deliberately has no time-based expiry (unlike that sibling +/// map): a large stream's real delete can legitimately run for many minutes, +/// and an expiry short enough to be useful would risk starting a duplicate +/// job while the original is still healthily in progress. See +/// `StreamDeletionGuard` for how an entry is still guaranteed to be cleared. +pub static ACTIVE_STREAM_DELETIONS: Lazy, String), Instant>> = + Lazy::new(DashMap::new); + +/// RAII guard that removes a stream's entry from `ACTIVE_STREAM_DELETIONS` on +/// drop, including on an unexpected panic inside the deletion task -- so a +/// single bad run can't wedge all future retries for that stream by leaving +/// its dedup entry stuck forever. +struct StreamDeletionGuard(Option<(Option, String)>); + +impl Drop for StreamDeletionGuard { + fn drop(&mut self) { + if let Some(key) = self.0.take() { + ACTIVE_STREAM_DELETIONS.remove(&key); + } + } +} + +/// Deletes a stream's data in the background and clears its tombstone once +/// done, so the synchronous DELETE handler can respond before a TB-scale +/// prefix delete completes. Safe to call more than once for the same +/// stream: a job already in flight is skipped rather than duplicated. +pub fn spawn_stream_deletion(stream_name: String, tenant_id: Option) { + let key = (tenant_id, stream_name); + match ACTIVE_STREAM_DELETIONS.entry(key.clone()) { + Entry::Occupied(_) => return, + Entry::Vacant(entry) => { + entry.insert(Instant::now()); + } + } + tokio::spawn(async move { + let _guard = StreamDeletionGuard(Some(key.clone())); + let (tenant_id, stream_name) = &key; + let storage = PARSEABLE.storage.get_object_store(); + match storage.delete_stream(stream_name, tenant_id).await { + Ok(()) => { + if let Err(e) = storage + .delete_object(&tombstone_path(stream_name, tenant_id), tenant_id) + .await + { + warn!( + "background deletion of {stream_name} finished but failed to clear its tombstone: {e}" + ); + } + PARSEABLE.streams.delete(stream_name, tenant_id); + if let Err(e) = stats::delete_stats(stream_name, "json", tenant_id) { + warn!("failed to clear stats for deleted stream {stream_name}: {e:?}"); + } + } + Err(e) => error!( + "background deletion failed for {stream_name}: {e}. tombstone left in place, retried on next restart or repeat DELETE" + ), + } + }); +} + pub fn sync_all_streams(joinset: &mut JoinSet>) { let object_store = PARSEABLE.storage().get_object_store(); let tenants = if let Some(tenants) = PARSEABLE.list_tenants() { @@ -1344,11 +1411,62 @@ pub fn sync_all_streams(joinset: &mut JoinSet>) { let handle = FLUSH_AND_CONVERT_RUNTIME.handle(); for tenant_id in tenants { for stream_name in PARSEABLE.streams.list(&tenant_id) { - if let Ok(stream) = PARSEABLE.get_stream(&stream_name, &tenant_id) - && stream.parquet_files().is_empty() - && stream.schema_files().is_empty() - { - continue; + if let Ok(stream) = PARSEABLE.get_stream(&stream_name, &tenant_id) { + if stream.is_deleting() { + // Fast path (the DELETE handler's own `for_each_live_node` + // push) already reaches most nodes immediately. This is + // the fallback for one that missed it, e.g. a node that + // was down or partitioned at the time: bounded to at most + // one sync interval instead of running forever. + let object_store = object_store.clone(); + let tenant_id = tenant_id.clone(); + let stream_name = stream_name.clone(); + // Only a node type that can actually receive a client's + // original DELETE request (query/standalone) ever resumes + // the physical delete here. An ingestor only ever gets + // is_deleting()=true via the delete handler's own + // fan-out push, which never starts a job on the ingestor + // itself -- letting it also spawn one here would mean + // every ingestor independently runs a redundant, + // uncoordinated bulk delete against the same prefix for + // the entire (possibly long) duration of every deletion, + // not just the rare case of a genuinely missed + // notification. + let is_deletion_owner = PARSEABLE.options.mode != Mode::Ingest; + joinset.spawn_on( + async move { + match is_tombstoned(object_store.as_ref(), &stream_name, &tenant_id) + .await + { + Ok(true) => { + if is_deletion_owner { + spawn_stream_deletion(stream_name, tenant_id); + } + } + Ok(false) => { + // Deletion already finished elsewhere and + // the tombstone is gone, but this node's + // resident entry was never dropped -- e.g. + // an ingestor, which doesn't run the + // background deletion job itself. Reap it + // so a stream recreated under the same + // name doesn't inherit a stuck + // deleting=true state. + PARSEABLE.streams.delete(&stream_name, &tenant_id); + } + Err(e) => error!( + "failed to check tombstone status for {stream_name}: {e}" + ), + } + Ok(()) + }, + handle, + ); + continue; + } + if stream.parquet_files().is_empty() && stream.schema_files().is_empty() { + continue; + } } let object_store = object_store.clone(); let id = tenant_id.clone(); @@ -1442,6 +1560,81 @@ pub fn stream_json_path(stream_name: &str, tenant_id: &Option) -> Relati } } +/// Path to a stream's deletion marker. Deliberately lives under +/// `TOMBSTONE_ROOT_DIRECTORY`, outside the `{tenant}/{stream_name}` prefix +/// that a bulk stream delete walks, so a mid-deletion crash can never lose +/// the marker before the deletion it records has actually finished. +/// +/// The marker is nested one level under `{stream_name}/`, not stored as a +/// bare key named after the stream: `list_dirs_relative` (used to discover +/// tombstoned streams on restart) only surfaces child directories on every +/// backend, so `{stream_name}` must itself resolve to a directory for that +/// scan to find it. +#[inline(always)] +pub fn tombstone_path(stream_name: &str, tenant_id: &Option) -> RelativePathBuf { + let tenant = tenant_id.as_deref().unwrap_or(""); + RelativePathBuf::from_iter([ + TOMBSTONE_ROOT_DIRECTORY, + tenant, + stream_name, + TOMBSTONE_MARKER_FILE_NAME, + ]) +} + +/// Whether a stream has a deletion marker present, i.e. a deletion was +/// started (possibly by a node that has since crashed or restarted) and has +/// not yet completed. +pub async fn is_tombstoned( + storage: &(impl ObjectStorage + ?Sized), + stream_name: &str, + tenant_id: &Option, +) -> Result { + match storage + .head(&tombstone_path(stream_name, tenant_id), tenant_id) + .await + { + Ok(_) => Ok(true), + // NoSuchKey is the object-store backends' not-found; LocalFS instead + // surfaces a plain io::Error, so both must be treated as "absent" + // here (see ObjectStoreMetastore::is_missing_optional_dir for the + // same not-found reconciliation across backends). + Err(ObjectStorageError::NoSuchKey(_)) => Ok(false), + Err(ObjectStorageError::IoError(e)) if e.kind() == std::io::ErrorKind::NotFound => { + Ok(false) + } + Err(e) => Err(e), + } +} + +/// Stream names with a deletion marker for the given tenant, discovered by +/// listing `TOMBSTONE_ROOT_DIRECTORY` rather than checking one name at a +/// time. Used to resume deletions left unfinished by a crashed or restarted +/// node, since a tombstoned stream whose `.stream.json` is already gone +/// would otherwise never surface via `list_streams`. See `is_tombstoned` for +/// the equivalent single-name check. +/// +/// `list_dirs_relative` only proves a directory exists under the tombstone +/// root, not that it actually holds a `.tombstone` marker (e.g. a partial or +/// interrupted write could leave an empty one behind) -- each candidate is +/// re-checked with `is_tombstoned` before being returned, so a directory +/// without a real marker is silently skipped rather than misreported. +pub async fn list_tombstoned_streams( + storage: &(impl ObjectStorage + ?Sized), + tenant_id: &Option, +) -> Result, ObjectStorageError> { + let tenant = tenant_id.as_deref().unwrap_or(""); + let root = RelativePathBuf::from_iter([TOMBSTONE_ROOT_DIRECTORY, tenant]); + let candidates = storage.list_dirs_relative(&root, tenant_id).await?; + + let mut confirmed = Vec::with_capacity(candidates.len()); + for stream_name in candidates { + if is_tombstoned(storage, &stream_name, tenant_id).await? { + confirmed.push(stream_name); + } + } + Ok(confirmed) +} + /// if filter_id is an empty str it should not append it to the rel path #[inline(always)] pub fn filter_path( @@ -1567,6 +1760,128 @@ pub fn manifest_segment_matches(manifest_path_str: &str, file_name: &str) -> boo manifest_path_str.rsplit('/').next() == Some(file_name) } +#[cfg(test)] +mod tombstone_tests { + use super::{ObjectStorage, is_tombstoned, list_tombstoned_streams, to_bytes, tombstone_path}; + use crate::storage::LocalFS; + use temp_dir::TempDir; + + #[tokio::test] + async fn no_marker_means_not_tombstoned() { + let dir = TempDir::new().unwrap(); + let storage = LocalFS::new(dir.path().to_path_buf()); + + assert!(!is_tombstoned(&storage, "test_stream", &None).await.unwrap()); + } + + #[tokio::test] + async fn marker_present_means_tombstoned() { + let dir = TempDir::new().unwrap(); + let storage = LocalFS::new(dir.path().to_path_buf()); + + storage + .put_object(&tombstone_path("test_stream", &None), to_bytes(&()), &None) + .await + .unwrap(); + + assert!(is_tombstoned(&storage, "test_stream", &None).await.unwrap()); + } + + #[tokio::test] + async fn no_markers_means_empty_discovery_list() { + let dir = TempDir::new().unwrap(); + let storage = LocalFS::new(dir.path().to_path_buf()); + + assert!( + list_tombstoned_streams(&storage, &None) + .await + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn marker_present_means_discoverable_by_listing() { + let dir = TempDir::new().unwrap(); + let storage = LocalFS::new(dir.path().to_path_buf()); + + storage + .put_object(&tombstone_path("test_stream", &None), to_bytes(&()), &None) + .await + .unwrap(); + + let discovered = list_tombstoned_streams(&storage, &None).await.unwrap(); + assert_eq!(discovered, vec!["test_stream".to_string()]); + } + + #[tokio::test] + async fn directory_without_the_actual_marker_is_not_reported_as_tombstoned() { + let dir = TempDir::new().unwrap(); + let storage = LocalFS::new(dir.path().to_path_buf()); + + // A directory can exist under the tombstone root without ever + // containing the marker itself (e.g. an interrupted write) -- + // list_dirs_relative alone can't tell the difference, so + // list_tombstoned_streams must re-verify via is_tombstoned. + let sibling_path = tombstone_path("test_stream", &None) + .parent() + .unwrap() + .join("not-the-marker"); + storage + .put_object(&sibling_path, to_bytes(&()), &None) + .await + .unwrap(); + + assert!(!is_tombstoned(&storage, "test_stream", &None).await.unwrap()); + assert!( + list_tombstoned_streams(&storage, &None) + .await + .unwrap() + .is_empty() + ); + } +} + +#[cfg(test)] +mod stream_deletion_dedup_tests { + use super::ACTIVE_STREAM_DELETIONS; + + // `spawn_stream_deletion` itself isn't unit-testable here: it reads + // PARSEABLE.storage/PARSEABLE.streams, and the global PARSEABLE static + // isn't initialized under `cargo test`. This instead exercises the + // contains_key-then-insert guard directly against the same map the real + // function uses, since that guard is the actual dedup mechanism. + #[test] + fn duplicate_key_is_recognized_as_already_running() { + let key = (Some("tenant-a".to_string()), "stream-a".to_string()); + ACTIVE_STREAM_DELETIONS.remove(&key); + + assert!(!ACTIVE_STREAM_DELETIONS.contains_key(&key)); + ACTIVE_STREAM_DELETIONS.insert(key.clone(), std::time::Instant::now()); + assert!(ACTIVE_STREAM_DELETIONS.contains_key(&key)); + // A second caller sees the job as already in flight and would skip + // re-inserting -- this is the exact check spawn_stream_deletion + // makes before spawning its background task. + assert!(ACTIVE_STREAM_DELETIONS.contains_key(&key)); + + ACTIVE_STREAM_DELETIONS.remove(&key); + assert!(!ACTIVE_STREAM_DELETIONS.contains_key(&key)); + } + + #[test] + fn same_stream_name_different_tenant_is_a_distinct_key() { + let key_a = (Some("tenant-a".to_string()), "shared-name".to_string()); + let key_b = (Some("tenant-b".to_string()), "shared-name".to_string()); + ACTIVE_STREAM_DELETIONS.remove(&key_a); + ACTIVE_STREAM_DELETIONS.remove(&key_b); + + ACTIVE_STREAM_DELETIONS.insert(key_a.clone(), std::time::Instant::now()); + assert!(!ACTIVE_STREAM_DELETIONS.contains_key(&key_b)); + + ACTIVE_STREAM_DELETIONS.remove(&key_a); + } +} + #[cfg(test)] mod manifest_ownership_tests { use super::manifest_segment_matches;