From 3b927080110e18fa849fbe49362c4c70247a151e Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Wed, 7 Oct 2026 14:53:53 +0800 Subject: [PATCH] feat(table-catalog): add durable catalog backups (#8266) * feat(table-catalog): add durable catalog backups * fix(table-catalog): preserve migration lock ordering * fix(table-catalog): preserve backup recovery fences * fix(table-catalog): order backup publication locks * fix(table-catalog): preserve recovery publication fences * test(table-catalog): use the storage facade for recovery fencing * test(ecstore): repair catalog publication fixtures * fix(table-catalog): address catalog backup clippy failures * test(table-catalog): adapt generation races to publication fencing --------- Co-authored-by: Henry Guo Co-authored-by: overtrue --- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 239 ++- .../lifecycle/object_handlers_common.rs | 41 +- crates/ecstore/src/object_api/types.rs | 3 +- .../src/object_api/write_commit_context.rs | 6 + crates/ecstore/src/store/bucket.rs | 112 +- docs/architecture/s3-tables-support-matrix.md | 5 +- docs/operations/s3-tables-cutover-runbook.md | 26 + .../admin/handlers/table_catalog/config.rs | 49 + .../src/admin/handlers/table_catalog/mod.rs | 74 +- .../admin/handlers/table_catalog/routes.rs | 10 + .../src/admin/handlers/table_catalog/tests.rs | 280 ++- rustfs/src/admin/route_policy.rs | 30 +- rustfs/src/admin/route_registration_test.rs | 4 + rustfs/src/app/multipart_usecase.rs | 5 +- rustfs/src/app/object/copy.rs | 3 + rustfs/src/app/object/delete.rs | 35 +- rustfs/src/app/object/extract.rs | 11 +- rustfs/src/app/object/internal_put.rs | 49 + rustfs/src/app/object/mod.rs | 9 +- rustfs/src/app/object/put.rs | 35 +- rustfs/src/app/object/restore.rs | 7 +- rustfs/src/app/storage_api.rs | 6 +- rustfs/src/storage/access.rs | 155 +- rustfs/src/storage/storage_api.rs | 9 +- .../src/table_catalog/iceberg/validation.rs | 82 +- rustfs/src/table_catalog/mod.rs | 9 + rustfs/src/table_catalog/store/backup.rs | 1837 +++++++++++++++++ rustfs/src/table_catalog/store/migration.rs | 11 +- rustfs/src/table_catalog/store/mod.rs | 379 +++- rustfs/src/table_catalog/store/object.rs | 97 +- rustfs/src/table_catalog/store/strong.rs | 122 +- rustfs/src/table_catalog/test_support.rs | 44 + rustfs/src/table_catalog/tests.rs | 966 ++++++++- scripts/table-catalog/README.md | 6 + 34 files changed, 4519 insertions(+), 237 deletions(-) create mode 100644 rustfs/src/table_catalog/store/backup.rs diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index a8f519734..e2f3b6588 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -4748,7 +4748,7 @@ async fn expire_transitioned_object_with_lock_lost_signal( bucket_incarnation_id: Uuid, lock_lost_signal: Option>, ) -> Result { - let publication_guard = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id) + let publication_guard = lifecycle_expiry_publication_guard(&api, &oi.bucket, &oi.name, bucket_incarnation_id) .await .ok_or_else(|| std::io::Error::other("lifecycle expiry is not allowed for this bucket"))?; let snapshot = lifecycle_delete_config_snapshot(&api, oi) @@ -4838,10 +4838,14 @@ pub fn gen_transition_objname(bucket: &str) -> Result { pub async fn transition_object(api: Arc, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> { let time_ilm = Metrics::time_ilm(lae.event.action); + let (publication_guard, bucket_incarnation_id) = lifecycle_transition_publication_guard(&api, oi) + .await + .ok_or_else(|| Error::other("lifecycle transition is not allowed for this bucket"))?; + let etag = if let Some(etag) = &oi.etag { etag } else { "" }; let etag = etag.to_string(); - let opts = ObjectOptions { + let mut opts = ObjectOptions { transition: TransitionOptions { status: lifecycle::TRANSITION_PENDING.to_string(), tier: lae.event.storage_class, @@ -4853,8 +4857,10 @@ pub async fn transition_object(api: Arc, oi: &ObjectInfo, lae: LcAuditE versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await, version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await, mod_time: oi.mod_time, + expected_bucket_incarnation_id: Some(bucket_incarnation_id), ..Default::default() }; + opts.add_namespace_lock_guard(&publication_guard); let result = api.transition_object(&oi.bucket, &oi.name, &opts).await; time_ilm(1)(); result @@ -5458,20 +5464,26 @@ pub async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, o .await } -async fn lifecycle_expiry_publication_guard( +async fn acquire_lifecycle_publication_guard(api: &ECStore, bucket: &str) -> Result { + let lock = api + .new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await?; + let guard = lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(Error::other)?; + if guard.is_lock_lost() { + return Err(Error::other("table-bucket publication lock was lost before lifecycle admission")); + } + Ok(guard) +} + +pub(crate) async fn lifecycle_expiry_publication_guard( api: &ECStore, - oi: &ObjectInfo, + bucket: &str, + object: &str, bucket_incarnation_id: Uuid, ) -> Option { - let result = async { - let lock = api - .new_ns_lock(&oi.bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) - .await?; - let guard = lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(Error::other)?; - if guard.is_lock_lost() { - return Err(Error::other("table-bucket publication lock was lost before lifecycle delete admission")); - } - if !metadata_boundary::lifecycle_expiry_allowed(api, &oi.bucket, bucket_incarnation_id).await? { + let result: std::result::Result, Error> = async { + let guard = acquire_lifecycle_publication_guard(api, bucket).await?; + if !metadata_boundary::lifecycle_expiry_allowed(api, bucket, bucket_incarnation_id).await? { return Ok(None); } Ok(Some(guard)) @@ -5484,8 +5496,8 @@ async fn lifecycle_expiry_publication_guard( event = EVENT_LIFECYCLE_DELETE_FAILED, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_LIFECYCLE, - bucket = %oi.bucket, - object = %oi.name, + bucket, + object, operation = "authorize_lifecycle_expiry", error = %err, "Lifecycle delete admission failed" @@ -5495,6 +5507,34 @@ async fn lifecycle_expiry_publication_guard( } } +async fn lifecycle_transition_publication_guard( + api: &ECStore, + oi: &ObjectInfo, +) -> Option<(rustfs_lock::NamespaceLockGuard, Uuid)> { + let result: std::result::Result<(rustfs_lock::NamespaceLockGuard, Uuid), Error> = async { + let guard = acquire_lifecycle_publication_guard(api, &oi.bucket).await?; + let configs = metadata_boundary::get_expiry_configs(api, &oi.bucket).await?; + Ok((guard, configs.bucket_incarnation_id)) + } + .await; + match result { + Ok(state) => Some(state), + Err(err) => { + warn!( + event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + bucket = %oi.bucket, + object = %oi.name, + operation = "authorize_lifecycle_transition", + error = %err, + "Lifecycle transition admission failed" + ); + None + } + } +} + pub async fn apply_expiry_on_transitioned_object( api: Arc, oi: &ObjectInfo, @@ -5551,7 +5591,8 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal( bucket_incarnation_id: Uuid, lock_lost_signal: Option>, ) -> bool { - let Some(publication_guard) = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id).await else { + let Some(publication_guard) = lifecycle_expiry_publication_guard(&api, &oi.bucket, &oi.name, bucket_incarnation_id).await + else { return false; }; let snapshot = match lifecycle_delete_config_snapshot(&api, oi).await { @@ -5844,6 +5885,7 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, #[cfg(test)] mod tests { use super::expiry_worker_count; + use super::lifecycle_transition_publication_guard; use super::{ DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX, DEFAULT_TRANSITION_WORKERS_CAP, EVENT_LIFECYCLE_EVALUATION_FAILED, EVENT_LIFECYCLE_EXPIRED_DETECTED, @@ -13028,6 +13070,171 @@ mod tests { } } + #[tokio::test] + #[serial] + async fn queued_noncurrent_batch_expiry_does_not_delete_from_table_bucket() { + let (_disk_paths, ecstore) = setup_test_env().await; + let bucket = format!("table-noncurrent-lifecycle-{}", Uuid::new_v4().simple()); + let object = "tables/table-id/data/part-00001.parquet"; + create_test_bucket(&ecstore, &bucket).await; + metadata_sys::update_in( + &ecstore.ctx, + &bucket, + BUCKET_VERSIONING_CONFIG, + b"Enabled".to_vec(), + ) + .await + .expect("bucket versioning should be enabled"); + + let mut old_reader = PutObjReader::from_vec(b"old table data".to_vec()); + let old = ecstore + .put_object( + &bucket, + object, + &mut old_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("the historical table version should be created"); + let old_version_id = old.version_id.expect("the historical version should have an identity"); + let mut current_reader = PutObjReader::from_vec(b"current table data".to_vec()); + ecstore + .put_object( + &bucket, + object, + &mut current_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("the current table version should be created"); + + let incarnation = ecstore + .bucket_incarnation_id_from_disk(&bucket) + .await + .expect("bucket incarnation should be available"); + let publication_lock = ecstore + .new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await + .expect("table-bucket publication lock should be created"); + let enable_guard = publication_lock + .get_write_lock(get_lock_acquire_timeout()) + .await + .expect("table-bucket enablement should acquire the publication lock"); + + let delete_store = ecstore.clone(); + let delete_bucket = bucket.clone(); + let delete_object = object.to_string(); + let mut delete = tokio::spawn(async move { + let target = ObjectToDelete { + object_name: delete_object, + version_id: Some(old_version_id), + ..Default::default() + }; + crate::bucket::lifecycle::object_handlers_common::delete_object_versions( + &delete_store, + &delete_bucket, + std::slice::from_ref(&target), + lifecycle::Event::default(), + incarnation, + ) + .await + }); + tokio::task::yield_now().await; + assert!( + tokio::time::timeout(StdDuration::from_millis(100), &mut delete) + .await + .is_err(), + "noncurrent lifecycle cleanup must wait for the table-bucket publication fence" + ); + + let sys = metadata_sys::bucket_metadata_sys_of(&ecstore.ctx).expect("metadata system should be initialized"); + let sys = sys.read().await.clone(); + let mut metadata = (*sys.get(&bucket).await.expect("bucket metadata should exist")).clone(); + metadata.table_bucket_config_json = br#"{"enabled":true}"#.to_vec(); + sys.persist_and_set(metadata) + .await + .expect("table bucket marker should be persisted"); + sys.reload_from_store(&bucket) + .await + .expect("table bucket marker should become authoritative"); + drop(enable_guard); + + let failed = tokio::time::timeout(StdDuration::from_secs(2), delete) + .await + .expect("noncurrent lifecycle cleanup should resume after the fence is released") + .expect("noncurrent lifecycle cleanup task should join"); + assert_eq!(failed, 1, "table-bucket noncurrent cleanup must be rejected"); + assert!( + ecstore + .get_object_info( + &bucket, + object, + &ObjectOptions { + version_id: Some(old_version_id.to_string()), + versioned: true, + ..Default::default() + }, + ) + .await + .is_ok(), + "table data must remain after queued noncurrent cleanup is rejected" + ); + } + + #[tokio::test] + #[serial] + async fn lifecycle_transition_admission_holds_publication_reader() { + let (_disk_paths, ecstore) = setup_test_env().await; + let bucket = format!("lifecycle-transition-{}", Uuid::new_v4().simple()); + let object = "tables/table-id/data/part-00001.parquet"; + create_test_bucket(&ecstore, &bucket).await; + let object_info = ObjectInfo { + bucket: bucket.clone(), + name: object.to_string(), + ..Default::default() + }; + + let publication_lock = ecstore + .new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await + .expect("table-bucket publication lock should be created"); + let enable_guard = publication_lock + .get_write_lock(get_lock_acquire_timeout()) + .await + .expect("publication writer should be acquired"); + let admission_store = ecstore.clone(); + let admission_object = object_info.clone(); + let mut admission = + tokio::spawn(async move { lifecycle_transition_publication_guard(&admission_store, &admission_object).await }); + assert!( + tokio::time::timeout(StdDuration::from_millis(100), &mut admission) + .await + .is_err(), + "transition admission must wait for an in-progress table publication" + ); + drop(enable_guard); + + let (transition_guard, incarnation) = tokio::time::timeout(StdDuration::from_secs(2), admission) + .await + .expect("transition admission should resume after publication") + .expect("transition admission task should join") + .expect("transition admission should succeed"); + assert_ne!(incarnation, Uuid::nil(), "transition admission must bind a bucket incarnation"); + let blocked_writer = publication_lock.get_write_lock(StdDuration::from_millis(100)).await; + assert!(blocked_writer.is_err(), "transition admission must retain the publication read lock"); + drop(transition_guard); + publication_lock + .get_write_lock(get_lock_acquire_timeout()) + .await + .expect("publication writer should proceed after transition admission releases its guard"); + } + #[tokio::test] #[serial] async fn queued_delete_all_rechecks_a_same_id_rule_moved_into_the_future() { diff --git a/crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs b/crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs index 6b57afcec..5df0d848e 100644 --- a/crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs +++ b/crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs @@ -36,6 +36,10 @@ pub async fn delete_object_versions( _lc_event: lifecycle::Event, bucket_incarnation_id: Uuid, ) -> usize { + if to_del.is_empty() { + return 0; + } + if to_del.iter().any(|target| { target.version_id.is_none() || (target.version_id.is_some_and(|version_id| version_id.is_nil()) && target.expected_identity.is_none()) @@ -52,6 +56,25 @@ pub async fn delete_object_versions( return to_del.len(); } + let Some(publication_guard) = super::bucket_lifecycle_ops::lifecycle_expiry_publication_guard( + api, + bucket, + to_del[0].object_name.as_str(), + bucket_incarnation_id, + ) + .await + else { + debug!( + event = EVENT_LIFECYCLE_CLEANUP_SKIPPED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + bucket, + target_count = to_del.len(), + reason = "publication_admission_rejected", + "Skipped lifecycle noncurrent version cleanup" + ); + return to_del.len(); + }; let delete_config_snapshot = match ReplicationObjectBridge::delete_request_config(api, bucket).await { Ok(snapshot) => Arc::new(snapshot), Err(err) => { @@ -78,17 +101,13 @@ pub async fn delete_object_versions( remaining = &[]; } - let (mut deleted_objs, errors) = api - .delete_objects( - bucket, - to_del.to_vec(), - ObjectOptions { - delete_replication_config_snapshot: Some(Arc::clone(&delete_config_snapshot)), - expected_bucket_incarnation_id: Some(bucket_incarnation_id), - ..Default::default() - }, - ) - .await; + let mut delete_options = ObjectOptions { + delete_replication_config_snapshot: Some(Arc::clone(&delete_config_snapshot)), + expected_bucket_incarnation_id: Some(bucket_incarnation_id), + ..Default::default() + }; + delete_options.add_namespace_lock_guard(&publication_guard); + let (mut deleted_objs, errors) = api.delete_objects(bucket, to_del.to_vec(), delete_options).await; failed += errors.iter().filter(|err| err.is_some()).count(); for (i, deleted_obj) in deleted_objs.iter_mut().enumerate() { diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 7666ef2b7..78f8df7a7 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -1237,7 +1237,8 @@ impl ObjectOptions { self.version_id.is_some() || !self.versioned || self.version_suspended } - pub(crate) fn add_namespace_lock_lost_signal(&mut self, signal: Arc) { + #[doc(hidden)] + pub fn add_namespace_lock_lost_signal(&mut self, signal: Arc) { #[cfg(test)] let test_fence = NAMESPACE_LOCK_SIGNAL_TEST_FENCES .get_or_init(|| std::sync::Mutex::new(Vec::new())) diff --git a/crates/ecstore/src/object_api/write_commit_context.rs b/crates/ecstore/src/object_api/write_commit_context.rs index 571664265..769255c1d 100644 --- a/crates/ecstore/src/object_api/write_commit_context.rs +++ b/crates/ecstore/src/object_api/write_commit_context.rs @@ -58,6 +58,12 @@ impl WriteCommitGuard { } } + /// Return the loss signal of the namespace owner retained by acquisition. + #[doc(hidden)] + pub fn lock_lost_signal(&self) -> Option> { + self.guards.first().and_then(|guard| guard.lock_lost_signal()) + } + pub fn is_lock_lost(&self) -> bool { self.guards.iter().any(|guard| guard.is_lock_lost() || guard.is_released()) } diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 55d78d84a..b2de0cbaf 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -121,9 +121,7 @@ async fn table_catalog_metadata_exists(ctx: &crate::runtime::instance::InstanceC } async fn validate_table_bucket_delete_guard(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<()> { - let table_bucket_enabled = metadata_sys::get_in(ctx, bucket) - .await - .is_ok_and(|metadata| metadata.table_bucket_enabled()); + let table_bucket_enabled = table_bucket_enabled_in(ctx, bucket).await?; if table_bucket_enabled { validate_table_bucket_delete_allowed(bucket, true, table_catalog_metadata_exists(ctx, bucket).await?)?; } @@ -131,6 +129,21 @@ async fn validate_table_bucket_delete_guard(ctx: &crate::runtime::instance::Inst Ok(()) } +async fn table_bucket_enabled_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result { + match metadata_sys::get_in(ctx, bucket).await { + Ok(metadata) => Ok(metadata.table_bucket_enabled()), + // A live bucket may be absent from the metadata cache during lazy + // startup. Re-read persisted metadata before deciding that it is an + // ordinary bucket; an actual metadata backend failure must never turn + // a protected-bucket check into a best-effort operation. + Err(Error::ConfigNotFound) => { + let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(ctx, bucket).await?; + Ok(persisted && metadata.table_bucket_enabled()) + } + Err(err) => Err(err), + } +} + fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] { [ table_bucket_catalog_metadata_prefix(bucket), @@ -298,6 +311,10 @@ impl ECStore { Ok(tags) } + pub async fn table_bucket_enabled(&self, bucket: &str) -> Result { + table_bucket_enabled_in(&self.ctx, bucket).await + } + pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> { let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?; sys.read().await.get_bucket_policy(bucket).await @@ -436,6 +453,26 @@ impl ECStore { }) } + async fn acquire_bucket_publication_write_lock(&self, bucket: &str) -> Result { + let lock = self + .new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await?; + lock.get_write_lock(get_lock_acquire_timeout()) + .await + .map_err(|err| match err { + rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => { + StorageError::NamespaceLockQuorumUnavailable { + mode: "table_bucket_publication_write", + bucket: bucket.to_string(), + object: rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH.to_string(), + required, + achieved, + } + } + other => StorageError::Lock(other), + }) + } + async fn mark_bucket_deleted(&self, bucket: &str) -> Result<()> { let marker_volume = bucket_deleted_marker_volume(bucket); @@ -1056,7 +1093,7 @@ impl ECStore { &self, bucket: &str, opts: &DeleteBucketOptions, - mut diagnostic_budget: BucketDeleteDiagnosticBudget, + diagnostic_budget: BucketDeleteDiagnosticBudget, ) -> Result<()> { if is_meta_bucketname(bucket) { return Err(StorageError::BucketNameInvalid(bucket.to_string())); @@ -1066,6 +1103,33 @@ impl ECStore { return Err(StorageError::BucketNameInvalid(err.to_string())); } + // Bucket deletion is a publication mutation too: a durable catalog + // backup must not enumerate or verify table objects while their + // containing bucket is being physically removed. Acquire this fence + // before the lifecycle and namespace locks to match object writers. + // `no_lock` is reserved for callers that already own the enclosing + // mutation locks (for example, failed bucket-creation rollback). + let publication_guard = if !opts.no_lock { + Some(self.acquire_bucket_publication_write_lock(bucket).await?) + } else { + None + }; + + await_bucket_namespace_operation( + publication_guard.as_ref(), + bucket, + "table-bucket publication fence during bucket deletion", + self.handle_delete_bucket_under_publication_lock(bucket, opts, diagnostic_budget), + ) + .await + } + + async fn handle_delete_bucket_under_publication_lock( + &self, + bucket: &str, + opts: &DeleteBucketOptions, + mut diagnostic_budget: BucketDeleteDiagnosticBudget, + ) -> Result<()> { let bucket_lifecycle_guard = if !opts.no_lock { Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?) } else { @@ -1227,9 +1291,9 @@ mod tests { BUCKET_DELETE_DIAGNOSTIC_MAX_ELAPSED, BUCKET_DELETE_DIAGNOSTIC_MAX_ENTRIES, BUCKET_DELETE_XLMETA_DIAGNOSTIC_MAX_BYTES, BucketDeleteBlockerKind, BucketDeleteDiagnosticBudget, BucketMetadataLessResidue, SCANNER_BUCKET_LIST_SET_CONCURRENCY, await_bucket_namespace_operation, bucket_delete_metadata_cleanup_prefixes, bucket_deleted_marker_prefix, - bucket_deleted_marker_volume, bucket_list_set_concurrency, record_bucket_delete_blocker, run_bucket_usage_cleanup, - run_physical_bucket_deletion, scan_metadata_less_residue, scan_metadata_less_residue_with_budget, - should_override_created_from_metadata, validate_table_bucket_delete_allowed, + bucket_deleted_marker_volume, bucket_list_set_concurrency, get_lock_acquire_timeout, record_bucket_delete_blocker, + run_bucket_usage_cleanup, run_physical_bucket_deletion, scan_metadata_less_residue, + scan_metadata_less_residue_with_budget, should_override_created_from_metadata, validate_table_bucket_delete_allowed, }; use crate::bucket::metadata::{BucketMetadata, table_bucket_catalog_metadata_prefix}; use crate::bucket::metadata_sys; @@ -3296,6 +3360,40 @@ mod tests { assert!(any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await); } + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn bucket_delete_waits_for_table_publication_readers() { + let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await; + let bucket = format!("bucket-delete-publication-{}", Uuid::new_v4().simple()); + ecstore + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + + let publication_lock = ecstore + .new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH) + .await + .expect("publication lock should be created"); + let publication_reader = publication_lock + .get_read_lock(get_lock_acquire_timeout()) + .await + .expect("publication reader should be acquired"); + + let delete_store = Arc::clone(&ecstore); + let mut delete = tokio::spawn(async move { delete_store.delete_bucket(&bucket, &DeleteBucketOptions::default()).await }); + assert!( + tokio::time::timeout(Duration::from_millis(250), &mut delete).await.is_err(), + "bucket deletion must remain behind the table publication reader" + ); + + drop(publication_reader); + tokio::time::timeout(Duration::from_secs(10), &mut delete) + .await + .expect("bucket deletion should proceed after publication reader release") + .expect("bucket deletion task should join") + .expect("empty bucket deletion should succeed"); + } + #[tokio::test] #[serial] async fn bucket_delete_mark_delete_rejects_non_empty_bucket_without_force() { diff --git a/docs/architecture/s3-tables-support-matrix.md b/docs/architecture/s3-tables-support-matrix.md index d103220f1..620f7dae4 100644 --- a/docs/architecture/s3-tables-support-matrix.md +++ b/docs/architecture/s3-tables-support-matrix.md @@ -68,7 +68,8 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t | Snapshot refs | Supported | List, create/replace, delete via commits; `main` is protected; refs with explicit retention need forced delete. | | Iceberg views | Supported | Create, list, load, replace, exists, drop with view-scoped authorization; only view format version 1. | | LoadTable and table credentials endpoint | Supported | Vending only on negotiated `vended-credentials`; one temporary session scoped to the warehouse prefix and current metadata location; missing credential permission falls back to metadata-only with an explicit reason; responses carry `Cache-Control: no-store, private`. | -| Catalog diagnostics and export | Supported | Object-backed and durable-strong stores expose recovery state, consistency, backing manifest, WAL state, migration target, single-active-writer policy, and scale validation matrix. Durable-strong responses also identify the snapshot path, ETag, and format version; diagnostics re-check the snapshot observation before returning and follow the persisted metadata directory. Until strong maintenance configuration is persisted, no count-based retention is added; reachable metadata remains protected and orphan candidates do not authorize deletion. | +| Catalog diagnostics and export | Supported | Object-backed and durable-strong stores expose recovery state, consistency, backing manifest, WAL state, migration target, single-active-writer policy, and scale validation matrix. Durable-strong responses also identify the snapshot path, ETag, and format version; diagnostics re-check the snapshot observation before returning and use zero-file retention until strong maintenance configuration is persisted. | +| Durable catalog backup and restore | Preview / controlled | Durable-strong only. `POST /iceberg/v1/{warehouse}/catalog/backup` seals a table-bucket snapshot behind the publication and migration fences, records catalog/object watermarks, and stores an immutable backup artifact. `POST /iceberg/v1/{warehouse}/catalog/restore` verifies those watermarks, requires an explicit replacement decision for an existing target, applies the catalog snapshot with ETag-CAS, and leaves a recoverable restore intent if maintenance-state finalization fails. It does not copy data files or claim automatic cross-region failover. | | Catalog import and rollback | Supported | Import/register and online rollback go through validation and commit paths. Online rollback accepts only forward-safe targets; restoring an older target that lowers watermarks is an offline disaster-recovery operation with all writers stopped. | | External catalog bridge | Supported operator path | Operator-supplied metadata pointer sync/import. Vendor SDK polling and policy mirroring not claimed. | | Multi-table transactions | Not claimed | Single-table commit atomicity only. | @@ -118,7 +119,7 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t | Durable backing migration preflight | Supported | `GET /iceberg/v1/{warehouse}/catalog/migration` (and the alias) reports inventory, recovery blockers, prefix-index readiness, identifier collisions, fence state, target agreement, and per-bucket cutover readiness. | | Durable backing migration execution | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/migration` fences registry changes, acquires a persistent per-bucket write fence, drains in-flight mutations, materializes the snapshot, and reports `ready_to_enable_durable_strong`. `DELETE` cancels only while the target has not advanced. Both mutations require `admin:MigrateTableCatalog`. Procedure: [s3-tables-cutover-runbook.md](../operations/s3-tables-cutover-runbook.md). | | Strong snapshot rolling compatibility | Supported | Reads snapshot versions 1 and 2; writes version 1 by default and version 2 only after both `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2` and `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED` are set. A process rejects a lower format after observing a higher one; once v2 is fleet-confirmed, data-plane resolution fails closed until the persisted snapshot is v2. | -| Disaster recovery rehearsal | Manual/live harness | `failure_coverage.py --print-disaster-recovery-rehearsal`. | +| Disaster recovery rehearsal | Manual/live harness | `failure_coverage.py --print-disaster-recovery-rehearsal`; the controlled catalog backup/restore path additionally requires referenced table objects to remain available and does not provide cross-region object replication. | | Scale and fault rehearsal | Manual/live harness | `failure_coverage.py --print-scale-fault-rehearsal`. | | Durable strong snapshot backing cutover | Preview / controlled | `RUSTFS_TABLE_CATALOG_BACKING=durable-strong` only after every table bucket reports `SNAPSHOT_MATERIALIZED` and `ready_to_enable_durable_strong: true`. No separate external KV/WAL service is claimed; the legacy `STRONG_KV_WAL` and `CUT_OVER_LINEARIZABLE_READS` labels in v1 manifests are wire-compatibility labels, not claims. | | Single active writer region | Supported policy | Diagnostics publish single-active-writer semantics and read-only replica limits. | diff --git a/docs/operations/s3-tables-cutover-runbook.md b/docs/operations/s3-tables-cutover-runbook.md index 7863090c1..bc2fe8b0c 100644 --- a/docs/operations/s3-tables-cutover-runbook.md +++ b/docs/operations/s3-tables-cutover-runbook.md @@ -42,6 +42,32 @@ DELETE /iceberg/v1/{warehouse}/catalog/migration After the durable-strong state advances, cancellation fails closed; recovery requires an operator-selected restore or reverse migration. +## Controlled Catalog Backup And Restore + +The durable catalog backup and restore endpoints are a controlled, same-table-bucket recovery mechanism. They are available only with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong`; they protect the catalog snapshot and maintenance state, but do not copy or replicate table data, metadata, manifest, or delete objects. + +Backup artifacts and restore intents use a strict internal format version. A binary rejects artifacts or pending intents from another format version; create a new backup after upgrading instead of assuming cross-version restore compatibility. + +1. Stop or drain catalog writers and maintenance workers for the table bucket. The backup operation acquires the table-bucket publication fence and the durable migration fence, rejects pending commit recovery and pending restore recovery, rejects active maintenance leases, and records the current snapshot ETag, format version, catalog fingerprint, and referenced object watermarks. +2. Create a backup with a principal authorized for `MigrateTableCatalogAction`. Backup creation persists an immutable recovery artifact and is therefore not covered by read-only catalog permission. The optional `expected-snapshot-etag` provides an additional compare-and-capture guard. + + ```text + POST /iceberg/v1/{warehouse}/catalog/backup + {"expected-snapshot-etag":""} + ``` + +3. Keep every referenced table object available for the retention period. A successful backup is immutable and idempotent by content-derived backup ID; it is not a copy of those objects and cannot restore an object that was deleted or changed after capture. +4. Before restoring, stop writers and maintenance workers again. Verify the target table bucket is still table-enabled, obtain its current snapshot ETag, and use `expected-snapshot-etag` whenever replacing an existing catalog. `allow-replace` must be explicitly `true` to replace a non-identical existing bucket snapshot. + + ```text + POST /iceberg/v1/{warehouse}/catalog/restore + {"backup-id":"","expected-snapshot-etag":"","allow-replace":true} + ``` + +5. Treat a restore conflict as a stale-target or changed-object condition, not as permission to retry blindly. The service verifies all recorded object watermarks before applying an ETag-CAS snapshot replacement. If the process fails after catalog replacement, retrying the same backup ID is safe only while maintenance state is exactly the persisted source state or already exactly the backup target state. A partial multi-object reconciliation fails closed with a conflict and requires operator inspection before manual cleanup. + +This procedure is not cross-region failover. Cross-region recovery requires an independent object replication/backup system and an operator-selected catalog import or restore procedure. + ## Strong Snapshot Version 1 to Version 2 1. Keep snapshot writes on version 1 during a rolling binary upgrade. Current binaries read both versions. diff --git a/rustfs/src/admin/handlers/table_catalog/config.rs b/rustfs/src/admin/handlers/table_catalog/config.rs index 56e3feeb0..0ee110bc3 100644 --- a/rustfs/src/admin/handlers/table_catalog/config.rs +++ b/rustfs/src/admin/handlers/table_catalog/config.rs @@ -137,6 +137,55 @@ impl Operation for CancelTableCatalogMigrationHandler { } } +pub struct CreateTableCatalogBackupHandler {} + +#[async_trait::async_trait] +impl Operation for CreateTableCatalogBackupHandler { + async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let resource = TableCatalogResource::warehouse(&warehouse); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::MigrateTableCatalogAction).await?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let request = read_json_body_or_default::(std::mem::take(&mut req.input)).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; + let started = Instant::now(); + let result = store + .create_durable_catalog_backup(&warehouse, request.expected_snapshot_etag.as_deref()) + .await + .map_err(catalog_store_error); + record_table_catalog_admin_operation_result("catalog-backup", &warehouse, "", "", started, &result); + let response = result?; + build_json_response(StatusCode::OK, &response) + } +} + +pub struct RestoreTableCatalogBackupHandler {} + +#[async_trait::async_trait] +impl Operation for RestoreTableCatalogBackupHandler { + async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { + let warehouse = warehouse_from_params(¶ms)?; + let resource = TableCatalogResource::warehouse(&warehouse); + authorize_table_catalog_resource_request(&req, &resource, AdminAction::MigrateTableCatalogAction).await?; + ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?; + let request = read_json_body::(std::mem::take(&mut req.input)).await?; + let store = table_catalog_store_from_extensions(&req.extensions)?; + let started = Instant::now(); + let result = store + .restore_durable_catalog_backup( + &warehouse, + &request.backup_id, + request.expected_snapshot_etag.as_deref(), + request.allow_replace, + ) + .await + .map_err(catalog_store_error); + record_table_catalog_admin_operation_result("catalog-restore", &warehouse, "", "", started, &result); + let response = result?; + build_json_response(StatusCode::OK, &response) + } +} + pub struct BackfillTableWarehouseIndexHandler {} #[async_trait::async_trait] diff --git a/rustfs/src/admin/handlers/table_catalog/mod.rs b/rustfs/src/admin/handlers/table_catalog/mod.rs index 9d564afe3..73825122c 100644 --- a/rustfs/src/admin/handlers/table_catalog/mod.rs +++ b/rustfs/src/admin/handlers/table_catalog/mod.rs @@ -177,7 +177,8 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[ "POST /v1/{prefix}/namespaces/{namespace}/views/{view}", "DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}", ]; -const TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS: &[&str] = &[]; +const TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS: &[&str] = + &["POST /v1/{prefix}/catalog/backup", "POST /v1/{prefix}/catalog/restore"]; static GET_CONFIG_HANDLER: GetCatalogConfigHandler = GetCatalogConfigHandler {}; static ENABLE_TABLE_BUCKET_HANDLER: EnableTableBucketHandler = EnableTableBucketHandler {}; @@ -187,6 +188,8 @@ static GET_TABLE_CATALOG_MIGRATION_HANDLER: GetTableCatalogMigrationHandler = Ge static MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER: MaterializeTableCatalogMigrationHandler = MaterializeTableCatalogMigrationHandler {}; static CANCEL_TABLE_CATALOG_MIGRATION_HANDLER: CancelTableCatalogMigrationHandler = CancelTableCatalogMigrationHandler {}; +static CREATE_TABLE_CATALOG_BACKUP_HANDLER: CreateTableCatalogBackupHandler = CreateTableCatalogBackupHandler {}; +static RESTORE_TABLE_CATALOG_BACKUP_HANDLER: RestoreTableCatalogBackupHandler = RestoreTableCatalogBackupHandler {}; static BACKFILL_TABLE_WAREHOUSE_INDEX_HANDLER: BackfillTableWarehouseIndexHandler = BackfillTableWarehouseIndexHandler {}; static LIST_NAMESPACES_HANDLER: RestListNamespacesHandler = RestListNamespacesHandler {}; static CREATE_NAMESPACE_HANDLER: RestCreateNamespaceHandler = RestCreateNamespaceHandler {}; @@ -463,6 +466,24 @@ struct CatalogImportRequest { properties: BTreeMap, } +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct CatalogBackupRequest { + #[serde(default, rename = "expected-snapshot-etag")] + expected_snapshot_etag: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct CatalogRestoreRequest { + #[serde(rename = "backup-id")] + backup_id: String, + #[serde(default, rename = "expected-snapshot-etag")] + expected_snapshot_etag: Option, + #[serde(default, rename = "allow-replace")] + allow_replace: bool, +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct ExternalCatalogBridgeRequest { @@ -1289,6 +1310,7 @@ struct TableCommitPublicationState { phase: TableCommitPublicationPhase, bucket_fence: Option, table_fence: Option<(String, String, String)>, + catalog_migration_read_permit: bool, observed_objects: BTreeMap<(String, String), TableCommitObservedObject>, guards: Vec, } @@ -1490,15 +1512,27 @@ where } publication.bucket_fence = Some(table_bucket.to_string()); } - let publication_lock = crate::table_catalog::default_table_bucket_publication_lock_path(); - let guard = match self.backend.acquire_write_lock(table_bucket, &publication_lock).await { - Ok(guard) => guard, + let migration_guards = match self.backend.acquire_catalog_migration_read_guards(table_bucket).await { + Ok(guards) => guards, Err(err) => { self.publication.lock().bucket_fence = None; return Err(err); } }; - self.publication.lock().guards.push(guard); + let publication_lock = crate::table_catalog::default_table_bucket_publication_lock_path(); + let guard = match self.backend.acquire_write_lock(table_bucket, &publication_lock).await { + Ok(guard) => guard, + Err(err) => { + let mut publication = self.publication.lock(); + publication.bucket_fence = None; + publication.catalog_migration_read_permit = false; + return Err(err); + } + }; + let mut publication = self.publication.lock(); + publication.catalog_migration_read_permit = !migration_guards.is_empty(); + publication.guards.extend(migration_guards); + publication.guards.push(guard); Ok(()) } @@ -1634,6 +1668,7 @@ where publication.observed_objects.clear(); publication.bucket_fence = None; publication.table_fence = None; + publication.catalog_migration_read_permit = false; publication.phase = TableCommitPublicationPhase::Complete; } @@ -1749,6 +1784,17 @@ where self.backend.delete_object(bucket, object).await } + async fn delete_object_if_match( + &self, + bucket: &str, + object: &str, + expected_etag: &str, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.authorize(bucket, object, S3Action::DeleteObjectAction).await?; + self.ensure_observation_allowed(bucket, object)?; + self.backend.delete_object_if_match(bucket, object, expected_etag).await + } + async fn delete_object_unlocked(&self, bucket: &str, object: &str) -> crate::table_catalog::TableCatalogStoreResult<()> { self.delete_object(bucket, object).await } @@ -1805,6 +1851,17 @@ where && publication.guards.iter().all(|guard| !guard.is_lock_lost()) } + fn acquires_catalog_migration_read_permit(&self) -> bool { + self.backend.acquires_catalog_migration_read_permit() + } + + fn catalog_migration_read_permit_status(&self) -> Option { + let publication = self.publication.lock(); + publication + .catalog_migration_read_permit + .then(|| publication.guards.iter().all(|guard| !guard.is_lock_lost())) + } + fn complete_table_commit_publication(&self) { self.complete_publication(); } @@ -2299,11 +2356,10 @@ fn table_catalog_object_store_from_extensions(extensions: &http::Extensions) -> async fn table_bucket_enabled_from_extensions(extensions: &http::Extensions, bucket: &str) -> S3Result { let store = runtime_sources::object_store_from_extensions(extensions) .ok_or_else(|| table_catalog_internal_error("request object store is not initialized"))?; - let metadata = store - .get_bucket_metadata(bucket) + store + .table_bucket_enabled(bucket) .await - .map_err(|err| s3_error!(InvalidRequest, "failed to load table bucket metadata for {bucket}: {}", err))?; - Ok(metadata.table_bucket_enabled()) + .map_err(|err| s3_error!(InvalidRequest, "failed to load table bucket metadata for {bucket}: {}", err)) } async fn ensure_table_bucket_enabled_from_extensions(extensions: &http::Extensions, bucket: &str) -> S3Result<()> { diff --git a/rustfs/src/admin/handlers/table_catalog/routes.rs b/rustfs/src/admin/handlers/table_catalog/routes.rs index 88dbac3eb..5ba191123 100644 --- a/rustfs/src/admin/handlers/table_catalog/routes.rs +++ b/rustfs/src/admin/handlers/table_catalog/routes.rs @@ -54,6 +54,16 @@ fn register_table_catalog_prefix_routes(r: &mut S3Router, prefix format!("{prefix}/{{warehouse}}/catalog/migration").as_str(), AdminOperation(&CANCEL_TABLE_CATALOG_MIGRATION_HANDLER), )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/catalog/backup").as_str(), + AdminOperation(&CREATE_TABLE_CATALOG_BACKUP_HANDLER), + )?; + r.insert( + Method::POST, + format!("{prefix}/{{warehouse}}/catalog/restore").as_str(), + AdminOperation(&RESTORE_TABLE_CATALOG_BACKUP_HANDLER), + )?; r.insert( Method::POST, format!("{prefix}/{{warehouse}}/catalog/warehouse-index/backfill").as_str(), diff --git a/rustfs/src/admin/handlers/table_catalog/tests.rs b/rustfs/src/admin/handlers/table_catalog/tests.rs index 2b5376ed2..2f806d2c6 100644 --- a/rustfs/src/admin/handlers/table_catalog/tests.rs +++ b/rustfs/src/admin/handlers/table_catalog/tests.rs @@ -540,7 +540,11 @@ fn catalog_config_response_reports_durable_strong_backing_override() { .contains(&"POST /v1/{prefix}/namespaces/{namespace}/properties") ); assert!(response.endpoints.contains(&"POST /v1/{prefix}/tables/rename")); - assert_eq!(response.endpoints.as_slice(), TABLE_CATALOG_ENDPOINTS); + assert!(response.endpoints.ends_with(TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS)); + assert_eq!( + response.endpoints.len(), + TABLE_CATALOG_ENDPOINTS.len() + TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS.len() + ); } #[test] @@ -687,6 +691,8 @@ fn table_catalog_handlers_require_table_admin_actions() { ("DisableTableBucketHandler", "AdminAction::SetTableBucketAction"), ("GetTableBucketHandler", "AdminAction::GetTableBucketAction"), ("GetTableCatalogMigrationHandler", "AdminAction::GetTableCatalogAction"), + ("CreateTableCatalogBackupHandler", "AdminAction::MigrateTableCatalogAction"), + ("RestoreTableCatalogBackupHandler", "AdminAction::MigrateTableCatalogAction"), ("RestListNamespacesHandler", "AdminAction::GetTableNamespaceAction"), ("RestCreateNamespaceHandler", "AdminAction::SetTableNamespaceAction"), ("RestGetNamespaceHandler", "AdminAction::GetTableNamespaceAction"), @@ -1003,6 +1009,23 @@ fn enable_table_bucket_response_fences_before_marker_and_catalog_entry() { ); } +#[test] +fn table_commit_publication_acquires_migration_permits_before_publication_lock() { + let src = table_catalog_handler_source(); + let block = function_block(&src, "async fn begin_bucket_publication"); + let migration_permits = block + .find("acquire_catalog_migration_read_guards(table_bucket)") + .expect("publication should acquire catalog migration permits"); + let publication_lock = block + .find("acquire_write_lock(table_bucket, &publication_lock)") + .expect("publication should acquire its bucket lock"); + + assert!( + migration_permits < publication_lock, + "catalog migration permits must be acquired before the table-bucket publication lock" + ); +} + #[test] fn table_catalog_resource_builds_policy_object_scope() { let namespace = crate::table_catalog::Namespace::parse("analytics.daily_events").expect("namespace should parse"); @@ -1066,6 +1089,8 @@ fn rest_catalog_mvp_routes_use_implemented_handlers() { let _: &GetTableCatalogMigrationHandler = &GET_TABLE_CATALOG_MIGRATION_HANDLER; let _: &MaterializeTableCatalogMigrationHandler = &MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER; let _: &CancelTableCatalogMigrationHandler = &CANCEL_TABLE_CATALOG_MIGRATION_HANDLER; + let _: &CreateTableCatalogBackupHandler = &CREATE_TABLE_CATALOG_BACKUP_HANDLER; + let _: &RestoreTableCatalogBackupHandler = &RESTORE_TABLE_CATALOG_BACKUP_HANDLER; let _: &RestListNamespacesHandler = &LIST_NAMESPACES_HANDLER; let _: &RestCreateNamespaceHandler = &CREATE_NAMESPACE_HANDLER; let _: &RestGetNamespaceHandler = &GET_NAMESPACE_HANDLER; @@ -5735,11 +5760,15 @@ async fn commit_publication_lock_order_remains_compatible_with_old_maintenance_n ) .await }); - metadata_backend.wait_for_lock_attempts(2).await; + metadata_backend.wait_for_lock_attempts(3).await; let migration_lock = crate::table_catalog::TableCatalogObjectPaths::default().backing_migration_fence_lock_path("warehouse"); assert_eq!( metadata_backend.lock_attempts.lock().await.as_slice(), &[ + ( + "warehouse".to_string(), + crate::table_catalog::default_table_bucket_publication_lock_path() + ), (crate::admin::storage_api::RUSTFS_META_BUCKET.to_string(), migration_lock), (crate::admin::storage_api::RUSTFS_META_BUCKET.to_string(), table_path), ] @@ -13509,6 +13538,253 @@ async fn legacy_commit_rejects_mismatched_table_uuid_before_commit() { assert_eq!(unchanged.generation, current.generation); } +#[derive(Clone)] +struct ColdStrongRegistrationBackend { + inner: B, + armed: Arc, + reader_ready: Arc, + resume: Arc, +} + +impl ColdStrongRegistrationBackend { + async fn pause_after_reader(&self) { + if self.armed.swap(false, std::sync::atomic::Ordering::SeqCst) { + self.reader_ready.notify_one(); + self.resume.notified().await; + } + } +} + +#[async_trait::async_trait] +impl crate::table_catalog::TableCatalogObjectBackend + for ColdStrongRegistrationBackend +{ + fn strong_catalog_runtime(&self) -> Option { + self.inner.strong_catalog_runtime() + } + + fn acquires_catalog_migration_read_permit(&self) -> bool { + self.inner.acquires_catalog_migration_read_permit() + } + + async fn acquire_catalog_migration_read_guards( + &self, + bucket: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + let guards = self.inner.acquire_catalog_migration_read_guards(bucket).await?; + self.pause_after_reader().await; + Ok(guards) + } + + async fn acquire_read_lock( + &self, + bucket: &str, + object: &str, + ) -> crate::table_catalog::TableCatalogStoreResult { + let guard = self.inner.acquire_read_lock(bucket, object).await?; + if object == crate::table_catalog::TableCatalogObjectPaths::default().backing_migration_global_fence_lock_path() { + self.pause_after_reader().await; + } + Ok(guard) + } + + async fn acquire_write_lock( + &self, + bucket: &str, + object: &str, + ) -> crate::table_catalog::TableCatalogStoreResult { + self.inner.acquire_write_lock(bucket, object).await + } + + async fn read_object( + &self, + bucket: &str, + object: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + self.inner.read_object(bucket, object).await + } + + async fn read_object_unlocked( + &self, + bucket: &str, + object: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + self.inner.read_object_unlocked(bucket, object).await + } + + async fn read_object_limited( + &self, + bucket: &str, + object: &str, + max_size: usize, + ) -> crate::table_catalog::TableCatalogStoreResult> { + self.inner.read_object_limited(bucket, object, max_size).await + } + + async fn object_metadata( + &self, + bucket: &str, + object: &str, + ) -> crate::table_catalog::TableCatalogStoreResult> { + self.inner.object_metadata(bucket, object).await + } + + async fn object_exists(&self, bucket: &str, object: &str) -> crate::table_catalog::TableCatalogStoreResult { + self.inner.object_exists(bucket, object).await + } + + async fn put_object( + &self, + bucket: &str, + object: &str, + data: Vec, + precondition: crate::table_catalog::TableCatalogPutPrecondition, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.inner.put_object(bucket, object, data, precondition).await + } + + async fn put_object_unlocked( + &self, + bucket: &str, + object: &str, + data: Vec, + precondition: crate::table_catalog::TableCatalogPutPrecondition, + guards: Vec, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.inner + .put_object_unlocked(bucket, object, data, precondition, guards) + .await + } + + async fn delete_object(&self, bucket: &str, object: &str) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.inner.delete_object(bucket, object).await + } + + async fn delete_object_if_match( + &self, + bucket: &str, + object: &str, + expected_etag: &str, + ) -> crate::table_catalog::TableCatalogStoreResult<()> { + self.inner.delete_object_if_match(bucket, object, expected_etag).await + } + + async fn list_objects(&self, bucket: &str, prefix: &str) -> crate::table_catalog::TableCatalogStoreResult> { + self.inner.list_objects(bucket, prefix).await + } +} + +#[tokio::test] +async fn cold_strong_registration_completes_while_other_bucket_backup_waits() { + use crate::table_catalog::{ + EcStoreTableCatalogObjectBackend, StrongTableCatalogRuntime, StrongTableCatalogStore, TableCatalogPutPrecondition, + }; + let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await; + let bucket = format!("cold-register-{}", Uuid::new_v4().simple()); + let backup_bucket = format!("other-backup-{}", Uuid::new_v4().simple()); + for bucket in [&bucket, &backup_bucket] { + object_store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create table bucket"); + enable_table_bucket_marker(&object_store, bucket) + .await + .expect("enable table bucket marker"); + } + let inner = EcStoreTableCatalogObjectBackend::new_with_strong_runtime(object_store, StrongTableCatalogRuntime::default()); + let backend = ColdStrongRegistrationBackend { + inner: inner.clone(), + armed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + reader_ready: Arc::new(tokio::sync::Notify::new()), + resume: Arc::new(tokio::sync::Notify::new()), + }; + let catalog = StrongTableCatalogStore::new(backend.clone()); + for bucket in [&bucket, &backup_bucket] { + catalog + .put_table_bucket(table_bucket_entry_from_metadata_marker(bucket)) + .await + .expect("seed durable table bucket"); + } + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace"); + catalog + .create_namespace(crate::table_catalog::NamespaceEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.clone(), + namespace: namespace.public_name(), + namespace_id: namespace.storage_id(), + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::new(), + created_at: None, + updated_at: None, + }) + .await + .expect("seed namespace"); + let table = crate::table_catalog::IdentifierSegment::parse("events").expect("table"); + let location = crate::table_catalog::default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + inner + .put_object( + &bucket, + &location, + serde_json::to_vec(&test_table_metadata_json( + "metadata-table-uuid", + &format!("s3://{bucket}/tables/table-id"), + )) + .expect("metadata JSON"), + TableCatalogPutPrecondition::IfAbsent, + ) + .await + .expect("seed valid metadata"); + let publication = TableCommitObjectBackend::trusted(backend.clone()); + backend.armed.store(true, std::sync::atomic::Ordering::SeqCst); + let registration_bucket = bucket.clone(); + let mut registration = tokio::spawn(async move { + let result = register_table_response( + &catalog, + &publication, + ®istration_bucket, + &namespace, + RegisterTableRequest { + name: "events".to_string(), + metadata_location: location, + overwrite: false, + }, + true, + ) + .await; + publication.finish(result).await + }); + tokio::time::timeout(StdDuration::from_secs(5), backend.reader_ready.notified()) + .await + .expect("registration holds first real global reader"); + // The backup targets another bucket, so its documented writer-drain precondition is met. + let backup_catalog = StrongTableCatalogStore::new(inner); + let mut backup = Box::pin(backup_catalog.create_durable_catalog_backup(&backup_bucket, None)); + assert!( + futures::poll!(backup.as_mut()).is_pending(), + "actual backup writer must queue behind held registration reader" + ); + backend.resume.notify_one(); + let result = tokio::time::timeout(StdDuration::from_secs(2), &mut registration).await; + if result.is_err() { + // Prove that the queued backup, rather than ECStore I/O latency, caused the stall. + drop(backup); + tokio::time::timeout(StdDuration::from_secs(5), &mut registration) + .await + .expect("registration must resume once only the queued backup is cancelled") + .expect("registration control joins") + .expect("registration control succeeds"); + panic!("cold registration stalled behind a queued backup while retaining its own global reader"); + } + result + .expect("registration should complete without a recursive global read") + .expect("registration joins") + .expect("registration succeeds"); + tokio::time::timeout(StdDuration::from_secs(5), backup) + .await + .expect("the other bucket backup should resume when registration releases its reader") + .expect("the other bucket backup succeeds"); +} + #[tokio::test] #[serial_test::serial] async fn disable_table_bucket_preserves_objects_and_can_be_reenabled() { diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 641c88b73..4808f561d 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -1049,6 +1049,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ MIGRATE_TABLE_CATALOG, RouteRiskLevel::High, ), + admin( + HttpMethod::Post, + "/iceberg/v1/{warehouse}/catalog/backup", + MIGRATE_TABLE_CATALOG, + RouteRiskLevel::High, + ), + admin( + HttpMethod::Post, + "/iceberg/v1/{warehouse}/catalog/restore", + MIGRATE_TABLE_CATALOG, + RouteRiskLevel::High, + ), admin( HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/warehouse-index/backfill", @@ -1345,6 +1357,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ MIGRATE_TABLE_CATALOG, RouteRiskLevel::High, ), + admin( + HttpMethod::Post, + "/_iceberg/v1/{warehouse}/catalog/backup", + MIGRATE_TABLE_CATALOG, + RouteRiskLevel::High, + ), + admin( + HttpMethod::Post, + "/_iceberg/v1/{warehouse}/catalog/restore", + MIGRATE_TABLE_CATALOG, + RouteRiskLevel::High, + ), admin( HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/warehouse-index/backfill", @@ -1929,7 +1953,7 @@ mod tests { let table_specs = ADMIN_ROUTE_POLICY_SPECS .iter() .filter(|spec| spec.path().starts_with("/iceberg/v1") || spec.path().starts_with("/_iceberg/v1")); - assert_eq!(table_specs.count(), 102); + assert_eq!(table_specs.count(), 106); assert_action(HttpMethod::Put, "/iceberg/v1/buckets/{warehouse}", SET_TABLE_BUCKET); assert_action(HttpMethod::Delete, "/iceberg/v1/buckets/{warehouse}", SET_TABLE_BUCKET); assert_action(HttpMethod::Delete, "/_iceberg/v1/buckets/{warehouse}", SET_TABLE_BUCKET); @@ -2114,6 +2138,10 @@ mod tests { assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG); assert_action(HttpMethod::Delete, "/iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG); assert_action(HttpMethod::Delete, "/_iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG); + assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/backup", MIGRATE_TABLE_CATALOG); + assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/backup", MIGRATE_TABLE_CATALOG); + assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/restore", MIGRATE_TABLE_CATALOG); + assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/restore", MIGRATE_TABLE_CATALOG); assert_action( HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/warehouse-index/backfill", diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index f99dc525e..1c2bc2165 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -462,6 +462,8 @@ fn expected_admin_route_matrix() -> Vec { table_route_sample(Method::GET, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"), table_route_sample(Method::POST, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"), table_route_sample(Method::DELETE, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"), + table_route_sample(Method::POST, "/{warehouse}/catalog/backup", "/analytics/catalog/backup"), + table_route_sample(Method::POST, "/{warehouse}/catalog/restore", "/analytics/catalog/restore"), table_route_sample( Method::POST, "/{warehouse}/catalog/warehouse-index/backfill", @@ -665,6 +667,8 @@ fn expected_admin_route_matrix() -> Vec { compat_table_route_sample(Method::GET, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"), compat_table_route_sample(Method::POST, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"), compat_table_route_sample(Method::DELETE, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"), + compat_table_route_sample(Method::POST, "/{warehouse}/catalog/backup", "/analytics/catalog/backup"), + compat_table_route_sample(Method::POST, "/{warehouse}/catalog/restore", "/analytics/catalog/restore"), compat_table_route_sample( Method::POST, "/{warehouse}/catalog/warehouse-index/backfill", diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 47b99b9ef..b65921624 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -17,7 +17,7 @@ use super::storage_api::multipart_usecase::ECStore; use super::storage_api::multipart_usecase::access::{ TableDataPlaneListAccess, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, - has_bypass_governance_header, replication_request_authorized, + has_bypass_governance_header, replication_request_authorized, retained_table_data_plane_publication_guards, }; use super::storage_api::multipart_usecase::bucket::quota::checker::QuotaChecker; use super::storage_api::multipart_usecase::bucket::{ @@ -710,6 +710,7 @@ impl DefaultMultipartUsecase { let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await; let upload_user_defined = multipart_info.as_ref().map(|info| info.user_defined.clone()); + let table_data_plane_publication_guards = retained_table_data_plane_publication_guards(&req); let quota_metadata_sys = self.bucket_metadata_sys(); let quota_tracking = quota_metadata_sys.is_some(); @@ -778,7 +779,9 @@ impl DefaultMultipartUsecase { let key = key.clone(); let upload_id = upload_id.clone(); let opts = opts.clone(); + let table_data_plane_publication_guards = table_data_plane_publication_guards; async move { + let _table_data_plane_publication_guards = table_data_plane_publication_guards; let obj_info = store .clone() .complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, &opts) diff --git a/rustfs/src/app/object/copy.rs b/rustfs/src/app/object/copy.rs index e674375cd..0b3cb0526 100644 --- a/rustfs/src/app/object/copy.rs +++ b/rustfs/src/app/object/copy.rs @@ -787,6 +787,7 @@ impl DefaultObjectUsecase { let has_bucket_metadata = self.bucket_metadata_sys().is_some(); let cache_adapter = self.object_data_cache(); let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await; + let table_data_plane_publication_guards = retained_table_data_plane_publication_guards(&req); let copy_commit = spawn_traced_join({ let store = Arc::clone(&store); @@ -796,10 +797,12 @@ impl DefaultObjectUsecase { let key = key.clone(); let src_opts = src_opts.clone(); let dst_opts = dst_opts.clone(); + let table_data_plane_publication_guards = table_data_plane_publication_guards; async move { let _source_bucket_lifecycle_guard = source_bucket_lifecycle_guard; let _destination_bucket_lifecycle_guard_storage = destination_bucket_lifecycle_guard_storage; let _self_copy_lock_guard = _self_copy_lock_guard; + let _table_data_plane_publication_guards = table_data_plane_publication_guards; let oi = store .copy_object(&src_bucket, &src_key, &bucket, &key, &mut src_info, &src_opts, &dst_opts) diff --git a/rustfs/src/app/object/delete.rs b/rustfs/src/app/object/delete.rs index 124740f03..351d78b7c 100644 --- a/rustfs/src/app/object/delete.rs +++ b/rustfs/src/app/object/delete.rs @@ -120,7 +120,7 @@ static DELETE_SNAPSHOT_TEST_HOOK: OnceLock> static DELETE_SOURCE_TEST_HOOK: OnceLock>> = OnceLock::new(); #[cfg(test)] -static DELETE_OBJECTS_AUTH_TEST_HOOK: OnceLock>> = OnceLock::new(); +static DELETE_OBJECTS_GENERATION_TEST_HOOK: OnceLock>> = OnceLock::new(); #[cfg(test)] pub(crate) fn install_delete_snapshot_test_hook( @@ -185,24 +185,24 @@ async fn wait_for_delete_source_test_hook(bucket: &str) { } #[cfg(test)] -pub(crate) fn install_delete_objects_auth_test_hook( +pub(crate) fn install_delete_objects_generation_test_hook( bucket: String, loaded: Arc, resume: Arc, ) { - *DELETE_OBJECTS_AUTH_TEST_HOOK + *DELETE_OBJECTS_GENERATION_TEST_HOOK .get_or_init(|| Mutex::new(None)) .lock() - .expect("delete objects auth test hook lock should not be poisoned") = Some((bucket, loaded, resume)); + .expect("delete objects generation test hook lock should not be poisoned") = Some((bucket, loaded, resume)); } #[cfg(test)] -async fn wait_for_delete_objects_auth_test_hook(bucket: &str) { +async fn wait_for_delete_objects_generation_test_hook(bucket: &str) { let hook = { - let mut slot = DELETE_OBJECTS_AUTH_TEST_HOOK + let mut slot = DELETE_OBJECTS_GENERATION_TEST_HOOK .get_or_init(|| Mutex::new(None)) .lock() - .expect("delete objects auth test hook lock should not be poisoned"); + .expect("delete objects generation test hook lock should not be poisoned"); if slot.as_ref().is_some_and(|(expected_bucket, _, _)| expected_bucket == bucket) { slot.take() } else { @@ -481,6 +481,8 @@ impl DefaultObjectUsecase { // Capture the bucket generation before per-object authorization, but // do not expose a bucket-state error unless at least one object is authorized. let bucket_generation = load_bucket_generation_from_store(store.as_ref(), &req, &bucket).await; + #[cfg(test)] + wait_for_delete_objects_generation_test_hook(&bucket).await; let bypass_governance = has_bypass_governance_header(&req.headers); @@ -595,8 +597,6 @@ impl DefaultObjectUsecase { let _ = helper.complete(&result); return result; } - #[cfg(test)] - wait_for_delete_objects_auth_test_hook(&bucket).await; req.extensions.insert(bucket_generation?); let bucket_lock_enabled = object_lock_checks_required(&bucket).await; @@ -2098,7 +2098,7 @@ mod tests { #[tokio::test] #[serial_test::serial] - async fn execute_delete_objects_rejects_bucket_recreated_after_authorization() { + async fn execute_delete_objects_rejects_bucket_recreated_before_publication_admission() { use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions}; let store = crate::app::gating_test_env::shared_gating_ecstore().await; @@ -2125,7 +2125,7 @@ mod tests { .expect("authorized bucket metadata should be cached")) .clone(); metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy should parse")); - metadata.policy_config_json = policy_json.into_bytes(); + metadata.policy_config_json = policy_json.clone().into_bytes(); crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata) .await .expect("publish test bucket policy"); @@ -2146,7 +2146,7 @@ mod tests { req.extensions.insert(crate::storage::access::ReqInfo::default()); let loaded = Arc::new(tokio::sync::Barrier::new(2)); let resume = Arc::new(tokio::sync::Barrier::new(2)); - install_delete_objects_auth_test_hook(bucket.clone(), Arc::clone(&loaded), Arc::clone(&resume)); + install_delete_objects_generation_test_hook(bucket.clone(), Arc::clone(&loaded), Arc::clone(&resume)); let usecase = DefaultObjectUsecase::with_context(Some(context)); let delete = tokio::spawn(async move { usecase.execute_delete_objects(req).await }); @@ -2166,6 +2166,15 @@ mod tests { .make_bucket(&bucket, &MakeBucketOptions::default()) .await .expect("recreate same bucket name"); + let mut metadata = (*crate::storage::get_bucket_metadata(&bucket) + .await + .expect("recreated bucket metadata should be cached")) + .clone(); + metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy should parse")); + metadata.policy_config_json = policy_json.into_bytes(); + crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata) + .await + .expect("republish the same policy for admission"); let mut reader = PutObjReader::from_vec(b"new generation".to_vec()); store .put_object(&bucket, "object", &mut reader, &ObjectOptions::default()) @@ -2176,7 +2185,7 @@ mod tests { let err = delete .await .expect("delete objects task should join") - .expect_err("old authorization must not delete from the recreated bucket"); + .expect_err("the captured generation must not delete from the recreated bucket"); assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket); store .get_object_info(&bucket, "object", &ObjectOptions::default()) diff --git a/rustfs/src/app/object/extract.rs b/rustfs/src/app/object/extract.rs index 26f19c432..c6080f143 100644 --- a/rustfs/src/app/object/extract.rs +++ b/rustfs/src/app/object/extract.rs @@ -627,6 +627,7 @@ struct ExtractPreparedMember { body: R, write_plan: WritePlan, opts: ObjectOptions, + table_data_plane_publication_guards: Option, replication: ReplicateDecision, producer_identity: rustfs_scanner::SegmentInvalidationProducerIdentity, staging_permit: OwnedSemaphorePermit, @@ -765,7 +766,8 @@ where actual_size, body, write_plan, - opts, + mut opts, + table_data_plane_publication_guards, replication, producer_identity, staging_permit, @@ -775,6 +777,9 @@ where let hrd = write_plan.apply(hrd, actual_size).map_err(ApiError::from)?; let (hrd, member_read_failed) = track_extract_member_read_errors(hrd).map_err(ApiError::from)?; let mut reader = PutObjReader::new(hrd); + if let Some(guards) = table_data_plane_publication_guards.as_ref() { + guards.add_lock_loss_fences(&mut opts); + } let _ = invalidate_object_data_cache_before_mutation(&context.cache_adapter, &context.bucket, &key).await; let (obj_info, backfilled_old_current_size) = match context @@ -2460,6 +2465,7 @@ impl DefaultObjectUsecase { if iam_requirements.replication { extract_try!(authorize_request(&mut auth_req, Action::S3Action(S3Action::ReplicateObjectAction)).await); } + let table_data_plane_publication_guards = retained_table_data_plane_publication_guards(&auth_req); drop(auth_req); if archive_entry_mod_time.is_some() { opts.mod_time = archive_entry_mod_time; @@ -2605,6 +2611,7 @@ impl DefaultObjectUsecase { body: std::io::Cursor::new(Vec::new()), write_plan, opts, + table_data_plane_publication_guards, replication, producer_identity: rustfs_scanner::SegmentInvalidationProducerIdentity::DirectoryObject, staging_permit, @@ -2630,6 +2637,7 @@ impl DefaultObjectUsecase { body: f, write_plan, opts, + table_data_plane_publication_guards, replication, producer_identity: rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject, staging_permit, @@ -2660,6 +2668,7 @@ impl DefaultObjectUsecase { body, write_plan, opts, + table_data_plane_publication_guards, replication, producer_identity: if is_dir { rustfs_scanner::SegmentInvalidationProducerIdentity::DirectoryObject diff --git a/rustfs/src/app/object/internal_put.rs b/rustfs/src/app/object/internal_put.rs index af5945002..45dca295a 100644 --- a/rustfs/src/app/object/internal_put.rs +++ b/rustfs/src/app/object/internal_put.rs @@ -24,6 +24,7 @@ use super::*; use crate::app::object_data_cache::invalidate_object_data_cache_after_complete_multipart_success; +use crate::app::runtime_sources::current_app_context; use crate::app::storage_api::multipart_usecase::contract::multipart::{CompletePart, MultipartOperations as _}; use crate::app::storage_api::object_usecase::compression::is_multipart_disk_compression_enabled; use crate::app::storage_api::object_usecase::io::WriteEncryption; @@ -35,6 +36,10 @@ use crate::app::storage_api::object_usecase::sse::{ }; use crate::capacity::record_capacity_write; use crate::runtime_sources::NotifyInterface; +use crate::table_catalog::{ + EcStoreTableCatalogObjectBackend, TableCatalogLockGuard, TableCatalogObjectBackend, TableCatalogStoreError, + default_table_bucket_publication_lock_path, +}; use http::HeaderName; /// Inputs of an internal object write. Content and user metadata follow the @@ -227,6 +232,34 @@ impl InternalPutObjectEvent { } impl DefaultObjectUsecase { + async fn acquire_table_bucket_publication_guard(&self, bucket: &str) -> Result, ApiError> { + let store = self.object_store().ok_or_else(not_initialized)?; + if !store.table_bucket_enabled(bucket).await.map_err(ApiError::from)? { + return Ok(None); + } + + let context = self + .context + .clone() + .or_else(current_app_context) + .ok_or_else(not_initialized)?; + let backend = EcStoreTableCatalogObjectBackend::new_with_strong_runtime( + context.object_store(), + context.table_catalog_strong_runtime(), + ); + let guard = backend + .acquire_read_lock(bucket, &default_table_bucket_publication_lock_path()) + .await + .map_err(|err: TableCatalogStoreError| { + let code = match err { + TableCatalogStoreError::Unavailable(_) => S3ErrorCode::ServiceUnavailable, + _ => S3ErrorCode::InternalError, + }; + api_error_from_s3(S3Error::with_message(code, "failed to acquire table publication guard")) + })?; + Ok(Some(guard)) + } + /// Write one object through the ordinary PutObject path on behalf of a /// trusted internal caller. /// @@ -265,6 +298,7 @@ impl DefaultObjectUsecase { headers.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("*")); } validate_internal_write_target(&key, &bucket, &headers).await?; + let table_publication_guard = self.acquire_table_bucket_publication_guard(&bucket).await?; remove_source_replication_bookkeeping(&mut internal_metadata); let write = PutObjectWriteRequest { @@ -300,6 +334,7 @@ impl DefaultObjectUsecase { emit_events, preserve_delete_marker, expected_bucket_incarnation_id, + table_publication_guard, }, }; let committed = self @@ -319,6 +354,7 @@ impl DefaultObjectUsecase { pub(crate) async fn internal_create_multipart_upload(&self, ctx: &InternalPutContext) -> Result { let headers = internal_put_headers(&ctx.content_headers)?; validate_internal_write_target(&ctx.key, &ctx.bucket, &headers).await?; + let table_publication_guard = self.acquire_table_bucket_publication_guard(&ctx.bucket).await?; let store = self.object_store().ok_or_else(not_initialized)?; let mut metadata = ctx.user_metadata.clone(); @@ -372,6 +408,11 @@ impl DefaultObjectUsecase { .map_err(ApiError::from)?; opts.expected_bucket_incarnation_id = ctx.expected_bucket_incarnation_id; + if let Some(guard) = table_publication_guard.as_ref() + && let Some(signal) = guard.lock_lost_signal() + { + opts.add_namespace_lock_lost_signal(signal); + } let dsc = must_replicate_object( &ctx.bucket, @@ -543,6 +584,7 @@ impl DefaultObjectUsecase { validate_table_catalog_object_mutation(&bucket, &key) .await .map_err(api_error_from_s3)?; + let table_publication_guard = self.acquire_table_bucket_publication_guard(&bucket).await?; let store = self.object_store().ok_or_else(not_initialized)?; let mut headers = HeaderMap::new(); @@ -552,6 +594,11 @@ impl DefaultObjectUsecase { let mut opts = get_complete_multipart_upload_opts_with_replication_authorization(&headers, false).map_err(ApiError::from)?; opts.expected_bucket_incarnation_id = ctx.expected_bucket_incarnation_id; + if let Some(guard) = table_publication_guard.as_ref() + && let Some(signal) = guard.lock_lost_signal() + { + opts.add_namespace_lock_lost_signal(signal); + } opts.preserve_etag = ctx.preserve_etag.clone(); opts.preserve_delete_marker = ctx.preserve_delete_marker; let (versioned, version_suspended) = BucketVersioningSys::write_state(&bucket, &key) @@ -667,7 +714,9 @@ impl DefaultObjectUsecase { let key = key.clone(); let upload_id = upload_id.to_string(); let opts = opts.clone(); + let table_publication_guard = table_publication_guard; async move { + let _table_publication_guard = table_publication_guard; let obj_info = store .clone() .complete_multipart_upload(&bucket, &key, &upload_id, parts, &opts) diff --git a/rustfs/src/app/object/mod.rs b/rustfs/src/app/object/mod.rs index a0b4f1aea..27d02fd2d 100644 --- a/rustfs/src/app/object/mod.rs +++ b/rustfs/src/app/object/mod.rs @@ -20,10 +20,11 @@ use rustfs_io_metrics::buffered_write; use crate::storage_api::table::get_bucket_metadata; use super::storage_api::object_usecase::access::{ - PostObjectRequestMarker, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorize_request, - delete_object_authorize_action, has_bypass_governance_header, load_bucket_generation_from_store, odm_read_generation, - prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller, replication_request_authorized, req_info_mut, - req_info_ref, + PostObjectRequestMarker, TableDataPlanePublicationGuards, apply_bucket_generation_guard, + apply_copy_source_bucket_generation_guard, authorize_request, delete_object_authorize_action, has_bypass_governance_header, + load_bucket_generation_from_store, odm_read_generation, prepare_odm_read_generation, + recursive_force_delete_has_authenticated_caller, replication_request_authorized, req_info_mut, req_info_ref, + retained_table_data_plane_publication_guards, }; #[cfg(test)] use super::storage_api::object_usecase::bucket::quota::BucketQuota; diff --git a/rustfs/src/app/object/put.rs b/rustfs/src/app/object/put.rs index 5584678cc..0bfe61edd 100644 --- a/rustfs/src/app/object/put.rs +++ b/rustfs/src/app/object/put.rs @@ -971,6 +971,7 @@ pub(super) enum PutObjectOrigin<'a> { emit_events: bool, preserve_delete_marker: bool, expected_bucket_incarnation_id: Option, + table_publication_guard: Option, }, } @@ -995,6 +996,33 @@ impl PutObjectOrigin<'_> { } } + fn add_lock_loss_fence(&self, opts: &mut ObjectOptions) { + if let Self::Internal { + table_publication_guard: Some(guard), + .. + } = self + && let Some(signal) = guard.lock_lost_signal() + { + opts.add_namespace_lock_lost_signal(signal); + } + } + + fn take_table_publication_guard(&mut self) -> Option { + match self { + Self::Internal { + table_publication_guard, .. + } => table_publication_guard.take(), + Self::S3 { .. } => None, + } + } + + fn table_data_plane_publication_guards(&self) -> Option { + match self { + Self::S3 { req, .. } => retained_table_data_plane_publication_guards(req), + Self::Internal { .. } => None, + } + } + fn sse_principal(&self) -> Option { match self { Self::S3 { req, .. } => SseKmsPrincipal::from_request(req), @@ -1439,7 +1467,7 @@ impl DefaultObjectUsecase { object_lock, content_md5, preserve_etag, - origin, + mut origin, } = write; let PutObjectSseInput { server_side_encryption, @@ -1648,6 +1676,7 @@ impl DefaultObjectUsecase { } rustfs_io_metrics::record_put_object_stage_duration_from("app_put_opts_build", put_opts_stage_start); origin.apply_bucket_generation_guard(&bucket, &mut opts)?; + origin.add_lock_loss_fence(&mut opts); apply_put_request_object_lock_opts( &bucket, &object_lock_config_state, @@ -1966,7 +1995,11 @@ impl DefaultObjectUsecase { let request_id = request_id.clone(); let put_path = put_path.to_string(); let put_admission = put_admission; + let table_publication_guard = origin.take_table_publication_guard(); + let table_data_plane_publication_guards = origin.table_data_plane_publication_guards(); async move { + let _table_publication_guard = table_publication_guard; + let _table_data_plane_publication_guards = table_data_plane_publication_guards; let _put_admission = put_admission; let object_traffic_progress = object_traffic_health .as_deref() diff --git a/rustfs/src/app/object/restore.rs b/rustfs/src/app/object/restore.rs index 59445ec98..412c1a1fd 100644 --- a/rustfs/src/app/object/restore.rs +++ b/rustfs/src/app/object/restore.rs @@ -472,9 +472,11 @@ impl DefaultObjectUsecase { } let restore_worker_guard = restore_worker_guard.take(); + let table_data_plane_publication_guards = retained_table_data_plane_publication_guards(&req); spawn_traced(async move { let _restore_worker_guard = restore_worker_guard; - let opts = ObjectOptions { + let _table_data_plane_publication_guards = table_data_plane_publication_guards; + let mut opts = ObjectOptions { transition: TransitionOptions { restore_request: rreq_clone, restore_expiry, @@ -487,6 +489,9 @@ impl DefaultObjectUsecase { user_defined: restore_operation_metadata, ..Default::default() }; + if let Some(guards) = _table_data_plane_publication_guards.as_ref() { + guards.add_lock_loss_fences(&mut opts); + } if let Err(err) = store_clone .restore_transitioned_object(&bucket_clone, &object_clone, &opts) diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 4e15fc1e6..68e1e4ffa 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -272,11 +272,11 @@ pub(crate) mod access { pub(crate) use crate::storage::storage_api::access_consumer::ReqInfo; pub(crate) use crate::storage::storage_api::access_consumer::{ PostObjectRequestMarker, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess, TableDataPlaneListCursorPosition, - apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorize_request, - bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header, + TableDataPlanePublicationGuards, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, + authorize_request, bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header, load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation, prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller, - replication_request_authorized, req_info_mut, req_info_ref, + replication_request_authorized, req_info_mut, req_info_ref, retained_table_data_plane_publication_guards, }; } diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index b566eb2b4..2f59124ae 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -138,7 +138,7 @@ struct InternalObjectAuthorization; struct StagedMultipartPartAuthorization; #[derive(Clone, Default)] -struct TableDataPlanePublicationGuards { +pub(crate) struct TableDataPlanePublicationGuards { state: Arc>, } @@ -146,10 +146,20 @@ struct TableDataPlanePublicationGuards { struct TableDataPlanePublicationState { keys: BTreeSet<(String, String)>, guards: Vec>, + lock_lost_signals: Vec>, resources: HashMap<(String, String), crate::table_catalog::TableDataPlaneResource>, missing_resources: BTreeSet<(String, String)>, } +impl TableDataPlanePublicationGuards { + pub(crate) fn add_lock_loss_fences(&self, options: &mut ObjectOptions) { + let state = self.state.lock(); + for signal in &state.lock_lost_signals { + options.add_namespace_lock_lost_signal(Arc::clone(signal)); + } + } +} + pub(crate) const TABLE_DATA_PLANE_LIST_CURSOR_PREFIX: &str = "rustfs-table-list:v1:"; const TABLE_DATA_PLANE_LIST_CURSOR_VERSION: u8 = 1; const TABLE_DATA_PLANE_LIST_CURSOR_KEY_CONTEXT: &[u8] = b"rustfs-table-list-cursor-key:v1"; @@ -400,38 +410,38 @@ struct CopySourceBucketGenerationGuard { } #[cfg(test)] -type RestoreAuthorizationTestHook = (String, tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>); +type ObjectGenerationTestHook = (String, tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>); #[cfg(test)] -static RESTORE_AUTHORIZATION_TEST_HOOK: OnceLock>> = OnceLock::new(); +static OBJECT_GENERATION_TEST_HOOK: OnceLock>> = OnceLock::new(); #[cfg(test)] -fn install_restore_authorization_test_hook( +fn install_object_generation_test_hook( bucket: String, - authorized: tokio::sync::oneshot::Sender<()>, + loaded: tokio::sync::oneshot::Sender<()>, resume: tokio::sync::oneshot::Receiver<()>, ) { - *RESTORE_AUTHORIZATION_TEST_HOOK + *OBJECT_GENERATION_TEST_HOOK .get_or_init(|| std::sync::Mutex::new(None)) .lock() - .expect("restore authorization test hook lock should not be poisoned") = Some((bucket, authorized, resume)); + .expect("object generation test hook lock should not be poisoned") = Some((bucket, loaded, resume)); } #[cfg(test)] -async fn wait_for_restore_authorization_test_hook(bucket: &str) { +async fn wait_for_object_generation_test_hook(bucket: &str) { let hook = { - let mut slot = RESTORE_AUTHORIZATION_TEST_HOOK + let mut slot = OBJECT_GENERATION_TEST_HOOK .get_or_init(|| std::sync::Mutex::new(None)) .lock() - .expect("restore authorization test hook lock should not be poisoned"); + .expect("object generation test hook lock should not be poisoned"); if slot.as_ref().is_some_and(|(expected_bucket, _, _)| expected_bucket == bucket) { slot.take() } else { None } }; - if let Some((_bucket, authorized, resume)) = hook { - let _ = authorized.send(()); + if let Some((_bucket, loaded, resume)) = hook { + let _ = loaded.send(()); let _ = resume.await; } } @@ -569,15 +579,25 @@ pub(crate) fn apply_bucket_generation_guard(req: &S3Request, bucket: &str, if req.extensions.get::>().is_some() { return Err(s3_error!(InternalError, "bucket generation guard is missing")); } + if let Some(publication_guards) = req.extensions.get::() { + publication_guards.add_lock_loss_fences(opts); + } return Ok(()); }; if guard.bucket != bucket { return Err(s3_error!(InternalError, "bucket generation guard does not match request bucket")); } opts.expected_bucket_incarnation_id = Some(guard.incarnation_id); + if let Some(publication_guards) = req.extensions.get::() { + publication_guards.add_lock_loss_fences(opts); + } Ok(()) } +pub(crate) fn retained_table_data_plane_publication_guards(req: &S3Request) -> Option { + req.extensions.get::().cloned() +} + pub(crate) fn apply_copy_source_bucket_generation_guard( req: &S3Request, bucket: &str, @@ -587,6 +607,9 @@ pub(crate) fn apply_copy_source_bucket_generation_guard( if req.extensions.get::>().is_some() { return Err(s3_error!(InternalError, "copy source bucket generation guard is missing")); } + if let Some(publication_guards) = req.extensions.get::() { + publication_guards.add_lock_loss_fences(opts); + } return Ok(()); }; if guard.bucket != bucket { @@ -596,6 +619,9 @@ pub(crate) fn apply_copy_source_bucket_generation_guard( )); } opts.expected_bucket_incarnation_id = Some(guard.incarnation_id); + if let Some(publication_guards) = req.extensions.get::() { + publication_guards.add_lock_loss_fences(opts); + } Ok(()) } @@ -1654,9 +1680,13 @@ async fn retain_table_data_plane_publication_guard( let guard = crate::table_catalog::TableCatalogObjectBackend::acquire_read_lock(&backend, table_bucket, lock_object) .await .map_err(table_publication_guard_error)?; + let lock_lost_signal = guard.lock_lost_signal(); let mut state = retained.state.lock(); state.keys.insert(key); state.guards.push(Box::new(guard)); + if let Some(signal) = lock_lost_signal { + state.lock_lost_signals.push(signal); + } drop(state); req.extensions.insert(retained); Ok(()) @@ -1702,9 +1732,8 @@ async fn table_bucket_enabled_for_data_plane(req: &S3Request, bucket: &str return Ok(false); } - match request_object_store(req)?.get_bucket_metadata(bucket).await { - Ok(metadata) => Ok(metadata.table_bucket_enabled()), - Err(StorageError::ConfigNotFound) => Ok(false), + match request_object_store(req)?.table_bucket_enabled(bucket).await { + Ok(table_bucket_enabled) => Ok(table_bucket_enabled), Err(err) if is_err_bucket_not_found(&err) => Ok(false), Err(err) => { tracing::warn!( @@ -2411,6 +2440,8 @@ impl S3Access for FS { async fn delete_object(&self, req: &mut S3Request) -> S3Result<()> { let bucket = req.input.bucket.clone(); let bucket_generation = load_bucket_generation(self, req, &bucket).await; + #[cfg(test)] + wait_for_object_generation_test_hook(&bucket).await; // Preserve DeleteObject's established NoSuchBucket response instead of // letting policy lookup turn a missing bucket into AccessDenied. if let Err(err) = &bucket_generation @@ -3283,14 +3314,14 @@ impl S3Access for FS { async fn restore_object(&self, req: &mut S3Request) -> S3Result<()> { let bucket = req.input.bucket.clone(); let bucket_generation = load_bucket_generation(self, req, &bucket).await; + #[cfg(test)] + wait_for_object_generation_test_hook(&bucket).await; let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); authorize_request(req, Action::S3Action(S3Action::RestoreObjectAction)).await?; - #[cfg(test)] - wait_for_restore_authorization_test_hook(&bucket).await; req.extensions.insert(bucket_generation?); Ok(()) } @@ -3377,7 +3408,7 @@ mod tests { apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorization_conditions, bucket_policy_needs_existing_object_tag_from_hint, bucket_website_config_authorize_action, classify_bucket_policy_raw_load_error, complete_multipart_upload_authorize_action, delete_object_authorize_action, - get_bucket_policy_authorize_action, has_write_offset_bytes_header, install_restore_authorization_test_hook, + get_bucket_policy_authorize_action, has_write_offset_bytes_header, install_object_generation_test_hook, legal_hold_write_requested, list_parts_authorize_action, load_bucket_policy_existing_object_tag_hint, maybe_merge_object_tag_conditions, merge_list_bucket_query_conditions, merge_request_object_tag_conditions, owner_can_bypass_policy_deny, post_object_authorize_action, put_bucket_policy_authorize_action, request_context_from_req, @@ -4987,9 +5018,16 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::InternalError); } - #[tokio::test] + #[test] #[serial] - async fn delete_object_access_captures_authorized_bucket_incarnation() { + fn delete_object_access_keeps_snapshot_across_recreation_before_admission() { + crate::app::gating_test_env::run_large_stack_test( + "delete-object-generation-guard", + delete_object_access_keeps_snapshot_across_recreation_before_admission_inner, + ); + } + + async fn delete_object_access_keeps_snapshot_across_recreation_before_admission_inner() { let store = crate::app::gating_test_env::shared_gating_ecstore().await; let server_ctx = ServerContextSlot::new(); let app_context = Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms))); @@ -5021,7 +5059,7 @@ mod tests { .expect("new bucket metadata should be cached")) .clone(); metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test bucket policy should parse")); - metadata.policy_config_json = policy_json.into_bytes(); + metadata.policy_config_json = policy_json.clone().into_bytes(); crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata) .await .expect("test bucket policy should be published"); @@ -5035,20 +5073,21 @@ mod tests { ensure_req_info(&mut req); req.extensions.insert(fs.server_ctx().clone()); - fs.delete_object(&mut req) + let captured_incarnation_id = store + .bucket_incarnation_id(&bucket) .await - .expect("anonymous DeleteObject should be authorized by the test policy"); - let mut opts = crate::storage::ObjectOptions::default(); - apply_bucket_generation_guard(&req, &bucket, &mut opts).expect("request snapshot should apply to DeleteObject options"); - assert_eq!( - opts.expected_bucket_incarnation_id, - Some( - store - .bucket_incarnation_id(&bucket) - .await - .expect("bucket incarnation should remain readable") - ) - ); + .expect("read the bucket generation before admission"); + let (loaded_tx, loaded_rx) = tokio::sync::oneshot::channel(); + let (resume_tx, resume_rx) = tokio::sync::oneshot::channel(); + install_object_generation_test_hook(bucket.clone(), loaded_tx, resume_rx); + let access = tokio::spawn(async move { + let result = fs.delete_object(&mut req).await; + (result, req) + }); + tokio::time::timeout(std::time::Duration::from_secs(10), loaded_rx) + .await + .expect("DeleteObject should reach the generation snapshot hook") + .expect("DeleteObject access should remain paused before admission"); store .delete_bucket(&bucket, &DeleteBucketOptions::default()) @@ -5058,12 +5097,37 @@ mod tests { .make_bucket(&bucket, &MakeBucketOptions::default()) .await .expect("recreate the same bucket name"); + let mut metadata = (*crate::storage::get_bucket_metadata(&bucket) + .await + .expect("recreated bucket metadata should be cached")) + .clone(); + metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy should parse")); + metadata.policy_config_json = policy_json.into_bytes(); + crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata) + .await + .expect("republish the same policy for admission"); let mut reader = crate::storage::PutObjReader::from_vec(b"new generation".to_vec()); store .put_object(&bucket, "object", &mut reader, &crate::storage::ObjectOptions::default()) .await .expect("put the new-generation object"); + assert_ne!( + captured_incarnation_id, + store + .bucket_incarnation_id(&bucket) + .await + .expect("read the recreated bucket generation") + ); + resume_tx + .send(()) + .expect("DeleteObject should still be paused before admission"); + let (result, req) = access.await.expect("DeleteObject access task should join"); + result.expect("the equivalent policy should authorize DeleteObject"); + let mut opts = crate::storage::ObjectOptions::default(); + apply_bucket_generation_guard(&req, &bucket, &mut opts).expect("request snapshot should apply to DeleteObject options"); + assert_eq!(opts.expected_bucket_incarnation_id, Some(captured_incarnation_id)); + let err = crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(app_context)) .execute_delete_object(req) .await @@ -5206,14 +5270,14 @@ mod tests { #[test] #[serial] - fn restore_object_access_keeps_authorized_bucket_incarnation_across_recreation() { + fn restore_object_access_keeps_snapshot_across_recreation_before_admission() { crate::app::gating_test_env::run_large_stack_test( "restore-object-generation-guard", - restore_object_access_keeps_authorized_bucket_incarnation_across_recreation_inner, + restore_object_access_keeps_snapshot_across_recreation_before_admission_inner, ); } - async fn restore_object_access_keeps_authorized_bucket_incarnation_across_recreation_inner() { + async fn restore_object_access_keeps_snapshot_across_recreation_before_admission_inner() { let store = crate::app::gating_test_env::shared_gating_ecstore().await; let server_ctx = ServerContextSlot::new(); let app_context = Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms))); @@ -5237,7 +5301,7 @@ mod tests { .expect("authorized bucket metadata should be cached")) .clone(); metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy should parse")); - metadata.policy_config_json = policy_json.into_bytes(); + metadata.policy_config_json = policy_json.clone().into_bytes(); crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata) .await .expect("publish the RestoreObject policy"); @@ -5261,7 +5325,7 @@ mod tests { req.extensions.insert(fs.server_ctx().clone()); let (authorized_tx, authorized_rx) = tokio::sync::oneshot::channel(); let (resume_tx, resume_rx) = tokio::sync::oneshot::channel(); - install_restore_authorization_test_hook(bucket.clone(), authorized_tx, resume_rx); + install_object_generation_test_hook(bucket.clone(), authorized_tx, resume_rx); let access = tokio::spawn(async move { let result = fs.restore_object(&mut req).await; @@ -5269,7 +5333,7 @@ mod tests { }); tokio::time::timeout(std::time::Duration::from_secs(10), authorized_rx) .await - .expect("RestoreObject authorization should reach the test hook") + .expect("RestoreObject should reach the generation snapshot hook") .expect("RestoreObject access must not fail before reaching the test hook"); store @@ -5280,6 +5344,15 @@ mod tests { .make_bucket(&bucket, &MakeBucketOptions::default()) .await .expect("recreate the bucket with the same name"); + let mut metadata = (*crate::storage::get_bucket_metadata(&bucket) + .await + .expect("recreated bucket metadata should be cached")) + .clone(); + metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy should parse")); + metadata.policy_config_json = policy_json.into_bytes(); + crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata) + .await + .expect("republish the same policy for admission"); let recreated_incarnation_id = store .bucket_incarnation_id(&bucket) .await @@ -5290,7 +5363,7 @@ mod tests { .expect("RestoreObject access should still be waiting at the test hook"); let (result, req) = access.await.expect("RestoreObject access task should join"); - result.expect("the already-authorized request should retain its generation guard"); + result.expect("the equivalent policy should authorize RestoreObject without refreshing its snapshot"); let mut opts = crate::storage::ObjectOptions::default(); apply_bucket_generation_guard(&req, &bucket, &mut opts).expect("apply the RestoreObject authorization guard"); assert_eq!(opts.expected_bucket_incarnation_id, Some(authorized_incarnation_id)); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index ba09225b8..1632fbf92 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -125,11 +125,12 @@ pub(crate) use super::sse::{ pub(crate) mod access_consumer { pub(crate) use super::super::access::{ PostObjectRequestMarker, ReqInfo, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess, - TableDataPlaneListCursorPosition, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, - authorize_internal_object_request, authorize_request, bucket_config_mutation_incarnation, delete_object_authorize_action, - has_bypass_governance_header, load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation, + TableDataPlaneListCursorPosition, TableDataPlanePublicationGuards, apply_bucket_generation_guard, + apply_copy_source_bucket_generation_guard, authorize_internal_object_request, authorize_request, + bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header, + load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation, prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller, - replication_request_authorized, req_info_mut, req_info_ref, + replication_request_authorized, req_info_mut, req_info_ref, retained_table_data_plane_publication_guards, }; } diff --git a/rustfs/src/table_catalog/iceberg/validation.rs b/rustfs/src/table_catalog/iceberg/validation.rs index f8d748d30..e2066b696 100644 --- a/rustfs/src/table_catalog/iceberg/validation.rs +++ b/rustfs/src/table_catalog/iceberg/validation.rs @@ -976,6 +976,13 @@ pub(crate) enum IcebergStatisticsFileKind { Partition, } +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct IcebergStatisticsObjectReference { + pub(crate) snapshot_id: i64, + pub(crate) location: String, + pub(crate) kind: IcebergStatisticsFileKind, +} + pub(crate) fn validate_iceberg_statistics_file( value: &serde_json::Value, label: &str, @@ -1073,10 +1080,16 @@ fn statistics_non_negative_i64( Ok(value) } -fn validate_table_statistics_references( +fn is_supported_partition_statistics_file(data: &[u8]) -> bool { + data.starts_with(b"ORC") + || data.starts_with(b"Obj\x01") + || (data.len() >= 8 && data.starts_with(b"PAR1") && data.ends_with(b"PAR1")) +} + +pub(crate) fn table_statistics_object_references( metadata: &serde_json::Value, - snapshot_ids: &BTreeSet, -) -> TableCatalogStoreResult<()> { +) -> TableCatalogStoreResult> { + let mut references = Vec::new(); for (field, kind) in [ ("statistics", IcebergStatisticsFileKind::Table), ("partition-statistics", IcebergStatisticsFileKind::Partition), @@ -1087,19 +1100,49 @@ fn validate_table_statistics_references( let values = values .as_array() .ok_or_else(|| TableCatalogStoreError::Invalid(format!("table metadata field {field} must be an array")))?; - let mut snapshot_ids_with_statistics = BTreeSet::new(); for value in values { let snapshot_id = validate_iceberg_statistics_file(value, field, kind)?; - if !snapshot_ids.contains(&snapshot_id) { - return Err(TableCatalogStoreError::Invalid(format!( - "{field} references missing snapshot {snapshot_id}" - ))); - } - if !snapshot_ids_with_statistics.insert(snapshot_id) { - return Err(TableCatalogStoreError::Invalid(format!( - "{field} contains duplicate entries for snapshot {snapshot_id}" - ))); - } + let location = value + .get("statistics-path") + .and_then(serde_json::Value::as_str) + .filter(|location| !location.is_empty()) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("{field}.statistics-path must be a non-empty string")))?; + references.push(IcebergStatisticsObjectReference { + snapshot_id, + location: location.to_string(), + kind, + }); + } + } + Ok(references) +} + +fn validate_table_statistics_references( + metadata: &serde_json::Value, + snapshot_ids: &BTreeSet, +) -> TableCatalogStoreResult<()> { + let mut table_statistics_snapshot_ids = BTreeSet::new(); + let mut partition_statistics_snapshot_ids = BTreeSet::new(); + for reference in table_statistics_object_references(metadata)? { + if !snapshot_ids.contains(&reference.snapshot_id) { + return Err(TableCatalogStoreError::Invalid(format!( + "{} references missing snapshot {}", + match reference.kind { + IcebergStatisticsFileKind::Table => "statistics", + IcebergStatisticsFileKind::Partition => "partition-statistics", + }, + reference.snapshot_id + ))); + } + let (field, snapshot_ids_with_statistics) = match reference.kind { + IcebergStatisticsFileKind::Table => ("statistics", &mut table_statistics_snapshot_ids), + IcebergStatisticsFileKind::Partition => ("partition-statistics", &mut partition_statistics_snapshot_ids), + }; + if !snapshot_ids_with_statistics.insert(reference.snapshot_id) { + return Err(TableCatalogStoreError::Invalid(format!( + "{field} contains duplicate entries for snapshot {}", + reference.snapshot_id + ))); } } Ok(()) @@ -2798,7 +2841,7 @@ fn snapshot_is_retained_v1_history(current_metadata: Option<&serde_json::Value>, current_snapshot == target_snapshot } -async fn validate_table_statistics_objects( +pub(crate) async fn validate_table_statistics_objects( context: &TableSnapshotGraphValidationContext<'_, B>, metadata: &serde_json::Value, ) -> TableCatalogStoreResult<()> @@ -2879,12 +2922,17 @@ where } let valid_magic = match kind { IcebergStatisticsFileKind::Table => object.data.starts_with(b"PFA1") && object.data.ends_with(b"PFA1"), - IcebergStatisticsFileKind::Partition => object.data.starts_with(b"PAR1") && object.data.ends_with(b"PAR1"), + // Iceberg partition statistics use the table's data-file format, not a + // fixed Parquet encoding. Validate the format envelope without parsing + // the optional statistics rows here. + IcebergStatisticsFileKind::Partition => is_supported_partition_statistics_file(&object.data), }; if !valid_magic { return Err(TableCatalogStoreError::Invalid(match kind { IcebergStatisticsFileKind::Table => "table statistics object is not a Puffin file".to_string(), - IcebergStatisticsFileKind::Partition => "partition statistics object is not a Parquet file".to_string(), + IcebergStatisticsFileKind::Partition => { + "partition statistics object is not a supported table data file".to_string() + } })); } Ok(()) diff --git a/rustfs/src/table_catalog/mod.rs b/rustfs/src/table_catalog/mod.rs index 8c14ca145..85b4d0105 100644 --- a/rustfs/src/table_catalog/mod.rs +++ b/rustfs/src/table_catalog/mod.rs @@ -207,6 +207,15 @@ const TABLE_CATALOG_MIGRATION_FENCE_FILE: &str = "durable-strong-fence.json"; const TABLE_CATALOG_MIGRATION_FENCE_LOCK: &str = "durable-strong-fence.lock"; const TABLE_CATALOG_MIGRATION_GLOBAL_FENCE_FILE: &str = "durable-strong-global-fence.json"; const TABLE_CATALOG_MIGRATION_GLOBAL_FENCE_LOCK: &str = "durable-strong-global-fence.lock"; +pub(crate) const TABLE_CATALOG_BACKUP_VERSION: u16 = 2; +const TABLE_CATALOG_BACKUP_ROOT: &str = "backups"; +const TABLE_CATALOG_BACKUP_RESTORE_ROOT: &str = "restore-intents"; +const TABLE_CATALOG_BACKUP_RESTORE_INTENT_FILE: &str = "intent.json"; +const TABLE_CATALOG_BACKUP_MAX_SIZE: usize = 64 * 1024 * 1024; +const TABLE_CATALOG_BACKUP_MAX_OBJECTS: usize = 1_000_000; +const TABLE_CATALOG_BACKUP_HASH_MAX_SIZE: u64 = 16 * 1024 * 1024; +const TABLE_CATALOG_BACKUP_MAX_MAINTENANCE_OBJECT_SIZE: usize = 4 * 1024 * 1024; +const TABLE_CATALOG_BACKUP_MAX_MAINTENANCE_BYTES: usize = 32 * 1024 * 1024; type CatalogListObjectsV2Info = StorageListObjectsV2Info; type CatalogListObjectVersionsInfo = StorageListObjectVersionsInfo; diff --git a/rustfs/src/table_catalog/store/backup.rs b/rustfs/src/table_catalog/store/backup.rs new file mode 100644 index 000000000..fcc1a6559 --- /dev/null +++ b/rustfs/src/table_catalog/store/backup.rs @@ -0,0 +1,1837 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::object::validate_table_maintenance_report_owner; +use super::strong::{ + StrongSnapshotWritePostcondition, StrongTableCatalogBucketSnapshot, StrongTableCatalogSnapshot, StrongTableCatalogStore, + table_catalog_bucket_snapshot_fingerprint, +}; +use super::*; +use std::collections::{BTreeMap, BTreeSet}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct StrongTableCatalogBackup { + version: u16, + backup_id: String, + table_bucket: String, + created_at: String, + source_snapshot_etag: String, + source_snapshot_version: u16, + catalog_fingerprint: String, + snapshot: StrongTableCatalogBucketSnapshot, + objects: Vec, + maintenance_objects: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct StrongTableCatalogBackupObject { + bucket: String, + object: String, + kind: StrongTableCatalogBackupObjectKind, + size_bytes: u64, + etag: Option, + sha256: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +enum StrongTableCatalogBackupObjectKind { + TableMetadata, + ViewMetadata, + ManifestList, + ManifestFile, + DataFile, + DeleteFile, + TableStatistics, + PartitionStatistics, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct StrongTableCatalogBackupMaintenanceObject { + object: String, + size_bytes: u64, + etag: Option, + sha256: String, + data: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct StrongTableCatalogRestoreIntent { + version: u16, + table_bucket: String, + backup_id: String, + expected_snapshot_etag: Option, + #[serde(default)] + target_snapshot_etag: Option, + #[serde(default = "missing_restore_target_bucket_fingerprint")] + target_bucket_fingerprint: String, + #[serde(default)] + target_bucket_present: Option, + #[serde(default)] + allow_replace: Option, + #[serde(default)] + source_maintenance_fingerprint: Option, + restored_snapshot_etag: Option, + state: StrongTableCatalogRestoreIntentState, +} + +fn missing_restore_target_bucket_fingerprint() -> String { + String::new() +} + +const ABSENT_RESTORE_TARGET_BUCKET_FINGERPRINT: &str = "ABSENT"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +enum StrongTableCatalogRestoreIntentState { + Prepared, + CatalogApplied, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogBackupStatus { + Created, + AlreadyPresent, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogBackupReport { + pub table_bucket: String, + pub backup_id: String, + pub status: TableCatalogBackupStatus, + pub backup_path: String, + pub source_snapshot_etag: String, + pub source_snapshot_version: u16, + pub catalog_fingerprint: String, + pub object_count: usize, + pub verified_object_count: usize, + pub verified_bytes: u64, + pub maintenance_object_count: usize, + pub created_at: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableCatalogRestoreStatus { + Restored, + AlreadyRestored, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct TableCatalogRestoreReport { + pub table_bucket: String, + pub backup_id: String, + pub status: TableCatalogRestoreStatus, + pub previous_snapshot_etag: Option, + pub restored_snapshot_etag: String, + pub object_count: usize, + pub maintenance_object_count: usize, +} + +impl StrongTableCatalogBackupObjectKind { + fn from_maintenance_kind(kind: &TableMetadataMaintenanceObjectKind) -> Self { + match kind { + TableMetadataMaintenanceObjectKind::ManifestList => Self::ManifestList, + TableMetadataMaintenanceObjectKind::ManifestFile => Self::ManifestFile, + TableMetadataMaintenanceObjectKind::DataFile => Self::DataFile, + TableMetadataMaintenanceObjectKind::DeleteFile => Self::DeleteFile, + TableMetadataMaintenanceObjectKind::MetadataFile => Self::TableMetadata, + } + } + + fn from_statistics_kind(kind: IcebergStatisticsFileKind) -> Self { + match kind { + IcebergStatisticsFileKind::Table => Self::TableStatistics, + IcebergStatisticsFileKind::Partition => Self::PartitionStatistics, + } + } +} + +fn backup_sha256(data: &[u8]) -> String { + hex_simd::encode_to_string(Sha256::digest(data), hex_simd::AsciiCase::Lower) +} + +fn has_usable_etag(etag: Option<&str>) -> bool { + etag.is_some_and(|etag| !etag.is_empty()) +} + +fn catalog_backup_id( + table_bucket: &str, + backup_version: u16, + source_snapshot_etag: &str, + source_snapshot_version: u16, + catalog_fingerprint: &str, + objects: &[StrongTableCatalogBackupObject], + maintenance_objects: &[StrongTableCatalogBackupMaintenanceObject], +) -> TableCatalogStoreResult { + let payload = serde_json::to_vec(&( + table_bucket, + backup_version, + source_snapshot_etag, + source_snapshot_version, + catalog_fingerprint, + objects, + maintenance_objects, + )) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to encode backup identity: {err}")))?; + Ok(backup_sha256(&payload)) +} + +fn maintenance_objects_fingerprint(objects: &[StrongTableCatalogBackupMaintenanceObject]) -> TableCatalogStoreResult { + let mut identity = objects + .iter() + .map(|object| (object.object.as_str(), object.size_bytes, object.sha256.as_str())) + .collect::>(); + identity.sort_unstable(); + let payload = serde_json::to_vec(&identity) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to encode maintenance backup identity: {err}")))?; + Ok(backup_sha256(&payload)) +} + +fn backup_object_key_is_safe(object: &str) -> bool { + !object.is_empty() + && !object.starts_with('/') + && !object.contains("..") + && !object.contains('\\') + && !object.bytes().any(|byte| byte.is_ascii_control()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MaintenanceBackupObjectType { + Config, + Job, +} + +#[derive(Debug)] +struct MaintenanceBackupObjectPath { + object_type: MaintenanceBackupObjectType, + namespace: Option, + table: Option, + table_id_hash: Option, + job_id_hash: Option, +} + +#[derive(Debug, Clone, Default)] +struct MaintenanceBackupPaths { + exact_paths: BTreeSet, + config_paths: BTreeSet, + job_prefixes: BTreeSet, + allow_all_table_paths: bool, +} + +impl MaintenanceBackupPaths { + fn extend(&mut self, other: Self) { + self.exact_paths.extend(other.exact_paths); + self.config_paths.extend(other.config_paths); + self.job_prefixes.extend(other.job_prefixes); + self.allow_all_table_paths |= other.allow_all_table_paths; + } +} + +fn is_table_catalog_path_hash(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn maintenance_backup_object_path(table_bucket: &str, object: &str) -> Option { + let root = TableCatalogObjectPaths::default().table_bucket_root_prefix(table_bucket); + let relative = object.strip_prefix(&root)?; + if relative == format!("{MAINTENANCE_ROOT}/{MAINTENANCE_CONFIG_FILE}") { + return Some(MaintenanceBackupObjectPath { + object_type: MaintenanceBackupObjectType::Config, + namespace: None, + table: None, + table_id_hash: None, + job_id_hash: None, + }); + } + + let relative = relative.strip_prefix(&format!("{NAMESPACE_ROOT}/"))?; + let (namespace_storage_id, table_path) = relative.rsplit_once(&format!("/{TABLE_ROOT}/"))?; + let namespace = Namespace::from_segments(namespace_storage_id.split('/').map(str::to_string).collect()).ok()?; + let (table_name, maintenance_path) = table_path.split_once(&format!("/{MAINTENANCE_ROOT}/"))?; + let table = IdentifierSegment::parse(table_name.to_string()).ok()?; + let mut segments = maintenance_path.split('/'); + let table_id_hash = segments.next()?.to_string(); + if !is_table_catalog_path_hash(&table_id_hash) { + return None; + } + match (segments.next(), segments.next(), segments.next()) { + (Some(file), None, None) + if matches!(file, MAINTENANCE_CONFIG_FILE | MAINTENANCE_LATEST_JOB_FILE | MAINTENANCE_CURRENT_JOB_FILE) => + { + Some(MaintenanceBackupObjectPath { + object_type: if file == MAINTENANCE_CONFIG_FILE { + MaintenanceBackupObjectType::Config + } else { + MaintenanceBackupObjectType::Job + }, + namespace: Some(namespace), + table: Some(table), + table_id_hash: Some(table_id_hash), + job_id_hash: None, + }) + } + (Some(MAINTENANCE_JOB_ROOT), Some(job_file), None) => { + let job_hash = job_file.strip_suffix(".json")?; + is_table_catalog_path_hash(job_hash).then_some(MaintenanceBackupObjectPath { + object_type: MaintenanceBackupObjectType::Job, + namespace: Some(namespace), + table: Some(table), + table_id_hash: Some(table_id_hash), + job_id_hash: Some(job_hash.to_string()), + }) + } + _ => None, + } +} + +fn maintenance_backup_structural_object_type(table_bucket: &str, object: &str) -> Option { + maintenance_backup_object_path(table_bucket, object).map(|path| path.object_type) +} + +fn maintenance_backup_object_type( + table_bucket: &str, + object: &str, + paths: &MaintenanceBackupPaths, +) -> Option { + if paths.config_paths.contains(object) { + return Some(MaintenanceBackupObjectType::Config); + } + if paths.exact_paths.contains(object) + || paths.job_prefixes.iter().any(|prefix| { + object + .strip_prefix(prefix) + .and_then(|suffix| suffix.strip_suffix(".json")) + .is_some_and(is_table_catalog_path_hash) + }) + { + return Some(MaintenanceBackupObjectType::Job); + } + paths + .allow_all_table_paths + .then(|| maintenance_backup_structural_object_type(table_bucket, object)) + .flatten() +} + +fn maintenance_restore_paths() -> MaintenanceBackupPaths { + MaintenanceBackupPaths { + allow_all_table_paths: true, + ..Default::default() + } +} + +fn maintenance_backup_paths( + table_bucket: &str, + snapshot: Option<&StrongTableCatalogBucketSnapshot>, +) -> TableCatalogStoreResult { + let paths = TableCatalogObjectPaths::default(); + let bucket_config_path = paths.table_bucket_maintenance_config_path(table_bucket); + let mut result = MaintenanceBackupPaths { + exact_paths: BTreeSet::from([bucket_config_path.clone()]), + config_paths: BTreeSet::from([bucket_config_path]), + job_prefixes: BTreeSet::new(), + allow_all_table_paths: false, + }; + if let Some(snapshot) = snapshot { + for entry in &snapshot.tables { + if entry.table_bucket != table_bucket { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup maintenance snapshot contains table {} from another table bucket", + entry.table + ))); + } + let namespace = parse_namespace_for_store(&entry.namespace)?; + let table = parse_table_for_store(&entry.table)?; + let config_path = paths.table_maintenance_config_path(table_bucket, &namespace, &table, &entry.table_id); + result.exact_paths.insert(config_path.clone()); + result.config_paths.insert(config_path); + result + .exact_paths + .insert(paths.table_maintenance_latest_job_path(table_bucket, &namespace, &table, &entry.table_id)); + result.exact_paths.insert(paths.table_maintenance_current_job_path( + table_bucket, + &namespace, + &table, + &entry.table_id, + )); + result + .job_prefixes + .insert(paths.table_maintenance_jobs_prefix(table_bucket, &namespace, &table, &entry.table_id)); + } + } + Ok(result) +} + +fn maintenance_backup_object_key_is_safe(table_bucket: &str, object: &str, paths: &MaintenanceBackupPaths) -> bool { + backup_object_key_is_safe(object) && maintenance_backup_object_type(table_bucket, object, paths).is_some() +} + +fn maintenance_object_is_active( + table_bucket: &str, + object: &str, + data: &[u8], + paths: &MaintenanceBackupPaths, +) -> TableCatalogStoreResult { + let object_type = maintenance_backup_object_type(table_bucket, object, paths) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("maintenance backup object has an invalid path: {object}")))?; + if object_type == MaintenanceBackupObjectType::Config { + let config = serde_json::from_slice::(data) + .map_err(|err| TableCatalogStoreError::Invalid(format!("maintenance backup config is not valid: {object}: {err}")))?; + validate_table_maintenance_config(&config)?; + return Ok(false); + } + let path = maintenance_backup_object_path(table_bucket, object) + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("maintenance backup report has an invalid path: {object}")))?; + let namespace = path + .namespace + .as_ref() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("maintenance backup report has no table owner: {object}")))?; + let table = path + .table + .as_ref() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("maintenance backup report has no table owner: {object}")))?; + let table_id_hash = path + .table_id_hash + .as_deref() + .ok_or_else(|| TableCatalogStoreError::Invalid(format!("maintenance backup report has no table identity: {object}")))?; + let report = serde_json::from_slice::(data) + .map_err(|err| TableCatalogStoreError::Invalid(format!("maintenance backup report is not valid: {object}: {err}")))?; + validate_table_maintenance_report_owner(&report, table_bucket, namespace, table, &report.job.table_id)?; + if table_catalog_path_hash(&report.job.table_id) != table_id_hash { + return Err(TableCatalogStoreError::Invalid(format!( + "maintenance backup report table identity does not match its object path: {object}" + ))); + } + if let Some(job_id_hash) = path.job_id_hash.as_deref() + && table_catalog_path_hash(&report.job.job_id) != job_id_hash + { + return Err(TableCatalogStoreError::Invalid(format!( + "maintenance backup report job identity does not match its object path: {object}" + ))); + } + Ok(matches!( + report.job.status, + TableMetadataMaintenanceJobStatus::Queued | TableMetadataMaintenanceJobStatus::Running + )) +} + +fn backup_metadata_log_locations(metadata: &serde_json::Value, entry: &TableEntry) -> TableCatalogStoreResult> { + let Some(metadata_log) = metadata.get("metadata-log") else { + return Ok(BTreeSet::new()); + }; + let entries = metadata_log + .as_array() + .ok_or_else(|| TableCatalogStoreError::Invalid("table metadata-log must be an array for a durable backup".to_string()))?; + let mut locations = BTreeSet::new(); + for (index, metadata_log_entry) in entries.iter().enumerate() { + let location = metadata_log_entry + .get("metadata-file") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + TableCatalogStoreError::Invalid(format!( + "table metadata-log entry {index} must identify a metadata-file for a durable backup" + )) + })?; + let object = table_catalog_object_key_from_location(&entry.table_bucket, location).ok_or_else(|| { + TableCatalogStoreError::Invalid(format!( + "table metadata-log entry {index} points outside table bucket {}", + entry.table_bucket + )) + })?; + if !is_valid_table_metadata_location_for_entry(entry, &object) { + return Err(TableCatalogStoreError::Invalid(format!( + "table metadata-log entry {index} points outside the table metadata directory: {object}" + ))); + } + locations.insert(object); + } + Ok(locations) +} + +fn validate_backup_table_metadata( + entry: &TableEntry, + metadata_location: &str, + metadata: &serde_json::Value, + current: bool, +) -> TableCatalogStoreResult<()> { + validate_supported_table_metadata(metadata)?; + if table_metadata_uuid(metadata)? != entry.table_uuid { + return Err(TableCatalogStoreError::Invalid(format!( + "table metadata UUID does not match catalog entry for {}/{}/{}", + entry.table_bucket, entry.namespace, entry.table + ))); + } + let location = table_metadata_location(metadata)?; + validate_table_warehouse_location(&entry.table_bucket, location)?; + if current && location != entry.warehouse_location { + return Err(TableCatalogStoreError::Invalid(format!( + "current table metadata location does not match catalog warehouse location for {}/{}/{}", + entry.table_bucket, entry.namespace, entry.table + ))); + } + if current && table_metadata_format_version(metadata)? != entry.format_version { + return Err(TableCatalogStoreError::Invalid(format!( + "current table metadata format version does not match catalog entry for {}/{}/{}", + entry.table_bucket, entry.namespace, entry.table + ))); + } + if !is_valid_table_metadata_location_for_entry(entry, metadata_location) { + return Err(TableCatalogStoreError::Invalid(format!( + "table metadata object is outside the catalog metadata directory: {metadata_location}" + ))); + } + Ok(()) +} + +fn validate_backup_view_metadata( + entry: &ViewEntry, + namespace: &Namespace, + view: &IdentifierSegment, + metadata_location: &str, + metadata: &serde_json::Value, +) -> TableCatalogStoreResult<()> { + validate_supported_view_metadata(metadata)?; + let view_uuid = metadata + .get("view-uuid") + .and_then(serde_json::Value::as_str) + .filter(|uuid| !uuid.is_empty()) + .ok_or_else(|| TableCatalogStoreError::Invalid("view metadata is missing view-uuid".to_string()))?; + if view_uuid != entry.view_uuid { + return Err(TableCatalogStoreError::Invalid(format!( + "view metadata UUID does not match catalog entry for {}/{}/{}", + entry.table_bucket, entry.namespace, entry.view + ))); + } + let location = metadata + .get("location") + .and_then(serde_json::Value::as_str) + .filter(|location| !location.is_empty()) + .ok_or_else(|| TableCatalogStoreError::Invalid("view metadata is missing location".to_string()))?; + validate_view_warehouse_location(&entry.table_bucket, location)?; + if location != entry.warehouse_location { + return Err(TableCatalogStoreError::Invalid(format!( + "current view metadata location does not match catalog warehouse location for {}/{}/{}", + entry.table_bucket, entry.namespace, entry.view + ))); + } + let format_version = metadata + .get("format-version") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| TableCatalogStoreError::Invalid("view metadata is missing format-version".to_string()))?; + if format_version != i64::from(entry.format_version) { + return Err(TableCatalogStoreError::Invalid(format!( + "view metadata format version does not match catalog entry for {}/{}/{}", + entry.table_bucket, entry.namespace, entry.view + ))); + } + if !is_valid_view_metadata_location(namespace, view, metadata_location) { + return Err(TableCatalogStoreError::Invalid(format!( + "view metadata object is outside the catalog metadata directory: {metadata_location}" + ))); + } + Ok(()) +} + +fn backup_metadata_graph_context( + entry: &TableEntry, + metadata: &serde_json::Value, +) -> TableCatalogStoreResult<(TableEntry, String)> { + let mut metadata_entry = entry.clone(); + metadata_entry.warehouse_location = table_metadata_location(metadata)?.to_string(); + let warehouse_prefix = table_warehouse_object_prefix(&metadata_entry)?; + Ok((metadata_entry, warehouse_prefix)) +} + +fn validate_backup_metadata_warehouse_owner( + snapshot: &StrongTableCatalogBucketSnapshot, + entry: &TableEntry, + warehouse_prefix: &str, +) -> TableCatalogStoreResult<()> { + for other in &snapshot.tables { + if other.state != TableCatalogEntryState::Active || other.table_id == entry.table_id { + continue; + } + let other_prefix = table_warehouse_object_prefix(other)?; + if warehouse_object_prefixes_overlap(warehouse_prefix, &other_prefix) { + return Err(TableCatalogStoreError::Conflict(format!( + "table backup metadata warehouse overlaps another active table: {warehouse_prefix}" + ))); + } + } + Ok(()) +} + +fn insert_backup_object_kind( + keys: &mut BTreeMap<(String, String), StrongTableCatalogBackupObjectKind>, + bucket: &str, + object: &str, + kind: StrongTableCatalogBackupObjectKind, +) -> TableCatalogStoreResult<()> { + let key = (bucket.to_string(), object.to_string()); + if let Some(existing) = keys.get(&key) { + if existing != &kind { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup object has conflicting kinds: {bucket}/{object}" + ))); + } + return Ok(()); + } + keys.insert(key, kind); + Ok(()) +} + +struct StrongTableCatalogBackupFence { + table_bucket_publication: TableCatalogLockGuard, + table_bucket_migration: TableCatalogLockGuard, + global_migration: TableCatalogLockGuard, +} + +impl StrongTableCatalogBackupFence { + fn ensure_held(&self) -> TableCatalogStoreResult<()> { + if self.table_bucket_publication.is_lock_lost() + || self.global_migration.is_lock_lost() + || self.table_bucket_migration.is_lock_lost() + { + return Err(TableCatalogStoreError::Unavailable( + "catalog backup or restore lost its fencing lock".to_string(), + )); + } + Ok(()) + } + + fn mutation_fence(&self) -> TableCatalogObjectMutationFence { + TableCatalogObjectMutationFence::from_signals([ + self.table_bucket_publication.lock_lost_signal(), + self.global_migration.lock_lost_signal(), + self.table_bucket_migration.lock_lost_signal(), + ]) + } +} + +impl StrongTableCatalogStore +where + B: TableCatalogObjectBackend, +{ + async fn acquire_backup_fence(&self, table_bucket: &str) -> TableCatalogStoreResult { + // Keep global migration -> per-bucket migration -> publication ordering. + let global_migration = self + .object_backend + .acquire_write_lock( + RUSTFS_META_BUCKET, + &TableCatalogObjectPaths::default().backing_migration_global_fence_lock_path(), + ) + .await?; + let table_bucket_migration = self + .object_backend + .acquire_write_lock( + RUSTFS_META_BUCKET, + &TableCatalogObjectPaths::default().backing_migration_fence_lock_path(table_bucket), + ) + .await?; + let table_bucket_publication = self + .object_backend + .acquire_write_lock(table_bucket, &default_table_bucket_publication_lock_path()) + .await?; + Ok(StrongTableCatalogBackupFence { + table_bucket_publication, + global_migration, + table_bucket_migration, + }) + } + + async fn backup_object_fingerprint( + &self, + bucket: &str, + object: &str, + kind: StrongTableCatalogBackupObjectKind, + ) -> TableCatalogStoreResult { + if !backup_object_key_is_safe(object) { + return Err(TableCatalogStoreError::Invalid(format!("backup object key is invalid: {object}"))); + } + let metadata = self + .object_backend + .object_metadata(bucket, object) + .await? + .ok_or_else(|| TableCatalogStoreError::NotFound(format!("backup object {bucket}/{object}")))?; + let sha256 = if metadata.size <= TABLE_CATALOG_BACKUP_HASH_MAX_SIZE { + let object_data = self + .object_backend + .read_object_limited(bucket, object, usize::try_from(TABLE_CATALOG_BACKUP_HASH_MAX_SIZE).unwrap_or(usize::MAX)) + .await? + .ok_or_else(|| TableCatalogStoreError::NotFound(format!("backup object {bucket}/{object}")))?; + if object_data.data.len() as u64 != metadata.size { + return Err(TableCatalogStoreError::Conflict(format!( + "backup object {bucket}/{object} changed while its watermark was collected" + ))); + } + if object_data.etag != metadata.etag { + return Err(TableCatalogStoreError::Conflict(format!( + "backup object {bucket}/{object} changed while its watermark was collected" + ))); + } + Some(backup_sha256(&object_data.data)) + } else if !has_usable_etag(metadata.etag.as_deref()) { + return Err(TableCatalogStoreError::Invalid(format!( + "backup object {bucket}/{object} is too large for a digest and has no etag watermark" + ))); + } else { + None + }; + let final_metadata = self.object_backend.object_metadata(bucket, object).await?.ok_or_else(|| { + TableCatalogStoreError::Conflict(format!( + "backup object {bucket}/{object} disappeared while its watermark was collected" + )) + })?; + if final_metadata.size != metadata.size || final_metadata.etag != metadata.etag { + return Err(TableCatalogStoreError::Conflict(format!( + "backup object {bucket}/{object} changed while its watermark was collected" + ))); + } + Ok(StrongTableCatalogBackupObject { + bucket: bucket.to_string(), + object: object.to_string(), + kind, + size_bytes: metadata.size, + etag: metadata.etag, + sha256, + }) + } + + async fn collect_maintenance_objects( + &self, + table_bucket: &str, + allowed_paths: &MaintenanceBackupPaths, + ) -> TableCatalogStoreResult> { + let object_paths = TableCatalogObjectPaths::default(); + let maintenance_marker = format!("/{MAINTENANCE_ROOT}/"); + let table_bucket_prefix = object_paths.table_bucket_root_prefix(table_bucket); + let mut total_bytes = 0usize; + let mut objects = Vec::new(); + for object in self + .object_backend + .list_objects(RUSTFS_META_BUCKET, &table_bucket_prefix) + .await? + .into_iter() + { + if !object.contains(&maintenance_marker) { + continue; + } + if !maintenance_backup_object_key_is_safe(table_bucket, &object, allowed_paths) { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup maintenance object is outside the catalog snapshot paths: {object}" + ))); + } + if objects.len() >= TABLE_CATALOG_BACKUP_MAX_OBJECTS { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup contains more than {TABLE_CATALOG_BACKUP_MAX_OBJECTS} maintenance objects" + ))); + } + let value = self + .object_backend + .read_object_limited(RUSTFS_META_BUCKET, &object, TABLE_CATALOG_BACKUP_MAX_MAINTENANCE_OBJECT_SIZE) + .await? + .ok_or_else(|| TableCatalogStoreError::NotFound(format!("maintenance backup object {object}")))?; + let etag = value.etag.clone().filter(|etag| !etag.is_empty()).ok_or_else(|| { + TableCatalogStoreError::Invalid(format!( + "maintenance backup object {object} has no etag watermark required for conditional restore" + )) + })?; + let final_metadata = self + .object_backend + .object_metadata(RUSTFS_META_BUCKET, &object) + .await? + .ok_or_else(|| { + TableCatalogStoreError::Conflict(format!( + "maintenance backup object {object} disappeared while its watermark was collected" + )) + })?; + if final_metadata.size != value.data.len() as u64 || final_metadata.etag.as_deref() != Some(etag.as_str()) { + return Err(TableCatalogStoreError::Conflict(format!( + "maintenance backup object {object} changed while its watermark was collected" + ))); + } + if maintenance_object_is_active(table_bucket, &object, &value.data, allowed_paths)? { + return Err(TableCatalogStoreError::Conflict( + "maintenance backup requires all scheduler and worker leases to be drained".to_string(), + )); + } + total_bytes = total_bytes.saturating_add(value.data.len()); + if total_bytes > TABLE_CATALOG_BACKUP_MAX_MAINTENANCE_BYTES { + return Err(TableCatalogStoreError::Invalid(format!( + "maintenance backup exceeds the maximum of {TABLE_CATALOG_BACKUP_MAX_MAINTENANCE_BYTES} bytes" + ))); + } + objects.push(StrongTableCatalogBackupMaintenanceObject { + object, + size_bytes: value.data.len() as u64, + etag: Some(etag), + sha256: backup_sha256(&value.data), + data: value.data, + }); + } + objects.sort_by(|left, right| left.object.cmp(&right.object)); + Ok(objects) + } + + async fn collect_maintenance_backup_objects( + &self, + table_bucket: &str, + snapshot: Option<&StrongTableCatalogBucketSnapshot>, + ) -> TableCatalogStoreResult> { + let paths = maintenance_backup_paths(table_bucket, snapshot)?; + self.collect_maintenance_objects(table_bucket, &paths).await + } + + async fn collect_backup_objects( + &self, + snapshot: &StrongTableCatalogBucketSnapshot, + ) -> TableCatalogStoreResult> { + let mut keys = BTreeMap::<(String, String), StrongTableCatalogBackupObjectKind>::new(); + for entry in &snapshot.tables { + match &entry.state { + TableCatalogEntryState::Deleted => continue, + TableCatalogEntryState::Renaming | TableCatalogEntryState::Deleting => { + return Err(TableCatalogStoreError::Conflict(format!( + "catalog backup requires table {} to leave its transient state before sealing", + entry.table + ))); + } + TableCatalogEntryState::Active => {} + } + let namespace = parse_namespace_for_store(&entry.namespace)?; + let table = parse_table_for_store(&entry.table)?; + if !is_valid_table_metadata_location_for_entry(entry, &entry.metadata_location) { + return Err(TableCatalogStoreError::Invalid(format!( + "table backup metadata location is invalid for {}/{}/{}", + entry.table_bucket, entry.namespace, entry.table + ))); + } + let current_metadata_location = table_catalog_object_key_from_location(&entry.table_bucket, &entry.metadata_location) + .ok_or_else(|| { + TableCatalogStoreError::Invalid(format!( + "table backup metadata location cannot be converted to an object key for {}/{}/{}", + entry.table_bucket, entry.namespace, entry.table + )) + })?; + let current_metadata = + read_table_metadata_value(&self.object_backend, &entry.table_bucket, ¤t_metadata_location) + .await? + .ok_or_else(|| { + TableCatalogStoreError::NotFound(format!("current metadata object {current_metadata_location}")) + })?; + validate_backup_table_metadata(entry, ¤t_metadata_location, ¤t_metadata, true)?; + let mut metadata_locations = backup_metadata_log_locations(¤t_metadata, entry)?; + metadata_locations.insert(current_metadata_location.clone()); + for metadata_location in metadata_locations { + if !is_valid_table_metadata_location_for_entry(entry, &metadata_location) { + return Err(TableCatalogStoreError::Invalid(format!( + "table backup metadata log location is outside the table: {metadata_location}" + ))); + } + insert_backup_object_kind( + &mut keys, + &entry.table_bucket, + &metadata_location, + StrongTableCatalogBackupObjectKind::TableMetadata, + )?; + let metadata = if metadata_location == current_metadata_location { + current_metadata.clone() + } else { + read_table_metadata_value(&self.object_backend, &entry.table_bucket, &metadata_location) + .await? + .ok_or_else(|| TableCatalogStoreError::NotFound(format!("metadata log object {metadata_location}")))? + }; + validate_backup_table_metadata( + entry, + &metadata_location, + &metadata, + metadata_location == current_metadata_location, + )?; + let (metadata_entry, warehouse_prefix) = backup_metadata_graph_context(entry, &metadata)?; + validate_backup_metadata_warehouse_owner(snapshot, entry, &warehouse_prefix)?; + let context = + TableSnapshotGraphValidationContext::new(&self.object_backend, &entry.table_bucket, &metadata_entry); + // A durable backup must be able to replay the complete Iceberg + // snapshot graph, not merely preserve the metadata locations. + // The maintenance reachability walk is intentionally tolerant + // of unsupported objects so it can report candidates; backup + // sealing must reject malformed or incomplete graphs instead. + validate_table_snapshot_changes(&context, None, &metadata).await?; + for reference in table_statistics_object_references(&metadata)? { + let object_key = table_catalog_object_key_from_location(&entry.table_bucket, &reference.location) + .ok_or_else(|| { + TableCatalogStoreError::Invalid(format!( + "table backup statistics location cannot be converted to an object key: {}", + reference.location + )) + })?; + if !object_key.starts_with(&warehouse_prefix) { + return Err(TableCatalogStoreError::Invalid(format!( + "table backup statistics object is outside the table warehouse: {object_key}" + ))); + } + let kind = StrongTableCatalogBackupObjectKind::from_statistics_kind(reference.kind); + insert_backup_object_kind(&mut keys, &entry.table_bucket, &object_key, kind)?; + } + let referenced = metadata_maintenance_referenced_object_reports( + &self.object_backend, + &entry.table_bucket, + &namespace, + &table, + Some(warehouse_prefix.as_str()), + &metadata, + std::slice::from_ref(&metadata_location), + ) + .await?; + for report in referenced { + if report.state == TableMetadataMaintenanceObjectState::ManualReviewRequired { + return Err(TableCatalogStoreError::Conflict(format!( + "table backup cannot seal an unresolved object reference: {}", + report.object_location + ))); + } + insert_backup_object_kind( + &mut keys, + &entry.table_bucket, + &report.object_location, + StrongTableCatalogBackupObjectKind::from_maintenance_kind(&report.object_kind), + )?; + } + } + } + for entry in &snapshot.views { + match &entry.state { + TableCatalogEntryState::Deleted => continue, + TableCatalogEntryState::Renaming | TableCatalogEntryState::Deleting => { + return Err(TableCatalogStoreError::Conflict(format!( + "catalog backup requires view {} to leave its transient state before sealing", + entry.view + ))); + } + TableCatalogEntryState::Active => {} + } + let namespace = parse_namespace_for_store(&entry.namespace)?; + let view = parse_table_for_store(&entry.view)?; + let current_metadata_location = table_catalog_object_key_from_location(&entry.table_bucket, &entry.metadata_location) + .ok_or_else(|| { + TableCatalogStoreError::Invalid(format!( + "view backup metadata location cannot be converted to an object key for {}/{}/{}", + entry.table_bucket, entry.namespace, entry.view + )) + })?; + if !is_valid_view_metadata_location(&namespace, &view, ¤t_metadata_location) { + return Err(TableCatalogStoreError::Invalid(format!( + "view backup metadata location is invalid for {}/{}/{}", + entry.table_bucket, entry.namespace, entry.view + ))); + } + let metadata = read_table_metadata_value(&self.object_backend, &entry.table_bucket, ¤t_metadata_location) + .await? + .ok_or_else(|| { + TableCatalogStoreError::NotFound(format!("current view metadata object {current_metadata_location}")) + })?; + validate_backup_view_metadata(entry, &namespace, &view, ¤t_metadata_location, &metadata)?; + insert_backup_object_kind( + &mut keys, + &entry.table_bucket, + ¤t_metadata_location, + StrongTableCatalogBackupObjectKind::ViewMetadata, + )?; + } + if keys.len() > TABLE_CATALOG_BACKUP_MAX_OBJECTS { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup references more than {TABLE_CATALOG_BACKUP_MAX_OBJECTS} objects" + ))); + } + let mut objects = Vec::with_capacity(keys.len()); + for ((bucket, object), kind) in keys { + objects.push(self.backup_object_fingerprint(&bucket, &object, kind).await?); + } + Ok(objects) + } + + async fn read_backup(&self, table_bucket: &str, backup_id: &str) -> TableCatalogStoreResult { + let path = TableCatalogObjectPaths::default().catalog_backup_path(table_bucket, backup_id); + let object = self + .object_backend + .read_object_limited(RUSTFS_META_BUCKET, &path, TABLE_CATALOG_BACKUP_MAX_SIZE) + .await? + .ok_or_else(|| TableCatalogStoreError::NotFound(format!("catalog backup {backup_id}")))?; + let backup = serde_json::from_slice::(&object.data) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to decode catalog backup: {err}")))?; + if backup.version != TABLE_CATALOG_BACKUP_VERSION || backup.backup_id != backup_id || backup.table_bucket != table_bucket + { + return Err(TableCatalogStoreError::Invalid( + "catalog backup identity or version does not match the restore request".to_string(), + )); + } + if backup.source_snapshot_etag.is_empty() { + return Err(TableCatalogStoreError::Invalid( + "catalog backup has no source snapshot ETag watermark".to_string(), + )); + } + if backup.snapshot.table_bucket.table_bucket != table_bucket { + return Err(TableCatalogStoreError::Invalid( + "catalog backup snapshot belongs to a different table bucket".to_string(), + )); + } + if backup.snapshot.table_bucket.state != TableCatalogEntryState::Active + || backup.snapshot.table_bucket.active_rename_id.is_some() + { + return Err(TableCatalogStoreError::Invalid( + "catalog backup snapshot must contain an active table bucket without a pending rename".to_string(), + )); + } + if backup.objects.len() > TABLE_CATALOG_BACKUP_MAX_OBJECTS { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup references more than {TABLE_CATALOG_BACKUP_MAX_OBJECTS} objects" + ))); + } + if backup.maintenance_objects.len() > TABLE_CATALOG_BACKUP_MAX_OBJECTS { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup contains more than {TABLE_CATALOG_BACKUP_MAX_OBJECTS} maintenance objects" + ))); + } + let fingerprint = table_catalog_bucket_snapshot_fingerprint(&backup.snapshot)?; + if fingerprint != backup.catalog_fingerprint { + return Err(TableCatalogStoreError::Invalid( + "catalog backup snapshot fingerprint does not match its payload".to_string(), + )); + } + let expected_backup_id = catalog_backup_id( + &backup.table_bucket, + backup.version, + &backup.source_snapshot_etag, + backup.source_snapshot_version, + &backup.catalog_fingerprint, + &backup.objects, + &backup.maintenance_objects, + )?; + if expected_backup_id != backup.backup_id { + return Err(TableCatalogStoreError::Invalid( + "catalog backup content does not match its backup identity".to_string(), + )); + } + StrongTableCatalogStore::::state_from_snapshot( + StrongTableCatalogSnapshot { + version: backup.source_snapshot_version, + table_buckets: vec![backup.snapshot.table_bucket.clone()], + namespaces: backup.snapshot.namespaces.clone(), + tables: backup.snapshot.tables.clone(), + views: backup.snapshot.views.clone(), + commits: backup.snapshot.commits.clone(), + idempotency: backup.snapshot.idempotency.clone(), + }, + None, + )?; + Ok(backup) + } + + fn validate_maintenance_backup_objects( + table_bucket: &str, + snapshot: &StrongTableCatalogBucketSnapshot, + objects: &[StrongTableCatalogBackupMaintenanceObject], + ) -> TableCatalogStoreResult<()> { + let paths = maintenance_backup_paths(table_bucket, Some(snapshot))?; + if objects + .windows(2) + .any(|pair| pair[0].object.as_str() >= pair[1].object.as_str()) + { + return Err(TableCatalogStoreError::Invalid( + "catalog backup maintenance objects must be sorted and unique".to_string(), + )); + } + let mut total_bytes = 0usize; + for object in objects { + if !maintenance_backup_object_key_is_safe(table_bucket, &object.object, &paths) + || object.data.len() > TABLE_CATALOG_BACKUP_MAX_MAINTENANCE_OBJECT_SIZE + { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup maintenance object is outside the catalog snapshot paths: {}", + object.object + ))); + } + if object.etag.as_deref().is_none_or(str::is_empty) { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup maintenance object has no etag watermark required for conditional restore: {}", + object.object + ))); + } + if object.data.len() as u64 != object.size_bytes || backup_sha256(&object.data) != object.sha256 { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup maintenance object checksum is invalid: {}", + object.object + ))); + } + if maintenance_object_is_active(table_bucket, &object.object, &object.data, &paths)? { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup contains active maintenance state: {}", + object.object + ))); + } + total_bytes = total_bytes.saturating_add(object.data.len()); + if total_bytes > TABLE_CATALOG_BACKUP_MAX_MAINTENANCE_BYTES { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup exceeds the maximum of {TABLE_CATALOG_BACKUP_MAX_MAINTENANCE_BYTES} maintenance bytes" + ))); + } + } + Ok(()) + } + + async fn verify_backup_objects(&self, backup: &StrongTableCatalogBackup) -> TableCatalogStoreResult<()> { + for object in &backup.objects { + if !backup_object_key_is_safe(&object.object) || object.bucket != backup.table_bucket { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup contains an object outside its protected buckets: {}/{}", + object.bucket, object.object + ))); + } + if object.size_bytes <= TABLE_CATALOG_BACKUP_HASH_MAX_SIZE && object.sha256.is_none() { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup is missing a content watermark: {}/{}", + object.bucket, object.object + ))); + } + if object.size_bytes > TABLE_CATALOG_BACKUP_HASH_MAX_SIZE && !has_usable_etag(object.etag.as_deref()) { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup is missing an etag watermark for a large object: {}/{}", + object.bucket, object.object + ))); + } + let metadata = self + .object_backend + .object_metadata(&object.bucket, &object.object) + .await? + .ok_or_else(|| { + TableCatalogStoreError::Conflict(format!( + "catalog backup object is missing: {}/{}", + object.bucket, object.object + )) + })?; + if metadata.size != object.size_bytes || metadata.etag != object.etag { + return Err(TableCatalogStoreError::Conflict(format!( + "catalog backup object watermark changed: {}/{}", + object.bucket, object.object + ))); + } + if let Some(expected_sha256) = object.sha256.as_deref() { + let data = self + .object_backend + .read_object_limited( + &object.bucket, + &object.object, + usize::try_from(TABLE_CATALOG_BACKUP_HASH_MAX_SIZE).unwrap_or(usize::MAX), + ) + .await? + .ok_or_else(|| { + TableCatalogStoreError::Conflict(format!("catalog backup object disappeared: {}", object.object)) + })?; + if data.data.len() as u64 != object.size_bytes + || data.etag != object.etag + || backup_sha256(&data.data) != expected_sha256 + { + return Err(TableCatalogStoreError::Conflict(format!( + "catalog backup object checksum changed: {}/{}", + object.bucket, object.object + ))); + } + } + let final_metadata = self + .object_backend + .object_metadata(&object.bucket, &object.object) + .await? + .ok_or_else(|| { + TableCatalogStoreError::Conflict(format!( + "catalog backup object disappeared while it was verified: {}/{}", + object.bucket, object.object + )) + })?; + if final_metadata.size != object.size_bytes || final_metadata.etag != object.etag { + return Err(TableCatalogStoreError::Conflict(format!( + "catalog backup object watermark changed: {}/{}", + object.bucket, object.object + ))); + } + } + Ok(()) + } + + async fn read_restore_intent( + &self, + table_bucket: &str, + ) -> TableCatalogStoreResult)>> { + let path = TableCatalogObjectPaths::default().catalog_backup_restore_intent_path(table_bucket); + let Some(object) = self + .object_backend + .read_object_limited(RUSTFS_META_BUCKET, &path, 64 * 1024) + .await? + else { + return Ok(None); + }; + let intent = serde_json::from_slice::(&object.data) + .map_err(|err| TableCatalogStoreError::Invalid(format!("failed to decode catalog restore intent: {err}")))?; + if intent.version != TABLE_CATALOG_BACKUP_VERSION || intent.table_bucket != table_bucket { + return Err(TableCatalogStoreError::Invalid( + "catalog restore intent has an invalid identity or version".to_string(), + )); + } + let intent_etag = object + .etag + .filter(|etag| !etag.is_empty()) + .ok_or_else(|| TableCatalogStoreError::Invalid("catalog restore intent has no usable ETag watermark".to_string()))?; + for (name, etag) in [ + ("expected snapshot", intent.expected_snapshot_etag.as_deref()), + ("target snapshot", intent.target_snapshot_etag.as_deref()), + ("restored snapshot", intent.restored_snapshot_etag.as_deref()), + ] { + if etag.is_some_and(str::is_empty) { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog restore intent has an empty {name} ETag watermark" + ))); + } + } + Ok(Some((intent, Some(intent_etag)))) + } + + async fn write_restore_intent( + &self, + table_bucket: &str, + intent: &StrongTableCatalogRestoreIntent, + precondition: TableCatalogPutPrecondition, + mutation_fence: Option<&TableCatalogObjectMutationFence>, + ) -> TableCatalogStoreResult<()> { + let data = serde_json::to_vec(intent) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to encode catalog restore intent: {err}")))?; + if data.len() > 64 * 1024 { + return Err(TableCatalogStoreError::Invalid( + "catalog restore intent exceeds the maximum encoded size of 65536 bytes".to_string(), + )); + } + let path = TableCatalogObjectPaths::default().catalog_backup_restore_intent_path(table_bucket); + match mutation_fence { + Some(fence) => { + self.object_backend + .put_object_fenced(RUSTFS_META_BUCKET, &path, data, precondition, fence) + .await + } + None => { + self.object_backend + .put_object(RUSTFS_META_BUCKET, &path, data, precondition) + .await + } + } + } + + async fn delete_restore_intent_if_unchanged( + &self, + table_bucket: &str, + expected_etag: &str, + mutation_fence: Option<&TableCatalogObjectMutationFence>, + ) -> TableCatalogStoreResult<()> { + let path = TableCatalogObjectPaths::default().catalog_backup_restore_intent_path(table_bucket); + match mutation_fence { + Some(fence) => { + self.object_backend + .delete_object_if_match_fenced(RUSTFS_META_BUCKET, &path, expected_etag, fence) + .await + } + None => { + self.object_backend + .delete_object_if_match(RUSTFS_META_BUCKET, &path, expected_etag) + .await + } + } + } + + fn validate_restore_intent_replay( + intent: &StrongTableCatalogRestoreIntent, + expected_snapshot_etag: Option<&str>, + allow_replace: bool, + current_snapshot_etag: Option<&str>, + current_bucket: Option<&StrongTableCatalogBucketSnapshot>, + backup: &StrongTableCatalogBackup, + ) -> TableCatalogStoreResult<()> { + let Some(target_bucket_present) = intent.target_bucket_present else { + return Err(TableCatalogStoreError::Invalid( + "catalog restore intent lacks a persisted target-presence precondition; manual recovery is required".to_string(), + )); + }; + let Some(intent_allow_replace) = intent.allow_replace else { + return Err(TableCatalogStoreError::Invalid( + "catalog restore intent lacks a persisted replacement policy; manual recovery is required".to_string(), + )); + }; + if intent.source_maintenance_fingerprint.is_none() { + return Err(TableCatalogStoreError::Invalid( + "catalog restore intent lacks a persisted maintenance precondition; manual recovery is required".to_string(), + )); + } + if intent.expected_snapshot_etag.as_deref() != expected_snapshot_etag { + return Err(TableCatalogStoreError::Conflict( + "catalog restore retry does not match the persisted request precondition".to_string(), + )); + } + if intent_allow_replace != allow_replace { + return Err(TableCatalogStoreError::Conflict( + "catalog restore retry does not match the persisted allow-replace policy".to_string(), + )); + } + if target_bucket_present && intent.target_snapshot_etag.is_none() { + return Err(TableCatalogStoreError::Invalid( + "catalog restore intent has a bucket target without a snapshot precondition".to_string(), + )); + } + if intent.target_bucket_fingerprint.is_empty() { + return Err(TableCatalogStoreError::Invalid( + "catalog restore intent lacks a persisted target bucket fingerprint; manual recovery is required".to_string(), + )); + } + let catalog_matches_backup = current_bucket == Some(&backup.snapshot); + if intent.state == StrongTableCatalogRestoreIntentState::CatalogApplied { + if intent.restored_snapshot_etag.is_none() { + return Err(TableCatalogStoreError::Invalid( + "catalog restore intent marks the catalog applied without a restored snapshot ETag".to_string(), + )); + } + if !catalog_matches_backup { + return Err(TableCatalogStoreError::Conflict( + "catalog restore intent records an applied snapshot but the target catalog changed".to_string(), + )); + } + return Ok(()); + } + // The catalog CAS may have succeeded immediately before a crash or an + // intent-finalization error. Once the exact target bucket is present, + // the target preconditions describe the state before publication and + // must not reject replay of the already-applied restore. + if catalog_matches_backup { + return Ok(()); + } + let current_target_bucket_fingerprint = current_bucket + .map(table_catalog_bucket_snapshot_fingerprint) + .transpose()? + .unwrap_or_else(|| ABSENT_RESTORE_TARGET_BUCKET_FINGERPRINT.to_string()); + if intent.target_bucket_fingerprint != current_target_bucket_fingerprint { + return Err(TableCatalogStoreError::Conflict( + "catalog restore target bucket changed while the restore intent was pending".to_string(), + )); + } + if intent.expected_snapshot_etag.is_some() && intent.target_snapshot_etag.as_deref() != current_snapshot_etag { + return Err(TableCatalogStoreError::Conflict( + "catalog restore global snapshot changed while the expected-snapshot-etag precondition was pending".to_string(), + )); + } + if target_bucket_present != current_bucket.is_some() { + return Err(TableCatalogStoreError::Conflict( + "catalog restore target changed while the restore intent was pending".to_string(), + )); + } + Ok(()) + } + + async fn restore_maintenance_objects( + &self, + table_bucket: &str, + snapshot: &StrongTableCatalogBucketSnapshot, + objects: &[StrongTableCatalogBackupMaintenanceObject], + mutation_fence: &TableCatalogObjectMutationFence, + ) -> TableCatalogStoreResult<()> { + Self::validate_maintenance_backup_objects(table_bucket, snapshot, objects)?; + let allowed_paths = maintenance_restore_paths(); + let existing_objects = self + .collect_maintenance_objects(table_bucket, &allowed_paths) + .await? + .into_iter() + .map(|object| (object.object.clone(), object)) + .collect::>(); + let target_paths = objects.iter().map(|object| object.object.as_str()).collect::>(); + + // Publish target objects before removing stale state. Each replacement is + // conditional so a concurrent change cannot be overwritten silently. + for object in objects { + mutation_fence.ensure_held()?; + match existing_objects.get(&object.object) { + Some(existing) if existing.data == object.data => {} + Some(existing) => { + let expected_etag = existing.etag.as_deref().ok_or_else(|| { + TableCatalogStoreError::Invalid(format!( + "catalog restore maintenance object has no etag watermark required for conditional replacement: {}", + object.object + )) + })?; + self.object_backend + .put_object_fenced( + RUSTFS_META_BUCKET, + &object.object, + object.data.clone(), + TableCatalogPutPrecondition::IfMatch(expected_etag.to_string()), + mutation_fence, + ) + .await?; + } + None => { + self.object_backend + .put_object_fenced( + RUSTFS_META_BUCKET, + &object.object, + object.data.clone(), + TableCatalogPutPrecondition::IfAbsent, + mutation_fence, + ) + .await?; + } + } + mutation_fence.ensure_held()?; + } + + // Only remove objects that are outside the complete target set. The + // ETag check makes a stale inventory fail closed instead of deleting a + // newer maintenance record. + for (path, object) in existing_objects { + if target_paths.contains(path.as_str()) { + continue; + } + mutation_fence.ensure_held()?; + let expected_etag = object.etag.as_deref().ok_or_else(|| { + TableCatalogStoreError::Invalid(format!( + "catalog restore maintenance object has no etag watermark required for conditional deletion: {path}" + )) + })?; + self.object_backend + .delete_object_if_match_fenced(RUSTFS_META_BUCKET, &path, expected_etag, mutation_fence) + .await?; + mutation_fence.ensure_held()?; + } + Ok(()) + } + + pub(crate) async fn create_durable_catalog_backup( + &self, + table_bucket: &str, + expected_snapshot_etag: Option<&str>, + ) -> TableCatalogStoreResult { + let fence = self.acquire_backup_fence(table_bucket).await?; + fence.ensure_held()?; + let mutation_fence = fence.mutation_fence(); + let _write_guard = self.write_lock.lock().await; + self.hydrate_state().await?; + if self.read_restore_intent(table_bucket).await?.is_some() { + return Err(TableCatalogStoreError::Conflict( + "catalog backup requires pending restore recovery before sealing".to_string(), + )); + } + let (snapshot, source_snapshot_etag, source_snapshot_version) = { + let state = self.state.lock().await; + let current_etag = state + .snapshot_etag + .clone() + .filter(|etag| !etag.is_empty()) + .ok_or_else(|| TableCatalogStoreError::Internal("durable strong catalog snapshot has no etag".to_string()))?; + if expected_snapshot_etag.is_some_and(|expected| expected != current_etag) { + return Err(TableCatalogStoreError::Conflict( + "catalog snapshot changed before backup capture".to_string(), + )); + } + let snapshot = StrongTableCatalogStore::::bucket_snapshot_from_state_locked(&state, table_bucket) + .ok_or_else(|| TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}")))?; + if snapshot.table_bucket.state != TableCatalogEntryState::Active || snapshot.table_bucket.active_rename_id.is_some() { + return Err(TableCatalogStoreError::Conflict( + "catalog backup requires an active table bucket without a pending rename".to_string(), + )); + } + for entry in &snapshot.tables { + if entry.state == TableCatalogEntryState::Active { + let recovery = StrongTableCatalogStore::::table_commit_recovery_report_for_entry_locked(&state, entry); + if recovery.staged_before_table_update_count > 0 + || recovery.finalization_required_count > 0 + || recovery.idempotency_repair_required_count > 0 + || recovery.manual_review_count > 0 + { + return Err(TableCatalogStoreError::Conflict(format!( + "table backup requires commit recovery before sealing table {}", + entry.table + ))); + } + } + } + let version = state + .snapshot_version + .ok_or_else(|| TableCatalogStoreError::Internal("durable strong catalog snapshot has no version".to_string()))?; + (snapshot, current_etag, version) + }; + let catalog_fingerprint = table_catalog_bucket_snapshot_fingerprint(&snapshot)?; + let objects = self.collect_backup_objects(&snapshot).await?; + let maintenance_objects = self.collect_maintenance_backup_objects(table_bucket, Some(&snapshot)).await?; + fence.ensure_held()?; + let backup_id = catalog_backup_id( + table_bucket, + TABLE_CATALOG_BACKUP_VERSION, + &source_snapshot_etag, + source_snapshot_version, + &catalog_fingerprint, + &objects, + &maintenance_objects, + )?; + let created_at = maintenance_timestamp(OffsetDateTime::now_utc()); + let backup = StrongTableCatalogBackup { + version: TABLE_CATALOG_BACKUP_VERSION, + backup_id: backup_id.clone(), + table_bucket: table_bucket.to_string(), + created_at: created_at.clone(), + source_snapshot_etag: source_snapshot_etag.clone(), + source_snapshot_version, + catalog_fingerprint: catalog_fingerprint.clone(), + snapshot, + objects, + maintenance_objects, + }; + let data = serde_json::to_vec(&backup) + .map_err(|err| TableCatalogStoreError::Internal(format!("failed to encode catalog backup: {err}")))?; + if data.len() > TABLE_CATALOG_BACKUP_MAX_SIZE { + return Err(TableCatalogStoreError::Invalid(format!( + "catalog backup exceeds the maximum encoded size of {TABLE_CATALOG_BACKUP_MAX_SIZE} bytes" + ))); + } + let path = TableCatalogObjectPaths::default().catalog_backup_path(table_bucket, &backup_id); + fence.ensure_held()?; + let (status, persisted_created_at) = match self + .object_backend + .put_object_fenced(RUSTFS_META_BUCKET, &path, data, TableCatalogPutPrecondition::IfAbsent, &mutation_fence) + .await + { + Ok(()) => (TableCatalogBackupStatus::Created, created_at.clone()), + Err(TableCatalogStoreError::Conflict(_)) => { + let existing = self + .object_backend + .read_object_limited(RUSTFS_META_BUCKET, &path, TABLE_CATALOG_BACKUP_MAX_SIZE) + .await? + .ok_or_else(|| { + TableCatalogStoreError::Conflict("catalog backup disappeared after a write conflict".to_string()) + })?; + let existing = serde_json::from_slice::(&existing.data) + .map_err(|err| TableCatalogStoreError::Invalid(format!("existing catalog backup is invalid: {err}")))?; + let mut comparable = backup.clone(); + comparable.created_at = existing.created_at.clone(); + if existing != comparable { + return Err(TableCatalogStoreError::Conflict( + "catalog backup identity is already occupied by different content".to_string(), + )); + } + (TableCatalogBackupStatus::AlreadyPresent, existing.created_at) + } + Err(err) => return Err(err), + }; + fence.ensure_held()?; + let verified_bytes = backup + .objects + .iter() + .fold(0u64, |total, object| total.saturating_add(object.size_bytes)); + Ok(TableCatalogBackupReport { + table_bucket: table_bucket.to_string(), + backup_id, + status, + backup_path: path, + source_snapshot_etag, + source_snapshot_version, + catalog_fingerprint, + object_count: backup.objects.len(), + verified_object_count: backup.objects.len(), + verified_bytes, + maintenance_object_count: backup.maintenance_objects.len(), + created_at: persisted_created_at, + }) + } + + pub(crate) async fn restore_durable_catalog_backup( + &self, + table_bucket: &str, + backup_id: &str, + expected_snapshot_etag: Option<&str>, + allow_replace: bool, + ) -> TableCatalogStoreResult { + let fence = self.acquire_backup_fence(table_bucket).await?; + fence.ensure_held()?; + let mutation_fence = fence.mutation_fence(); + let _write_guard = self.write_lock.lock().await; + let backup = self.read_backup(table_bucket, backup_id).await?; + let current_objects = self.collect_backup_objects(&backup.snapshot).await?; + if current_objects != backup.objects { + return Err(TableCatalogStoreError::Conflict( + "catalog backup object graph no longer matches the saved catalog snapshot".to_string(), + )); + } + self.verify_backup_objects(&backup).await?; + Self::validate_maintenance_backup_objects(table_bucket, &backup.snapshot, &backup.maintenance_objects)?; + + self.hydrate_state().await?; + let existing_intent = self.read_restore_intent(table_bucket).await?; + if let Some((intent, _)) = existing_intent.as_ref() + && intent.backup_id != backup_id + { + return Err(TableCatalogStoreError::Conflict( + "another catalog restore is already pending for this table bucket".to_string(), + )); + } + let (current_snapshot_etag, current_bucket) = { + let state = self.state.lock().await; + ( + state.snapshot_etag.clone(), + StrongTableCatalogStore::::bucket_snapshot_from_state_locked(&state, table_bucket), + ) + }; + let already_restored = current_bucket.as_ref() == Some(&backup.snapshot); + if let Some((intent, _)) = existing_intent.as_ref() { + Self::validate_restore_intent_replay( + intent, + expected_snapshot_etag, + allow_replace, + current_snapshot_etag.as_deref(), + current_bucket.as_ref(), + &backup, + )?; + } + if !already_restored && current_bucket.is_some() && !allow_replace { + return Err(TableCatalogStoreError::Conflict( + "restoring over an existing table bucket requires allow-replace".to_string(), + )); + } + if !already_restored && current_bucket.is_none() && expected_snapshot_etag.is_some() { + return Err(TableCatalogStoreError::Conflict( + "expected-snapshot-etag cannot be used when the target table bucket is absent".to_string(), + )); + } + + let source_paths = maintenance_restore_paths(); + let source_objects = self.collect_maintenance_objects(table_bucket, &source_paths).await?; + let source_fingerprint = maintenance_objects_fingerprint(&source_objects)?; + let target_fingerprint = maintenance_objects_fingerprint(&backup.maintenance_objects)?; + let fully_restored = already_restored && source_fingerprint == target_fingerprint; + if existing_intent.is_none() + && !fully_restored + && expected_snapshot_etag.is_some_and(|expected| current_snapshot_etag.as_deref() != Some(expected)) + { + return Err(TableCatalogStoreError::Conflict( + "target catalog snapshot does not match expected-snapshot-etag".to_string(), + )); + } + let target_bucket_fingerprint = current_bucket + .as_ref() + .map(table_catalog_bucket_snapshot_fingerprint) + .transpose()? + .unwrap_or_else(|| ABSENT_RESTORE_TARGET_BUCKET_FINGERPRINT.to_string()); + if let Some((intent, _)) = existing_intent.as_ref() { + let persisted_source = intent.source_maintenance_fingerprint.as_deref().ok_or_else(|| { + TableCatalogStoreError::Invalid( + "catalog restore intent lacks a persisted maintenance precondition; manual recovery is required".to_string(), + ) + })?; + if source_fingerprint != persisted_source && source_fingerprint != target_fingerprint { + return Err(TableCatalogStoreError::Conflict( + "catalog restore maintenance state changed while the restore was pending".to_string(), + )); + } + } + + let mut intent = + existing_intent + .as_ref() + .map(|(intent, _)| intent.clone()) + .unwrap_or_else(|| StrongTableCatalogRestoreIntent { + version: TABLE_CATALOG_BACKUP_VERSION, + table_bucket: table_bucket.to_string(), + backup_id: backup_id.to_string(), + expected_snapshot_etag: expected_snapshot_etag.map(str::to_string), + target_snapshot_etag: current_snapshot_etag.clone(), + target_bucket_fingerprint, + target_bucket_present: Some(current_bucket.is_some()), + allow_replace: Some(allow_replace), + source_maintenance_fingerprint: Some(source_fingerprint.clone()), + restored_snapshot_etag: already_restored.then(|| current_snapshot_etag.clone()).flatten(), + state: if already_restored { + StrongTableCatalogRestoreIntentState::CatalogApplied + } else { + StrongTableCatalogRestoreIntentState::Prepared + }, + }); + let mut intent_etag = existing_intent.as_ref().and_then(|(_, etag)| etag.clone()); + if existing_intent.is_none() { + fence.ensure_held()?; + self.write_restore_intent(table_bucket, &intent, TableCatalogPutPrecondition::IfAbsent, Some(&mutation_fence)) + .await?; + fence.ensure_held()?; + let (persisted, etag) = self.read_restore_intent(table_bucket).await?.ok_or_else(|| { + TableCatalogStoreError::Internal("catalog restore intent disappeared after it was written".to_string()) + })?; + if persisted != intent { + return Err(TableCatalogStoreError::Invalid( + "catalog restore intent changed while it was being persisted".to_string(), + )); + } + intent = persisted; + intent_etag = Some(etag.ok_or_else(|| { + TableCatalogStoreError::Internal("catalog restore intent has no etag after it was written".to_string()) + })?); + } + + let initially_already_restored = already_restored; + let mut restored_snapshot_etag = current_snapshot_etag.clone(); + if intent.state == StrongTableCatalogRestoreIntentState::Prepared { + if !already_restored { + let persisted_source = intent.source_maintenance_fingerprint.as_deref().ok_or_else(|| { + TableCatalogStoreError::Invalid( + "catalog restore intent lacks a persisted maintenance precondition; manual recovery is required" + .to_string(), + ) + })?; + if source_fingerprint != persisted_source { + return Err(TableCatalogStoreError::Conflict( + "catalog restore source maintenance state changed before catalog publication".to_string(), + )); + } + } + if !already_restored { + let (snapshot, precondition) = { + let state = self.state.lock().await; + let precondition = state + .snapshot_etag + .clone() + .map_or(TableCatalogPutPrecondition::IfAbsent, TableCatalogPutPrecondition::IfMatch); + let mut draft_state = state.clone(); + StrongTableCatalogStore::::remove_bucket_from_state_locked(&mut draft_state, table_bucket); + StrongTableCatalogStore::::insert_bucket_snapshot_locked(&mut draft_state, backup.snapshot.clone())?; + // A restore must never downgrade a snapshot format that this process has already observed. + // The backup version is part of the source snapshot contract even though the bucket payload + // itself is version-independent. + draft_state.snapshot_version = Some( + draft_state + .snapshot_version + .unwrap_or(STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION) + .max(backup.source_snapshot_version), + ); + let snapshot = StrongTableCatalogStore::::snapshot_from_mutated_state_locked( + &mut draft_state, + self.snapshot_write_version, + )?; + (snapshot, precondition) + }; + fence.ensure_held()?; + self.verify_backup_objects(&backup).await?; + self.finalize_snapshot_write_with_fence( + snapshot, + precondition, + StrongSnapshotWritePostcondition::BucketSnapshotPresent(backup.snapshot.clone()), + Some(&mutation_fence), + ) + .await?; + fence.ensure_held()?; + self.hydrate_state().await?; + restored_snapshot_etag = + Some( + self.state.lock().await.snapshot_etag.clone().ok_or_else(|| { + TableCatalogStoreError::Internal("restored catalog snapshot has no etag".to_string()) + })?, + ); + } else { + restored_snapshot_etag = current_snapshot_etag.clone(); + } + intent.restored_snapshot_etag = restored_snapshot_etag.clone(); + intent.state = StrongTableCatalogRestoreIntentState::CatalogApplied; + let current_intent_etag = intent_etag.clone().ok_or_else(|| { + TableCatalogStoreError::Internal("catalog restore intent has no etag before finalization".to_string()) + })?; + fence.ensure_held()?; + self.write_restore_intent( + table_bucket, + &intent, + TableCatalogPutPrecondition::IfMatch(current_intent_etag), + Some(&mutation_fence), + ) + .await?; + fence.ensure_held()?; + let (persisted, etag) = self.read_restore_intent(table_bucket).await?.ok_or_else(|| { + TableCatalogStoreError::Internal("catalog restore intent disappeared after finalization".to_string()) + })?; + if persisted != intent { + return Err(TableCatalogStoreError::Invalid( + "finalized catalog restore intent changed while it was being persisted".to_string(), + )); + } + intent_etag = Some(etag.ok_or_else(|| { + TableCatalogStoreError::Internal("catalog restore intent has no etag after finalization".to_string()) + })?); + } else if restored_snapshot_etag.is_none() { + return Err(TableCatalogStoreError::Internal( + "already-restored catalog snapshot has no etag".to_string(), + )); + } + + let mut allowed_paths = source_paths; + allowed_paths.extend(maintenance_backup_paths(table_bucket, Some(&backup.snapshot))?); + let current_after_catalog = self.collect_maintenance_objects(table_bucket, &allowed_paths).await?; + let current_after_catalog_fingerprint = maintenance_objects_fingerprint(¤t_after_catalog)?; + if current_after_catalog_fingerprint != target_fingerprint { + let persisted_source = intent.source_maintenance_fingerprint.as_deref().ok_or_else(|| { + TableCatalogStoreError::Invalid( + "catalog restore intent lacks a persisted maintenance precondition; manual recovery is required".to_string(), + ) + })?; + if current_after_catalog_fingerprint != persisted_source { + return Err(TableCatalogStoreError::Conflict( + "catalog restore maintenance state changed while the restore was pending".to_string(), + )); + } + fence.ensure_held()?; + self.restore_maintenance_objects(table_bucket, &backup.snapshot, &backup.maintenance_objects, &mutation_fence) + .await?; + fence.ensure_held()?; + } + let restored_maintenance = self + .collect_maintenance_backup_objects(table_bucket, Some(&backup.snapshot)) + .await?; + if maintenance_objects_fingerprint(&restored_maintenance)? != target_fingerprint { + return Err(TableCatalogStoreError::Conflict( + "catalog restore maintenance state does not match the backup after publication".to_string(), + )); + } + let intent_etag = intent_etag + .ok_or_else(|| TableCatalogStoreError::Internal("catalog restore intent has no etag before cleanup".to_string()))?; + fence.ensure_held()?; + self.delete_restore_intent_if_unchanged(table_bucket, &intent_etag, Some(&mutation_fence)) + .await?; + fence.ensure_held()?; + Ok(TableCatalogRestoreReport { + table_bucket: table_bucket.to_string(), + backup_id: backup_id.to_string(), + status: if initially_already_restored { + TableCatalogRestoreStatus::AlreadyRestored + } else { + TableCatalogRestoreStatus::Restored + }, + previous_snapshot_etag: current_snapshot_etag, + restored_snapshot_etag: restored_snapshot_etag + .ok_or_else(|| TableCatalogStoreError::Internal("restored catalog snapshot etag disappeared".to_string()))?, + object_count: backup.objects.len(), + maintenance_object_count: backup.maintenance_objects.len(), + }) + } +} diff --git a/rustfs/src/table_catalog/store/migration.rs b/rustfs/src/table_catalog/store/migration.rs index 872fdc9fe..e61f15a9a 100644 --- a/rustfs/src/table_catalog/store/migration.rs +++ b/rustfs/src/table_catalog/store/migration.rs @@ -298,12 +298,21 @@ where ) -> TableCatalogStoreResult { let lock_path = self.paths.backing_migration_fence_lock_path(table_bucket); let guard = self.backend.acquire_read_lock(self.catalog_bucket(), &lock_path).await?; + self.ensure_object_backed_catalog_write_permit_after_lock(table_bucket) + .await?; + Ok(guard) + } + + pub(super) async fn ensure_object_backed_catalog_write_permit_after_lock( + &self, + table_bucket: &str, + ) -> TableCatalogStoreResult<()> { if self.read_backing_migration_fence(table_bucket).await?.is_some() { return Err(TableCatalogStoreError::Conflict(format!( "object-backed catalog writes are fenced while table bucket {table_bucket} is prepared for durable strong cutover" ))); } - Ok(guard) + Ok(()) } async fn ensure_global_backing_migration_fence( diff --git a/rustfs/src/table_catalog/store/mod.rs b/rustfs/src/table_catalog/store/mod.rs index 9ab0ee6ce..378b01f66 100644 --- a/rustfs/src/table_catalog/store/mod.rs +++ b/rustfs/src/table_catalog/store/mod.rs @@ -16,10 +16,14 @@ use super::identifier::is_valid_table_metadata_file_name; use super::*; use crate::storage_api::table::{WriteCommitGuard, WriteCompletion}; +mod backup; mod migration; mod object; mod strong; +pub(crate) use backup::{TableCatalogBackupReport, TableCatalogRestoreReport}; +#[cfg(test)] +pub(crate) use backup::{TableCatalogBackupStatus, TableCatalogRestoreStatus}; use migration::{durable_strong_table_catalog_backing_manifest, table_catalog_backing_manifest}; pub(crate) use object::ObjectTableCatalogStore; #[cfg(test)] @@ -455,6 +459,15 @@ pub(crate) trait TableCommitPublication: Send + Sync { fn holds_table(&self, table_bucket: &str, namespace: &str, table: &str) -> bool; + /// Whether bucket publication acquires its own migration read permits. + fn acquires_catalog_migration_read_permit(&self) -> bool { + false + } + + fn catalog_migration_read_permit_status(&self) -> Option { + None + } + fn complete(&self); } @@ -583,6 +596,7 @@ pub(crate) struct TableCatalogObject { pub(crate) struct TableCatalogObjectMetadata { pub etag: Option, pub mod_time: Option, + pub size: u64, } pub(crate) struct TableCatalogLockGuard { @@ -614,9 +628,10 @@ impl TableCatalogLockGuard { } fn write_namespace(guard: WriteCommitGuard) -> Self { + let lock_lost = guard.lock_lost_signal(); Self { _guard: Box::new(()), - lock_lost: None, + lock_lost, write_commit_guard: Some(guard), } } @@ -629,6 +644,40 @@ impl TableCatalogLockGuard { self.lock_lost.as_ref().is_some_and(|signal| signal.is_lost()) || self.write_commit_guard.as_ref().is_some_and(WriteCommitGuard::is_lock_lost) } + + pub(crate) fn lock_lost_signal(&self) -> Option> { + self.lock_lost.clone() + } +} + +#[derive(Clone, Default)] +pub(crate) struct TableCatalogObjectMutationFence { + signals: Vec>, +} + +impl TableCatalogObjectMutationFence { + pub(crate) fn from_signals( + signals: impl IntoIterator>>, + ) -> Self { + Self { + signals: signals.into_iter().flatten().collect(), + } + } + + pub(crate) fn ensure_held(&self) -> TableCatalogStoreResult<()> { + if self.signals.iter().any(|signal| signal.is_lost()) { + return Err(TableCatalogStoreError::Unavailable( + "catalog backup or restore lost its fencing lock".to_string(), + )); + } + Ok(()) + } + + pub(crate) fn add_to_object_options(&self, options: &mut ObjectOptions) { + for signal in &self.signals { + options.add_namespace_lock_lost_signal(Arc::clone(signal)); + } + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -669,6 +718,13 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static { None } + async fn acquire_catalog_migration_read_guards( + &self, + _table_bucket: &str, + ) -> TableCatalogStoreResult> { + Ok(Vec::new()) + } + async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult>; async fn read_object_limited( @@ -712,6 +768,7 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static { .map(|object| TableCatalogObjectMetadata { etag: object.etag, mod_time: object.mod_time, + size: u64::try_from(object.data.len()).unwrap_or(u64::MAX), })) } @@ -726,6 +783,7 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static { .map(|object| TableCatalogObjectMetadata { etag: object.etag, mod_time: object.mod_time, + size: u64::try_from(object.data.len()).unwrap_or(u64::MAX), })) } @@ -758,8 +816,35 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static { self.put_object(bucket, object, data, precondition).await } + async fn put_object_fenced( + &self, + bucket: &str, + object: &str, + data: Vec, + precondition: TableCatalogPutPrecondition, + fence: &TableCatalogObjectMutationFence, + ) -> TableCatalogStoreResult<()> { + fence.ensure_held()?; + self.put_object(bucket, object, data, precondition).await?; + fence.ensure_held() + } + async fn delete_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()>; + async fn delete_object_if_match(&self, bucket: &str, object: &str, expected_etag: &str) -> TableCatalogStoreResult<()>; + + async fn delete_object_if_match_fenced( + &self, + bucket: &str, + object: &str, + expected_etag: &str, + fence: &TableCatalogObjectMutationFence, + ) -> TableCatalogStoreResult<()> { + fence.ensure_held()?; + self.delete_object_if_match(bucket, object, expected_etag).await?; + fence.ensure_held() + } + async fn delete_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> { self.delete_object(bucket, object).await } @@ -813,6 +898,15 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static { false } + /// Whether bucket publication acquires its own migration read permits. + fn acquires_catalog_migration_read_permit(&self) -> bool { + false + } + + fn catalog_migration_read_permit_status(&self) -> Option { + None + } + fn complete_table_commit_publication(&self) {} } @@ -931,6 +1025,14 @@ where self.table_commit_publication_is_held(table_bucket, namespace, table) } + fn acquires_catalog_migration_read_permit(&self) -> bool { + TableCatalogObjectBackend::acquires_catalog_migration_read_permit(self) + } + + fn catalog_migration_read_permit_status(&self) -> Option { + TableCatalogObjectBackend::catalog_migration_read_permit_status(self) + } + fn complete(&self) { self.complete_table_commit_publication(); } @@ -1195,6 +1297,26 @@ impl TableCatalogObjectPaths { ) } + pub fn catalog_backup_path(&self, table_bucket: &str, backup_id: &str) -> String { + format!( + "{}/{}/{}/{}.json", + self.catalog_root, + TABLE_CATALOG_BACKUP_ROOT, + table_catalog_path_hash(table_bucket), + table_catalog_path_hash(backup_id) + ) + } + + pub fn catalog_backup_restore_intent_path(&self, table_bucket: &str) -> String { + format!( + "{}/{}/{}/{}", + self.catalog_root, + TABLE_CATALOG_BACKUP_RESTORE_ROOT, + table_catalog_path_hash(table_bucket), + TABLE_CATALOG_BACKUP_RESTORE_INTENT_FILE + ) + } + fn table_bucket_root_prefix(&self, table_bucket: &str) -> String { format!("{}/{}/{}/", self.catalog_root, TABLE_BUCKET_ROOT, table_catalog_path_hash(table_bucket)) } @@ -1839,6 +1961,42 @@ where Self::DurableStrong(_) => Err(Self::unsupported_for_durable_strong("external catalog bridge")), } } + + pub(crate) async fn create_durable_catalog_backup( + &self, + table_bucket: &str, + expected_snapshot_etag: Option<&str>, + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(_) => Err(TableCatalogStoreError::Unsupported( + "durable catalog backup requires durable-strong backing".to_string(), + )), + Self::DurableStrong(store) => { + store + .create_durable_catalog_backup(table_bucket, expected_snapshot_etag) + .await + } + } + } + + pub(crate) async fn restore_durable_catalog_backup( + &self, + table_bucket: &str, + backup_id: &str, + expected_snapshot_etag: Option<&str>, + allow_replace: bool, + ) -> TableCatalogStoreResult { + match self { + Self::ObjectBacked(_) => Err(TableCatalogStoreError::Unsupported( + "durable catalog restore requires durable-strong backing".to_string(), + )), + Self::DurableStrong(store) => { + store + .restore_durable_catalog_backup(table_bucket, backup_id, expected_snapshot_etag, allow_replace) + .await + } + } + } } pub(crate) struct EcStoreTableCatalogObjectBackend { @@ -1875,6 +2033,26 @@ where Some(self.strong_runtime.clone()) } + fn acquires_catalog_migration_read_permit(&self) -> bool { + true + } + + async fn acquire_catalog_migration_read_guards( + &self, + table_bucket: &str, + ) -> TableCatalogStoreResult> { + // Handlers discover and publish objects through one publication object. Keep the + // migration read permits ahead of its publication lock, matching store mutations. + let paths = TableCatalogObjectPaths::default(); + let global = self + .acquire_read_lock(RUSTFS_META_BUCKET, &paths.backing_migration_global_fence_lock_path()) + .await?; + let bucket = self + .acquire_read_lock(RUSTFS_META_BUCKET, &paths.backing_migration_fence_lock_path(table_bucket)) + .await?; + Ok(vec![global, bucket]) + } + async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult> { self.read_object_with_options(bucket, object, ObjectOptions::default(), None) .await @@ -1926,6 +2104,7 @@ where Ok(info) => Ok(Some(TableCatalogObjectMetadata { etag: info.etag, mod_time: info.mod_time, + size: u64::try_from(info.size.max(0)).unwrap_or(u64::MAX), })), Err(err) if is_missing_storage_error(&err) => Ok(None), Err(err) => Err(storage_error_to_catalog("stat catalog object", err)), @@ -1952,6 +2131,7 @@ where Ok(info) => Ok(Some(TableCatalogObjectMetadata { etag: info.etag, mod_time: info.mod_time, + size: u64::try_from(info.size.max(0)).unwrap_or(u64::MAX), })), Err(err) if is_missing_storage_error(&err) => Ok(None), Err(err) => Err(storage_error_to_catalog("stat catalog object", err)), @@ -1992,7 +2172,8 @@ where data: Vec, precondition: TableCatalogPutPrecondition, ) -> TableCatalogStoreResult<()> { - self.put_object_with_options(bucket, object, data, precondition, None).await + self.put_object_with_options(bucket, object, data, precondition, None, None) + .await } async fn put_object_unlocked( @@ -2003,7 +2184,20 @@ where precondition: TableCatalogPutPrecondition, write_guards: Vec, ) -> TableCatalogStoreResult<()> { - self.put_object_with_options(bucket, object, data, precondition, Some(write_guards)) + self.put_object_with_options(bucket, object, data, precondition, Some(write_guards), None) + .await + } + + async fn put_object_fenced( + &self, + bucket: &str, + object: &str, + data: Vec, + precondition: TableCatalogPutPrecondition, + fence: &TableCatalogObjectMutationFence, + ) -> TableCatalogStoreResult<()> { + fence.ensure_held()?; + self.put_object_with_options(bucket, object, data, precondition, None, Some(fence)) .await } @@ -2015,6 +2209,23 @@ where } } + async fn delete_object_if_match(&self, bucket: &str, object: &str, expected_etag: &str) -> TableCatalogStoreResult<()> { + self.delete_object_if_match_with_fence(bucket, object, expected_etag, None) + .await + } + + async fn delete_object_if_match_fenced( + &self, + bucket: &str, + object: &str, + expected_etag: &str, + fence: &TableCatalogObjectMutationFence, + ) -> TableCatalogStoreResult<()> { + fence.ensure_held()?; + self.delete_object_if_match_with_fence(bucket, object, expected_etag, Some(fence)) + .await + } + async fn delete_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> { match self .store @@ -2115,6 +2326,37 @@ impl EcStoreTableCatalogObjectBackend where S: TableCatalogStorage, { + async fn delete_object_if_match_with_fence( + &self, + bucket: &str, + object: &str, + expected_etag: &str, + fence: Option<&TableCatalogObjectMutationFence>, + ) -> TableCatalogStoreResult<()> { + let mut options = ObjectOptions { + http_preconditions: Some(HTTPPreconditions { + if_match: Some(expected_etag.to_string()), + ..Default::default() + }), + ..Default::default() + }; + if let Some(fence) = fence { + fence.add_to_object_options(&mut options); + } + match self.store.delete_object(bucket, object, options).await { + Ok(_) => { + if let Some(fence) = fence { + fence.ensure_held()?; + } + Ok(()) + } + Err(err) if is_missing_storage_error(&err) => Err(TableCatalogStoreError::Conflict(format!( + "delete catalog object: {bucket}/{object} disappeared before conditional cleanup" + ))), + Err(err) => Err(storage_error_to_catalog("conditionally delete catalog object", err)), + } + } + async fn read_object_with_options( &self, bucket: &str, @@ -2168,12 +2410,16 @@ where data: Vec, precondition: TableCatalogPutPrecondition, write_guards: Option>, + fence: Option<&TableCatalogObjectMutationFence>, ) -> TableCatalogStoreResult<()> { let mut reader = PutObjReader::from_vec(data); let mut opts = ObjectOptions { http_preconditions: http_preconditions_for_catalog_put(precondition), ..Default::default() }; + if let Some(fence) = fence { + fence.add_to_object_options(&mut opts); + } if let Some(write_guards) = write_guards { if write_guards.is_empty() { return Err(TableCatalogStoreError::Internal("catalog write has no namespace owner".to_string())); @@ -2183,10 +2429,133 @@ where } opts.write_completion = WriteCompletion::TailDrained; } - self.store + let result = self + .store .put_object(bucket, object, &mut reader, &opts) .await .map(|_| ()) - .map_err(|err| storage_error_to_catalog("write catalog object", err)) + .map_err(|err| storage_error_to_catalog("write catalog object", err)); + if result.is_ok() + && let Some(fence) = fence + { + fence.ensure_held()?; + } + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage_api::protocols::client::{BucketOperations as _, MakeBucketOptions}; + use rustfs_lock::{LockClient, LockId, LockInfo, LockRequest, LockResponse, LockStats}; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + #[derive(Debug)] + struct RecordingLeaseClient { + inner: rustfs_lock::LocalClient, + acquired: Mutex>, + } + + #[async_trait::async_trait] + impl LockClient for RecordingLeaseClient { + async fn acquire_lock(&self, request: &LockRequest) -> rustfs_lock::Result { + let response = self.inner.acquire_lock(request).await?; + if let Some(info) = response.lock_info.as_ref() { + *self.acquired.lock().expect("record lease") = Some(info.id.clone()); + } + Ok(response) + } + async fn release(&self, id: &LockId) -> rustfs_lock::Result { + self.inner.release(id).await + } + async fn refresh(&self, id: &LockId) -> rustfs_lock::Result { + self.inner.refresh(id).await + } + async fn force_release(&self, id: &LockId) -> rustfs_lock::Result { + self.inner.force_release(id).await + } + async fn check_status(&self, id: &LockId) -> rustfs_lock::Result> { + self.inner.check_status(id).await + } + async fn get_stats(&self) -> rustfs_lock::Result { + self.inner.get_stats().await + } + async fn close(&self) -> rustfs_lock::Result<()> { + self.inner.close().await + } + async fn is_online(&self) -> bool { + self.inner.is_online().await + } + async fn is_local(&self) -> bool { + self.inner.is_local().await + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn catalog_writer_loss_prevents_recovery_storage_commit() { + let (_temp, _paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await; + let bucket = "catalog-fence-loss-probe"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("make probe bucket"); + let backend = EcStoreTableCatalogObjectBackend::new_with_strong_runtime(store, StrongTableCatalogRuntime::default()); + let client = Arc::new(RecordingLeaseClient { + inner: rustfs_lock::LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new( + rustfs_lock::FastObjectLockManager::new(), + )))), + acquired: Mutex::new(None), + }); + let lock = rustfs_lock::NamespaceLockWrapper::new( + rustfs_lock::NamespaceLock::with_clients_and_quorum( + "catalog-probe".to_string(), + vec![client.clone() as Arc], + 1, + ), + rustfs_lock::ObjectKey::new(bucket, default_table_bucket_publication_lock_path()), + "catalog-backup-probe".to_string(), + ); + // These are the exact acquisition and conversion used by the production + // EcStoreTableCatalogObjectBackend::acquire_write_lock implementation. + let owner = WriteCommitGuard::acquire(&lock, Duration::from_secs(5)) + .await + .expect("acquire distributed writer"); + let guard = TableCatalogLockGuard::write_namespace(owner); + let fence = TableCatalogObjectMutationFence::from_signals([guard.lock_lost_signal()]); + assert!(!guard.is_lock_lost(), "models backup's last outer fencing check"); + let id = client + .acquired + .lock() + .expect("lease recorded") + .clone() + .expect("actual lease acquired"); + assert!(client.force_release(&id).await.expect("force-release lease")); + tokio::time::timeout(Duration::from_secs(15), async { + while !guard.is_lock_lost() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("actual heartbeat must observe released distributed lease"); + let result = backend + .put_object_fenced( + bucket, + "recovery-artifact.json", + b"stale recovery artifact".to_vec(), + TableCatalogPutPrecondition::IfAbsent, + &fence, + ) + .await; + let persisted = backend + .read_object(bucket, "recovery-artifact.json") + .await + .expect("read real persisted object"); + assert!( + result.is_err() && persisted.is_none(), + "lost publication writer must prevent storage commit; result={result:?}, persisted={}", + persisted.is_some() + ); } } diff --git a/rustfs/src/table_catalog/store/object.rs b/rustfs/src/table_catalog/store/object.rs index 9ed39f40d..e54b06804 100644 --- a/rustfs/src/table_catalog/store/object.rs +++ b/rustfs/src/table_catalog/store/object.rs @@ -185,7 +185,7 @@ fn validate_external_catalog_bridge_entry_object( Ok(()) } -fn validate_table_maintenance_report_owner( +pub(super) fn validate_table_maintenance_report_owner( report: &TableMetadataMaintenanceReport, table_bucket: &str, namespace: &Namespace, @@ -2168,6 +2168,39 @@ where .await } + async fn begin_table_bucket_with_migration_permit( + &self, + table_bucket: &str, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult> { + let permit_before_publication = publication.catalog_migration_read_permit_status(); + if permit_before_publication == Some(false) { + return Err(TableCatalogStoreError::Conflict( + "table-bucket catalog migration read permit was lost".to_string(), + )); + } + + publication.begin_table_bucket(table_bucket).await?; + let result = match (permit_before_publication, publication.catalog_migration_read_permit_status()) { + (Some(true), Some(true)) | (None, Some(true)) => self + .ensure_object_backed_catalog_write_permit_after_lock(table_bucket) + .await + .map(|()| None), + (None, None) => self.acquire_object_backed_catalog_write_permit(table_bucket).await.map(Some), + (Some(true), Some(false)) | (None, Some(false)) => Err(TableCatalogStoreError::Conflict( + "table-bucket catalog migration read permit was lost".to_string(), + )), + (Some(true), None) => Err(TableCatalogStoreError::Internal( + "table-bucket catalog migration read permit disappeared during publication".to_string(), + )), + (Some(false), _) => unreachable!("a lost migration permit is rejected before publication"), + }; + if result.is_err() { + publication.complete(); + } + result + } + async fn write_table_entry_with_publication( &self, entry: TableEntry, @@ -2178,15 +2211,16 @@ where let namespace = parse_namespace_for_store(&entry.namespace)?; let table = parse_table_for_store(&entry.table)?; validate_table_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?; - publication.begin_table_bucket(&entry.table_bucket).await?; + self.require_table_bucket(&entry.table_bucket).await?; + let _migration_guard = self + .begin_table_bucket_with_migration_permit(&entry.table_bucket, publication) + .await?; if !publication.holds_table_bucket(&entry.table_bucket) { return Err(TableCatalogStoreError::Internal( "table registration requires a table-bucket publication fence".to_string(), )); } let _publication_completion = TableCommitPublicationCompletion::new(publication); - self.require_table_bucket(&entry.table_bucket).await?; - let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; self.recover_active_table_rename(&entry.table_bucket, publication).await?; let namespace_path = self.paths.namespace_entry_path(&entry.table_bucket, &namespace); let _namespace_guard = self @@ -2274,18 +2308,19 @@ where publication: &(dyn TableCommitPublication + Sync), ) -> TableCatalogStoreResult<()> { validate_view_entry_version_and_id(&entry)?; - publication.begin_table_bucket(&entry.table_bucket).await?; + self.require_table_bucket(&entry.table_bucket).await?; + let _migration_guard = self + .begin_table_bucket_with_migration_permit(&entry.table_bucket, publication) + .await?; if !publication.holds_table_bucket(&entry.table_bucket) { return Err(TableCatalogStoreError::Internal( "view creation requires a table-bucket publication fence".to_string(), )); } let _publication_completion = TableCommitPublicationCompletion::new(publication); - self.require_table_bucket(&entry.table_bucket).await?; let namespace = parse_namespace_for_store(&entry.namespace)?; let view = parse_table_for_store(&entry.view)?; validate_view_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?; - let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; self.recover_active_table_rename(&entry.table_bucket, publication).await?; let namespace_path = self.paths.namespace_entry_path(&entry.table_bucket, &namespace); let _namespace_guard = self @@ -2527,9 +2562,9 @@ where let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; let publication = TableCommitLockPublication::new(&self.backend); + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; publication.begin_table_bucket(table_bucket).await?; let _publication_completion = TableCommitPublicationCompletion::new(&publication); - let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; self.recover_active_table_rename(table_bucket, &publication).await?; let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; @@ -5321,6 +5356,7 @@ where let destination_namespace = parse_namespace_for_store(destination_namespace)?; let destination_table = parse_table_for_store(destination_table)?; let publication = TableCommitLockPublication::new(&self.backend); + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; publication.begin_table_bucket(table_bucket).await?; if !publication.holds_table_bucket(table_bucket) { return Err(TableCatalogStoreError::Internal( @@ -5328,7 +5364,6 @@ where )); } let _publication_completion = TableCommitPublicationCompletion::new(&publication); - let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; self.recover_active_table_rename(table_bucket, &publication).await?; { @@ -5564,7 +5599,6 @@ where async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { let publication = TableCommitLockPublication::new(&self.backend); - publication.begin_table_bucket(&request.table_bucket).await?; self.commit_table_with_publication(request, &publication).await } @@ -5577,7 +5611,14 @@ where record_table_commit_attempt(&request.operation); let namespace = parse_namespace_for_store(&request.namespace)?; let table = parse_table_for_store(&request.table)?; - let _migration_guard = self.acquire_object_backed_catalog_write_permit(&request.table_bucket).await?; + let _migration_guard = self + .begin_table_bucket_with_migration_permit(&request.table_bucket, publication) + .await?; + if !publication.holds_table_bucket(&request.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "table commit requires a table-bucket publication fence".to_string(), + )); + } if publication.holds_table_bucket(&request.table_bucket) { self.recover_active_table_rename(&request.table_bucket, publication).await?; } else { @@ -5999,10 +6040,10 @@ where async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()> { let publication = TableCommitLockPublication::new(&self.backend); - publication.begin_table_bucket(table_bucket).await?; let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; + publication.begin_table_bucket(table_bucket).await?; self.recover_active_table_rename(table_bucket, &publication).await?; let namespace_path = self.paths.namespace_entry_path(table_bucket, &namespace); let _namespace_guard = self @@ -6148,15 +6189,29 @@ where ) -> TableCatalogStoreResult { let namespace = parse_namespace_for_store(&request.namespace)?; let view = parse_table_for_store(&request.view)?; - if table_bucket_fence_required { - publication.begin_table_bucket(&request.table_bucket).await?; - if !publication.holds_table_bucket(&request.table_bucket) { - return Err(TableCatalogStoreError::Internal( - "view replacement requires a table-bucket publication fence".to_string(), - )); + let _migration_guard = if table_bucket_fence_required { + self.begin_table_bucket_with_migration_permit(&request.table_bucket, publication) + .await? + } else { + match publication.catalog_migration_read_permit_status() { + Some(true) => { + self.ensure_object_backed_catalog_write_permit_after_lock(&request.table_bucket) + .await?; + None + } + Some(false) => { + return Err(TableCatalogStoreError::Conflict( + "table-bucket catalog migration read permit was lost".to_string(), + )); + } + None => Some(self.acquire_object_backed_catalog_write_permit(&request.table_bucket).await?), } + }; + if table_bucket_fence_required && !publication.holds_table_bucket(&request.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "view replacement requires a table-bucket publication fence".to_string(), + )); } - let _migration_guard = self.acquire_object_backed_catalog_write_permit(&request.table_bucket).await?; if publication.holds_table_bucket(&request.table_bucket) { self.recover_active_table_rename(&request.table_bucket, publication).await?; } else { @@ -6282,11 +6337,11 @@ where async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()> { let publication = TableCommitLockPublication::new(&self.backend); - publication.begin_table_bucket(table_bucket).await?; - let _publication_completion = TableCommitPublicationCompletion::new(&publication); let namespace = parse_namespace_for_store(namespace)?; let view = parse_table_for_store(view)?; let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; + publication.begin_table_bucket(table_bucket).await?; + let _publication_completion = TableCommitPublicationCompletion::new(&publication); self.recover_active_table_rename(table_bucket, &publication).await?; let namespace_path = self.paths.namespace_entry_path(table_bucket, &namespace); let _namespace_guard = self diff --git a/rustfs/src/table_catalog/store/strong.rs b/rustfs/src/table_catalog/store/strong.rs index 43979be8e..f70f180fe 100644 --- a/rustfs/src/table_catalog/store/strong.rs +++ b/rustfs/src/table_catalog/store/strong.rs @@ -51,7 +51,7 @@ pub(in crate::table_catalog) struct StrongTableCatalogState { pub(super) hydrated: bool, snapshot_required: bool, pub(in crate::table_catalog) snapshot_etag: Option, - snapshot_version: Option, + pub(super) snapshot_version: Option, pub(in crate::table_catalog) table_buckets: BTreeMap, pub(in crate::table_catalog) namespaces: BTreeMap, namespace_children: BTreeMap, @@ -102,7 +102,8 @@ pub(in crate::table_catalog) struct StrongTableCatalogSnapshot { pub(in crate::table_catalog) idempotency: Vec, } -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub(in crate::table_catalog) struct StrongTableCatalogBucketSnapshot { pub(super) table_bucket: TableBucketEntry, pub(super) namespaces: Vec, @@ -113,7 +114,7 @@ pub(in crate::table_catalog) struct StrongTableCatalogBucketSnapshot { } #[derive(Clone)] -enum StrongSnapshotWritePostcondition { +pub(super) enum StrongSnapshotWritePostcondition { TableBucketPresent(TableBucketEntry), TableBucketAbsent(String), NamespacePresent(NamespaceEntry), @@ -297,8 +298,8 @@ pub(in crate::table_catalog) fn table_catalog_bucket_snapshot_fingerprint( #[derive(Clone)] pub(crate) struct StrongTableCatalogStore { - object_backend: B, - snapshot_write_version: u16, + pub(super) object_backend: B, + pub(super) snapshot_write_version: u16, snapshot_required_on_start: bool, // Single mutex protecting all catalog state (table_buckets, namespaces, tables, views, commits, idempotency). // This is intentional: many operations require atomic read-modify-write across multiple fields. @@ -309,7 +310,7 @@ pub(crate) struct StrongTableCatalogStore { // 3. Using optimistic concurrency with version checks pub(in crate::table_catalog) state: Arc>, // Serializes local snapshot mutations; object ETags fence independent store instances. - write_lock: Arc>, + pub(super) write_lock: Arc>, // Coalesces reloads for clones of one store so only one task reads and decodes a changed snapshot. reload_lock: Arc>, #[cfg(test)] @@ -596,7 +597,7 @@ where } } - fn bucket_snapshot_from_state_locked( + pub(super) fn bucket_snapshot_from_state_locked( state: &StrongTableCatalogState, table_bucket: &str, ) -> Option { @@ -646,7 +647,7 @@ where }) } - fn remove_bucket_from_state_locked(state: &mut StrongTableCatalogState, table_bucket: &str) { + pub(super) fn remove_bucket_from_state_locked(state: &mut StrongTableCatalogState, table_bucket: &str) { state.table_buckets.remove(table_bucket); state.namespaces.retain(|(entry_bucket, _), _| entry_bucket != table_bucket); state @@ -960,7 +961,7 @@ where Ok(()) } - fn snapshot_from_mutated_state_locked( + pub(super) fn snapshot_from_mutated_state_locked( state: &mut StrongTableCatalogState, configured_write_version: u16, ) -> TableCatalogStoreResult { @@ -1000,7 +1001,7 @@ where Ok(snapshot) } - fn state_from_snapshot( + pub(super) fn state_from_snapshot( snapshot: StrongTableCatalogSnapshot, snapshot_etag: Option, ) -> TableCatalogStoreResult { @@ -1228,7 +1229,7 @@ where ); } - async fn hydrate_state(&self) -> TableCatalogStoreResult<()> { + pub(super) async fn hydrate_state(&self) -> TableCatalogStoreResult<()> { let Some((current_snapshot_etag, current_snapshot_required)) = ({ let state = self.state.lock().await; if state.hydrated { @@ -1330,11 +1331,22 @@ where )) } - async fn finalize_snapshot_write( + pub(super) async fn finalize_snapshot_write( &self, snapshot: StrongTableCatalogSnapshot, precondition: TableCatalogPutPrecondition, postcondition: StrongSnapshotWritePostcondition, + ) -> TableCatalogStoreResult<()> { + self.finalize_snapshot_write_with_fence(snapshot, precondition, postcondition, None) + .await + } + + pub(super) async fn finalize_snapshot_write_with_fence( + &self, + snapshot: StrongTableCatalogSnapshot, + precondition: TableCatalogPutPrecondition, + postcondition: StrongSnapshotWritePostcondition, + mutation_fence: Option<&TableCatalogObjectMutationFence>, ) -> TableCatalogStoreResult<()> { let data = serde_json::to_vec(&snapshot) .map_err(|err| TableCatalogStoreError::Internal(format!("failed to encode strong catalog snapshot: {err}")))?; @@ -1343,16 +1355,27 @@ where "durable strong catalog snapshot exceeds the maximum encoded size of {STRONG_TABLE_CATALOG_SNAPSHOT_MAX_SIZE} bytes" ))); } - match self - .object_backend - .put_object(RUSTFS_META_BUCKET, &Self::snapshot_object_path(), data, precondition) - .await - { + if let Some(fence) = mutation_fence { + fence.ensure_held()?; + } + let write_result = if let Some(fence) = mutation_fence { + self.object_backend + .put_object_fenced(RUSTFS_META_BUCKET, &Self::snapshot_object_path(), data, precondition, fence) + .await + } else { + self.object_backend + .put_object(RUSTFS_META_BUCKET, &Self::snapshot_object_path(), data, precondition) + .await + }; + match write_result { Ok(()) => { self.state.lock().await.snapshot_required = true; if let Err(err) = self.reload_state_from_durable().await { self.mark_snapshot_reload_failed(&err, "confirmed-write").await; } + if let Some(fence) = mutation_fence { + fence.ensure_held()?; + } Ok(()) } Err(err @ TableCatalogStoreError::Conflict(_)) => { @@ -1360,12 +1383,18 @@ where if let Err(reload_err) = self.reload_state_from_durable().await { self.mark_snapshot_reload_failed(&reload_err, "write-conflict").await; } + if let Some(fence) = mutation_fence { + fence.ensure_held()?; + } Err(err) } Err(err) => { self.state.lock().await.snapshot_required = true; match self.reload_state_from_durable().await { Ok(()) => { + if let Some(fence) = mutation_fence { + fence.ensure_held()?; + } let state = self.state.lock().await; if postcondition.is_satisfied_by::(&state) { Ok(()) @@ -1496,7 +1525,7 @@ where Ok(()) } - pub(super) async fn bucket_snapshot_observation( + pub(crate) async fn bucket_snapshot_observation( &self, table_bucket: &str, ) -> TableCatalogStoreResult<(Option, Option)> { @@ -1591,7 +1620,7 @@ where Ok(()) } - fn table_commit_recovery_report_for_entry_locked( + pub(super) fn table_commit_recovery_report_for_entry_locked( state: &StrongTableCatalogState, entry: &TableEntry, ) -> TableCommitRecoveryReport { @@ -2381,6 +2410,16 @@ where "table metadata location must be inside the table metadata directory".to_string(), )); } + let _migration_guard = match publication.catalog_migration_read_permit_status() { + Some(true) => None, + Some(false) => { + return Err(TableCatalogStoreError::Conflict( + "table-bucket catalog migration read permit was lost".to_string(), + )); + } + None if publication.acquires_catalog_migration_read_permit() => None, + None => Some(self.acquire_snapshot_write_permit().await?), + }; publication.begin_table_bucket(&entry.table_bucket).await?; if !publication.holds_table_bucket(&entry.table_bucket) { return Err(TableCatalogStoreError::Internal( @@ -2388,7 +2427,6 @@ where )); } let _publication_completion = TableCommitPublicationCompletion::new(publication); - let _migration_guard = self.acquire_snapshot_write_permit().await?; publication .prepare(&entry.table_bucket, &entry.namespace, &entry.table) .await?; @@ -2536,6 +2574,7 @@ where destination_table: &str, ) -> TableCatalogStoreResult<()> { let publication = TableCommitLockPublication::new(&self.object_backend); + let _migration_guard = self.acquire_snapshot_write_permit().await?; publication.begin_table_bucket(table_bucket).await?; if !publication.holds_table_bucket(table_bucket) { return Err(TableCatalogStoreError::Internal( @@ -2543,7 +2582,6 @@ where )); } let _publication_completion = TableCommitPublicationCompletion::new(&publication); - let _migration_guard = self.acquire_snapshot_write_permit().await?; let _write_guard = self.write_lock.lock().await; self.hydrate_state().await?; @@ -2677,7 +2715,6 @@ where async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { let publication = TableCommitLockPublication::new(&self.object_backend); - publication.begin_table_bucket(&request.table_bucket).await?; self.commit_table_with_publication(request, &publication).await } @@ -2686,11 +2723,26 @@ where request: TableCommitRequest, publication: &(dyn TableCommitPublication + Sync), ) -> TableCatalogStoreResult { - let _migration_guard = self.acquire_snapshot_write_permit().await?; + let _migration_guard = match publication.catalog_migration_read_permit_status() { + Some(true) => None, + Some(false) => { + return Err(TableCatalogStoreError::Conflict( + "table-bucket catalog migration read permit was lost".to_string(), + )); + } + None if publication.acquires_catalog_migration_read_permit() => None, + None => Some(self.acquire_snapshot_write_permit().await?), + }; let commit_started = Instant::now(); record_table_commit_attempt(&request.operation); let namespace = parse_namespace_for_store(&request.namespace)?; let table = parse_table_for_store(&request.table)?; + publication.begin_table_bucket(&request.table_bucket).await?; + if !publication.holds_table_bucket(&request.table_bucket) { + return Err(TableCatalogStoreError::Internal( + "table commit requires a table-bucket publication fence".to_string(), + )); + } publication .prepare(&request.table_bucket, &request.namespace, &request.table) .await?; @@ -2876,8 +2928,8 @@ where async fn drop_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult<()> { let publication = TableCommitLockPublication::new(&self.object_backend); - publication.begin_table_bucket(table_bucket).await?; let _migration_guard = self.acquire_snapshot_write_permit().await?; + publication.begin_table_bucket(table_bucket).await?; publication.prepare(table_bucket, namespace, table).await?; if !publication.holds_table_bucket(table_bucket) || !publication.holds_table(table_bucket, namespace, table) { return Err(TableCatalogStoreError::Internal( @@ -2959,6 +3011,16 @@ where "view metadata location must be inside the view metadata directory".to_string(), )); } + let _migration_guard = match publication.catalog_migration_read_permit_status() { + Some(true) => None, + Some(false) => { + return Err(TableCatalogStoreError::Conflict( + "table-bucket catalog migration read permit was lost".to_string(), + )); + } + None if publication.acquires_catalog_migration_read_permit() => None, + None => Some(self.acquire_snapshot_write_permit().await?), + }; publication.begin_table_bucket(&entry.table_bucket).await?; if !publication.holds_table_bucket(&entry.table_bucket) { return Err(TableCatalogStoreError::Internal( @@ -2966,7 +3028,6 @@ where )); } let _publication_completion = TableCommitPublicationCompletion::new(publication); - let _migration_guard = self.acquire_snapshot_write_permit().await?; publication .prepare(&entry.table_bucket, &entry.namespace, &entry.view) .await?; @@ -3083,6 +3144,16 @@ where table_bucket_fence_required: bool, publication: &(dyn TableCommitPublication + Sync), ) -> TableCatalogStoreResult { + let _migration_guard = match publication.catalog_migration_read_permit_status() { + Some(true) => None, + Some(false) => { + return Err(TableCatalogStoreError::Conflict( + "table-bucket catalog migration read permit was lost".to_string(), + )); + } + None if table_bucket_fence_required && publication.acquires_catalog_migration_read_permit() => None, + None => Some(self.acquire_snapshot_write_permit().await?), + }; if table_bucket_fence_required { publication.begin_table_bucket(&request.table_bucket).await?; if !publication.holds_table_bucket(&request.table_bucket) { @@ -3092,7 +3163,6 @@ where } } let _publication_completion = TableCommitPublicationCompletion::new(publication); - let _migration_guard = self.acquire_snapshot_write_permit().await?; let namespace = parse_namespace_for_store(&request.namespace)?; let view = parse_table_for_store(&request.view)?; publication diff --git a/rustfs/src/table_catalog/test_support.rs b/rustfs/src/table_catalog/test_support.rs index 4c9c466a6..3c0878b1c 100644 --- a/rustfs/src/table_catalog/test_support.rs +++ b/rustfs/src/table_catalog/test_support.rs @@ -819,6 +819,7 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { Ok(state.objects.get(&key).map(|record| TableCatalogObjectMetadata { etag: (!etagless).then(|| record.etag.clone()), mod_time: record.mod_time, + size: record.data.len() as u64, })) } @@ -958,6 +959,49 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend { Ok(()) } + async fn delete_object_if_match(&self, bucket: &str, object: &str, expected_etag: &str) -> TableCatalogStoreResult<()> { + let mut state = self.state.lock().await; + let key = (bucket.to_string(), object.to_string()); + let attempt = { + let attempts = state.delete_attempts.entry(key.clone()).or_default(); + *attempts += 1; + *attempts + }; + if state + .fail_delete_attempts + .get(&key) + .is_some_and(|attempts| attempts.contains(&attempt)) + { + return Err(TableCatalogStoreError::Internal(format!( + "injected delete failure for {object} attempt {attempt}" + ))); + } + match state.objects.get(&key) { + None => { + return Err(TableCatalogStoreError::Conflict(format!( + "conditionally delete catalog object: {bucket}/{object} disappeared before cleanup" + ))); + } + Some(record) if record.etag != expected_etag => { + return Err(TableCatalogStoreError::Conflict(format!( + "conditionally delete catalog object: {bucket}/{object} changed before cleanup" + ))); + } + Some(_) => {} + } + state.objects.remove(&key); + if state + .fail_after_delete_attempts + .get(&key) + .is_some_and(|attempts| attempts.contains(&attempt)) + { + return Err(TableCatalogStoreError::Internal(format!( + "injected post-commit delete failure for {object} attempt {attempt}" + ))); + } + Ok(()) + } + async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult> { let mut state = self.state.lock().await; state.list_calls += 1; diff --git a/rustfs/src/table_catalog/tests.rs b/rustfs/src/table_catalog/tests.rs index 9f4be77a6..55ee4bc28 100644 --- a/rustfs/src/table_catalog/tests.rs +++ b/rustfs/src/table_catalog/tests.rs @@ -492,6 +492,77 @@ impl TableCommitPublication for LosingTestPublication { fn complete(&self) {} } +struct PersistedMigrationFenceTestPublication; + +#[async_trait::async_trait] +impl TableCommitPublication for PersistedMigrationFenceTestPublication { + async fn begin_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + async fn prepare(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> { + Ok(()) + } + + fn holds_table_bucket(&self, _table_bucket: &str) -> bool { + true + } + + fn holds_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> bool { + true + } + + fn catalog_migration_read_permit_status(&self) -> Option { + Some(true) + } + + fn complete(&self) {} +} + +#[tokio::test] +async fn object_catalog_rechecks_persisted_migration_fence_when_reusing_read_permit() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + let table = IdentifierSegment::parse("returns").expect("table should parse"); + let existing_table = IdentifierSegment::parse("orders").expect("table should parse"); + seed_table_for_metadata_maintenance( + &store, + bucket, + &namespace, + &existing_table, + default_table_metadata_file_path(&namespace, &existing_table, "00001.metadata.json"), + ) + .await; + store + .materialize_durable_strong_backing_migration(bucket) + .await + .expect("migration should persist the object-write fence"); + + let entry = test_table_entry( + bucket, + &namespace, + &table, + default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), + ); + let error = store + .register_table_with_publication(entry, &PersistedMigrationFenceTestPublication) + .await + .expect_err("a persisted migration fence must reject object-backed registration"); + assert_matches!( + error, + TableCatalogStoreError::Conflict(message) if message.contains("writes are fenced") + ); + assert!( + store + .load_table(bucket, &namespace.public_name(), table.as_str()) + .await + .expect("table lookup should succeed") + .is_none() + ); +} + async fn assert_view_replacement_rechecks_publication_fence(store: &S, backend: &TestCatalogObjectBackend) where S: TableCatalogStore, @@ -568,7 +639,7 @@ async fn catalog_backings_stop_view_replacement_after_publication_fence_loss() { } #[tokio::test] -async fn strong_table_registration_and_drop_acquire_publication_before_migration_read_lock() { +async fn strong_table_registration_and_drop_acquire_migration_read_lock_before_publication() { let backend = TestCatalogObjectBackend::default(); let store = StrongTableCatalogStore::new(backend.clone()); let bucket = "analytics"; @@ -587,6 +658,10 @@ async fn strong_table_registration_and_drop_acquire_publication_before_migration .acquire_write_lock(bucket, &publication_lock) .await .expect("publication lock should be acquired"); + let migration_guard = backend + .acquire_write_lock(RUSTFS_META_BUCKET, &migration_lock) + .await + .expect("migration lock should be acquired"); let publication_attempts = backend.write_lock_acquisition_count(bucket, &publication_lock).await; let migration_reads = backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock).await; let register_store = store.clone(); @@ -595,35 +670,28 @@ async fn strong_table_registration_and_drop_acquire_publication_before_migration .register_table(test_table_entry(bucket, &namespace, &table, metadata_location)) .await }); - tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { - while backend.write_lock_acquisition_count(bucket, &publication_lock).await == publication_attempts { - tokio::task::yield_now().await; - } - }) - .await - .expect("registration should wait on the table-bucket publication lock"); - assert_eq!( - backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock).await, - migration_reads, - "registration must not retain a migration read lock while waiting for publication" - ); - let migration_guard = tokio::time::timeout( - TABLE_CATALOG_TEST_TIMEOUT, - backend.acquire_write_lock(RUSTFS_META_BUCKET, &migration_lock), - ) - .await - .expect("migration writer must not be blocked by registration waiting on publication") - .expect("migration write lock should be acquired"); - drop(publication_guard); tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { while backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock).await == migration_reads { tokio::task::yield_now().await; } }) .await - .expect("registration should request the migration read lock after publication"); - assert!(!register.is_finished()); + .expect("registration should request the migration read lock first"); + assert_eq!( + backend.write_lock_acquisition_count(bucket, &publication_lock).await, + publication_attempts, + "registration must not request publication while waiting for the migration fence" + ); drop(migration_guard); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { + while backend.write_lock_acquisition_count(bucket, &publication_lock).await == publication_attempts { + tokio::task::yield_now().await; + } + }) + .await + .expect("registration should request publication after the migration fence"); + assert!(!register.is_finished()); + drop(publication_guard); register .await .expect("registration task should join") @@ -633,39 +701,36 @@ async fn strong_table_registration_and_drop_acquire_publication_before_migration .acquire_read_lock(bucket, &publication_lock) .await .expect("publication reader should be acquired"); + let migration_guard = backend + .acquire_write_lock(RUSTFS_META_BUCKET, &migration_lock) + .await + .expect("migration lock should be acquired"); let publication_attempts = backend.write_lock_acquisition_count(bucket, &publication_lock).await; let migration_reads = backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock).await; let drop_store = store.clone(); let drop_task = tokio::spawn(async move { drop_store.drop_table(bucket, "sales", "orders").await }); - tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { - while backend.write_lock_acquisition_count(bucket, &publication_lock).await == publication_attempts { - tokio::task::yield_now().await; - } - }) - .await - .expect("drop should wait on the table-bucket publication lock"); - assert_eq!( - backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock).await, - migration_reads, - "drop must not retain a migration read lock while waiting for publication" - ); - let migration_guard = tokio::time::timeout( - TABLE_CATALOG_TEST_TIMEOUT, - backend.acquire_write_lock(RUSTFS_META_BUCKET, &migration_lock), - ) - .await - .expect("migration writer must not be blocked by drop waiting on publication") - .expect("migration write lock should be acquired"); - drop(publication_guard); tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { while backend.read_lock_acquisition_count(RUSTFS_META_BUCKET, &migration_lock).await == migration_reads { tokio::task::yield_now().await; } }) .await - .expect("drop should request the migration read lock after publication"); - assert!(!drop_task.is_finished()); + .expect("drop should request the migration read lock first"); + assert_eq!( + backend.write_lock_acquisition_count(bucket, &publication_lock).await, + publication_attempts, + "drop must not request publication while waiting for the migration fence" + ); drop(migration_guard); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { + while backend.write_lock_acquisition_count(bucket, &publication_lock).await == publication_attempts { + tokio::task::yield_now().await; + } + }) + .await + .expect("drop should request publication after the migration fence"); + assert!(!drop_task.is_finished()); + drop(publication_guard); drop_task.await.expect("drop task should join").expect("drop should succeed"); } @@ -1302,6 +1367,13 @@ fn table_metadata_json_for_validation() -> serde_json::Value { }) } +fn table_metadata_json_for_backup(bucket: &str, table_id: &str, table_uuid: &str) -> serde_json::Value { + let mut metadata = table_metadata_json_for_validation(); + metadata["table-uuid"] = serde_json::Value::String(table_uuid.to_string()); + metadata["location"] = serde_json::Value::String(format!("s3://{bucket}/tables/{table_id}")); + metadata +} + #[test] fn iceberg_metadata_validation_accepts_complete_v1_and_v2_shapes() { validate_supported_table_metadata(&table_metadata_json_for_validation()) @@ -2190,7 +2262,7 @@ async fn snapshot_validation_rejects_non_puffin_table_statistics() { } #[tokio::test] -async fn snapshot_validation_rejects_non_parquet_partition_statistics() { +async fn snapshot_validation_accepts_non_parquet_partition_statistics() { let backend = TestCatalogObjectBackend::default(); let namespace = Namespace::parse("analytics").expect("namespace should parse"); let table = IdentifierSegment::parse("events").expect("table should parse"); @@ -2207,7 +2279,7 @@ async fn snapshot_validation_rejects_non_parquet_partition_statistics() { current["current-snapshot-id"] = serde_json::Value::from(10); current["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); let mut target = current.clone(); - let statistics = b"notparquet".to_vec(); + let statistics = b"ORC".to_vec(); target["partition-statistics"] = serde_json::json!([{ "snapshot-id": 10, "statistics-path": "s3://warehouse/tables/table-id/metadata/partition-stats.parquet", @@ -2218,12 +2290,19 @@ async fn snapshot_validation_rejects_non_parquet_partition_statistics() { .await; let context = TableSnapshotGraphValidationContext::new(&backend, "warehouse", &entry); + validate_table_snapshot_changes(&context, Some(¤t), &target) + .await + .expect("Iceberg partition statistics are not restricted to Parquet"); + + backend + .seed_object("warehouse", "tables/table-id/metadata/partition-stats.parquet", b"BAD".to_vec()) + .await; let error = validate_table_snapshot_changes(&context, Some(¤t), &target) .await - .expect_err("partition statistics must be a Parquet file"); + .expect_err("partition statistics must use a supported table data-file envelope"); assert_eq!( error, - TableCatalogStoreError::Invalid("partition statistics object is not a Parquet file".to_string()) + TableCatalogStoreError::Invalid("partition statistics object is not a supported table data file".to_string()) ); } @@ -3757,6 +3836,790 @@ async fn configured_table_catalog_store_uses_durable_strong_snapshot() { assert!(reloaded.get_table_bucket(bucket).await.unwrap().is_some()); } +#[tokio::test] +async fn durable_catalog_backup_is_idempotent_and_restorable() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + backend + .seed_object( + bucket, + &metadata_location, + serde_json::to_vec(&table_metadata_json_for_backup(bucket, "table-id", "table-uuid")).unwrap(), + ) + .await; + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, metadata_location)) + .await + .unwrap(); + let maintenance_store = ObjectTableCatalogStore::new(backend.clone()); + maintenance_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + maintenance_store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + maintenance_store + .create_table(test_table_entry( + bucket, + &namespace, + &table, + default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), + )) + .await + .unwrap(); + let mut failed_report = maintenance_store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .unwrap(); + failed_report.job.status = TableMetadataMaintenanceJobStatus::Failed; + let latest_job_path = + TableCatalogObjectPaths::default().table_maintenance_latest_job_path(bucket, &namespace, &table, "table-id"); + backend + .seed_object(RUSTFS_META_BUCKET, &latest_job_path, serde_json::to_vec(&failed_report).unwrap()) + .await; + + let first = store.create_durable_catalog_backup(bucket, None).await.unwrap(); + assert_eq!(first.status, TableCatalogBackupStatus::Created); + assert_eq!(first.object_count, 1); + assert_eq!(first.verified_object_count, 1); + assert_eq!(first.maintenance_object_count, 1); + let paths = TableCatalogObjectPaths::default(); + assert!( + backend + .write_lock_acquisition_count(RUSTFS_META_BUCKET, &paths.backing_migration_fence_lock_path(bucket)) + .await + > 0, + "durable catalog backup must fence object-backed maintenance for the table bucket" + ); + + let second = store.create_durable_catalog_backup(bucket, None).await.unwrap(); + assert_eq!(second.status, TableCatalogBackupStatus::AlreadyPresent); + assert_eq!(second.backup_id, first.backup_id); + + let extra_namespace = Namespace::parse("marketing").unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &extra_namespace)) + .await + .unwrap(); + let (_, replacement_snapshot_etag) = store.bucket_snapshot_observation(bucket).await.unwrap(); + let replacement_snapshot_etag = replacement_snapshot_etag.expect("replacement snapshot should have an etag"); + let conflict = store + .restore_durable_catalog_backup(bucket, &first.backup_id, Some(&replacement_snapshot_etag), false) + .await + .expect_err("restore must require an explicit replacement decision"); + assert_matches!(conflict, TableCatalogStoreError::Conflict(message) if message.contains("allow-replace")); + + let replaced = store + .restore_durable_catalog_backup(bucket, &first.backup_id, Some(&replacement_snapshot_etag), true) + .await + .unwrap(); + assert_eq!(replaced.status, TableCatalogRestoreStatus::Restored); + assert!(store.get_namespace(bucket, "marketing").await.unwrap().is_none()); + let replayed_after_response_loss = store + .restore_durable_catalog_backup(bucket, &first.backup_id, Some(&replacement_snapshot_etag), true) + .await + .expect("a completed restore must accept an idempotent retry with its original precondition"); + assert_eq!(replayed_after_response_loss.status, TableCatalogRestoreStatus::AlreadyRestored); + + store + .remove_bucket_snapshot_if_unchanged(bucket, &first.catalog_fingerprint) + .await + .unwrap(); + assert!(store.get_table_bucket(bucket).await.unwrap().is_none()); + + let restored = store + .restore_durable_catalog_backup(bucket, &first.backup_id, None, false) + .await + .unwrap(); + assert_eq!(restored.status, TableCatalogRestoreStatus::Restored); + assert!(store.get_table_bucket(bucket).await.unwrap().is_some()); + assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_some()); + + let repeated = store + .restore_durable_catalog_backup(bucket, &first.backup_id, None, false) + .await + .unwrap(); + assert_eq!(repeated.status, TableCatalogRestoreStatus::AlreadyRestored); +} + +#[tokio::test] +async fn durable_catalog_restore_preserves_the_backup_snapshot_version() { + let backend = TestCatalogObjectBackend::default(); + let source_store = + StrongTableCatalogStore::new_with_snapshot_write_version(backend.clone(), STRONG_TABLE_CATALOG_SNAPSHOT_VERSION); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + backend + .seed_object( + bucket, + &metadata_location, + serde_json::to_vec(&table_metadata_json_for_backup(bucket, "table-id", "table-uuid")).unwrap(), + ) + .await; + source_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + source_store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + source_store + .create_table(test_table_entry(bucket, &namespace, &table, metadata_location)) + .await + .unwrap(); + let backup = source_store.create_durable_catalog_backup(bucket, None).await.unwrap(); + assert_eq!(read_strong_snapshot(&backend).await.version, STRONG_TABLE_CATALOG_SNAPSHOT_VERSION); + + let replacement_namespace = Namespace::parse("replacement").unwrap(); + let mut target_snapshot = test_strong_snapshot(bucket, &namespace, Vec::new(), Vec::new()); + target_snapshot + .namespaces + .push(test_namespace_entry(bucket, &replacement_namespace)); + seed_strong_snapshot(&backend, &target_snapshot).await; + + let target_store = + StrongTableCatalogStore::new_with_snapshot_write_version(backend.clone(), STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION); + target_store + .restore_durable_catalog_backup(bucket, &backup.backup_id, None, true) + .await + .unwrap(); + + assert_eq!(read_strong_snapshot(&backend).await.version, STRONG_TABLE_CATALOG_SNAPSHOT_VERSION); + assert!(target_store.get_namespace(bucket, "sales").await.unwrap().is_some()); + assert!(target_store.get_namespace(bucket, "replacement").await.unwrap().is_none()); +} + +#[tokio::test] +async fn durable_catalog_restore_rejects_changed_referenced_object() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + backend + .seed_object( + bucket, + &metadata_location, + serde_json::to_vec(&table_metadata_json_for_backup(bucket, "table-id", "table-uuid")).unwrap(), + ) + .await; + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, metadata_location.clone())) + .await + .unwrap(); + let backup = store.create_durable_catalog_backup(bucket, None).await.unwrap(); + + let mut changed_metadata = table_metadata_json_for_backup(bucket, "table-id", "table-uuid"); + changed_metadata["last-updated-ms"] = serde_json::Value::from(2); + backend + .seed_object(bucket, &metadata_location, serde_json::to_vec(&changed_metadata).unwrap()) + .await; + let error = store + .restore_durable_catalog_backup(bucket, &backup.backup_id, None, true) + .await + .expect_err("restore must reject changed referenced objects"); + assert_matches!(error, TableCatalogStoreError::Conflict(message) if message.contains("object graph no longer matches")); +} + +#[tokio::test] +async fn durable_catalog_backup_tracks_statistics_objects() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let manifest_list_location = "tables/table-id/metadata/snap-10.avro"; + let statistics_location = "tables/table-id/metadata/stats.puffin"; + let partition_statistics_location = "tables/table-id/metadata/partition-stats.orc"; + let mut metadata = table_metadata_json_for_backup(bucket, "table-id", "table-uuid"); + metadata["last-sequence-number"] = serde_json::Value::from(1); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": format!("s3://{bucket}/{manifest_list_location}"), + "summary": {"operation": "append"} + }]); + metadata["current-snapshot-id"] = serde_json::Value::from(10); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + metadata["statistics"] = serde_json::json!([{ + "snapshot-id": 10, + "statistics-path": format!("s3://{bucket}/{statistics_location}"), + "file-size-in-bytes": 12, + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + }]); + metadata["partition-statistics"] = serde_json::json!([{ + "snapshot-id": 10, + "statistics-path": format!("s3://{bucket}/{partition_statistics_location}"), + "file-size-in-bytes": 3 + }]); + + backend + .seed_object(bucket, &metadata_location, serde_json::to_vec(&metadata).unwrap()) + .await; + backend + .seed_object(bucket, manifest_list_location, manifest_list_avro_bytes(&[])) + .await; + backend + .seed_object(bucket, statistics_location, b"PFA1old!PFA1".to_vec()) + .await; + backend + .seed_object(bucket, partition_statistics_location, b"ORC".to_vec()) + .await; + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, metadata_location)) + .await + .unwrap(); + + let backup = store.create_durable_catalog_backup(bucket, None).await.unwrap(); + assert_eq!(backup.object_count, 4); + + backend + .seed_object(bucket, statistics_location, b"PFA1new!PFA1".to_vec()) + .await; + let error = store + .restore_durable_catalog_backup(bucket, &backup.backup_id, None, true) + .await + .expect_err("restore must reject changed statistics references"); + assert_matches!(error, TableCatalogStoreError::Conflict(message) if message.contains("object graph no longer matches")); +} + +#[tokio::test] +async fn durable_catalog_backup_rejects_incomplete_snapshot_graph() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let mut metadata = table_metadata_json_for_backup(bucket, "table-id", "table-uuid"); + metadata["last-sequence-number"] = serde_json::Value::from(1); + metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": format!("s3://{bucket}/tables/table-id/metadata/missing.avro"), + "summary": {"operation": "append"} + }]); + metadata["current-snapshot-id"] = serde_json::Value::from(10); + metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + + backend + .seed_object(bucket, &metadata_location, serde_json::to_vec(&metadata).unwrap()) + .await; + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, metadata_location)) + .await + .unwrap(); + + let error = store + .create_durable_catalog_backup(bucket, None) + .await + .expect_err("backup must reject a snapshot without a manifest graph"); + assert_eq!( + error, + TableCatalogStoreError::Invalid("snapshot manifest-list object is missing".to_string()) + ); +} + +#[tokio::test] +async fn durable_catalog_backup_accepts_historical_metadata_after_warehouse_relocation() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let historical_metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current_metadata_location = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let historical_manifest_list_location = "tables/old-table-id/metadata/snap-10.avro"; + let historical_statistics_location = "tables/old-table-id/metadata/stats.puffin"; + let historical_warehouse = format!("s3://{bucket}/tables/old-table-id"); + let current_warehouse = format!("s3://{bucket}/tables/table-id"); + + let mut historical_metadata = table_metadata_json_for_backup(bucket, "table-id", "table-uuid"); + historical_metadata["location"] = serde_json::Value::String(historical_warehouse); + historical_metadata["last-sequence-number"] = serde_json::Value::from(1); + historical_metadata["snapshots"] = serde_json::json!([{ + "snapshot-id": 10, + "sequence-number": 1, + "timestamp-ms": 1, + "manifest-list": format!("s3://{bucket}/{historical_manifest_list_location}"), + "summary": {"operation": "append"} + }]); + historical_metadata["current-snapshot-id"] = serde_json::Value::from(10); + historical_metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 10}}); + historical_metadata["statistics"] = serde_json::json!([{ + "snapshot-id": 10, + "statistics-path": format!("s3://{bucket}/{historical_statistics_location}"), + "file-size-in-bytes": 8, + "file-footer-size-in-bytes": 0, + "blob-metadata": [] + }]); + + let mut current_metadata = table_metadata_json_for_backup(bucket, "table-id", "table-uuid"); + current_metadata["location"] = serde_json::Value::String(current_warehouse); + current_metadata["metadata-log"] = serde_json::json!([{ + "timestamp-ms": 1, + "metadata-file": historical_metadata_location.clone() + }]); + + backend + .seed_object(bucket, &historical_metadata_location, serde_json::to_vec(&historical_metadata).unwrap()) + .await; + backend + .seed_object(bucket, ¤t_metadata_location, serde_json::to_vec(¤t_metadata).unwrap()) + .await; + backend + .seed_object(bucket, historical_manifest_list_location, manifest_list_avro_bytes(&[])) + .await; + backend + .seed_object(bucket, historical_statistics_location, b"PFA1PFA1".to_vec()) + .await; + + let mut entry = test_table_entry(bucket, &namespace, &table, current_metadata_location); + entry.warehouse_location = format!("s3://{bucket}/tables/table-id"); + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store.create_table(entry).await.unwrap(); + + let backup = store.create_durable_catalog_backup(bucket, None).await.unwrap(); + assert_eq!(backup.object_count, 4); +} + +#[tokio::test] +async fn durable_catalog_backup_rejects_historical_warehouse_owned_by_another_table() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let other_table = IdentifierSegment::parse("customers").unwrap(); + let historical_metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + let current_metadata_location = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json"); + let other_metadata_location = default_table_metadata_file_path(&namespace, &other_table, "00001.metadata.json"); + let historical_warehouse = format!("s3://{bucket}/tables/reused-prefix"); + + let mut historical_metadata = table_metadata_json_for_backup(bucket, "table-id", "table-uuid"); + historical_metadata["location"] = serde_json::Value::String(historical_warehouse.clone()); + let mut current_metadata = table_metadata_json_for_backup(bucket, "table-id", "table-uuid"); + current_metadata["metadata-log"] = serde_json::json!([{ + "timestamp-ms": 1, + "metadata-file": historical_metadata_location.clone() + }]); + let mut other_metadata = table_metadata_json_for_backup(bucket, "other-table-id", "other-table-uuid"); + other_metadata["location"] = serde_json::Value::String(historical_warehouse.clone()); + + backend + .seed_object(bucket, &historical_metadata_location, serde_json::to_vec(&historical_metadata).unwrap()) + .await; + backend + .seed_object(bucket, ¤t_metadata_location, serde_json::to_vec(¤t_metadata).unwrap()) + .await; + backend + .seed_object(bucket, &other_metadata_location, serde_json::to_vec(&other_metadata).unwrap()) + .await; + + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, current_metadata_location)) + .await + .unwrap(); + let mut other_entry = test_table_entry(bucket, &namespace, &other_table, other_metadata_location); + other_entry.table_id = "other-table-id".to_string(); + other_entry.table_uuid = "other-table-uuid".to_string(); + other_entry.warehouse_location = historical_warehouse; + store.create_table(other_entry).await.unwrap(); + + let error = store + .create_durable_catalog_backup(bucket, None) + .await + .expect_err("historical metadata must not claim another active table's warehouse"); + assert_matches!( + error, + TableCatalogStoreError::Conflict(message) + if message.contains("metadata warehouse overlaps another active table") + ); +} + +#[tokio::test] +async fn durable_catalog_backup_rejects_unknown_maintenance_status() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + backend + .seed_object( + bucket, + &metadata_location, + serde_json::to_vec(&table_metadata_json_for_backup(bucket, "table-id", "table-uuid")).unwrap(), + ) + .await; + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, metadata_location)) + .await + .unwrap(); + let maintenance_store = ObjectTableCatalogStore::new(backend.clone()); + maintenance_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + maintenance_store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + maintenance_store + .create_table(test_table_entry( + bucket, + &namespace, + &table, + default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), + )) + .await + .unwrap(); + let mut future_report = maintenance_store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .unwrap(); + future_report.job.status = TableMetadataMaintenanceJobStatus::Failed; + let maintenance_path = TableCatalogObjectPaths::default().table_maintenance_job_path( + bucket, + &namespace, + &table, + "table-id", + &future_report.job.job_id, + ); + let mut future_report_json = serde_json::to_value(future_report).unwrap(); + future_report_json["job"]["status"] = serde_json::Value::String("FUTURE_STATUS".to_string()); + backend + .seed_object(RUSTFS_META_BUCKET, &maintenance_path, serde_json::to_vec(&future_report_json).unwrap()) + .await; + + let error = store + .create_durable_catalog_backup(bucket, None) + .await + .expect_err("unknown maintenance status must fail closed"); + assert_matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("maintenance backup report is not valid")); +} + +#[tokio::test] +async fn durable_catalog_restore_rejects_changed_target_after_pending_intent() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + backend + .seed_object( + bucket, + &metadata_location, + serde_json::to_vec(&table_metadata_json_for_backup(bucket, "table-id", "table-uuid")).unwrap(), + ) + .await; + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, metadata_location)) + .await + .unwrap(); + let backup = store.create_durable_catalog_backup(bucket, None).await.unwrap(); + let replacement_namespace = Namespace::parse("replacement").unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &replacement_namespace)) + .await + .unwrap(); + backend + .fail_next_put( + RUSTFS_META_BUCKET, + &StrongTableCatalogStore::::snapshot_object_path(), + ) + .await; + let first_error = store + .restore_durable_catalog_backup(bucket, &backup.backup_id, None, true) + .await + .expect_err("a failed snapshot write must leave a recoverable pending intent"); + assert_matches!(first_error, TableCatalogStoreError::Internal(message) if message.contains("injected put failure")); + + let later_namespace = Namespace::parse("later").unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &later_namespace)) + .await + .unwrap(); + + let error = store + .restore_durable_catalog_backup(bucket, &backup.backup_id, None, true) + .await + .expect_err("a pending restore must not overwrite a changed target"); + assert_matches!(error, TableCatalogStoreError::Conflict(message) if message.contains("target bucket changed")); + assert!(store.get_namespace(bucket, "later").await.unwrap().is_some()); +} + +#[tokio::test] +async fn durable_catalog_restore_replays_after_unrelated_bucket_change() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + backend + .seed_object( + bucket, + &metadata_location, + serde_json::to_vec(&table_metadata_json_for_backup(bucket, "table-id", "table-uuid")).unwrap(), + ) + .await; + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, metadata_location)) + .await + .unwrap(); + let backup = store.create_durable_catalog_backup(bucket, None).await.unwrap(); + + let changed_namespace = Namespace::parse("changed-before-restore").unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &changed_namespace)) + .await + .unwrap(); + + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + backend.fail_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let first_error = store + .restore_durable_catalog_backup(bucket, &backup.backup_id, None, true) + .await + .expect_err("a failed snapshot write should preserve a prepared restore intent"); + assert_matches!(first_error, TableCatalogStoreError::Internal(message) if message.contains("injected put failure")); + + let backup_error = store + .create_durable_catalog_backup(bucket, None) + .await + .expect_err("a pending restore must be recovered before sealing another backup"); + assert_matches!(backup_error, TableCatalogStoreError::Conflict(message) if message.contains("pending restore recovery")); + + let unrelated_bucket = "unrelated"; + store.put_table_bucket(test_bucket_entry(unrelated_bucket)).await.unwrap(); + + let replay = store + .restore_durable_catalog_backup(bucket, &backup.backup_id, None, true) + .await + .expect("an unrelated bucket change must not invalidate a pending bucket restore"); + assert_eq!(replay.status, TableCatalogRestoreStatus::Restored); + assert!( + store + .get_namespace(bucket, &changed_namespace.public_name()) + .await + .unwrap() + .is_none() + ); + assert!(store.get_table_bucket(unrelated_bucket).await.unwrap().is_some()); +} + +#[tokio::test] +async fn durable_catalog_restore_replays_after_catalog_cas_before_intent_finalization() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + backend + .seed_object( + bucket, + &metadata_location, + serde_json::to_vec(&table_metadata_json_for_backup(bucket, "table-id", "table-uuid")).unwrap(), + ) + .await; + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, metadata_location)) + .await + .unwrap(); + let backup = store.create_durable_catalog_backup(bucket, None).await.unwrap(); + + let stale_table = IdentifierSegment::parse("stale").unwrap(); + let stale_metadata_location = default_table_metadata_file_path(&namespace, &stale_table, "00001.metadata.json"); + backend + .seed_object( + bucket, + &stale_metadata_location, + serde_json::to_vec(&table_metadata_json_for_backup(bucket, "stale-table-id", "stale-table-uuid")).unwrap(), + ) + .await; + let mut stale_entry = test_table_entry(bucket, &namespace, &stale_table, stale_metadata_location); + stale_entry.table_id = "stale-table-id".to_string(); + stale_entry.table_uuid = "stale-table-uuid".to_string(); + stale_entry.warehouse_location = format!("s3://{bucket}/tables/stale-table-id"); + store.create_table(stale_entry.clone()).await.unwrap(); + let stale_config_path = + TableCatalogObjectPaths::default().table_maintenance_config_path(bucket, &namespace, &stale_table, &stale_entry.table_id); + backend + .seed_object( + RUSTFS_META_BUCKET, + &stale_config_path, + serde_json::to_vec(&TableMaintenanceConfig::default()).unwrap(), + ) + .await; + + let snapshot_path = StrongTableCatalogStore::::snapshot_object_path(); + backend.fail_next_put(RUSTFS_META_BUCKET, &snapshot_path).await; + let first_error = store + .restore_durable_catalog_backup(bucket, &backup.backup_id, None, true) + .await + .expect_err("a failed snapshot write should preserve a prepared restore intent"); + assert_matches!(first_error, TableCatalogStoreError::Internal(message) if message.contains("injected put failure")); + + let intent_path = TableCatalogObjectPaths::default().catalog_backup_restore_intent_path(bucket); + backend.fail_next_put(RUSTFS_META_BUCKET, &intent_path).await; + let second_error = store + .restore_durable_catalog_backup(bucket, &backup.backup_id, None, true) + .await + .expect_err("a failed intent finalization should leave the catalog-applied intent for replay"); + assert_matches!(second_error, TableCatalogStoreError::Internal(message) if message.contains("injected put failure")); + assert!(store.load_table(bucket, "sales", "stale").await.unwrap().is_none()); + + let replay = store + .restore_durable_catalog_backup(bucket, &backup.backup_id, None, true) + .await + .expect("restore replay should reconcile the old maintenance path before cleanup"); + assert_eq!(replay.status, TableCatalogRestoreStatus::AlreadyRestored); + assert!(!backend.object_exists(RUSTFS_META_BUCKET, &stale_config_path).await.unwrap()); +} + +#[tokio::test] +async fn durable_catalog_restore_retries_after_ambiguous_maintenance_replacement() { + let backend = TestCatalogObjectBackend::default(); + let store = StrongTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").unwrap(); + let table = IdentifierSegment::parse("orders").unwrap(); + let metadata_location = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"); + + backend + .seed_object( + bucket, + &metadata_location, + serde_json::to_vec(&table_metadata_json_for_backup(bucket, "table-id", "table-uuid")).unwrap(), + ) + .await; + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry(bucket, &namespace, &table, metadata_location)) + .await + .unwrap(); + + let maintenance_store = ObjectTableCatalogStore::new(backend.clone()); + maintenance_store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + maintenance_store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .unwrap(); + maintenance_store + .create_table(test_table_entry( + bucket, + &namespace, + &table, + default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"), + )) + .await + .unwrap(); + let mut backup_report = maintenance_store + .plan_table_metadata_maintenance(bucket, "sales", "orders", 0) + .await + .unwrap(); + backup_report.job.status = TableMetadataMaintenanceJobStatus::Failed; + backup_report.job.failure_reason = Some("backup source".to_string()); + let latest_job_path = + TableCatalogObjectPaths::default().table_maintenance_latest_job_path(bucket, &namespace, &table, "table-id"); + backend + .seed_object(RUSTFS_META_BUCKET, &latest_job_path, serde_json::to_vec(&backup_report).unwrap()) + .await; + let backup = store.create_durable_catalog_backup(bucket, None).await.unwrap(); + + let mut changed_report = backup_report.clone(); + changed_report.job.failure_reason = Some("changed source".to_string()); + backend + .seed_object(RUSTFS_META_BUCKET, &latest_job_path, serde_json::to_vec(&changed_report).unwrap()) + .await; + backend.fail_after_next_put(RUSTFS_META_BUCKET, &latest_job_path).await; + + let first_error = store + .restore_durable_catalog_backup(bucket, &backup.backup_id, None, true) + .await + .expect_err("a post-commit maintenance replacement failure must remain observable"); + assert_matches!(first_error, TableCatalogStoreError::Internal(message) if message.contains("post-commit put failure")); + + let replay = store + .restore_durable_catalog_backup(bucket, &backup.backup_id, None, true) + .await + .expect("retry should recognize the fully published target object"); + assert_eq!(replay.status, TableCatalogRestoreStatus::AlreadyRestored); + let restored = backend + .read_object(RUSTFS_META_BUCKET, &latest_job_path) + .await + .unwrap() + .expect("target maintenance object should remain present"); + assert_eq!(restored.data, serde_json::to_vec(&backup_report).unwrap()); +} + #[tokio::test] async fn object_table_catalog_store_persists_view_entries_and_blocks_non_empty_namespace_drop() { let backend = TestCatalogObjectBackend { @@ -15760,6 +16623,11 @@ async fn strong_catalog_v1_inactive_resources_are_hidden_and_cleanup_only() { .expect("views should list") .is_empty() ); + let backup = store + .create_durable_catalog_backup(bucket, None) + .await + .expect("legacy inactive tombstones must not block a catalog backup"); + assert_eq!(backup.object_count, 0); store .drop_table(bucket, &namespace.public_name(), table.as_str()) .await diff --git a/scripts/table-catalog/README.md b/scripts/table-catalog/README.md index 5eae2a165..b1b42df83 100644 --- a/scripts/table-catalog/README.md +++ b/scripts/table-catalog/README.md @@ -312,6 +312,12 @@ The smoke test also probes catalog-backed advanced Iceberg surfaces: and snapshot refs remain protected. Orphan candidates are read-only observations, not deletion authorization. Object-backed reports share one recovery observation but do not provide a cross-object linearizable snapshot. + diagnostics fail closed if the strong snapshot changes during inspection; + until strong maintenance configuration is persisted, diagnostics use a + conservative zero-file retention default. Durable catalog backup and restore + are controlled durable-strong operations; backup artifacts record catalog and + referenced-object watermarks, while restore verifies them and retains a + recoverable intent when maintenance-state finalization is interrupted. ## Client Matrix