diff --git a/crates/ecstore/src/disk/local/commit.rs b/crates/ecstore/src/disk/local/commit.rs index 03e2051d1..f011a0511 100644 --- a/crates/ecstore/src/disk/local/commit.rs +++ b/crates/ecstore/src/disk/local/commit.rs @@ -111,46 +111,107 @@ async fn wait_for_heal_commit_test_barrier(root: &Path, bucket: &str, object: &s } } -fn rollback_committed_rename_std( - dst_file_path: &Path, - new_data_path: Option<&Path>, - rollback_data_dir: Option, -) -> std::io::Result<()> { - if let Some(old_data_dir) = rollback_data_dir { - let Some(dst_parent) = dst_file_path.parent() else { - return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent")); - }; - let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP); - std::fs::rename(backup_path, dst_file_path)?; - } else { - remove_file_if_exists(dst_file_path)?; - } - - if let Some(new_data_path) = new_data_path { - remove_dir_all_if_exists(new_data_path)?; - } - - Ok(()) +struct RenameCommitContext<'a> { + // Drop the directory identity guard before releasing mutation ownership. + directory_guard: os::RenameCommitGuard, + publication_root: &'a os::PublicationRoot, + mutation_lease: Arc, + source_metadata: &'a Path, + destination_metadata: &'a Path, + source_volume_dir: &'a Path, + data_paths: Option<&'a (PathBuf, PathBuf)>, } -fn rollback_inline_metadata_commit_std( - dst_file_path: &Path, - rollback_data_dir: Option, - local_rollback_path: Option<&Path>, -) -> std::io::Result<()> { - #[cfg(all(test, not(windows)))] - os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Rollback, dst_file_path); - if let Some(backup_path) = local_rollback_path { - // The commit immediately before this rollback renamed the staged - // xl.meta from the same directory as `backup_path` onto - // `dst_file_path`, proving both paths are on the same filesystem. - // Unix rename atomically replaces the committed destination; never - // unlink it first or an interrupted rollback could lose xl.meta. - std::fs::rename(backup_path, dst_file_path)?; - } else { - rollback_committed_rename_std(dst_file_path, None, rollback_data_dir)?; +struct StagedRollbackBackup(PathBuf); + +impl StagedRollbackBackup { + fn path(&self) -> &Path { + &self.0 + } +} + +impl RenameCommitContext<'_> { + fn rollback_metadata_std(&self, rollback_data_dir: Option) -> std::io::Result<()> { + if let Some(old_data_dir) = rollback_data_dir { + let Some(dst_parent) = self.destination_metadata.parent() else { + return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent")); + }; + let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP); + std::fs::rename(backup_path, self.destination_metadata)?; + } else { + remove_file_if_exists(self.destination_metadata)?; + } + Ok(()) + } + + fn rollback_non_inline_std(&self, rollback_data_dir: Option) -> std::io::Result<()> { + self.rollback_metadata_std(rollback_data_dir)?; + if let Some((_, new_data_path)) = self.data_paths { + remove_dir_all_if_exists(new_data_path)?; + } + Ok(()) + } + + fn rollback_inline_std( + &self, + rollback_data_dir: Option, + local_backup: Option<&StagedRollbackBackup>, + ) -> std::io::Result<()> { + #[cfg(all(test, not(windows)))] + os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Rollback, self.destination_metadata); + if let Some(backup) = local_backup { + // The preceding commit used this staged directory on the same + // filesystem. Never unlink xl.meta before its atomic replacement. + std::fs::rename(backup.path(), self.destination_metadata)?; + } else { + self.rollback_metadata_std(rollback_data_dir)?; + } + Ok(()) + } + + async fn restore_published_data_source(&self) -> Result<()> { + let Some((src_data_path, dst_data_path)) = self.data_paths else { + return Ok(()); + }; + let src_volume_dir = self.source_volume_dir; + let publication_root = self.publication_root; + let mutation_lease = self.mutation_lease.clone(); + if fs::symlink_metadata(src_data_path).await.is_ok() { + return Ok(()); + } + let result = + match os::rename_all_with_lease(dst_data_path, src_data_path, src_volume_dir, publication_root, mutation_lease).await + { + Ok(()) => Ok(()), + Err(DiskError::FileNotFound) => { + let source_exists = fs::symlink_metadata(src_data_path).await.is_ok(); + let destination_missing = matches!( + fs::symlink_metadata(dst_data_path).await, + Err(err) if err.kind() == ErrorKind::NotFound + ); + if source_exists && destination_missing { + Ok(()) + } else { + Err(DiskError::FileNotFound) + } + } + Err(err) => Err(err), + }; + if let Err(err) = &result { + warn!( + target: "rustfs_ecstore::disk::local", + event = EVENT_DISK_LOCAL_RENAME_REJECTED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + reason = "restore_staged_data_source_failed", + src_path = ?src_data_path, + dst_path = ?dst_data_path, + error = ?err, + "Failed to restore staged data after a metadata commit was rejected" + ); + } + result } - Ok(()) } #[cfg(any(not(windows), test))] @@ -249,61 +310,6 @@ async fn prepare_rename_destination_metadata( .map_err(DiskError::from) } -async fn restore_renamed_data_source( - src_volume_dir: &Path, - src_data_path: &Path, - dst_data_path: &Path, - publication_root: &os::PublicationRoot, - mutation_lease: Arc, -) -> Result<()> { - if fs::symlink_metadata(src_data_path).await.is_ok() { - return Ok(()); - } - let result = - match os::rename_all_with_lease(dst_data_path, src_data_path, src_volume_dir, publication_root, mutation_lease).await { - Ok(()) => Ok(()), - Err(DiskError::FileNotFound) => { - let source_exists = fs::symlink_metadata(src_data_path).await.is_ok(); - let destination_missing = matches!( - fs::symlink_metadata(dst_data_path).await, - Err(err) if err.kind() == ErrorKind::NotFound - ); - if source_exists && destination_missing { - Ok(()) - } else { - Err(DiskError::FileNotFound) - } - } - Err(err) => Err(err), - }; - if let Err(err) = &result { - warn!( - target: "rustfs_ecstore::disk::local", - event = EVENT_DISK_LOCAL_RENAME_REJECTED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_DISK_LOCAL, - reason = "restore_staged_data_source_failed", - src_path = ?src_data_path, - dst_path = ?dst_data_path, - error = ?err, - "Failed to restore staged data after a metadata commit was rejected" - ); - } - result -} - -async fn restore_published_data_source( - data_paths: Option<&(PathBuf, PathBuf)>, - src_volume_dir: &Path, - publication_root: &os::PublicationRoot, - mutation_lease: Arc, -) -> Result<()> { - let Some((src_data_path, dst_data_path)) = data_paths else { - return Ok(()); - }; - restore_renamed_data_source(src_volume_dir, src_data_path, dst_data_path, publication_root, mutation_lease).await -} - /// Proof produced only when the local rename returns at an existing access /// preflight, before metadata, backups, or object data can be published. #[derive(Debug)] @@ -493,6 +499,15 @@ impl LocalDisk { mutation_lease.clone(), ) .await?; + let mut commit = RenameCommitContext { + directory_guard: rename_commit_guard, + publication_root: &self.publication_root, + mutation_lease, + source_metadata: &src_file_path, + destination_metadata: &dst_file_path, + source_volume_dir: &src_volume_dir, + data_paths: has_data_dir_path.as_ref(), + }; if no_inline { // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta @@ -576,18 +591,18 @@ impl LocalDisk { // rename below to report through the existing rollback path. Payload // durability is kept by both strict and relaxed. let tmp_meta_write = { - let src_file_path = src_file_path.clone(); + let src_file_path = commit.source_metadata.to_path_buf(); let dst_file_path = dst_file_path.clone(); - let rename_commit_guard = rename_commit_guard.clone(); - let mutation_lease = mutation_lease.clone(); + let owned_directory_guard = commit.directory_guard.clone(); + let owned_mutation_lease = commit.mutation_lease.clone(); async move { - os::run_blocking_namespace_operation(mutation_lease, move || { + os::run_blocking_namespace_operation(owned_mutation_lease, move || { #[cfg(test)] run_owned_file_write_before_open(&src_file_path); let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard( &src_file_path, &dst_file_path, - &rename_commit_guard, + &owned_directory_guard, )?; prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?; Ok(prepared_metadata_source) @@ -613,14 +628,14 @@ impl LocalDisk { // sequential version did. let prepared_metadata_source = tmp_meta_res?; shard_sync_res?; - let rename_commit_guard = remove_dst_base_before_commit( + commit.directory_guard = remove_dst_base_before_commit( dst_path, - rename_commit_guard, + commit.directory_guard, src_file_parent, dst_file_parent, &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), + commit.publication_root, + commit.mutation_lease.clone(), ) .await?; if should_remove_staged_meta_before_commit(dst_path) { @@ -638,7 +653,7 @@ impl LocalDisk { if fi_healing && let Some((_, dst_data_path)) = has_data_dir_path.as_ref() && let Err(err) = self - .move_to_trash_with_namespace_owner(dst_data_path, true, false, Some(mutation_lease.clone())) + .move_to_trash_with_namespace_owner(dst_data_path, true, false, Some(commit.mutation_lease.clone())) .await { warn!( @@ -656,9 +671,9 @@ impl LocalDisk { src_data_path, dst_data_path, &skip_parent, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), + commit.publication_root, + &commit.directory_guard, + commit.mutation_lease.clone(), ) .await { @@ -673,13 +688,7 @@ impl LocalDisk { error = ?err, "Disk local rename flow failed" ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; + commit.restore_published_data_source().await?; return Err(err); } #[cfg(test)] @@ -704,13 +713,7 @@ impl LocalDisk { reason = "test_fail_before_old_metadata_backup", "Disk local rename flow failed before metadata commit" ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; + commit.restore_published_data_source().await?; return Err(DiskError::Unexpected); } @@ -728,25 +731,16 @@ impl LocalDisk { let backup_parent = dst_file_parent.join(old_data_dir.to_string()); #[cfg(not(windows))] if let Err(err) = os::make_dir_all(&backup_parent, &skip_parent).await { - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; + commit.restore_published_data_source().await?; return Err(err); } - let backup_path_guard = match rename_commit_guard.create_destination_directory_for_path_access(&backup_parent) { + let backup_path_guard = match commit + .directory_guard + .create_destination_directory_for_path_access(&backup_parent) + { Ok(guard) => guard, Err(err) => { - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; + commit.restore_published_data_source().await?; return Err(DiskError::from(to_file_error(err))); } }; @@ -754,13 +748,7 @@ impl LocalDisk { if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) { #[cfg(windows)] drop(backup_path_guard); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; + commit.restore_published_data_source().await?; return Err(err); } let backup_bytes = dst_buf.clone(); @@ -769,7 +757,7 @@ impl LocalDisk { // sync finish. A detached spawn_blocking writer could survive // cancellation and later truncate a newer transaction's // deterministic rollback backup. - let write_result = os::run_blocking_namespace_operation(mutation_lease.clone(), move || { + let write_result = os::run_blocking_namespace_operation(commit.mutation_lease.clone(), move || { #[cfg(test)] run_owned_file_write_before_open(&backup_path); backup_path_guard.write_file_for_path_access( @@ -792,13 +780,7 @@ impl LocalDisk { error = ?err, "Disk local rename flow failed" ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; + commit.restore_published_data_source().await?; return Err(err); } } @@ -816,9 +798,9 @@ impl LocalDisk { &src_file_path, &dst_file_path, &skip_parent, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), + commit.publication_root, + &commit.directory_guard, + commit.mutation_lease.clone(), ) .await { @@ -833,20 +815,11 @@ impl LocalDisk { error = ?err, "Disk local rename flow failed" ); - restore_published_data_source( - has_data_dir_path.as_ref(), - &src_volume_dir, - &self.publication_root, - mutation_lease.clone(), - ) - .await?; + commit.restore_published_data_source().await?; return Err(err); } - - let committed_new_data_path = has_data_dir_path.as_ref().map(|(_, dst_data_path)| dst_data_path.as_path()); if should_fail_after_metadata_commit(dst_path) { - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; + commit.rollback_non_inline_std(rollback_data_dir).map_err(to_file_error)?; return Err(DiskError::Unexpected); } @@ -866,13 +839,14 @@ impl LocalDisk { && let Some(parent) = dst_file_path.parent() { let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_commit_directory(parent, &rename_commit_guard, mutation_lease.clone(), None).await { + if let Err(err) = + os::fsync_commit_directory(parent, &commit.directory_guard, commit.mutation_lease.clone(), None).await + { rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, fsync_started, ); - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; + commit.rollback_non_inline_std(rollback_data_dir).map_err(to_file_error)?; // The commit rename changed the dst part inodes before this fsync // failed and rolled them back; drop any fd cached during that // window so readers re-open the restored inode (rustfs/backlog#1177). @@ -904,13 +878,12 @@ impl LocalDisk { break; } let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dir_with_owner(dir, Some(mutation_lease.clone())).await { + if let Err(err) = os::fsync_dir_with_owner(dir, Some(commit.mutation_lease.clone())).await { rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, fsync_started, ); - rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir) - .map_err(to_file_error)?; + commit.rollback_non_inline_std(rollback_data_dir).map_err(to_file_error)?; // Same post-commit rollback window as above — drop cached // dst part fds so readers re-open the restored inode // (rustfs/backlog#1177). @@ -934,7 +907,7 @@ impl LocalDisk { // complete. Do not retain the Windows object identity guard while // cleaning staging paths or invalidating cached descriptors. #[cfg(windows)] - drop(rename_commit_guard); + drop(commit.directory_guard); if let Some(src_file_path_parent) = src_file_path.parent() { if src_volume != super::super::RUSTFS_META_MULTIPART_BUCKET { @@ -986,7 +959,7 @@ impl LocalDisk { let dst_path_for_source = dst_path.to_string(); #[cfg(windows)] let source_parent = src_file_parent.to_path_buf(); - let rename_commit_guard_for_preparation = rename_commit_guard.clone(); + let rename_commit_guard_for_preparation = commit.directory_guard.clone(); let sync = durability.syncs_commit_metadata(); #[cfg(test)] run_inline_before_file_sync_admission(dst_path); @@ -1077,7 +1050,7 @@ impl LocalDisk { if sync { std::fs::File::open(&backup_path)?.sync_data()?; } - staged_rollback_path = Some(backup_path); + staged_rollback_path = Some(StagedRollbackBackup(backup_path)); } Ok::<_, std::io::Error>(( @@ -1093,9 +1066,10 @@ impl LocalDisk { )) }; let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() { - os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await + os::run_blocking_namespace_file_sync_operation(commit.mutation_lease.clone(), admission, prepare_inline_metadata) + .await } else { - os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await + os::run_blocking_namespace_operation(commit.mutation_lease.clone(), prepare_inline_metadata).await } .map_err(to_file_error) .map_err(DiskError::from); @@ -1120,28 +1094,30 @@ impl LocalDisk { } }; - let rename_commit_guard = remove_dst_base_before_commit( + commit.directory_guard = remove_dst_base_before_commit( dst_path, - rename_commit_guard, + commit.directory_guard, src_file_parent, dst_file_parent, &dst_volume_dir, - &self.publication_root, - mutation_lease.clone(), + commit.publication_root, + commit.mutation_lease.clone(), ) .await?; if should_remove_staged_meta_before_commit(dst_path) { drop(prepared_metadata_source); let remove_result = std::fs::remove_file(&src_file_path); - if let Some(backup_path) = local_rollback_path.as_deref() { + if let Some(backup_path) = local_rollback_path.as_ref().map(StagedRollbackBackup::path) { let _ = remove_file_if_exists(backup_path); } remove_result.map_err(to_file_error)?; return Err(DiskError::FileNotFound); } - if let (Some(rollback_data_dir), Some(staged_backup)) = (rollback_data_dir, local_rollback_path.as_deref()) { + if let (Some(rollback_data_dir), Some(staged_backup)) = + (rollback_data_dir, local_rollback_path.as_ref().map(StagedRollbackBackup::path)) + { let Some(dst_parent) = dst_file_path.parent() else { return Err(DiskError::other("missing object metadata parent")); }; @@ -1155,8 +1131,8 @@ impl LocalDisk { staged_backup, &backup_path, &dst_volume_dir, - &self.publication_root, - Some(mutation_lease.clone()), + commit.publication_root, + Some(commit.mutation_lease.clone()), ) .await { @@ -1178,7 +1154,8 @@ impl LocalDisk { { let fsync_started = rustfs_io_metrics::put_stage_timer(); if let Err(err) = - os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await + os::fsync_dir_with_namespace_file_sync_limit(backup_parent, commit.mutation_lease.clone(), admission) + .await { rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC, @@ -1202,14 +1179,14 @@ impl LocalDisk { &src_file_path, &dst_file_path, &dst_volume_dir, - &self.publication_root, - &rename_commit_guard, - mutation_lease.clone(), + commit.publication_root, + &commit.directory_guard, + commit.mutation_lease.clone(), ) .await }; if let Err(err) = commit_result { - if let Some(backup_path) = local_rollback_path.as_deref() { + if let Some(backup_path) = local_rollback_path.as_ref().map(StagedRollbackBackup::path) { let _ = remove_file_if_exists(backup_path); } for part_path in &invalidate_part_paths { @@ -1220,7 +1197,7 @@ impl LocalDisk { let post_commit = async { if should_fail_after_metadata_commit(dst_path) { - rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + commit.rollback_inline_std(rollback_data_dir, local_rollback_path.as_ref())?; return Err(std::io::Error::other("test fail after metadata commit")); } @@ -1229,15 +1206,19 @@ impl LocalDisk { && let Some(dst_parent) = dst_file_path.parent() { let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = - os::fsync_commit_directory(dst_parent, &rename_commit_guard, mutation_lease.clone(), Some(admission)) - .await + if let Err(err) = os::fsync_commit_directory( + dst_parent, + &commit.directory_guard, + commit.mutation_lease.clone(), + Some(admission), + ) + .await { rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, fsync_started, ); - rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?; + commit.rollback_inline_std(rollback_data_dir, local_rollback_path.as_ref())?; return Err(err); } rustfs_io_metrics::record_put_object_stage_duration_from( @@ -1263,17 +1244,14 @@ impl LocalDisk { } let fsync_started = rustfs_io_metrics::put_stage_timer(); if let Err(err) = - os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await + os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, commit.mutation_lease.clone(), admission) + .await { rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, fsync_started, ); - rollback_inline_metadata_commit_std( - &dst_file_path, - rollback_data_dir, - local_rollback_path.as_deref(), - )?; + commit.rollback_inline_std(rollback_data_dir, local_rollback_path.as_ref())?; return Err(err); } rustfs_io_metrics::record_put_object_stage_duration_from( @@ -1310,9 +1288,9 @@ impl LocalDisk { // The commit no longer has a rollback path. Release the Windows // object identity guard before best-effort staging cleanup. #[cfg(windows)] - drop(rename_commit_guard); + drop(commit.directory_guard); - if let Some(backup_path) = local_rollback_path.as_deref() { + if let Some(backup_path) = local_rollback_path.as_ref().map(StagedRollbackBackup::path) { let _ = remove_file_if_exists(backup_path); } diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index f47b99918..94f5374ae 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -16,11 +16,9 @@ //! operation families (P5 of the God-Object split, tracking backlog#815, //! issue backlog#820). //! -//! These are the metadata-fanout quorum accumulator, bitrot reader -//! scheduling/creation, shard-cost classification, and read-repair heal -//! dedup helpers relocated verbatim from `set_disk/read.rs`. Bodies are -//! byte-identical to the pre-move sources; only the module header -//! (`use super::*;` -> `use super::super::*;`) and item visibility change. +//! Bitrot reader setup, shard-cost classification, read repair and object +//! mutation primitives. Metadata scheduling and decisions have separate owners +//! in `metadata_read` and `metadata_quorum`; legacy callers retain adapters here. #[cfg(test)] use super::super::ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE; @@ -46,48 +44,61 @@ use super::super::{ GetCodecStreamingFallbackReason, GetObjectMetadataCacheEntry, HTTPPreconditions, HashAlgorithm, HealAdmissionResult, HealChannelPriority, HealRequestSource, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, MultipartWriteQuorumContext, OBJECT_OP_IGNORED_ERRS, ObjectOptions, ObjectPartInfo, OffsetDateTime, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, - RawFileInfo, ReadMultipleReq, ReadMultipleResp, ReadOptions, Result, SLASH_SEPARATOR, STORAGE_FORMAT_FILE, SetDisks, - SnapshotLeaseToken, StorageError, UpdateMetadataOpts, Uuid, build_inline_bitrot_readers_from_refs, - can_try_inline_data_shards_direct, capacity_scope_from_disks, codec_streaming_rollout_applies, coding, - collect_inline_data_shard_fileinfos_by_index_or_reason, current_dirty_generation, debug, disk, - file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info, inline_erasure_shard_file_offset, - inline_erasure_shard_size, is_get_metadata_data_read_early_stop_enabled, is_get_metadata_early_stop_bounded_fanout_enabled, - is_get_metadata_early_stop_enabled, is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling, - is_version_early_stop_enabled, issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure, - merge_file_meta_versions, object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs, - reduce_write_quorum_errs, send_heal_request_with_admission, should_prevent_write, to_object_err, - try_read_inline_data_shards_direct, warn, + RawFileInfo, ReadMultipleReq, ReadMultipleResp, Result, SLASH_SEPARATOR, STORAGE_FORMAT_FILE, SetDisks, SnapshotLeaseToken, + StorageError, UpdateMetadataOpts, Uuid, capacity_scope_from_disks, current_dirty_generation, debug, disk, + file_info_is_valid_for_metadata, info, is_object_dangling, issue3031_diag_enabled, join_all, join_errs, + log_multipart_write_quorum_failure, merge_file_meta_versions, path_join_buf, record_global_dirty_scope, + reduce_read_quorum_errs, reduce_write_quorum_errs, send_heal_request_with_admission, should_prevent_write, to_object_err, + warn, }; #[cfg(test)] +use super::super::{ReadOptions, coding}; +#[cfg(test)] pub(in crate::set_disk) use super::metadata_quorum::MetadataEarlyStopDecision; -pub(in crate::set_disk) use super::metadata_quorum::{ - MetadataQuorumAccumulator, is_metadata_fanout_ignored_error, metadata_early_stop_candidate_matches, +#[cfg(test)] +use super::metadata_quorum::MetadataQuorumAccumulator; +pub(in crate::set_disk) use super::metadata_quorum::metadata_early_stop_candidate_matches; +#[cfg(test)] +use super::metadata_read::MetadataReadResult; +#[cfg(test)] +use super::metadata_read::{ + ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, + ExpectedBatchReadVersionItem, data_read_early_stop_inline_body_miss_reason, map_batch_read_version_responses, + metadata_distribution_key, +}; +#[cfg(test)] +pub(in crate::set_disk) use super::metadata_read::{ + MetadataFanoutDiagnostics, MetadataFanoutObservation, bounded_metadata_fanout_order, non_inline_data_read_early_stop_allowed, }; #[cfg(test)] use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; #[cfg(test)] -use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH; -#[cfg(test)] use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD; +#[cfg(test)] use crate::diagnostics::get::{ GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, - GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, - GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, - GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, - GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_DIRECT_MEMORY, - GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READER_SETUP_DROP_PENDING, - GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT, - GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled, - record_get_stage_duration_if_enabled, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, GET_METADATA_RESPONSE_CORRUPT, + GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, + GET_METADATA_RESPONSE_VALID, +}; +#[cfg(test)] +use crate::diagnostics::get::{ + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH, GET_OBJECT_PATH_INTERNAL_META, + GET_OBJECT_PATH_LEGACY_DUPLEX, }; #[cfg(test)] use crate::diagnostics::get::{ GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, }; +use crate::diagnostics::get::{ + GET_OBJECT_PATH_DIRECT_MEMORY, GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, + GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, + GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled, record_get_stage_duration_if_enabled, +}; #[cfg(test)] use crate::disk::CHECK_PART_FILE_NOT_FOUND; use crate::disk::DiskAPI; @@ -95,7 +106,6 @@ use crate::disk::DiskAPI; use crate::disk::DiskOption; #[cfg(test)] use crate::disk::RUSTFS_META_TMP_BUCKET; -use crate::disk::disk_store::get_drive_metadata_timeout; #[cfg(test)] use crate::disk::endpoint::Endpoint; #[cfg(test)] @@ -103,10 +113,11 @@ use crate::disk::format::FormatV3; use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX; #[cfg(test)] use crate::disk::new_disk; +#[cfg(test)] +use crate::disk::{BatchReadVersionItem, BatchReadVersionResp}; use crate::disk::{ - BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk, - OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, - STORAGE_FORMAT_FILE_BACKUP, part_transaction_path, + DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, + PartTransactionAction, STORAGE_FORMAT_FILE_BACKUP, part_transaction_path, }; use crate::erasure::coding::BitrotReader; use crate::io_support::bitrot::ShardReader; @@ -122,9 +133,6 @@ use crate::storage_api_contracts::object::ObjectOperations; use futures::FutureExt as _; use futures::stream::{FuturesUnordered, StreamExt}; use metrics::counter; -use rustfs_io_metrics::internode_metrics::{ - INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP, -}; use std::{ collections::{HashMap, HashSet, VecDeque}, future::Future, @@ -136,353 +144,12 @@ use std::{ task::{Context, Poll}, time::{Duration, Instant}, }; - -fn metadata_metrics_path(bucket: &str) -> &'static str { - if crate::bucket::utils::is_meta_bucketname(bucket) { - GET_OBJECT_PATH_INTERNAL_META - } else { - GET_OBJECT_PATH_LEGACY_DUPLEX - } -} - -fn metadata_distribution_key(bucket: &str, object: &str) -> String { - [bucket, object].join("/") -} - -fn read_version_coalescing_enabled() -> bool { - let enabled = || { - rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE) - .is_some_and(|value| value.eq_ignore_ascii_case("auto") || value.eq_ignore_ascii_case("on")) - }; - - #[cfg(test)] - { - enabled() - } - - #[cfg(not(test))] - { - static ENABLED: OnceLock = OnceLock::new(); - *ENABLED.get_or_init(enabled) - } -} - -fn read_version_coalescing_delay() -> Duration { - #[cfg(test)] - { - let micros = rustfs_utils::get_env_u64( - ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, - DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, - ); - Duration::from_micros(micros) - } - - #[cfg(not(test))] - { - static DELAY: OnceLock = OnceLock::new(); - *DELAY.get_or_init(|| { - Duration::from_micros(rustfs_utils::get_env_u64( - ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, - DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, - )) - }) - } -} - -struct CoalescedReadVersionRequest { - item: BatchReadVersionItem, - tx: oneshot::Sender>, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -struct ExpectedBatchReadVersionItem { - path: String, - version_id: String, -} - -impl From<&BatchReadVersionItem> for ExpectedBatchReadVersionItem { - fn from(item: &BatchReadVersionItem) -> Self { - Self { - path: item.path.clone(), - version_id: item.version_id.clone(), - } - } -} - -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -struct ReadVersionCoalescerKey { - disk: usize, - incl_free_versions: bool, - read_data: bool, - healing: bool, -} - -impl ReadVersionCoalescerKey { - fn new(disk: &DiskStore, opts: &ReadOptions) -> Self { - Self { - disk: Arc::as_ptr(disk) as usize, - incl_free_versions: opts.incl_free_versions, - read_data: opts.read_data, - healing: opts.healing, - } - } -} - -#[derive(Default)] -struct ReadVersionCoalescer { - lanes: HashMap>, -} - -fn read_version_coalescer() -> &'static Mutex { - static COALESCER: OnceLock> = OnceLock::new(); - COALESCER.get_or_init(|| Mutex::new(ReadVersionCoalescer::default())) -} - -fn record_read_version_coalescer_event(event: &'static str, item_count: usize) { - counter!( - METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL, - "event" => event, - "item_count" => item_count.to_string() - ) - .increment(1); -} - -fn batch_read_version_stage_timer() -> Option { - rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now) -} - -fn record_batch_read_version_stage(stage: &'static str, started_at: Option) { - if let Some(started_at) = started_at { - crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed()); - } -} - -async fn read_version_via_coalescer( - disk: DiskStore, - org_bucket: &str, - bucket: &str, - object: &str, - version_id: &str, - opts: &ReadOptions, - allow_coalescing: bool, -) -> disk::error::Result { - if !allow_coalescing || !read_version_coalescing_enabled() { - return disk.read_version(org_bucket, bucket, object, version_id, opts).await; - } - if !matches!(disk.as_ref(), Disk::Remote(_)) { - record_read_version_coalescer_event("bypass_non_remote", 1); - return disk.read_version(org_bucket, bucket, object, version_id, opts).await; - } - - let (tx, rx) = oneshot::channel(); - let item = BatchReadVersionItem { - org_volume: org_bucket.to_string(), - volume: bucket.to_string(), - path: object.to_string(), - version_id: version_id.to_string(), - }; - let lane_key = ReadVersionCoalescerKey::new(&disk, opts); - let pending = { - let mut coalescer = read_version_coalescer().lock().await; - let lane = coalescer.lanes.entry(lane_key).or_default(); - let schedule_delayed_flush = lane.is_empty(); - lane.push(CoalescedReadVersionRequest { item, tx }); - if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS { - coalescer.lanes.remove(&lane_key) - } else if schedule_delayed_flush { - let disk = disk.clone(); - let task_opts = *opts; - tokio::spawn(async move { - tokio::time::sleep(read_version_coalescing_delay()).await; - flush_read_version_coalescer_lane(lane_key, disk, task_opts).await; - }); - None - } else { - None - } - }; - - if let Some(pending) = pending { - flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await; - } - - let wait_started = batch_read_version_stage_timer(); - let response = rx - .await - .unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed"))); - record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, wait_started); - response -} - -async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) { - let pending = { - let mut coalescer = read_version_coalescer().lock().await; - coalescer.lanes.remove(&lane_key).unwrap_or_default() - }; - flush_read_version_coalescer_pending(lane_key, disk, opts, pending).await; -} - -async fn flush_read_version_coalescer_pending( - lane_key: ReadVersionCoalescerKey, - disk: DiskStore, - opts: ReadOptions, - pending: Vec, -) { - if pending.is_empty() { - return; - } - - // Only the #[cfg(test)] counter-recording block below reads this. - #[cfg(not(test))] - let _ = lane_key; - #[cfg(test)] - { - let mut observed_paths = HashSet::new(); - for request in &pending { - if observed_paths.insert(request.item.path.as_str()) { - disk_call_counters::record(&request.item.path, disk_call_counters::KIND_BATCH_READ_VERSION, lane_key.disk); - } - } - } - - let mut senders = Vec::with_capacity(pending.len()); - let mut items = Vec::with_capacity(pending.len()); - for request in pending { - senders.push(request.tx); - items.push(request.item); - } - - let expected_items = items.iter().map(ExpectedBatchReadVersionItem::from).collect::>(); - record_read_version_coalescer_event("attempted_batch", items.len()); - let result = - match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts })) - .await - { - Ok(result) => result, - Err(_) => Err(DiskError::Timeout), - }; - match result { - Ok(responses) => { - let map_started = batch_read_version_stage_timer(); - let results = map_batch_read_version_responses(&expected_items, responses); - record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP, map_started); - for (tx, result) in senders.into_iter().zip(results) { - let _ = tx.send(result); - } - } - Err(err) => { - let message = err.to_string(); - for tx in senders { - let _ = tx.send(Err(DiskError::other(message.clone()))); - } - } - } -} - -fn map_batch_read_version_responses( - expected_items: &[ExpectedBatchReadVersionItem], - responses: Vec, -) -> Vec> { - let mut results = (0..expected_items.len()) - .map(|_| Err(DiskError::other("coalesced read_version response missing"))) - .collect::>(); - let mut seen = vec![false; expected_items.len()]; - for response in responses { - let Some(expected) = expected_items.get(response.index) else { - continue; - }; - let Some(slot) = results.get_mut(response.index) else { - continue; - }; - if seen[response.index] { - *slot = Err(DiskError::other("coalesced read_version response duplicate index")); - continue; - } - seen[response.index] = true; - if response.path != expected.path || response.version_id != expected.version_id { - *slot = Err(DiskError::other("coalesced read_version response identity mismatch")); - } else { - *slot = if response.success { - Ok(response.file_info) - } else { - Err(batch_read_version_response_error(response.error_code, response.error)) - }; - } - } - results -} - -fn batch_read_version_response_error(error_code: u32, error: String) -> DiskError { - match DiskError::from_u32(error_code) { - Some(DiskError::Io(_)) | None => DiskError::other(error), - Some(error) => error, - } -} - -pub(in crate::set_disk) fn bounded_metadata_fanout_order( - bucket: &str, - object: &str, - total_disks: usize, - default_parity_count: usize, -) -> Vec { - let fallback_order = || (0..total_disks).collect::>(); - if default_parity_count == 0 || default_parity_count >= total_disks { - return fallback_order(); - } - - let data_blocks = total_disks - default_parity_count; - let distribution_key = metadata_distribution_key(bucket, object); - let distribution = FileInfo::new(&distribution_key, data_blocks, default_parity_count) - .erasure - .distribution; - if distribution.len() != total_disks { - return fallback_order(); - } - - let mut order = Vec::with_capacity(total_disks); - for block_index in 1..=data_blocks { - let Some(disk_index) = distribution - .iter() - .position(|distributed_block| *distributed_block == block_index) - else { - return fallback_order(); - }; - order.push(disk_index); - } - order.extend( - distribution - .iter() - .enumerate() - .filter_map(|(disk_index, block_index)| (*block_index > data_blocks).then_some(disk_index)), - ); - order -} use tokio::io::{AsyncRead, ReadBuf}; -use tokio::sync::{Mutex, RwLock, oneshot}; +use tokio::sync::{RwLock, oneshot}; use tokio::task::JoinSet; -struct AbortOnDropJoinHandle(tokio::task::JoinHandle); - -impl Future for AbortOnDropJoinHandle { - type Output = std::result::Result; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - Pin::new(&mut self.0).poll(cx) - } -} - -impl Drop for AbortOnDropJoinHandle { - fn drop(&mut self) { - self.0.abort(); - } -} - pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read"; pub(in crate::set_disk) const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP"; -const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE"; -const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS"; -const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200; -const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total"; pub(crate) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE"; /// Default reader-setup strategy for the GET read path (rustfs/backlog#1215, /// #1159, #923). @@ -570,134 +237,6 @@ pub(in crate::set_disk) fn codec_streaming_reader_setup_fallback_reason( (missing_shards > 0).then_some(GetCodecStreamingFallbackReason::ReadQuorumNotSafe) } -#[derive(Clone, Copy, Debug)] -pub(in crate::set_disk) struct MetadataFanoutObservation { - pub(in crate::set_disk) outcome: &'static str, - pub(in crate::set_disk) elapsed: Duration, - pub(in crate::set_disk) valid: bool, - pub(in crate::set_disk) ignored: bool, -} - -impl MetadataFanoutObservation { - pub(in crate::set_disk) fn from_file_info(file_info: &FileInfo, elapsed: Duration) -> Self { - if file_info_is_valid_for_metadata(file_info) { - Self { - outcome: GET_METADATA_RESPONSE_VALID, - elapsed, - valid: true, - ignored: false, - } - } else { - Self { - outcome: GET_METADATA_RESPONSE_ERROR, - elapsed, - valid: false, - ignored: false, - } - } - } - - pub(in crate::set_disk) fn from_error(err: &DiskError, elapsed: Duration) -> Self { - Self { - outcome: classify_metadata_response_error(err), - elapsed, - valid: false, - ignored: is_metadata_fanout_ignored_error(err), - } - } -} - -#[derive(Clone, Debug, Default)] -pub(in crate::set_disk) struct MetadataFanoutDiagnostics { - pub(in crate::set_disk) fanout_duration: Duration, - pub(in crate::set_disk) observations: Vec, -} - -impl MetadataFanoutDiagnostics { - pub(in crate::set_disk) fn new(fanout_duration: Duration, observations: Vec) -> Self { - Self { - fanout_duration, - observations, - } - } - - pub(in crate::set_disk) fn total_responses(&self) -> usize { - self.observations.len() - } - - pub(in crate::set_disk) fn valid_responses(&self) -> usize { - self.observations.iter().filter(|observation| observation.valid).count() - } - - pub(in crate::set_disk) fn ignored_responses(&self) -> usize { - self.observations.iter().filter(|observation| observation.ignored).count() - } - - pub(in crate::set_disk) fn non_valid_responses(&self) -> usize { - self.total_responses().saturating_sub(self.valid_responses()) - } - - pub(in crate::set_disk) fn first_response_latency(&self) -> Option { - self.observations.iter().map(|observation| observation.elapsed).min() - } - - pub(in crate::set_disk) fn first_valid_response_latency(&self) -> Option { - self.observations - .iter() - .filter(|observation| observation.valid) - .map(|observation| observation.elapsed) - .min() - } - - pub(in crate::set_disk) fn slowest_response_latency(&self) -> Option { - self.observations.iter().map(|observation| observation.elapsed).max() - } - - pub(in crate::set_disk) fn quorum_candidate_latency(&self, read_quorum: usize) -> Option { - if read_quorum == 0 { - return Some(Duration::ZERO); - } - - let mut valid_latencies = self - .observations - .iter() - .filter(|observation| observation.valid) - .map(|observation| observation.elapsed) - .collect::>(); - valid_latencies.sort_unstable(); - valid_latencies.get(read_quorum.saturating_sub(1)).copied() - } - - pub(in crate::set_disk) fn record(&self, path: &'static str) { - rustfs_io_metrics::record_get_object_metadata_fanout_duration(path, self.fanout_duration.as_secs_f64()); - if let Some(latency) = self.first_response_latency() { - rustfs_io_metrics::record_get_object_first_metadata_response_latency(path, latency.as_secs_f64()); - } - if let Some(latency) = self.first_valid_response_latency() { - rustfs_io_metrics::record_get_object_first_valid_metadata_response_latency(path, latency.as_secs_f64()); - } - if let Some(latency) = self.slowest_response_latency() { - rustfs_io_metrics::record_get_object_slowest_metadata_response_latency(path, latency.as_secs_f64()); - } - rustfs_io_metrics::record_get_object_metadata_fanout_shape( - path, - self.total_responses(), - self.valid_responses(), - self.ignored_responses(), - self.non_valid_responses(), - ); - for observation in &self.observations { - rustfs_io_metrics::record_get_object_metadata_response(path, observation.outcome); - } - } - - pub(in crate::set_disk) fn record_quorum_candidate_latency(&self, path: &'static str, read_quorum: usize) { - if let Some(latency) = self.quorum_candidate_latency(read_quorum) { - rustfs_io_metrics::record_get_object_quorum_reached_latency(path, latency.as_secs_f64()); - } - } -} - #[derive(Clone, Debug)] pub(in crate::set_disk) enum MetadataCacheLookup { Hit(Arc), @@ -705,205 +244,6 @@ pub(in crate::set_disk) enum MetadataCacheLookup { RejectedInsufficientQuorum, } -pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason( - bucket: &str, - object: &str, - candidate: &FileInfo, - parts_metadata: &[FileInfo], - disks: &[Option], -) -> Option<&'static str> { - if let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(candidate) { - return Some(reason); - } - - let Ok(erasure) = coding::Erasure::try_new_with_options( - candidate.erasure.data_blocks, - candidate.erasure.parity_blocks, - candidate.erasure.block_size, - candidate.uses_legacy_checksum, - ) else { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY); - }; - let data_files = - match collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, candidate, erasure.data_shards, |index| { - disks.get(index).is_some_and(Option::is_some) - }) { - Ok(data_files) => data_files, - Err(reason) => return Some(reason), - }; - - let Some(part) = candidate.parts.first() else { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE); - }; - let Ok(object_size) = usize::try_from(candidate.size) else { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); - }; - let checksum_info = candidate.erasure.get_checksum_info(part.number); - let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S { - HashAlgorithm::HighwayHash256SLegacy - } else { - checksum_info.algorithm - }; - let read_length = inline_erasure_shard_file_offset( - 0, - object_size, - object_size, - candidate.erasure.block_size, - erasure.data_shards, - candidate.uses_legacy_checksum, - ); - let shard_size = inline_erasure_shard_size(candidate.erasure.block_size, erasure.data_shards, candidate.uses_legacy_checksum); - let Ok(mut readers) = - build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await - else { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY); - }; - - match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await { - Some(body) if body.len() == object_size => None, - _ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY), - } -} - -fn data_read_early_stop_inline_candidate_miss_reason(candidate: &FileInfo) -> Option<&'static str> { - // `inline_data` excludes remote objects; this diagnostic reports them separately. - if !rustfs_utils::http::contains_key_str(&candidate.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA) { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE); - } - if candidate.is_compressed() - || candidate - .metadata - .keys() - .any(|key| rustfs_utils::http::is_object_encryption_marker(key)) - { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED); - } - if candidate.is_remote() { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE); - } - if candidate.deleted { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED); - } - if candidate.size <= 0 { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); - } - if candidate.parts.len() != 1 { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE); - } - if !candidate.has_valid_erasure_geometry() { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY); - } - - let Ok(object_size) = usize::try_from(candidate.size) else { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); - }; - if candidate.parts.first().is_none_or(|part| part.size != object_size) { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE); - } - if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) { - return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); - } - None -} - -pub(in crate::set_disk) fn non_inline_data_read_candidate_is_safe(candidate: &FileInfo) -> bool { - if candidate.inline_data() - || candidate.is_compressed() - || candidate.is_remote() - || candidate - .metadata - .keys() - .any(|key| rustfs_utils::http::is_object_encryption_marker(key)) - || candidate.parts.len() != 1 - { - return false; - } - candidate.has_valid_erasure_geometry() -} - -pub(in crate::set_disk) fn late_materialization_candidate_is_safe(candidate: &FileInfo) -> bool { - non_inline_data_read_candidate_is_safe(candidate) - && candidate.size > 512 * 1024 - && object_fits_single_block(candidate.size, candidate.erasure.block_size) -} - -pub(in crate::set_disk) fn non_inline_data_read_early_stop_allowed(read_data: bool, bucket: &str, object: &str) -> bool { - read_data && is_get_metadata_non_inline_data_read_early_stop_enabled() && !codec_streaming_rollout_applies(bucket, object) -} - -const NON_INLINE_SINGLE_PENDING_HEDGE_DELAY: Duration = Duration::from_millis(100); - -fn data_read_inline_missing_shards_are_pending( - candidate: &FileInfo, - parts_metadata: &[FileInfo], - errors: &[Option], - disks: &[Option], - fanout_order: &[usize], - scheduled_fanout_len: usize, -) -> bool { - let Ok(erasure) = coding::Erasure::try_new_with_options( - candidate.erasure.data_blocks, - candidate.erasure.parity_blocks, - candidate.erasure.block_size, - candidate.uses_legacy_checksum, - ) else { - return false; - }; - let distribution = &candidate.erasure.distribution; - let mut data_shards_seen_or_pending = vec![false; erasure.data_shards]; - let mut missing_pending_data_shards = 0usize; - - for (disk_index, file_info) in parts_metadata.iter().enumerate() { - let Some(&block_index) = distribution.get(disk_index) else { - return false; - }; - if block_index == 0 || block_index > erasure.data_shards { - continue; - } - if !disks.get(disk_index).is_some_and(Option::is_some) { - return false; - } - - let data_slot = block_index - 1; - if file_info.name.is_empty() { - let scheduled_and_not_failed = fanout_order - .get(..scheduled_fanout_len) - .is_some_and(|scheduled_disks| scheduled_disks.contains(&disk_index)) - && errors.get(disk_index).is_some_and(Option::is_none); - if scheduled_and_not_failed { - data_shards_seen_or_pending[data_slot] = true; - missing_pending_data_shards = missing_pending_data_shards.saturating_add(1); - continue; - } - return false; - } - if file_info.erasure.index != block_index - || !file_info.has_valid_erasure_geometry() - || !metadata_early_stop_candidate_matches(file_info, candidate) - || file_info.data.as_ref().is_none_or(|data| data.is_empty()) - { - return false; - } - data_shards_seen_or_pending[data_slot] = true; - } - - missing_pending_data_shards > 0 && data_shards_seen_or_pending.into_iter().all(|seen_or_pending| seen_or_pending) -} - -pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str { - match err { - DiskError::FileNotFound | DiskError::VolumeNotFound => GET_METADATA_RESPONSE_NOT_FOUND, - DiskError::FileVersionNotFound => GET_METADATA_RESPONSE_VERSION_NOT_FOUND, - DiskError::DiskNotFound => GET_METADATA_RESPONSE_DISK_NOT_FOUND, - DiskError::FileCorrupt | DiskError::CorruptedFormat | DiskError::CorruptedBackend | DiskError::OutdatedXLMeta => { - GET_METADATA_RESPONSE_CORRUPT - } - DiskError::Timeout => GET_METADATA_RESPONSE_TIMEOUT, - DiskError::FaultyDisk | DiskError::FaultyRemoteDisk => GET_METADATA_RESPONSE_IGNORED, - _ => GET_METADATA_RESPONSE_ERROR, - } -} - pub(in crate::set_disk) fn is_confirmed_missing_part_error(err: Option<&str>) -> bool { let Some(err) = err else { return false; @@ -2451,37 +1791,6 @@ where // metadata cache) stays in read.rs and reaches these through the SetDisks core. // =========================================================================== -pub(in crate::set_disk) fn should_allow_metadata_early_stop( - read_data: bool, - version_id: &str, - healing: bool, - incl_free_versions: bool, -) -> bool { - if read_data && !is_get_metadata_data_read_early_stop_enabled() { - return false; - } - - (is_get_metadata_early_stop_enabled() && version_id.is_empty() && !healing && !incl_free_versions) - || (is_version_early_stop_enabled() && !version_id.is_empty() && !healing && !incl_free_versions) -} - -/// Final gate for the metadata early-stop fast path. -/// -/// `caller_allows_early_stop=false` unconditionally forces the full quorum -/// fanout so read-before-write callers (object tagging) get the complete -/// online-disk set as their write target; the early-stop subset would only -/// carry read quorum and fail write quorum (backlog#872 regression). -pub(in crate::set_disk) fn metadata_early_stop_permitted( - caller_allows_early_stop: bool, - observe: bool, - read_data: bool, - version_id: &str, - healing: bool, - incl_free_versions: bool, -) -> bool { - caller_allows_early_stop && observe && should_allow_metadata_early_stop(read_data, version_id, healing, incl_free_versions) -} - impl SetDisks { pub(in crate::set_disk) async fn read_parts( disks: &[Option], @@ -2548,7 +1857,7 @@ impl SetDisks { healing: bool, incl_free_versions: bool, ) -> disk::error::Result<(Vec, Vec>)> { - let (ress, errors, _) = Self::read_all_fileinfo_inner( + let result = Self::read_metadata_inner( disks, org_bucket, bucket, @@ -2563,9 +1872,11 @@ impl SetDisks { false, ) .await?; - Ok((ress, errors)) + let (metadata, errors, _) = result.into_legacy(); + Ok((metadata, errors)) } + #[cfg(test)] #[allow(clippy::too_many_arguments)] pub(in crate::set_disk) async fn read_all_fileinfo_observed( disks: &[Option], @@ -2579,7 +1890,7 @@ impl SetDisks { caller_allows_early_stop: bool, default_parity_count: usize, ) -> disk::error::Result<(Vec, Vec>, MetadataFanoutDiagnostics)> { - Self::read_all_fileinfo_inner( + Self::read_metadata_inner( disks, org_bucket, bucket, @@ -2594,8 +1905,10 @@ impl SetDisks { false, ) .await + .map(MetadataReadResult::into_legacy) } + #[cfg(test)] #[allow(clippy::too_many_arguments)] pub(in crate::set_disk) async fn read_all_fileinfo_observed_for_get_object( disks: &[Option], @@ -2608,7 +1921,7 @@ impl SetDisks { caller_allows_early_stop: bool, default_parity_count: usize, ) -> disk::error::Result<(Vec, Vec>, MetadataFanoutDiagnostics)> { - Self::read_all_fileinfo_inner( + Self::read_metadata_inner( disks, org_bucket, bucket, @@ -2623,497 +1936,7 @@ impl SetDisks { true, ) .await - } - - #[allow(clippy::too_many_arguments)] - async fn read_all_fileinfo_inner( - disks: &[Option], - org_bucket: &str, - bucket: &str, - object: &str, - version_id: &str, - read_data: bool, - healing: bool, - incl_free_versions: bool, - observe: bool, - // When false, the caller opts out of the early-stop fast path even for - // otherwise-eligible reads. Read-before-write callers (e.g. object - // tagging) must set this so the returned online-disk set reflects the - // full quorum fanout rather than the early-stop subset — writing to the - // subset would fail write quorum (backlog#872 regression). - caller_allows_early_stop: bool, - default_parity_count: usize, - allow_coalescing: bool, - ) -> disk::error::Result<(Vec, Vec>, MetadataFanoutDiagnostics)> { - let early_stop_enabled = - caller_allows_early_stop && observe && (is_get_metadata_early_stop_enabled() || is_version_early_stop_enabled()); - let allow_early_stop = - metadata_early_stop_permitted(caller_allows_early_stop, observe, read_data, version_id, healing, incl_free_versions); - if allow_early_stop { - return Self::read_all_fileinfo_early_stop( - disks, - org_bucket, - bucket, - object, - version_id, - read_data, - healing, - incl_free_versions, - non_inline_data_read_early_stop_allowed(read_data, bucket, object), - default_parity_count, - allow_coalescing, - ) - .await; - } - if early_stop_enabled { - let metrics_path = metadata_metrics_path(bucket); - rustfs_io_metrics::record_get_object_metadata_early_stop_miss( - metrics_path, - GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, - ); - rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0); - } - - Self::read_all_fileinfo_full_wait( - disks, - org_bucket, - bucket, - object, - version_id, - read_data, - healing, - incl_free_versions, - observe, - allow_coalescing, - ) - .await - } - - #[allow(clippy::too_many_arguments)] - async fn read_all_fileinfo_full_wait( - disks: &[Option], - org_bucket: &str, - bucket: &str, - object: &str, - version_id: &str, - read_data: bool, - healing: bool, - incl_free_versions: bool, - observe: bool, - allow_coalescing: bool, - ) -> disk::error::Result<(Vec, Vec>, MetadataFanoutDiagnostics)> { - let fanout_start = observe.then(Instant::now); - let mut ress = Vec::with_capacity(disks.len()); - let mut errors = Vec::with_capacity(disks.len()); - let mut observations = observe.then(|| Vec::with_capacity(disks.len())); - let scheduled_count = disks.len(); - let opts = ReadOptions { - incl_free_versions, - read_data, - healing, - }; - let org_bucket: Arc = Arc::from(org_bucket); - let bucket: Arc = Arc::from(bucket); - let object: Arc = Arc::from(object); - let version_id: Arc = Arc::from(version_id); - let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data); - let futures = disks.iter().enumerate().map(|(disk_index, disk)| { - let disk = disk.clone(); - let task_opts = opts; - let org_bucket = org_bucket.clone(); - let bucket = bucket.clone(); - let object = object.clone(); - let version_id = version_id.clone(); - let slowtail_fault = slowtail_fault.clone(); - AbortOnDropJoinHandle(tokio::spawn(async move { - let response_start = observe.then(Instant::now); - let result = if let Some(disk) = disk { - Self::record_read_version_call(&object, disk_index); - if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) { - Self::record_metadata_slowtail_fault(&object, disk_index); - tokio::time::sleep(delay).await; - } - read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing) - .await - } else { - Err(DiskError::DiskNotFound) - }; - let elapsed = response_start.map(|start| start.elapsed()); - (result, elapsed) - })) - }); - - // Wait for all futures to complete - let results = join_all(futures).await; - - for join_result in results { - match join_result { - Ok((res, elapsed)) => match res { - Ok(file_info) => { - if let (Some(observations), Some(elapsed)) = (&mut observations, elapsed) { - observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed)); - } - ress.push(file_info); - errors.push(None); - } - Err(e) => { - if let (Some(observations), Some(elapsed)) = (&mut observations, elapsed) { - observations.push(MetadataFanoutObservation::from_error(&e, elapsed)); - } - ress.push(FileInfo::default()); - errors.push(Some(e)); - } - }, - Err(_join_err) => { - // A spawned task panicked — treat as unexpected disk error - if let Some(observations) = &mut observations { - observations.push(MetadataFanoutObservation::from_error(&DiskError::Unexpected, Duration::ZERO)); - } - ress.push(FileInfo::default()); - errors.push(Some(DiskError::Unexpected)); - } - } - } - let diagnostics = match (fanout_start, observations) { - (Some(fanout_start), Some(observations)) => MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations), - _ => MetadataFanoutDiagnostics::default(), - }; - if observe { - rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle( - metadata_metrics_path(bucket.as_ref()), - scheduled_count, - scheduled_count, - 0, - ); - } - Ok((ress, errors, diagnostics)) - } - - #[allow(clippy::too_many_arguments)] - async fn read_all_fileinfo_early_stop( - disks: &[Option], - org_bucket: &str, - bucket: &str, - object: &str, - version_id: &str, - read_data: bool, - healing: bool, - incl_free_versions: bool, - allow_non_inline_data_read_early_stop: bool, - default_parity_count: usize, - allow_coalescing: bool, - ) -> disk::error::Result<(Vec, Vec>, MetadataFanoutDiagnostics)> { - let fanout_start = Instant::now(); - let mut ress = vec![FileInfo::default(); disks.len()]; - let mut errors = vec![None; disks.len()]; - let mut observations = Vec::with_capacity(disks.len()); - let mut accumulator = - MetadataQuorumAccumulator::new(disks.len(), default_parity_count, true).with_requested_version_id(version_id); - let opts = ReadOptions { - incl_free_versions, - read_data, - healing, - }; - let org_bucket: Arc = Arc::from(org_bucket); - let bucket: Arc = Arc::from(bucket); - let object: Arc = Arc::from(object); - let version_id: Arc = Arc::from(version_id); - let metrics_path = metadata_metrics_path(bucket.as_ref()); - let mut join_set = JoinSet::new(); - let bounded_fanout = is_get_metadata_early_stop_bounded_fanout_enabled(); - let fanout_order = if bounded_fanout { - bounded_metadata_fanout_order(bucket.as_ref(), object.as_ref(), disks.len(), default_parity_count) - } else { - Vec::new() - }; - let mut next_fanout_index = 0usize; - let mut scheduled_count = 0usize; - let mut force_full_wait = false; - let mut final_miss_reason_override = None; - let mut non_inline_candidate_eligible = None; - let mut single_pending_hedge_deadline = None; - let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data); - let spawn_read_version = - |join_set: &mut JoinSet<(usize, disk::error::Result, Duration)>, index: usize, disk: Option| { - let task_opts = opts; - let org_bucket = org_bucket.clone(); - let bucket = bucket.clone(); - let object = object.clone(); - let version_id = version_id.clone(); - let slowtail_fault = slowtail_fault.clone(); - join_set.spawn(async move { - let response_start = Instant::now(); - let result = if let Some(disk) = disk { - #[allow(clippy::let_unit_value)] - let _fanout_task_guard = Self::rename_fanout_task_guard(&object); - Self::record_read_version_call(&object, index); - #[cfg(test)] - Self::read_version_fanout_barrier(&object, index).await; - if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) { - Self::record_metadata_slowtail_fault(&object, index); - tokio::time::sleep(delay).await; - } - read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing) - .await - } else { - Err(DiskError::DiskNotFound) - }; - (index, result, response_start.elapsed()) - }); - }; - - if bounded_fanout { - let initial_target = accumulator.default_write_quorum().min(disks.len()); - while next_fanout_index < initial_target { - let disk_index = fanout_order[next_fanout_index]; - if let Some(disk) = disks.get(disk_index).cloned() { - spawn_read_version(&mut join_set, disk_index, disk); - scheduled_count = scheduled_count.saturating_add(1); - } - next_fanout_index = next_fanout_index.saturating_add(1); - } - } else { - for (index, disk) in disks.iter().cloned().enumerate() { - spawn_read_version(&mut join_set, index, disk); - scheduled_count = scheduled_count.saturating_add(1); - } - } - - loop { - let mut defer_pending_inline_data_shard = false; - let result = if let Some(deadline) = single_pending_hedge_deadline.take() { - tokio::select! { - result = join_set.join_next() => result, - _ = async { - tokio::time::sleep_until(deadline).await; - #[cfg(test)] - rename_fanout_barrier::checkpoint( - object.as_ref(), - 0, - rename_fanout_barrier::PHASE_NON_INLINE_HEDGE_TIMER, - ) - .await; - } => { - if bounded_fanout - && !force_full_wait - && join_set.len() == 1 - && non_inline_candidate_eligible == Some(true) - && !accumulator.candidate_has_read_reserve() - && next_fanout_index < disks.len() - { - while next_fanout_index < disks.len() { - let disk_index = fanout_order[next_fanout_index]; - next_fanout_index = next_fanout_index.saturating_add(1); - if let Some(disk) = disks.get(disk_index).cloned() { - spawn_read_version(&mut join_set, disk_index, disk); - scheduled_count = scheduled_count.saturating_add(1); - break; - } - } - } - continue; - } - } - } else { - join_set.join_next().await - }; - let Some(result) = result else { break }; - match result { - Ok((index, res, elapsed)) => match res { - Ok(file_info) => { - observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed)); - if allow_non_inline_data_read_early_stop { - accumulator.observe_file_info_at(index, &file_info); - } else { - accumulator.observe_file_info(&file_info); - } - if allow_non_inline_data_read_early_stop && non_inline_candidate_eligible.is_none() { - non_inline_candidate_eligible = - accumulator.candidate.as_ref().map(non_inline_data_read_candidate_is_safe); - } - if bounded_fanout - && read_data - && !force_full_wait - && let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(&file_info) - && !(non_inline_candidate_eligible == Some(true) - && reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE) - { - force_full_wait = true; - final_miss_reason_override.get_or_insert(reason); - } - if let Some(slot) = ress.get_mut(index) { - *slot = file_info; - } - } - Err(err) => { - observations.push(MetadataFanoutObservation::from_error(&err, elapsed)); - accumulator.observe_error(&err); - if let Some(slot) = errors.get_mut(index) { - *slot = Some(err); - } - } - }, - Err(_) => { - let err = DiskError::Unexpected; - observations.push(MetadataFanoutObservation::from_error(&err, fanout_start.elapsed())); - accumulator.observe_error(&err); - } - } - - if !force_full_wait - && let Some(decision) = accumulator - .early_stop_decision() - .or_else(|| accumulator.version_early_stop_decision()) - { - let should_return_early = if read_data { - match accumulator.candidate.as_ref() { - Some(_candidate) if non_inline_candidate_eligible == Some(true) => { - accumulator.candidate_has_read_reserve() - } - Some(candidate) => match data_read_early_stop_inline_body_miss_reason( - bucket.as_ref(), - object.as_ref(), - candidate, - &ress, - disks, - ) - .await - { - None => true, - Some(reason) => { - final_miss_reason_override = Some(reason); - if bounded_fanout - && reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD - && data_read_inline_missing_shards_are_pending( - candidate, - &ress, - &errors, - disks, - &fanout_order, - next_fanout_index, - ) - { - defer_pending_inline_data_shard = true; - } else { - force_full_wait = true; - } - false - } - }, - None => { - force_full_wait = true; - final_miss_reason_override = Some(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM); - false - } - } - } else { - true - }; - - if should_return_early { - let saved_responses = if bounded_fanout { - disks.len().saturating_sub(observations.len()) - } else { - join_set.len() - }; - join_set.abort_all(); - rustfs_io_metrics::record_get_object_metadata_early_stop_hit(metrics_path, decision.reason); - rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, saved_responses); - let mut cancelled_count = 0usize; - while let Some(join_result) = join_set.join_next().await { - match join_result { - Err(join_error) if join_error.is_cancelled() => { - cancelled_count = cancelled_count.saturating_add(1); - } - _ => {} - } - } - rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle( - metrics_path, - scheduled_count, - scheduled_count.saturating_sub(cancelled_count), - cancelled_count, - ); - let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations); - return Ok((ress, errors, diagnostics)); - } - } - - let pending_responses = join_set.len(); - // Inline verification can still depend on a missing data shard; - // issue one immediate spare when only that shard remains. The - // non-inline path keeps its delayed hedge below to avoid healthy - // reads paying speculative I/O before the candidate is classified. - let should_hedge_single_pending_inline_read = read_data - && !force_full_wait - && !defer_pending_inline_data_shard - && pending_responses == 1 - && non_inline_candidate_eligible != Some(true) - && accumulator.can_still_reach_early_stop_with_pending(pending_responses); - // A non-inline plan must retain one extra matching shard as a - // reconstruction reserve. Schedule that reserve only after the - // candidate is known to be eligible, so inline GETs do not pay an - // extra fanout and the healthy path remains allocation-free. - let needs_non_inline_read_reserve = non_inline_candidate_eligible == Some(true) - && !accumulator.candidate_has_read_reserve() - && accumulator - .candidate_read_reserve_target() - .is_some_and(|reserve_target| scheduled_count < reserve_target || pending_responses == 0); - if bounded_fanout - && !force_full_wait - && (needs_non_inline_read_reserve || should_hedge_single_pending_inline_read) - && next_fanout_index < disks.len() - { - let disk_index = fanout_order[next_fanout_index]; - if let Some(disk) = disks.get(disk_index).cloned() { - spawn_read_version(&mut join_set, disk_index, disk); - scheduled_count = scheduled_count.saturating_add(1); - } - next_fanout_index = next_fanout_index.saturating_add(1); - } else if bounded_fanout && force_full_wait { - while next_fanout_index < disks.len() { - let disk_index = fanout_order[next_fanout_index]; - if let Some(disk) = disks.get(disk_index).cloned() { - spawn_read_version(&mut join_set, disk_index, disk); - scheduled_count = scheduled_count.saturating_add(1); - } - next_fanout_index = next_fanout_index.saturating_add(1); - } - } else if bounded_fanout - && !defer_pending_inline_data_shard - && next_fanout_index < disks.len() - && !accumulator.can_still_reach_early_stop_with_pending(pending_responses) - { - let disk_index = fanout_order[next_fanout_index]; - if let Some(disk) = disks.get(disk_index).cloned() { - spawn_read_version(&mut join_set, disk_index, disk); - scheduled_count = scheduled_count.saturating_add(1); - } - next_fanout_index = next_fanout_index.saturating_add(1); - } - if bounded_fanout - && !force_full_wait - && !defer_pending_inline_data_shard - && join_set.len() == 1 - && non_inline_candidate_eligible == Some(true) - && !accumulator.candidate_has_read_reserve() - && accumulator.can_still_reach_early_stop_with_pending(join_set.len()) - && next_fanout_index < disks.len() - { - single_pending_hedge_deadline = Some(tokio::time::Instant::now() + NON_INLINE_SINGLE_PENDING_HEDGE_DELAY); - } - } - - let accumulator_miss_reason = accumulator.final_miss_reason(); - let final_miss_reason = match (final_miss_reason_override, accumulator_miss_reason) { - (Some(reason), GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM) => reason, - _ => accumulator_miss_reason, - }; - rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, final_miss_reason); - rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0); - rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0); - let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations); - Ok((ress, errors, diagnostics)) + .map(MetadataReadResult::into_legacy) } pub(in crate::set_disk) async fn read_all_xl( @@ -5995,37 +4818,6 @@ impl SetDisks { None } - /// Test-only seam that records one per-disk `read_version` metadata RPC for - /// the call-counter registry (backlog#1325). In production this is inlined to - /// nothing and adds no behavior; only the `#[cfg(test)]` variant touches the - /// registry. Placed inside the fan-out spawn tasks so counts are observed - /// even though the increments run on arbitrary runtime worker threads. - #[cfg(test)] - #[inline] - fn record_read_version_call(object: &str, disk_index: usize) { - disk_call_counters::record(object, disk_call_counters::KIND_READ_VERSION, disk_index); - } - - #[cfg(not(test))] - #[inline(always)] - fn record_read_version_call(_object: &str, _disk_index: usize) {} - - #[cfg(test)] - #[inline] - fn record_metadata_slowtail_fault(object: &str, disk_index: usize) { - disk_call_counters::record(object, disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT, disk_index); - } - - #[cfg(not(test))] - #[inline(always)] - fn record_metadata_slowtail_fault(_object: &str, _disk_index: usize) {} - - #[cfg(test)] - #[inline] - async fn read_version_fanout_barrier(object: &str, disk_index: usize) { - rename_fanout_barrier::checkpoint(object, disk_index, rename_fanout_barrier::PHASE_READ_VERSION).await; - } - /// Test-only awaitable pause point for the rename/commit fan-out (backlog#1325, /// serving the barrier-style acceptances of #1312 / #1319 / #1313). `phase` is /// [`rename_fanout_barrier::PHASE_RENAME`] or `PHASE_CLEANUP`. When a test has @@ -7960,7 +6752,7 @@ pub(crate) mod disk_call_counters { /// Record one call of `kind` against `object`'s `disk_index`. A no-op unless /// an observing scope for `object` is currently installed. - pub(super) fn record(object: &str, kind: &str, disk_index: usize) { + pub(in crate::set_disk) fn record(object: &str, kind: &str, disk_index: usize) { let mut reg = registry().lock().expect("disk call-counter registry poisoned"); if !reg.observed.contains(object) { return; @@ -8047,9 +6839,8 @@ pub(in crate::set_disk) mod rename_fanout_barrier_phase { /// disk_index, phase)`. The matching spawned fan-out task blocks at its /// [`checkpoint`] until the test releases it. The test awaits the pause via /// [`BarrierHandle::wait_until_paused`] (a deterministic `Notify` handshake — -/// no sleeps) and resumes it via [`BarrierHandle::release`]. At most one -/// barrier is armed per object at a time, matching the single-scope style of -/// `disk_call_counters`. +/// no sleeps) and resumes it via [`BarrierHandle::release`]. Different disks +/// and phases may be armed together for the same object. /// /// 2. **Background-task introspection.** A test [`observe_tasks`] for `object`; /// each instrumented fan-out task then holds a [`TaskGuard`] for its whole @@ -8075,17 +6866,38 @@ pub(crate) mod rename_fanout_barrier { use std::sync::{Arc, Mutex, OnceLock}; use tokio::sync::Notify; - pub use super::rename_fanout_barrier_phase::{ - CLEANUP as PHASE_CLEANUP, READ_VERSION as PHASE_READ_VERSION, RENAME as PHASE_RENAME, ROLLBACK as PHASE_ROLLBACK, - }; + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] + pub enum Phase { + Rename, + Cleanup, + Rollback, + ReadVersion, + NonInlineHedgeTimer, + } + + impl From<&'static str> for Phase { + fn from(label: &'static str) -> Self { + match label { + super::rename_fanout_barrier_phase::RENAME => Self::Rename, + super::rename_fanout_barrier_phase::CLEANUP => Self::Cleanup, + super::rename_fanout_barrier_phase::ROLLBACK => Self::Rollback, + super::rename_fanout_barrier_phase::READ_VERSION => Self::ReadVersion, + "non_inline_hedge_timer" => Self::NonInlineHedgeTimer, + _ => panic!("unknown commit fault phase: {label}"), + } + } + } + + pub const PHASE_RENAME: Phase = Phase::Rename; + pub const PHASE_CLEANUP: Phase = Phase::Cleanup; + pub const PHASE_ROLLBACK: Phase = Phase::Rollback; + pub const PHASE_READ_VERSION: Phase = Phase::ReadVersion; /// Object-scoped hedge timer checkpoint; slot zero identifies the timer, not a disk. - pub const PHASE_NON_INLINE_HEDGE_TIMER: &str = "non_inline_hedge_timer"; + pub const PHASE_NON_INLINE_HEDGE_TIMER: Phase = Phase::NonInlineHedgeTimer; /// One armed barrier: the fan-out task matching `(disk_index, phase)` pauses. struct Armed { - disk_index: usize, - phase: &'static str, /// Signalled (task -> test) when the target task reaches the checkpoint. arrived: Arc, /// Signalled (test -> task) to release the paused task. @@ -8096,8 +6908,8 @@ pub(crate) mod rename_fanout_barrier { #[derive(Default)] struct Registry { - /// object -> armed barrier (at most one per object). - armed: HashMap, + /// Independently armed checkpoints for an object's fault schedule. + armed: HashMap<(String, usize, Phase), Armed>, /// object -> live in-flight fan-out task count, only for observed objects. observed: HashMap>, } @@ -8116,7 +6928,7 @@ pub(crate) mod rename_fanout_barrier { /// leave a spawned fan-out task wedged. #[must_use] pub struct BarrierHandle { - object: String, + key: (String, usize, Phase), arrived: Arc, release: Arc, paused: Arc, @@ -8124,22 +6936,26 @@ pub(crate) mod rename_fanout_barrier { /// Arm a barrier: the fan-out task for `object` at `(disk_index, phase)` will /// pause at its checkpoint until the returned handle is released or dropped. - pub fn arm(object: &str, disk_index: usize, phase: &'static str) -> BarrierHandle { + pub fn arm(object: &str, disk_index: usize, phase: Phase) -> BarrierHandle { let arrived = Arc::new(Notify::new()); let release = Arc::new(Notify::new()); let paused = Arc::new(AtomicBool::new(false)); - lock().armed.insert( - object.to_string(), + let key = (object.to_string(), disk_index, phase); + let mut reg = lock(); + if reg.armed.contains_key(&key) { + drop(reg); + panic!("commit fault checkpoint already armed: {key:?}"); + } + reg.armed.insert( + key.clone(), Armed { - disk_index, - phase, arrived: arrived.clone(), release: release.clone(), paused: paused.clone(), }, ); BarrierHandle { - object: object.to_string(), + key, arrived, release, paused, @@ -8157,6 +6973,13 @@ pub(crate) mod rename_fanout_barrier { self.arrived.notified().await; } + /// Bound a fixture handshake and identify the unreached checkpoint. + pub async fn wait_until_paused_before(&self, deadline: tokio::time::Instant) { + tokio::time::timeout_at(deadline, self.wait_until_paused()) + .await + .unwrap_or_else(|_| panic!("commit fault checkpoint not reached before deadline: {:?}", self.key)); + } + /// Whether the target task is currently parked at the checkpoint. pub fn is_paused(&self) -> bool { self.paused.load(Ordering::SeqCst) @@ -8173,7 +6996,14 @@ pub(crate) mod rename_fanout_barrier { // Unblock any task still parked at the checkpoint before disarming, so // a dropped handle can never wedge a spawned fan-out task. self.release.notify_one(); - lock().armed.remove(&self.object); + let mut reg = lock(); + if reg + .armed + .get(&self.key) + .is_some_and(|armed| Arc::ptr_eq(&armed.release, &self.release)) + { + reg.armed.remove(&self.key); + } } } @@ -8181,15 +7011,13 @@ pub(crate) mod rename_fanout_barrier { /// unless a barrier is armed for exactly this `(object, disk_index, phase)`. /// The registry mutex is released before awaiting, so it is never held across /// the pause. - pub(in crate::set_disk) async fn checkpoint(object: &str, disk_index: usize, phase: &'static str) { + pub(in crate::set_disk) async fn checkpoint(object: &str, disk_index: usize, phase: impl Into) { + let key = (object.to_string(), disk_index, phase.into()); let hooks = { let reg = lock(); - match reg.armed.get(object) { - Some(a) if a.disk_index == disk_index && a.phase == phase => { - Some((a.arrived.clone(), a.release.clone(), a.paused.clone())) - } - _ => None, - } + reg.armed + .get(&key) + .map(|a| (a.arrived.clone(), a.release.clone(), a.paused.clone())) }; if let Some((arrived, release, paused)) = hooks { paused.store(true, Ordering::SeqCst); @@ -8272,6 +7100,45 @@ mod tests { use tempfile::TempDir; use tokio::io::AsyncReadExt; + #[tokio::test] + async fn commit_fault_schedule_keeps_multiple_disk_phases_independent() { + use rename_fanout_barrier::{PHASE_CLEANUP, PHASE_RENAME, arm, checkpoint}; + + let object = "commit-fault-schedule-multiple-phases"; + let rename = arm(object, 0, PHASE_RENAME); + let cleanup = arm(object, 1, PHASE_CLEANUP); + let rename_task = tokio::spawn(checkpoint(object, 0, PHASE_RENAME)); + let cleanup_task = tokio::spawn(checkpoint(object, 1, PHASE_CLEANUP)); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); + rename.wait_until_paused_before(deadline).await; + cleanup.wait_until_paused_before(deadline).await; + + rename.release(); + tokio::time::timeout_at(deadline, rename_task) + .await + .expect("rename checkpoint should release independently") + .expect("rename checkpoint task should finish"); + assert!(!cleanup_task.is_finished(), "releasing rename must not release cleanup"); + drop(rename); + assert!(cleanup.is_paused(), "dropping a different handle must preserve cleanup"); + drop(cleanup); + tokio::time::timeout_at(deadline, cleanup_task) + .await + .expect("dropping cleanup should release its task") + .expect("cleanup checkpoint task should finish"); + } + + #[test] + fn commit_fault_schedule_rejects_duplicate_checkpoint_without_disarming_owner() { + let object = "commit-fault-schedule-duplicate-checkpoint"; + let owner = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let duplicate = std::panic::catch_unwind(|| rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME)); + assert!(duplicate.is_err(), "a second handle cannot replace a live checkpoint"); + drop(owner); + let next = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + drop(next); + } + #[test] fn orphan_dir_entries_must_be_single_relative_components() { for entry in ["part.1", "child/", "delete-data.00000000-0000-0000-0000-000000000001"] { @@ -11740,7 +10607,7 @@ mod tests { prepare_rename_source_dirs(&dirs, &disks, "source").await; let receipt = RenameRollbackReceipt::default(); let _fault = rollback_fault_injection::arm(&object, 0, fault); - let barrier = rename_fanout_barrier::arm(&object, 0, rename_fanout_barrier_phase::RENAME); + let barrier = rename_fanout_barrier::arm(&object, 0, rename_fanout_barrier::PHASE_RENAME); let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence( &disks, (RUSTFS_META_TMP_BUCKET, "source"), @@ -11792,9 +10659,20 @@ mod tests { .await; } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] + #[serial_test::serial] #[serial_test::serial(capacity_dirty_scope)] async fn rename_rollback_incomplete_preserves_overwrite_data_dirs_and_staging() { + let heal_sender_state = match rustfs_heal_contracts::heal_channel::get_heal_channel_sender() { + None => "not_initialized", + Some(sender) => { + assert!(sender.is_closed(), "the global heal receiver must not admit rollback inspection"); + "closed" + } + }; + // Spawned rollback coordinators share this current-thread recorder. + let recorder = crate::test_metrics::CapturingRecorder::default(); + let _recorder_guard = metrics::set_default_local_recorder(&recorder); temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { for fault in [ rollback_fault_injection::Fault::Io, @@ -11846,6 +10724,8 @@ mod tests { let _rename_fault = rename_fault_injection::fail_rename_on(&object, &[2, 3]); let _undo_fault = rollback_fault_injection::arm(&object, 0, fault); let receipt = RenameRollbackReceipt::default(); + let rejected_inspections_before = + recorder.counter_value("rustfs_rename_rollback_inspection_total", &[("admission", "failed")]); assert!( SetDisks::rename_data_owned_with_fence( &disks, @@ -11859,6 +10739,16 @@ mod tests { .is_err() ); assert!(receipt.is_incomplete()); + let rejected_inspections = + recorder.counter_value("rustfs_rename_rollback_inspection_total", &[("admission", "failed")]); + assert_eq!( + rejected_inspections, + rejected_inspections_before + 1, + "the actual rollback inspection must retain its rejected admission evidence" + ); + eprintln!( + "rollback inspection sender={heal_sender_state}, fault={fault:?}, early_ack={early_ack}, failed_admission_total={rejected_inspections}" + ); if !matches!(fault, rollback_fault_injection::Fault::Io) { assert!( matches!( @@ -12295,7 +11185,7 @@ mod tests { } let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); let _undo_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::Io); - let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_ROLLBACK); let receipt = RenameRollbackReceipt::default(); let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence( &disks, @@ -12387,7 +11277,7 @@ mod tests { let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]); let _rollback_fault = rollback_fault_injection::arm(object, 0, rollback_fault_injection::Fault::RollbackCoordinatorPanic); - let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier_phase::ROLLBACK); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_ROLLBACK); let receipt = RenameRollbackReceipt::default(); let result = tokio::time::timeout( BARRIER_PAUSE_GUARD, diff --git a/crates/ecstore/src/set_disk/core/metadata_quorum.rs b/crates/ecstore/src/set_disk/core/metadata_quorum.rs index 2922b6fc6..1b97adec8 100644 --- a/crates/ecstore/src/set_disk/core/metadata_quorum.rs +++ b/crates/ecstore/src/set_disk/core/metadata_quorum.rs @@ -29,6 +29,27 @@ use crate::disk::error_reduce::OBJECT_OP_IGNORED_ERRS; use crate::set_disk::file_info_is_valid_for_metadata; use rustfs_filemeta::FileInfo; +/// One disk's observation. Pending is neither a successful vote nor an offline +/// disk: the scheduler may not have issued this slot or may still be waiting. +// Option> keeps FileInfo inline without a per-response Box. +// None is pending; Some(Ok(_)) is success; Some(Err(_)) retains a disk error. +pub(in crate::set_disk) type MetadataDiskResult = Option>; + +#[derive(Debug)] +pub(in crate::set_disk) struct MetadataDiskObservation { + pub(in crate::set_disk) disk_index: usize, + pub(in crate::set_disk) result: MetadataDiskResult, +} + +impl MetadataDiskObservation { + pub(in crate::set_disk) fn file_info(&self) -> Option<&FileInfo> { + match &self.result { + Some(Ok(metadata)) => Some(metadata), + None | Some(Err(_)) => None, + } + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(in crate::set_disk) struct MetadataEarlyStopDecision { pub(in crate::set_disk) reason: &'static str, @@ -59,6 +80,14 @@ pub(in crate::set_disk) struct MetadataQuorumAccumulator { } impl MetadataQuorumAccumulator { + pub(in crate::set_disk) fn observe(&mut self, observation: &MetadataDiskObservation) { + match &observation.result { + None => {} + Some(Ok(metadata)) => self.observe_file_info_at(observation.disk_index, metadata), + Some(Err(error)) => self.observe_error(error), + } + } + pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self { Self { total_disks, @@ -86,6 +115,7 @@ impl MetadataQuorumAccumulator { self } + #[cfg(test)] pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) { self.observe_file_info_with_index(None, file_info); } @@ -383,3 +413,186 @@ pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool { OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::set_disk::SetDisks; + use time::OffsetDateTime; + use uuid::Uuid; + + fn payload(disk_index: usize) -> FileInfo { + let mut metadata = FileInfo::new("object", 2, 2); + metadata.volume = "bucket".to_string(); + metadata.name = "object".to_string(); + metadata.size = 1; + metadata.mod_time = Some(OffsetDateTime::UNIX_EPOCH); + metadata.data_dir = Some(Uuid::from_u128(1)); + metadata.erasure.index = metadata.erasure.distribution[disk_index]; + metadata.metadata.insert("etag".to_string(), "etag".to_string()); + metadata.add_object_part(1, "etag".to_string(), 1, None, 1, None, None); + assert!(file_info_is_valid_for_metadata(&metadata), "the observation corpus needs a valid payload"); + metadata + } + + fn observation(disk_index: usize, state: usize) -> MetadataDiskObservation { + MetadataDiskObservation { + disk_index, + result: match state { + 0 => Some(Ok(payload(disk_index))), + 1 => Some(Err(DiskError::FileNotFound)), + 2 => Some(Err(DiskError::FileCorrupt)), + 3 => Some(Err(DiskError::DiskNotFound)), + 4 => None, + _ => panic!("unexpected test state"), + }, + } + } + + fn assert_same_reduction(typed: &MetadataQuorumAccumulator, legacy: &MetadataQuorumAccumulator) { + assert_eq!(typed.early_stop_decision(), legacy.early_stop_decision()); + assert_eq!(typed.version_early_stop_decision(), legacy.version_early_stop_decision()); + assert_eq!(typed.final_miss_reason(), legacy.final_miss_reason()); + assert_eq!(typed.valid_responses, legacy.valid_responses); + assert_eq!(typed.not_found_responses, legacy.not_found_responses); + assert_eq!(typed.version_not_found_responses, legacy.version_not_found_responses); + assert_eq!(typed.ignored_errors, legacy.ignored_errors); + assert_eq!(typed.hard_errors, legacy.hard_errors); + assert_eq!(typed.candidate_votes, legacy.candidate_votes); + assert_eq!(typed.candidate_shard_mask, legacy.candidate_shard_mask); + assert_eq!(typed.matching_version_votes, legacy.matching_version_votes); + assert_eq!(typed.delete_marker_votes, legacy.delete_marker_votes); + assert_eq!(typed.conflicting_metadata, legacy.conflicting_metadata); + assert_eq!( + typed.candidate.as_ref().map(SetDisks::file_info_quorum_hash), + legacy.candidate.as_ref().map(SetDisks::file_info_quorum_hash) + ); + } + + fn observe_legacy(accumulator: &mut MetadataQuorumAccumulator, observation: &MetadataDiskObservation) { + match &observation.result { + None => {} + Some(Ok(metadata)) => accumulator.observe_file_info_at(observation.disk_index, metadata), + Some(Err(error)) => accumulator.observe_error(error), + } + } + + #[test] + fn metadata_observation_all_four_slot_states_and_arrival_orders_preserve_reduction() { + let permutations = (0..4) + .flat_map(|a| (0..4).filter(move |b| *b != a).map(move |b| (a, b))) + .flat_map(|(a, b)| (0..4).filter(move |c| *c != a && *c != b).map(move |c| (a, b, c))) + .map(|(a, b, c)| [a, b, c, 6 - a - b - c]) + .collect::>(); + assert_eq!(permutations.len(), 24); + // 5^4 states and all 4! arrivals, both with early-stop enabled and disabled. + for encoded in 0..625usize { + let states = [encoded % 5, encoded / 5 % 5, encoded / 25 % 5, encoded / 125 % 5]; + let inputs = std::array::from_fn::<_, 4, _>(|index| observation(index, states[index])); + for enabled in [false, true] { + for order in &permutations { + let mut typed = MetadataQuorumAccumulator::new(4, 2, enabled); + let mut legacy = MetadataQuorumAccumulator::new(4, 2, enabled); + for &index in order { + typed.observe(&inputs[index]); + observe_legacy(&mut legacy, &inputs[index]); + assert_same_reduction(&typed, &legacy); + } + assert_eq!(typed.valid_responses, states.iter().filter(|&&state| state == 0).count()); + assert_eq!(typed.not_found_responses, states.iter().filter(|&&state| state == 1).count()); + assert_eq!(typed.hard_errors, states.iter().filter(|&&state| state == 2).count()); + assert_eq!(typed.ignored_errors, states.iter().filter(|&&state| state == 3).count()); + let expected_early_stop = + enabled && typed.valid_responses >= 3 && !states.iter().any(|&state| state == 1 || state == 2); + assert_eq!( + typed.early_stop_decision().is_some(), + expected_early_stop, + "states={states:?}, order={order:?}" + ); + } + } + } + } + + #[test] + fn metadata_observation_pending_newer_version_and_same_time_different_directory_force_full_wait() { + let mut accumulator = MetadataQuorumAccumulator::new(4, 2, true); + for index in 0..2 { + accumulator.observe(&observation(index, 0)); + } + accumulator.observe(&observation(2, 4)); + assert_eq!(accumulator.candidate_votes, 2, "an outstanding response cannot supply the deciding vote"); + assert_eq!(accumulator.early_stop_decision(), None); + + let mut newer = payload(2); + newer.mod_time = Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)); + accumulator.observe(&MetadataDiskObservation { + disk_index: 2, + result: Some(Ok(newer)), + }); + assert!(accumulator.conflicting_metadata); + assert_eq!(accumulator.early_stop_decision(), None); + + let mut same_time_different_directory = payload(3); + same_time_different_directory.data_dir = Some(Uuid::from_u128(2)); + accumulator.observe(&MetadataDiskObservation { + disk_index: 3, + result: Some(Ok(same_time_different_directory)), + }); + assert_eq!(accumulator.early_stop_decision(), None); + } + + #[test] + fn metadata_observation_duplicate_shard_cannot_supply_a_read_reserve() { + let mut accumulator = MetadataQuorumAccumulator::new(4, 2, true); + let first = payload(0); + for disk_index in 0..3 { + accumulator.observe(&MetadataDiskObservation { + disk_index, + result: Some(Ok(first.clone())), + }); + } + assert_eq!(accumulator.candidate_votes, 3); + assert!( + !accumulator.candidate_has_read_reserve(), + "copied erasure indexes cannot supply independent data shards" + ); + } + + #[test] + fn metadata_observation_null_versions_markers_and_invalid_success_keep_their_meaning() { + let mut null_version = MetadataQuorumAccumulator::new(4, 2, true); + for index in 0..3 { + null_version.observe(&observation(index, 0)); + } + assert!(null_version.early_stop_decision().is_some()); + assert_eq!(null_version.matching_version_votes, 0); + + let mut marker = MetadataQuorumAccumulator::new(4, 2, true); + for disk_index in 0..3 { + let metadata = FileInfo { + volume: "bucket".to_string(), + name: "object".to_string(), + deleted: true, + mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)), + ..Default::default() + }; + assert!(metadata.is_canonical_delete_marker(), "the null marker fixture must reach marker voting"); + marker.observe(&MetadataDiskObservation { + disk_index, + result: Some(Ok(metadata)), + }); + } + assert_eq!(marker.delete_marker_votes, 3); + assert!(marker.early_stop_decision().is_some()); + + let mut invalid = MetadataQuorumAccumulator::new(4, 2, true); + invalid.observe(&MetadataDiskObservation { + disk_index: 0, + result: Some(Ok(FileInfo::default())), + }); + assert_eq!(invalid.valid_responses, 0); + assert_eq!(invalid.hard_errors, 1); + assert_eq!(invalid.early_stop_decision(), None); + } +} diff --git a/crates/ecstore/src/set_disk/core/metadata_read.rs b/crates/ecstore/src/set_disk/core/metadata_read.rs new file mode 100644 index 000000000..82dd3b8a4 --- /dev/null +++ b/crates/ecstore/src/set_disk/core/metadata_read.rs @@ -0,0 +1,1443 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Metadata read scheduling, cancellation and coalescing owned by `SetDisks`. +//! Quorum decisions consume slot observations in `metadata_quorum`. + +#[cfg(test)] +use super::io_primitives::{disk_call_counters, rename_fanout_barrier}; +use super::metadata_quorum::{ + MetadataDiskObservation, MetadataQuorumAccumulator, is_metadata_fanout_ignored_error, metadata_early_stop_candidate_matches, +}; +use crate::diagnostics::get::{ + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, + GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, + GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, + GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, + GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_INTERNAL_META, + GET_OBJECT_PATH_LEGACY_DUPLEX, +}; +use crate::disk::disk_store::get_drive_metadata_timeout; +use crate::disk::{ + self, BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, Disk, DiskAPI, +}; +use crate::set_disk::{ + DiskError, DiskStore, FileInfo, HashAlgorithm, ReadOptions, SetDisks, build_inline_bitrot_readers_from_refs, + can_try_inline_data_shards_direct, codec_streaming_rollout_applies, coding, + collect_inline_data_shard_fileinfos_from_observations, file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, + inline_erasure_shard_file_offset, inline_erasure_shard_size, is_get_metadata_data_read_early_stop_enabled, + is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled, + is_get_metadata_non_inline_data_read_early_stop_enabled, is_version_early_stop_enabled, object_fits_single_block, + try_read_inline_data_shards_direct, +}; +use futures::future::join_all; +use metrics::counter; +use rustfs_io_metrics::internode_metrics::{ + INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP, +}; +#[cfg(test)] +use std::collections::HashSet; +use std::{ + collections::HashMap, + future::Future, + pin::Pin, + sync::{Arc, OnceLock}, + task::{Context, Poll}, + time::{Duration, Instant}, +}; +use tokio::sync::{Mutex, oneshot}; +use tokio::task::JoinSet; + +#[derive(Debug)] +pub(in crate::set_disk) struct MetadataReadResult { + pub(in crate::set_disk) slots: Vec, + pub(in crate::set_disk) diagnostics: MetadataFanoutDiagnostics, +} + +impl MetadataReadResult { + fn pending(total_disks: usize) -> Vec { + (0..total_disks) + .map(|disk_index| MetadataDiskObservation { + disk_index, + result: None, + }) + .collect() + } + + pub(in crate::set_disk) fn is_complete(&self) -> bool { + self.slots.iter().all(|slot| slot.result.is_some()) + } + + /// Legacy quorum/layout helpers still require aligned slices. Only this + /// boundary produces their empty placeholders; the scheduler and reducer + /// never infer a response from a default `FileInfo` or a missing error. + // RUSTFS_COMPAT_TODO(backlog-2253-metadata-observation-adapter): Existing layout, shard and exact-version writer/delete callers still consume aligned slices. Remove after those consumers use typed observations in a separate cleanup. + pub(in crate::set_disk) fn into_legacy(self) -> (Vec, Vec>, MetadataFanoutDiagnostics) { + let mut metadata = Vec::with_capacity(self.slots.len()); + let mut errors = Vec::with_capacity(self.slots.len()); + for slot in self.slots { + match slot.result { + None => { + metadata.push(FileInfo::default()); + errors.push(None); + } + Some(Ok(file_info)) => { + metadata.push(file_info); + errors.push(None); + } + Some(Err(error)) => { + metadata.push(FileInfo::default()); + errors.push(Some(error)); + } + } + } + (metadata, errors, self.diagnostics) + } +} + +pub(super) const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE"; +pub(super) const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = + "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS"; +const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200; +const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total"; + +fn metadata_metrics_path(bucket: &str) -> &'static str { + if crate::bucket::utils::is_meta_bucketname(bucket) { + GET_OBJECT_PATH_INTERNAL_META + } else { + GET_OBJECT_PATH_LEGACY_DUPLEX + } +} + +pub(super) fn metadata_distribution_key(bucket: &str, object: &str) -> String { + [bucket, object].join("/") +} + +fn read_version_coalescing_enabled() -> bool { + let enabled = || { + rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE) + .is_some_and(|value| value.eq_ignore_ascii_case("auto") || value.eq_ignore_ascii_case("on")) + }; + + #[cfg(test)] + { + enabled() + } + + #[cfg(not(test))] + { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(enabled) + } +} + +fn read_version_coalescing_delay() -> Duration { + #[cfg(test)] + { + let micros = rustfs_utils::get_env_u64( + ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, + DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, + ); + Duration::from_micros(micros) + } + + #[cfg(not(test))] + { + static DELAY: OnceLock = OnceLock::new(); + *DELAY.get_or_init(|| { + Duration::from_micros(rustfs_utils::get_env_u64( + ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, + DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, + )) + }) + } +} + +struct CoalescedReadVersionRequest { + item: BatchReadVersionItem, + tx: oneshot::Sender>, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct ExpectedBatchReadVersionItem { + pub(super) path: String, + pub(super) version_id: String, +} + +impl From<&BatchReadVersionItem> for ExpectedBatchReadVersionItem { + fn from(item: &BatchReadVersionItem) -> Self { + Self { + path: item.path.clone(), + version_id: item.version_id.clone(), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +struct ReadVersionCoalescerKey { + disk: usize, + incl_free_versions: bool, + read_data: bool, + healing: bool, +} + +impl ReadVersionCoalescerKey { + fn new(disk: &DiskStore, opts: &ReadOptions) -> Self { + Self { + disk: Arc::as_ptr(disk) as usize, + incl_free_versions: opts.incl_free_versions, + read_data: opts.read_data, + healing: opts.healing, + } + } +} + +#[derive(Default)] +struct ReadVersionCoalescer { + lanes: HashMap>, +} + +fn read_version_coalescer() -> &'static Mutex { + static COALESCER: OnceLock> = OnceLock::new(); + COALESCER.get_or_init(|| Mutex::new(ReadVersionCoalescer::default())) +} + +fn record_read_version_coalescer_event(event: &'static str, item_count: usize) { + counter!( + METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL, + "event" => event, + "item_count" => item_count.to_string() + ) + .increment(1); +} + +fn batch_read_version_stage_timer() -> Option { + rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now) +} + +fn record_batch_read_version_stage(stage: &'static str, started_at: Option) { + if let Some(started_at) = started_at { + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed()); + } +} + +async fn read_version_via_coalescer( + disk: DiskStore, + org_bucket: &str, + bucket: &str, + object: &str, + version_id: &str, + opts: &ReadOptions, + allow_coalescing: bool, +) -> disk::error::Result { + if !allow_coalescing || !read_version_coalescing_enabled() { + return disk.read_version(org_bucket, bucket, object, version_id, opts).await; + } + if !matches!(disk.as_ref(), Disk::Remote(_)) { + record_read_version_coalescer_event("bypass_non_remote", 1); + return disk.read_version(org_bucket, bucket, object, version_id, opts).await; + } + + let (tx, rx) = oneshot::channel(); + let item = BatchReadVersionItem { + org_volume: org_bucket.to_string(), + volume: bucket.to_string(), + path: object.to_string(), + version_id: version_id.to_string(), + }; + let lane_key = ReadVersionCoalescerKey::new(&disk, opts); + let pending = { + let mut coalescer = read_version_coalescer().lock().await; + let lane = coalescer.lanes.entry(lane_key).or_default(); + let schedule_delayed_flush = lane.is_empty(); + lane.push(CoalescedReadVersionRequest { item, tx }); + if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS { + coalescer.lanes.remove(&lane_key) + } else if schedule_delayed_flush { + let disk = disk.clone(); + let task_opts = *opts; + tokio::spawn(async move { + tokio::time::sleep(read_version_coalescing_delay()).await; + flush_read_version_coalescer_lane(lane_key, disk, task_opts).await; + }); + None + } else { + None + } + }; + + if let Some(pending) = pending { + flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await; + } + + let wait_started = batch_read_version_stage_timer(); + let response = rx + .await + .unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed"))); + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, wait_started); + response +} + +async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) { + let pending = { + let mut coalescer = read_version_coalescer().lock().await; + coalescer.lanes.remove(&lane_key).unwrap_or_default() + }; + flush_read_version_coalescer_pending(lane_key, disk, opts, pending).await; +} + +async fn flush_read_version_coalescer_pending( + lane_key: ReadVersionCoalescerKey, + disk: DiskStore, + opts: ReadOptions, + pending: Vec, +) { + if pending.is_empty() { + return; + } + + // Only the #[cfg(test)] counter-recording block below reads this. + #[cfg(not(test))] + let _ = lane_key; + #[cfg(test)] + { + let mut observed_paths = HashSet::new(); + for request in &pending { + if observed_paths.insert(request.item.path.as_str()) { + disk_call_counters::record(&request.item.path, disk_call_counters::KIND_BATCH_READ_VERSION, lane_key.disk); + } + } + } + + let mut senders = Vec::with_capacity(pending.len()); + let mut items = Vec::with_capacity(pending.len()); + for request in pending { + senders.push(request.tx); + items.push(request.item); + } + + let expected_items = items.iter().map(ExpectedBatchReadVersionItem::from).collect::>(); + record_read_version_coalescer_event("attempted_batch", items.len()); + let result = + match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts })) + .await + { + Ok(result) => result, + Err(_) => Err(DiskError::Timeout), + }; + match result { + Ok(responses) => { + let map_started = batch_read_version_stage_timer(); + let results = map_batch_read_version_responses(&expected_items, responses); + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP, map_started); + for (tx, result) in senders.into_iter().zip(results) { + let _ = tx.send(result); + } + } + Err(err) => { + let message = err.to_string(); + for tx in senders { + let _ = tx.send(Err(DiskError::other(message.clone()))); + } + } + } +} + +pub(super) fn map_batch_read_version_responses( + expected_items: &[ExpectedBatchReadVersionItem], + responses: Vec, +) -> Vec> { + let mut results = (0..expected_items.len()) + .map(|_| Err(DiskError::other("coalesced read_version response missing"))) + .collect::>(); + let mut seen = vec![false; expected_items.len()]; + for response in responses { + let Some(expected) = expected_items.get(response.index) else { + continue; + }; + let Some(slot) = results.get_mut(response.index) else { + continue; + }; + if seen[response.index] { + *slot = Err(DiskError::other("coalesced read_version response duplicate index")); + continue; + } + seen[response.index] = true; + if response.path != expected.path || response.version_id != expected.version_id { + *slot = Err(DiskError::other("coalesced read_version response identity mismatch")); + } else { + *slot = if response.success { + Ok(response.file_info) + } else { + Err(batch_read_version_response_error(response.error_code, response.error)) + }; + } + } + results +} + +fn batch_read_version_response_error(error_code: u32, error: String) -> DiskError { + match DiskError::from_u32(error_code) { + Some(DiskError::Io(_)) | None => DiskError::other(error), + Some(error) => error, + } +} + +pub(in crate::set_disk) fn bounded_metadata_fanout_order( + bucket: &str, + object: &str, + total_disks: usize, + default_parity_count: usize, +) -> Vec { + let fallback_order = || (0..total_disks).collect::>(); + if default_parity_count == 0 || default_parity_count >= total_disks { + return fallback_order(); + } + + let data_blocks = total_disks - default_parity_count; + let distribution_key = metadata_distribution_key(bucket, object); + let distribution = FileInfo::new(&distribution_key, data_blocks, default_parity_count) + .erasure + .distribution; + if distribution.len() != total_disks { + return fallback_order(); + } + + let mut order = Vec::with_capacity(total_disks); + for block_index in 1..=data_blocks { + let Some(disk_index) = distribution + .iter() + .position(|distributed_block| *distributed_block == block_index) + else { + return fallback_order(); + }; + order.push(disk_index); + } + order.extend( + distribution + .iter() + .enumerate() + .filter_map(|(disk_index, block_index)| (*block_index > data_blocks).then_some(disk_index)), + ); + order +} + +struct AbortOnDropJoinHandle(tokio::task::JoinHandle); + +impl Future for AbortOnDropJoinHandle { + type Output = std::result::Result; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Pin::new(&mut self.0).poll(cx) + } +} + +impl Drop for AbortOnDropJoinHandle { + fn drop(&mut self) { + self.0.abort(); + } +} + +#[derive(Clone, Copy, Debug)] +pub(in crate::set_disk) struct MetadataFanoutObservation { + pub(in crate::set_disk) outcome: &'static str, + pub(in crate::set_disk) elapsed: Duration, + pub(in crate::set_disk) valid: bool, + pub(in crate::set_disk) ignored: bool, +} + +impl MetadataFanoutObservation { + pub(in crate::set_disk) fn from_file_info(file_info: &FileInfo, elapsed: Duration) -> Self { + if file_info_is_valid_for_metadata(file_info) { + Self { + outcome: GET_METADATA_RESPONSE_VALID, + elapsed, + valid: true, + ignored: false, + } + } else { + Self { + outcome: GET_METADATA_RESPONSE_ERROR, + elapsed, + valid: false, + ignored: false, + } + } + } + + pub(in crate::set_disk) fn from_error(err: &DiskError, elapsed: Duration) -> Self { + Self { + outcome: classify_metadata_response_error(err), + elapsed, + valid: false, + ignored: is_metadata_fanout_ignored_error(err), + } + } +} + +#[derive(Clone, Debug, Default)] +pub(in crate::set_disk) struct MetadataFanoutDiagnostics { + pub(in crate::set_disk) fanout_duration: Duration, + pub(in crate::set_disk) observations: Vec, +} + +impl MetadataFanoutDiagnostics { + pub(in crate::set_disk) fn new(fanout_duration: Duration, observations: Vec) -> Self { + Self { + fanout_duration, + observations, + } + } + + pub(in crate::set_disk) fn total_responses(&self) -> usize { + self.observations.len() + } + + pub(in crate::set_disk) fn valid_responses(&self) -> usize { + self.observations.iter().filter(|observation| observation.valid).count() + } + + pub(in crate::set_disk) fn ignored_responses(&self) -> usize { + self.observations.iter().filter(|observation| observation.ignored).count() + } + + pub(in crate::set_disk) fn non_valid_responses(&self) -> usize { + self.total_responses().saturating_sub(self.valid_responses()) + } + + pub(in crate::set_disk) fn first_response_latency(&self) -> Option { + self.observations.iter().map(|observation| observation.elapsed).min() + } + + pub(in crate::set_disk) fn first_valid_response_latency(&self) -> Option { + self.observations + .iter() + .filter(|observation| observation.valid) + .map(|observation| observation.elapsed) + .min() + } + + pub(in crate::set_disk) fn slowest_response_latency(&self) -> Option { + self.observations.iter().map(|observation| observation.elapsed).max() + } + + pub(in crate::set_disk) fn quorum_candidate_latency(&self, read_quorum: usize) -> Option { + if read_quorum == 0 { + return Some(Duration::ZERO); + } + + let mut valid_latencies = self + .observations + .iter() + .filter(|observation| observation.valid) + .map(|observation| observation.elapsed) + .collect::>(); + valid_latencies.sort_unstable(); + valid_latencies.get(read_quorum.saturating_sub(1)).copied() + } + + pub(in crate::set_disk) fn record(&self, path: &'static str) { + rustfs_io_metrics::record_get_object_metadata_fanout_duration(path, self.fanout_duration.as_secs_f64()); + if let Some(latency) = self.first_response_latency() { + rustfs_io_metrics::record_get_object_first_metadata_response_latency(path, latency.as_secs_f64()); + } + if let Some(latency) = self.first_valid_response_latency() { + rustfs_io_metrics::record_get_object_first_valid_metadata_response_latency(path, latency.as_secs_f64()); + } + if let Some(latency) = self.slowest_response_latency() { + rustfs_io_metrics::record_get_object_slowest_metadata_response_latency(path, latency.as_secs_f64()); + } + rustfs_io_metrics::record_get_object_metadata_fanout_shape( + path, + self.total_responses(), + self.valid_responses(), + self.ignored_responses(), + self.non_valid_responses(), + ); + for observation in &self.observations { + rustfs_io_metrics::record_get_object_metadata_response(path, observation.outcome); + } + } + + pub(in crate::set_disk) fn record_quorum_candidate_latency(&self, path: &'static str, read_quorum: usize) { + if let Some(latency) = self.quorum_candidate_latency(read_quorum) { + rustfs_io_metrics::record_get_object_quorum_reached_latency(path, latency.as_secs_f64()); + } + } +} + +#[cfg(test)] +pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason( + bucket: &str, + object: &str, + candidate: &FileInfo, + parts_metadata: &[FileInfo], + disks: &[Option], +) -> Option<&'static str> { + inline_body_miss_reason_from_observations( + bucket, + object, + candidate, + parts_metadata + .iter() + .enumerate() + .map(|(index, metadata)| (index, Some(metadata))), + disks, + ) + .await +} + +async fn inline_body_miss_reason_from_observations<'a>( + bucket: &str, + object: &str, + candidate: &FileInfo, + observations: impl IntoIterator)>, + disks: &[Option], +) -> Option<&'static str> { + if let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(candidate) { + return Some(reason); + } + + let Ok(erasure) = coding::Erasure::try_new_with_options( + candidate.erasure.data_blocks, + candidate.erasure.parity_blocks, + candidate.erasure.block_size, + candidate.uses_legacy_checksum, + ) else { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY); + }; + let data_files = + match collect_inline_data_shard_fileinfos_from_observations(observations, candidate, erasure.data_shards, |index| { + disks.get(index).is_some_and(Option::is_some) + }) { + Ok(data_files) => data_files, + Err(reason) => return Some(reason), + }; + + let Some(part) = candidate.parts.first() else { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE); + }; + let Ok(object_size) = usize::try_from(candidate.size) else { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); + }; + let checksum_info = candidate.erasure.get_checksum_info(part.number); + let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S { + HashAlgorithm::HighwayHash256SLegacy + } else { + checksum_info.algorithm + }; + let read_length = inline_erasure_shard_file_offset( + 0, + object_size, + object_size, + candidate.erasure.block_size, + erasure.data_shards, + candidate.uses_legacy_checksum, + ); + let shard_size = inline_erasure_shard_size(candidate.erasure.block_size, erasure.data_shards, candidate.uses_legacy_checksum); + let Ok(mut readers) = + build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await + else { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY); + }; + + match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await { + Some(body) if body.len() == object_size => None, + _ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY), + } +} + +fn data_read_early_stop_inline_candidate_miss_reason(candidate: &FileInfo) -> Option<&'static str> { + // `inline_data` excludes remote objects; this diagnostic reports them separately. + if !rustfs_utils::http::contains_key_str(&candidate.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA) { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE); + } + if candidate.is_compressed() + || candidate + .metadata + .keys() + .any(|key| rustfs_utils::http::is_object_encryption_marker(key)) + { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED); + } + if candidate.is_remote() { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE); + } + if candidate.deleted { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED); + } + if candidate.size <= 0 { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); + } + if candidate.parts.len() != 1 { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE); + } + if !candidate.has_valid_erasure_geometry() { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY); + } + + let Ok(object_size) = usize::try_from(candidate.size) else { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); + }; + if candidate.parts.first().is_none_or(|part| part.size != object_size) { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE); + } + if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) { + return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE); + } + None +} + +pub(in crate::set_disk) fn non_inline_data_read_candidate_is_safe(candidate: &FileInfo) -> bool { + if candidate.inline_data() + || candidate.is_compressed() + || candidate.is_remote() + || candidate + .metadata + .keys() + .any(|key| rustfs_utils::http::is_object_encryption_marker(key)) + || candidate.parts.len() != 1 + { + return false; + } + candidate.has_valid_erasure_geometry() +} + +pub(in crate::set_disk) fn late_materialization_candidate_is_safe(candidate: &FileInfo) -> bool { + non_inline_data_read_candidate_is_safe(candidate) + && candidate.size > 512 * 1024 + && object_fits_single_block(candidate.size, candidate.erasure.block_size) +} + +pub(in crate::set_disk) fn non_inline_data_read_early_stop_allowed(read_data: bool, bucket: &str, object: &str) -> bool { + read_data && is_get_metadata_non_inline_data_read_early_stop_enabled() && !codec_streaming_rollout_applies(bucket, object) +} + +const NON_INLINE_SINGLE_PENDING_HEDGE_DELAY: Duration = Duration::from_millis(100); + +fn data_read_inline_missing_shards_are_pending( + candidate: &FileInfo, + slots: &[MetadataDiskObservation], + disks: &[Option], + fanout_order: &[usize], + scheduled_fanout_len: usize, +) -> bool { + let Ok(erasure) = coding::Erasure::try_new_with_options( + candidate.erasure.data_blocks, + candidate.erasure.parity_blocks, + candidate.erasure.block_size, + candidate.uses_legacy_checksum, + ) else { + return false; + }; + let distribution = &candidate.erasure.distribution; + let mut data_shards_seen_or_pending = vec![false; erasure.data_shards]; + let mut missing_pending_data_shards = 0usize; + + for slot in slots { + let disk_index = slot.disk_index; + let Some(&block_index) = distribution.get(disk_index) else { + return false; + }; + if block_index == 0 || block_index > erasure.data_shards { + continue; + } + if !disks.get(disk_index).is_some_and(Option::is_some) { + return false; + } + + let data_slot = block_index - 1; + if slot.file_info().is_none_or(|metadata| metadata.name.is_empty()) { + let scheduled_and_not_failed = fanout_order + .get(..scheduled_fanout_len) + .is_some_and(|scheduled_disks| scheduled_disks.contains(&disk_index)) + && slot.result.is_none(); + if scheduled_and_not_failed { + data_shards_seen_or_pending[data_slot] = true; + missing_pending_data_shards = missing_pending_data_shards.saturating_add(1); + continue; + } + return false; + } + let Some(file_info) = slot.file_info() else { return false }; + if file_info.erasure.index != block_index + || !file_info.has_valid_erasure_geometry() + || !metadata_early_stop_candidate_matches(file_info, candidate) + || file_info.data.as_ref().is_none_or(|data| data.is_empty()) + { + return false; + } + data_shards_seen_or_pending[data_slot] = true; + } + + missing_pending_data_shards > 0 && data_shards_seen_or_pending.into_iter().all(|seen_or_pending| seen_or_pending) +} + +pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str { + match err { + DiskError::FileNotFound | DiskError::VolumeNotFound => GET_METADATA_RESPONSE_NOT_FOUND, + DiskError::FileVersionNotFound => GET_METADATA_RESPONSE_VERSION_NOT_FOUND, + DiskError::DiskNotFound => GET_METADATA_RESPONSE_DISK_NOT_FOUND, + DiskError::FileCorrupt | DiskError::CorruptedFormat | DiskError::CorruptedBackend | DiskError::OutdatedXLMeta => { + GET_METADATA_RESPONSE_CORRUPT + } + DiskError::Timeout => GET_METADATA_RESPONSE_TIMEOUT, + DiskError::FaultyDisk | DiskError::FaultyRemoteDisk => GET_METADATA_RESPONSE_IGNORED, + _ => GET_METADATA_RESPONSE_ERROR, + } +} + +pub(in crate::set_disk) fn should_allow_metadata_early_stop( + read_data: bool, + version_id: &str, + healing: bool, + incl_free_versions: bool, +) -> bool { + if read_data && !is_get_metadata_data_read_early_stop_enabled() { + return false; + } + + (is_get_metadata_early_stop_enabled() && version_id.is_empty() && !healing && !incl_free_versions) + || (is_version_early_stop_enabled() && !version_id.is_empty() && !healing && !incl_free_versions) +} + +/// Final gate for the metadata early-stop fast path. +/// +/// `caller_allows_early_stop=false` unconditionally forces the full quorum +/// fanout so read-before-write callers (object tagging) get the complete +/// online-disk set as their write target; the early-stop subset would only +/// carry read quorum and fail write quorum (backlog#872 regression). +pub(in crate::set_disk) fn metadata_early_stop_permitted( + caller_allows_early_stop: bool, + observe: bool, + read_data: bool, + version_id: &str, + healing: bool, + incl_free_versions: bool, +) -> bool { + caller_allows_early_stop && observe && should_allow_metadata_early_stop(read_data, version_id, healing, incl_free_versions) +} + +impl SetDisks { + #[allow(clippy::too_many_arguments)] + #[tracing::instrument(level = "debug", skip(disks))] + pub(in crate::set_disk) async fn read_metadata( + disks: &[Option], + org_bucket: &str, + bucket: &str, + object: &str, + version_id: &str, + read_data: bool, + healing: bool, + incl_free_versions: bool, + ) -> disk::error::Result { + Self::read_metadata_inner( + disks, + org_bucket, + bucket, + object, + version_id, + read_data, + healing, + incl_free_versions, + false, + true, + 0, + false, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub(in crate::set_disk) async fn read_metadata_observed( + disks: &[Option], + org_bucket: &str, + bucket: &str, + object: &str, + version_id: &str, + read_data: bool, + healing: bool, + incl_free_versions: bool, + caller_allows_early_stop: bool, + default_parity_count: usize, + ) -> disk::error::Result { + Self::read_metadata_inner( + disks, + org_bucket, + bucket, + object, + version_id, + read_data, + healing, + incl_free_versions, + true, + caller_allows_early_stop, + default_parity_count, + false, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub(in crate::set_disk) async fn read_metadata_for_get_object( + disks: &[Option], + org_bucket: &str, + bucket: &str, + object: &str, + version_id: &str, + read_data: bool, + incl_free_versions: bool, + caller_allows_early_stop: bool, + default_parity_count: usize, + ) -> disk::error::Result { + Self::read_metadata_inner( + disks, + org_bucket, + bucket, + object, + version_id, + read_data, + false, + incl_free_versions, + true, + caller_allows_early_stop, + default_parity_count, + true, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub(in crate::set_disk) async fn read_metadata_inner( + disks: &[Option], + org_bucket: &str, + bucket: &str, + object: &str, + version_id: &str, + read_data: bool, + healing: bool, + incl_free_versions: bool, + observe: bool, + // When false, the caller opts out of the early-stop fast path even for + // otherwise-eligible reads. Read-before-write callers (e.g. object + // tagging) must set this so the returned online-disk set reflects the + // full quorum fanout rather than the early-stop subset — writing to the + // subset would fail write quorum (backlog#872 regression). + caller_allows_early_stop: bool, + default_parity_count: usize, + allow_coalescing: bool, + ) -> disk::error::Result { + let early_stop_enabled = + caller_allows_early_stop && observe && (is_get_metadata_early_stop_enabled() || is_version_early_stop_enabled()); + let allow_early_stop = + metadata_early_stop_permitted(caller_allows_early_stop, observe, read_data, version_id, healing, incl_free_versions); + if allow_early_stop { + return Self::read_metadata_early_stop( + disks, + org_bucket, + bucket, + object, + version_id, + read_data, + healing, + incl_free_versions, + non_inline_data_read_early_stop_allowed(read_data, bucket, object), + default_parity_count, + allow_coalescing, + ) + .await; + } + if early_stop_enabled { + let metrics_path = metadata_metrics_path(bucket); + rustfs_io_metrics::record_get_object_metadata_early_stop_miss( + metrics_path, + GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, + ); + rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0); + } + + Self::read_metadata_full_wait( + disks, + org_bucket, + bucket, + object, + version_id, + read_data, + healing, + incl_free_versions, + observe, + allow_coalescing, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn read_metadata_full_wait( + disks: &[Option], + org_bucket: &str, + bucket: &str, + object: &str, + version_id: &str, + read_data: bool, + healing: bool, + incl_free_versions: bool, + observe: bool, + allow_coalescing: bool, + ) -> disk::error::Result { + let fanout_start = observe.then(Instant::now); + let mut slots = MetadataReadResult::pending(disks.len()); + let mut observations = observe.then(|| Vec::with_capacity(disks.len())); + let scheduled_count = disks.len(); + let opts = ReadOptions { + incl_free_versions, + read_data, + healing, + }; + let org_bucket: Arc = Arc::from(org_bucket); + let bucket: Arc = Arc::from(bucket); + let object: Arc = Arc::from(object); + let version_id: Arc = Arc::from(version_id); + let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data); + let futures = disks.iter().enumerate().map(|(disk_index, disk)| { + let disk = disk.clone(); + let task_opts = opts; + let org_bucket = org_bucket.clone(); + let bucket = bucket.clone(); + let object = object.clone(); + let version_id = version_id.clone(); + let slowtail_fault = slowtail_fault.clone(); + AbortOnDropJoinHandle(tokio::spawn(async move { + let response_start = observe.then(Instant::now); + let result = if let Some(disk) = disk { + Self::record_read_version_call(&object, disk_index); + if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) { + Self::record_metadata_slowtail_fault(&object, disk_index); + tokio::time::sleep(delay).await; + } + read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing) + .await + } else { + Err(DiskError::DiskNotFound) + }; + let elapsed = response_start.map(|start| start.elapsed()); + (result, elapsed) + })) + }); + + // Wait for all futures to complete + let results = join_all(futures).await; + + for (index, join_result) in results.into_iter().enumerate() { + match join_result { + Ok((res, elapsed)) => match res { + Ok(file_info) => { + if let (Some(observations), Some(elapsed)) = (&mut observations, elapsed) { + observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed)); + } + slots[index].result = Some(Ok(file_info)); + } + Err(e) => { + if let (Some(observations), Some(elapsed)) = (&mut observations, elapsed) { + observations.push(MetadataFanoutObservation::from_error(&e, elapsed)); + } + slots[index].result = Some(Err(e)); + } + }, + Err(_join_err) => { + // A spawned task panicked — treat as unexpected disk error + if let Some(observations) = &mut observations { + observations.push(MetadataFanoutObservation::from_error(&DiskError::Unexpected, Duration::ZERO)); + } + slots[index].result = Some(Err(DiskError::Unexpected)); + } + } + } + let diagnostics = match (fanout_start, observations) { + (Some(fanout_start), Some(observations)) => MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations), + _ => MetadataFanoutDiagnostics::default(), + }; + if observe { + rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle( + metadata_metrics_path(bucket.as_ref()), + scheduled_count, + scheduled_count, + 0, + ); + } + Ok(MetadataReadResult { slots, diagnostics }) + } + + #[allow(clippy::too_many_arguments)] + async fn read_metadata_early_stop( + disks: &[Option], + org_bucket: &str, + bucket: &str, + object: &str, + version_id: &str, + read_data: bool, + healing: bool, + incl_free_versions: bool, + allow_non_inline_data_read_early_stop: bool, + default_parity_count: usize, + allow_coalescing: bool, + ) -> disk::error::Result { + let fanout_start = Instant::now(); + let mut slots = MetadataReadResult::pending(disks.len()); + let mut task_ids = vec![None; disks.len()]; + let mut observations = Vec::with_capacity(disks.len()); + let mut accumulator = + MetadataQuorumAccumulator::new(disks.len(), default_parity_count, true).with_requested_version_id(version_id); + let opts = ReadOptions { + incl_free_versions, + read_data, + healing, + }; + let org_bucket: Arc = Arc::from(org_bucket); + let bucket: Arc = Arc::from(bucket); + let object: Arc = Arc::from(object); + let version_id: Arc = Arc::from(version_id); + let metrics_path = metadata_metrics_path(bucket.as_ref()); + let mut join_set = JoinSet::new(); + let bounded_fanout = is_get_metadata_early_stop_bounded_fanout_enabled(); + let fanout_order = if bounded_fanout { + bounded_metadata_fanout_order(bucket.as_ref(), object.as_ref(), disks.len(), default_parity_count) + } else { + Vec::new() + }; + let mut next_fanout_index = 0usize; + let mut scheduled_count = 0usize; + let mut force_full_wait = false; + let mut final_miss_reason_override = None; + let mut non_inline_candidate_eligible = None; + let mut single_pending_hedge_deadline = None; + let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data); + let spawn_read_version = |join_set: &mut JoinSet<(usize, disk::error::Result, Duration)>, + task_ids: &mut [Option], + index: usize, + disk: Option| { + let task_opts = opts; + let org_bucket = org_bucket.clone(); + let bucket = bucket.clone(); + let object = object.clone(); + let version_id = version_id.clone(); + let slowtail_fault = slowtail_fault.clone(); + let task = join_set.spawn(async move { + let response_start = Instant::now(); + let result = if let Some(disk) = disk { + #[cfg(test)] + let _fanout_task_guard = rename_fanout_barrier::task_guard(&object); + Self::record_read_version_call(&object, index); + #[cfg(test)] + Self::read_version_fanout_barrier(&object, index).await; + if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) { + Self::record_metadata_slowtail_fault(&object, index); + tokio::time::sleep(delay).await; + } + read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing) + .await + } else { + Err(DiskError::DiskNotFound) + }; + (index, result, response_start.elapsed()) + }); + task_ids[index] = Some(task.id()); + }; + + if bounded_fanout { + let initial_target = accumulator.default_write_quorum().min(disks.len()); + while next_fanout_index < initial_target { + let disk_index = fanout_order[next_fanout_index]; + if let Some(disk) = disks.get(disk_index).cloned() { + spawn_read_version(&mut join_set, &mut task_ids, disk_index, disk); + scheduled_count = scheduled_count.saturating_add(1); + } + next_fanout_index = next_fanout_index.saturating_add(1); + } + } else { + for (index, disk) in disks.iter().cloned().enumerate() { + spawn_read_version(&mut join_set, &mut task_ids, index, disk); + scheduled_count = scheduled_count.saturating_add(1); + } + } + + loop { + let mut defer_pending_inline_data_shard = false; + let result = if let Some(deadline) = single_pending_hedge_deadline.take() { + tokio::select! { + result = join_set.join_next_with_id() => result, + _ = async { + tokio::time::sleep_until(deadline).await; + #[cfg(test)] + rename_fanout_barrier::checkpoint( + object.as_ref(), + 0, + rename_fanout_barrier::PHASE_NON_INLINE_HEDGE_TIMER, + ) + .await; + } => { + if bounded_fanout + && !force_full_wait + && join_set.len() == 1 + && non_inline_candidate_eligible == Some(true) + && !accumulator.candidate_has_read_reserve() + && next_fanout_index < disks.len() + { + while next_fanout_index < disks.len() { + let disk_index = fanout_order[next_fanout_index]; + next_fanout_index = next_fanout_index.saturating_add(1); + if let Some(disk) = disks.get(disk_index).cloned() { + spawn_read_version(&mut join_set, &mut task_ids, disk_index, disk); + scheduled_count = scheduled_count.saturating_add(1); + break; + } + } + } + continue; + } + } + } else { + join_set.join_next_with_id().await + }; + let Some(result) = result else { break }; + match result { + Ok((_task_id, (index, res, elapsed))) => match res { + Ok(file_info) => { + observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed)); + let inline_candidate_miss_reason = if bounded_fanout && read_data && !force_full_wait { + data_read_early_stop_inline_candidate_miss_reason(&file_info) + } else { + None + }; + slots[index].result = Some(Ok(file_info)); + accumulator.observe(&slots[index]); + if allow_non_inline_data_read_early_stop && non_inline_candidate_eligible.is_none() { + non_inline_candidate_eligible = + accumulator.candidate.as_ref().map(non_inline_data_read_candidate_is_safe); + } + if bounded_fanout + && read_data + && !force_full_wait + && let Some(reason) = inline_candidate_miss_reason + && !(non_inline_candidate_eligible == Some(true) + && reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE) + { + force_full_wait = true; + final_miss_reason_override.get_or_insert(reason); + } + } + Err(err) => { + observations.push(MetadataFanoutObservation::from_error(&err, elapsed)); + slots[index].result = Some(Err(err)); + accumulator.observe(&slots[index]); + } + }, + Err(join_error) => { + let err = DiskError::Unexpected; + observations.push(MetadataFanoutObservation::from_error(&err, fanout_start.elapsed())); + if let Some(index) = task_ids.iter().position(|id| *id == Some(join_error.id())) { + slots[index].result = Some(Err(err)); + accumulator.observe(&slots[index]); + } + } + } + + if !force_full_wait + && let Some(decision) = accumulator + .early_stop_decision() + .or_else(|| accumulator.version_early_stop_decision()) + { + let should_return_early = if read_data { + match accumulator.candidate.as_ref() { + Some(_candidate) if non_inline_candidate_eligible == Some(true) => { + accumulator.candidate_has_read_reserve() + } + Some(candidate) => match inline_body_miss_reason_from_observations( + bucket.as_ref(), + object.as_ref(), + candidate, + slots.iter().map(|slot| (slot.disk_index, slot.file_info())), + disks, + ) + .await + { + None => true, + Some(reason) => { + final_miss_reason_override = Some(reason); + if bounded_fanout + && reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD + && data_read_inline_missing_shards_are_pending( + candidate, + &slots, + disks, + &fanout_order, + next_fanout_index, + ) + { + defer_pending_inline_data_shard = true; + } else { + force_full_wait = true; + } + false + } + }, + None => { + force_full_wait = true; + final_miss_reason_override = Some(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM); + false + } + } + } else { + true + }; + + if should_return_early { + let saved_responses = if bounded_fanout { + disks.len().saturating_sub(observations.len()) + } else { + join_set.len() + }; + join_set.abort_all(); + rustfs_io_metrics::record_get_object_metadata_early_stop_hit(metrics_path, decision.reason); + rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, saved_responses); + let mut cancelled_count = 0usize; + while let Some(join_result) = join_set.join_next_with_id().await { + match join_result { + Err(join_error) if join_error.is_cancelled() => { + cancelled_count = cancelled_count.saturating_add(1); + } + _ => {} + } + } + rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle( + metrics_path, + scheduled_count, + scheduled_count.saturating_sub(cancelled_count), + cancelled_count, + ); + let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations); + return Ok(MetadataReadResult { slots, diagnostics }); + } + } + + let pending_responses = join_set.len(); + // Inline verification can still depend on a missing data shard; + // issue one immediate spare when only that shard remains. The + // non-inline path keeps its delayed hedge below to avoid healthy + // reads paying speculative I/O before the candidate is classified. + let should_hedge_single_pending_inline_read = read_data + && !force_full_wait + && !defer_pending_inline_data_shard + && pending_responses == 1 + && non_inline_candidate_eligible != Some(true) + && accumulator.can_still_reach_early_stop_with_pending(pending_responses); + // A non-inline plan must retain one extra matching shard as a + // reconstruction reserve. Schedule that reserve only after the + // candidate is known to be eligible, so inline GETs do not pay an + // extra fanout and the healthy path remains allocation-free. + let needs_non_inline_read_reserve = non_inline_candidate_eligible == Some(true) + && !accumulator.candidate_has_read_reserve() + && accumulator + .candidate_read_reserve_target() + .is_some_and(|reserve_target| scheduled_count < reserve_target || pending_responses == 0); + if bounded_fanout + && !force_full_wait + && (needs_non_inline_read_reserve || should_hedge_single_pending_inline_read) + && next_fanout_index < disks.len() + { + let disk_index = fanout_order[next_fanout_index]; + if let Some(disk) = disks.get(disk_index).cloned() { + spawn_read_version(&mut join_set, &mut task_ids, disk_index, disk); + scheduled_count = scheduled_count.saturating_add(1); + } + next_fanout_index = next_fanout_index.saturating_add(1); + } else if bounded_fanout && force_full_wait { + while next_fanout_index < disks.len() { + let disk_index = fanout_order[next_fanout_index]; + if let Some(disk) = disks.get(disk_index).cloned() { + spawn_read_version(&mut join_set, &mut task_ids, disk_index, disk); + scheduled_count = scheduled_count.saturating_add(1); + } + next_fanout_index = next_fanout_index.saturating_add(1); + } + } else if bounded_fanout + && !defer_pending_inline_data_shard + && next_fanout_index < disks.len() + && !accumulator.can_still_reach_early_stop_with_pending(pending_responses) + { + let disk_index = fanout_order[next_fanout_index]; + if let Some(disk) = disks.get(disk_index).cloned() { + spawn_read_version(&mut join_set, &mut task_ids, disk_index, disk); + scheduled_count = scheduled_count.saturating_add(1); + } + next_fanout_index = next_fanout_index.saturating_add(1); + } + if bounded_fanout + && !force_full_wait + && !defer_pending_inline_data_shard + && join_set.len() == 1 + && non_inline_candidate_eligible == Some(true) + && !accumulator.candidate_has_read_reserve() + && accumulator.can_still_reach_early_stop_with_pending(join_set.len()) + && next_fanout_index < disks.len() + { + single_pending_hedge_deadline = Some(tokio::time::Instant::now() + NON_INLINE_SINGLE_PENDING_HEDGE_DELAY); + } + } + + let accumulator_miss_reason = accumulator.final_miss_reason(); + let final_miss_reason = match (final_miss_reason_override, accumulator_miss_reason) { + (Some(reason), GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM) => reason, + _ => accumulator_miss_reason, + }; + rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, final_miss_reason); + rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0); + rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0); + let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations); + Ok(MetadataReadResult { slots, diagnostics }) + } + + /// Test-only seam that records one per-disk `read_version` metadata RPC for + /// the call-counter registry (backlog#1325). In production this is inlined to + /// nothing and adds no behavior; only the `#[cfg(test)]` variant touches the + /// registry. Placed inside the fan-out spawn tasks so counts are observed + /// even though the increments run on arbitrary runtime worker threads. + #[cfg(test)] + #[inline] + fn record_read_version_call(object: &str, disk_index: usize) { + disk_call_counters::record(object, disk_call_counters::KIND_READ_VERSION, disk_index); + } + + #[cfg(not(test))] + #[inline(always)] + fn record_read_version_call(_object: &str, _disk_index: usize) {} + + #[cfg(test)] + #[inline] + fn record_metadata_slowtail_fault(object: &str, disk_index: usize) { + disk_call_counters::record(object, disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT, disk_index); + } + + #[cfg(not(test))] + #[inline(always)] + fn record_metadata_slowtail_fault(_object: &str, _disk_index: usize) {} + + #[cfg(test)] + #[inline] + async fn read_version_fanout_barrier(object: &str, disk_index: usize) { + rename_fanout_barrier::checkpoint(object, disk_index, rename_fanout_barrier::PHASE_READ_VERSION).await; + } +} diff --git a/crates/ecstore/src/set_disk/core/mod.rs b/crates/ecstore/src/set_disk/core/mod.rs index b8507603e..8ecce941c 100644 --- a/crates/ecstore/src/set_disk/core/mod.rs +++ b/crates/ecstore/src/set_disk/core/mod.rs @@ -18,4 +18,5 @@ //! duplicating read/write/erasure logic. pub(crate) mod io_primitives; -mod metadata_quorum; +pub(in crate::set_disk) mod metadata_quorum; +pub(in crate::set_disk) mod metadata_read; diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index bce019a22..2bddc7734 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -5886,12 +5886,29 @@ fn collect_inline_data_shard_fileinfos_by_index_or_reason<'a>( parts_metadata: &'a [FileInfo], fi: &FileInfo, data_shards: usize, + disk_is_online: impl FnMut(usize) -> bool, +) -> std::result::Result, &'static str> { + collect_inline_data_shard_fileinfos_from_observations( + parts_metadata + .iter() + .enumerate() + .map(|(index, metadata)| (index, Some(metadata))), + fi, + data_shards, + disk_is_online, + ) +} + +fn collect_inline_data_shard_fileinfos_from_observations<'a>( + observations: impl IntoIterator)>, + fi: &FileInfo, + data_shards: usize, mut disk_is_online: impl FnMut(usize) -> bool, ) -> std::result::Result, &'static str> { let distribution = &fi.erasure.distribution; let mut data_files = vec![None; data_shards]; - for (disk_index, file_info) in parts_metadata.iter().enumerate() { + for (disk_index, file_info) in observations { if !disk_is_online(disk_index) { continue; } @@ -5901,9 +5918,9 @@ fn collect_inline_data_shard_fileinfos_by_index_or_reason<'a>( if block_index == 0 || block_index > data_shards { continue; } - if file_info.name.is_empty() { + let Some(file_info) = file_info.filter(|metadata| !metadata.name.is_empty()) else { return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD); - } + }; if file_info.erasure.index != block_index { return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH); } diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 7ca7e0916..b8265ef60 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -602,17 +602,18 @@ impl SetDisks { } scope.check()?; - let (_, errors) = - Self::read_all_fileinfo(disks.as_slice(), "", bucket, object, &version.to_string(), false, false, false).await?; + let metadata_read = + Self::read_metadata(disks.as_slice(), "", bucket, object, &version.to_string(), false, false, false).await?; + let all_absent = metadata_read + .slots + .iter() + .all(|slot| matches!(&slot.result, Some(Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)))); + let selected = self.get_disks_internal().await; let same_targets = selected.len() == disks.len() && selected.iter().zip(&disks).all(|(current, original)| { matches!((current, original), (Some(current), Some(original)) if std::sync::Arc::ptr_eq(current, original)) }); - if !same_targets - || !errors - .iter() - .all(|error| matches!(error, Some(DiskError::FileNotFound | DiskError::FileVersionNotFound))) - { + if !same_targets || !all_absent { return Err(StorageError::SlowDown); } scope.check()?; @@ -741,7 +742,7 @@ impl SetDisks { object: &str, version_id: &str, ) -> disk::error::Result { - let (parts_metadata, errs, _) = Self::read_all_fileinfo_observed( + let metadata_read = Self::read_metadata_observed( disks, "", bucket, @@ -754,6 +755,7 @@ impl SetDisks { self.default_parity_count, ) .await?; + let (parts_metadata, errs, _) = metadata_read.into_legacy(); let (read_quorum, _) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)?; let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?; let (_, quorum_mod_time, quorum_etag) = Self::list_online_disks(disks, &parts_metadata, &errs, read_quorum); @@ -938,8 +940,9 @@ impl SetDisks { } }; - let (mut parts_metadata, errs) = - Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true, false).await?; + let (mut parts_metadata, errs, _) = Self::read_metadata(&disks, "", bucket, object, version_id, true, true, false) + .await? + .into_legacy(); trace!( event = EVENT_SET_DISK_HEAL, @@ -1291,8 +1294,10 @@ impl SetDisks { // in `parts_metadata` to defaults. Re-read only before // destructive cleanup so the guard sees every original // identity. - let (delete_guard_metadata, delete_guard_errs) = - Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true, false).await?; + let (delete_guard_metadata, delete_guard_errs, _) = + Self::read_metadata(&disks, "", bucket, object, version_id, true, true, false) + .await? + .into_legacy(); if self .dangling_delete_safety(bucket, object, &delete_guard_metadata, &delete_guard_errs, &disks) .await? @@ -2327,16 +2332,16 @@ impl SetDisks { return Err(DiskError::retired_marker_deferred(format!("conditional marker deletion failed: {error}"))); } } - let (_, errors) = Self::read_all_fileinfo(disks, "", bucket, object, &version.to_string(), false, false, false).await?; + let metadata_read = Self::read_metadata(disks, "", bucket, object, &version.to_string(), false, false, false).await?; + let all_absent = metadata_read + .slots + .iter() + .all(|slot| matches!(&slot.result, Some(Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)))); let selected = self.get_disks_internal().await; let same_targets = selected.len() == disks.len() && selected.iter().zip(disks).all(|(current, original)| { matches!((current, original), (Some(current), Some(original)) if std::sync::Arc::ptr_eq(current, original)) }); - if !same_targets - || !errors - .iter() - .all(|error| matches!(error, Some(DiskError::FileNotFound | DiskError::FileVersionNotFound))) - { + if !same_targets || !all_absent { return Err(DiskError::retired_marker_deferred( "complete marker absence could not be verified under the original target set", )); @@ -3421,9 +3426,10 @@ impl SetDisks { // The inner heal and missing-object report read the registry again; // release this snapshot guard before a topology writer can queue between reads. let disks = self.get_disks_internal().await; - let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false) + let (_, errs, _) = Self::read_metadata(&disks, "", bucket, object, version_id, false, false, false) .await - .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?; + .map_err(|e| to_object_err(e.into(), vec![bucket, object]))? + .into_legacy(); if DiskError::is_all_not_found(&errs) { debug!( event = EVENT_SET_DISK_HEAL, diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 68edacb67..2cf010f63 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -115,7 +115,28 @@ use super::GetCodecStreamingObjectClass; use super::GetCodecStreamingRollout; #[cfg(test)] use super::classify_get_codec_streaming_object_class; -use super::core::io_primitives::*; +use super::core::io_primitives::{ + BitrotReaderSetup, BitrotReaderSetupAttribution, BitrotReaderSetupMode, DeferredReaderReopener, EVENT_SET_DISK_READ, + GetCodecStreamingReaderBuildOutcome, MetadataCacheLookup, ObjectBitrotReader, ReadRepairAdmissionSubmitter, + ReadRepairHealSubmission, SLOW_OBJECT_READ_LOG_THRESHOLD, codec_streaming_reader_setup_fallback_reason, + create_bitrot_readers_until_quorum_with_preference, create_data_block_bitrot_readers, resolved_read_repair_version_id, + send_read_repair_heal_request, shard_read_costs_for_disks, submit_read_repair_heal, submit_read_repair_heal_with_submitter, +}; +#[cfg(test)] +use super::core::io_primitives::{ + ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP, ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, + MultipartCodecStreamingReader, ReadRepairAdmissionFuture, ReadRepairAdmissionOutcome, collect_read_multiple_results, + collect_read_parts_results, create_bitrot_readers_until_quorum, create_bitrot_readers_until_quorum_all_shards, + release_read_repair_heal_reservation, reserve_read_repair_heal, resolve_read_part_from_responses, shard_read_cost_for_disk, + shard_read_cost_for_endpoint, +}; +#[cfg(test)] +use super::core::metadata_quorum::{MetadataEarlyStopDecision, MetadataQuorumAccumulator}; +#[cfg(test)] +use super::core::metadata_read::{ + MetadataFanoutDiagnostics, MetadataFanoutObservation, metadata_early_stop_permitted, should_allow_metadata_early_stop, +}; +use super::core::metadata_read::{late_materialization_candidate_is_safe, non_inline_data_read_early_stop_allowed}; #[cfg(test)] use super::get_codec_streaming_config_cached_core; #[cfg(test)] @@ -672,11 +693,11 @@ impl SetDisks { }; // Early-stop for safe metadata reads is handled inside - // read_all_fileinfo_observed (see read_all_fileinfo_early_stop in - // core/io_primitives.rs); unsafe requests and callers that opt out + // read_metadata_observed (see read_all_fileinfo_early_stop in + // core/metadata_read.rs); unsafe requests and callers that opt out // (allow_early_stop=false) fall back to full-wait. - let (mut parts_metadata, errs, metadata_fanout_diagnostics) = if allow_read_version_coalescing { - Self::read_all_fileinfo_observed_for_get_object( + let metadata_read = if allow_read_version_coalescing { + Self::read_metadata_for_get_object( &disks, "", bucket, @@ -689,7 +710,7 @@ impl SetDisks { ) .await? } else { - Self::read_all_fileinfo_observed( + Self::read_metadata_observed( &disks, "", bucket, @@ -703,13 +724,14 @@ impl SetDisks { ) .await? }; + let metadata_fanout_complete = metadata_read.is_complete(); + let (mut parts_metadata, errs, metadata_fanout_diagnostics) = metadata_read.into_legacy(); let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) { GET_OBJECT_PATH_INTERNAL_META } else { GET_OBJECT_PATH_LEGACY_DUPLEX }; metadata_fanout_diagnostics.record(metadata_metrics_path); - let metadata_fanout_complete = metadata_fanout_diagnostics.total_responses() >= disks.len(); // warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata); // warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs); @@ -2190,7 +2212,7 @@ impl SetDisks { expected: &LateMetadataIdentity, metrics_path: &'static str, ) -> Result<(FileInfo, Vec, Vec>)> { - let (mut parts_metadata, errs, diagnostics) = SetDisks::read_all_fileinfo_observed( + let metadata_read = SetDisks::read_metadata_observed( fallback_disks, "", bucket, @@ -2203,6 +2225,7 @@ impl SetDisks { expected.parity_blocks, ) .await?; + let (mut parts_metadata, errs, diagnostics) = metadata_read.into_legacy(); diagnostics.record(metrics_path); let (read_quorum, write_quorum) = SetDisks::object_quorum_from_meta(&parts_metadata, &errs, expected.parity_blocks) @@ -2631,6 +2654,7 @@ fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOpti #[cfg(test)] mod metadata_cache_tests { use super::*; + use crate::set_disk::core::io_primitives::disk_call_counters; use rustfs_heal_contracts::heal_channel::HealAdmissionDropReason; use serial_test::serial; use std::sync::atomic::{AtomicUsize, Ordering}; diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 19fc2260d..6ecabdad2 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -354,6 +354,15 @@ fn ensure_producer_limit_state(options: &mut ListPathOptions) -> Arc state } +fn list_path_folds_common_prefixes(options: &ListPathOptions) -> bool { + !options.versioned + && (options.include_directories + || options + .separator + .as_deref() + .is_some_and(|separator| !separator.is_empty() && separator != SLASH_SEPARATOR)) +} + async fn can_skip_hidden_prefix_check(options: &ListPathOptions) -> bool { if options.recursive || options.incl_deleted || options.versioned { return false; @@ -2440,6 +2449,12 @@ where let suffix = key.trim_start_matches(prefix); if let Some((common_prefix, _)) = suffix.split_once(separator) { let common_prefix = format!("{prefix}{common_prefix}{separator}"); + if marker.is_some_and(|marker| common_prefix.as_str() <= marker) { + if collect_stats { + stats.skipped_keys += 1; + } + continue; + } if prefix_set.insert(common_prefix.clone()) { if collect_stats { stats.common_prefixes += 1; @@ -2542,6 +2557,10 @@ fn list_objects_from_metadata_snapshot_candidates( let suffix = object.name.trim_start_matches(prefix); if let Some((common_prefix, _)) = suffix.split_once(separator) { let common_prefix = format!("{prefix}{common_prefix}{separator}"); + if marker.is_some_and(|marker| common_prefix.as_str() <= marker) { + stats.skipped_keys += 1; + continue; + } if prefix_set.insert(common_prefix.clone()) { stats.common_prefixes += 1; visible_entries.push(VerifiedIndexVisibleEntry::Prefix(common_prefix)); @@ -4962,6 +4981,8 @@ async fn gather_results( let gather_started = list_metrics_enabled.then(std::time::Instant::now); let mut scanned_entries = 0usize; let mut candidate_entries = 0usize; + let mut previous_prefix = String::new(); + let folds_prefixes = list_path_folds_common_prefixes(&opts); while let Some(mut entry) = recv.recv().await { scanned_entries += 1; @@ -5001,6 +5022,34 @@ async fn gather_results( continue; } + // Version LIST projections omit cleanup-only free versions. They must + // not consume the lookahead budget before a later visible version. + // Cleanup walks consume the raw set stream without this collector. + if opts.versioned + && is_object + && entry + .xl_meta() + .is_ok_and(|meta| !meta.versions.is_empty() && meta.versions.iter().all(|version| version.header.free_version())) + { + continue; + } + + if folds_prefixes + && let Some(separator) = opts.separator.as_deref().filter(|separator| !separator.is_empty()) + && let Some(offset) = entry.name[opts.prefix.len()..].find(separator) + { + let common_prefix = &entry.name[..opts.prefix.len() + offset + separator.len()]; + if common_prefix == previous_prefix + || opts.marker.as_deref().is_some_and(|marker| { + (!opts.include_marker && common_prefix <= marker) || (opts.include_marker && common_prefix < marker) + }) + { + continue; + } + previous_prefix.clear(); + previous_prefix.push_str(common_prefix); + } + // TODO(backlog): integrate lifecycle evaluation during object listing entries.push(Some(entry)); @@ -7023,7 +7072,9 @@ impl SetDisks { ); let limit = { - if opts.limit > 0 && opts.stop_disk_at_limit { + // A raw scan budget can be consumed entirely by keys folding into + // one common prefix. Let the logical collector stop these walks. + if opts.limit > 0 && opts.stop_disk_at_limit && !list_path_folds_common_prefixes(&opts) { opts.limit + 4 + (opts.limit / 16) } else { 0 @@ -7248,6 +7299,9 @@ fn calc_common_counter(infos: &[DiskInfo], read_quorum: usize) -> u64 { // list_path_raw +#[cfg(test)] +mod differential_tests; + #[cfg(test)] mod test { use super::{ @@ -7926,6 +7980,117 @@ mod test { assert!(cancel.is_cancelled()); } + #[tokio::test] + async fn list_path_gather_results_counts_common_prefixes_before_page_limit() { + for separator in ["/", "-"] { + for extra_object in [false, true] { + let prefix_name = format!("a{separator}"); + let first = if separator == "/" { + prefix_name.clone() + } else { + format!("{prefix_name}first") + }; + let mut input = vec![test_object_meta_entry(&first)]; + if separator == "/" { + input.push(test_dir_meta_entry(&prefix_name)); + } else { + input.push(test_object_meta_entry(&format!("{prefix_name}second"))); + } + input.push(test_object_meta_entry("b")); + if extra_object { + input.push(test_object_meta_entry("c")); + } + let (entry_tx, entry_rx) = mpsc::channel(input.len()); + let (result_tx, mut result_rx) = mpsc::channel(1); + for entry in input { + entry_tx.send(entry).await.expect("delimiter candidates should queue"); + } + drop(entry_tx); + let state = gather_results( + CancellationToken::new(), + ListPathOptions { + bucket: "bucket".to_owned(), + separator: Some(separator.to_owned()), + recursive: separator != "/", + include_directories: separator == "/", + limit: 3, + ..Default::default() + }, + entry_rx, + result_tx, + ) + .await + .expect("delimiter collection should succeed"); + let result = result_rx.recv().await.expect("delimiter page should arrive"); + let mut expected = vec![first, "b".to_owned()]; + if extra_object { + expected.push("c".to_owned()); + } + assert_eq!( + result + .entries + .expect("delimiter entries should exist") + .entries() + .into_iter() + .map(|entry| entry.name.clone()) + .collect::>(), + expected, + "delimiter={separator}, extra={extra_object}" + ); + assert_eq!(result.err.is_none(), extra_object, "n=max must reach EOF; max+1 must retain lookahead"); + assert_eq!( + state, + if extra_object { + GatherResultsState::LimitReached + } else { + GatherResultsState::InputClosed + } + ); + } + } + for include_marker in [false, true] { + let (entry_tx, entry_rx) = mpsc::channel(2); + let (result_tx, mut result_rx) = mpsc::channel(1); + for name in ["a/child", "b"] { + entry_tx + .send(test_object_meta_entry(name)) + .await + .expect("marker candidates should queue"); + } + drop(entry_tx); + gather_results( + CancellationToken::new(), + ListPathOptions { + bucket: "bucket".to_owned(), + separator: Some("/".to_owned()), + recursive: true, + include_directories: true, + marker: Some("a/".to_owned()), + include_marker, + limit: 3, + ..Default::default() + }, + entry_rx, + result_tx, + ) + .await + .expect("logical marker collection should succeed"); + let entries = result_rx + .recv() + .await + .expect("logical marker page should arrive") + .entries + .expect("logical marker entries should exist"); + let projected = ObjectInfo::from_meta_cache_entries_sorted_infos(&entries, "bucket", "", Some("/".to_owned())).await; + let names: Vec<_> = projected.into_iter().map(|object| object.name).collect(); + assert_eq!( + names, + if include_marker { vec!["a/", "b"] } else { vec!["b"] }, + "a folded prefix equal to the marker must follow include_marker={include_marker}" + ); + } + } + #[test] fn list_versions_pagination_scan_limit_boundaries() { for has_version_marker in [false, true] { @@ -8977,6 +9142,29 @@ mod test { assert_eq!(result.info.objects[0].etag.as_deref(), Some("etag-z")); assert_eq!(result.info.prefixes, vec!["photos/2026/nested/".to_string()]); assert!(!result.info.is_truncated); + + let resumed = list_objects_from_metadata_snapshot_candidates( + "bucket", + "photos/2026/", + Some("photos/2026/nested/"), + &Some("/".to_string()), + 1, + &objects, + ); + assert!( + resumed.info.prefixes.is_empty(), + "metadata-fast cursors must not replay an emitted prefix" + ); + assert_eq!( + resumed + .info + .objects + .iter() + .map(|object| object.name.as_str()) + .collect::>(), + ["photos/2026/z.jpg"] + ); + assert!(!resumed.info.is_truncated); } #[test] @@ -10089,6 +10277,23 @@ mod test { assert_eq!(result.prefixes, vec!["photos/2026/archive/".to_string()]); assert!(result.is_truncated); assert_eq!(result.next_marker.as_deref(), Some("photos/2026/archive/")); + + let resumed = list_objects_from_verified_index_candidates( + "photos/2026/", + result.next_marker.as_deref(), + &Some("/".to_string()), + 1, + &candidates, + |key| async move { Ok(Some(test_live_object_info(&key, "live-etag"))) }, + ) + .await + .expect("verified prefix continuation should succeed"); + assert!(resumed.prefixes.is_empty(), "verified cursors must not replay an emitted prefix"); + assert_eq!( + resumed.objects.iter().map(|object| object.name.as_str()).collect::>(), + ["photos/2026/d.jpg"] + ); + assert!(!resumed.is_truncated); } #[tokio::test] diff --git a/crates/ecstore/src/store/list_objects/differential_tests.rs b/crates/ecstore/src/store/list_objects/differential_tests.rs new file mode 100644 index 000000000..4fef15b27 --- /dev/null +++ b/crates/ecstore/src/store/list_objects/differential_tests.rs @@ -0,0 +1,1129 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; +use crate::disk::STORAGE_FORMAT_FILE; +use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions}; +use rustfs_filemeta::{FileInfo, ObjectPartInfo}; +use std::collections::BTreeMap; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::io::{AsyncWrite, DuplexStream}; +use tokio::sync::Notify; +use tokio::time::timeout; + +fn shared_keys() -> Vec { + serde_json::from_str(include_str!("../../../tests/fixtures/list_namespace_keys.json")) + .expect("shared LIST namespace corpus should decode") +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum VersionKind { + Object, + Delete, + Free, +} + +#[derive(Clone, Debug)] +struct Version { + id: Option, + modified: i64, + kind: VersionKind, +} + +#[derive(Clone, Debug)] +struct Object { + key: String, + // Expected S3 version order is part of the input, including equal-time ties. + versions: Vec, +} + +fn namespace(keys: &[String]) -> Vec { + keys.iter() + .map(|key| Object { + key: key.clone(), + versions: vec![Version { + id: None, + modified: 1_705_312_300, + kind: VersionKind::Object, + }], + }) + .collect() +} + +#[derive(Debug, PartialEq, Eq)] +struct Page { + objects: Vec, + prefixes: Vec, + next_key: Option, + truncated: bool, +} + +// This oracle sees only the input graph. It never reads FileMeta or calls the +// production filtering, pagination, merge, or prefix helpers. +fn reference_page(objects: &[Object], prefix: &str, delimiter: Option<&str>, after: Option<&str>, max: usize) -> Page { + let mut entries = BTreeMap::new(); + for object in objects { + if !object.key.starts_with(prefix) + || object + .versions + .iter() + .find(|version| version.kind != VersionKind::Free) + .is_none_or(|version| version.kind == VersionKind::Delete) + { + continue; + } + let (key, is_prefix) = match delimiter.filter(|delimiter| !delimiter.is_empty()).and_then(|delimiter| { + object.key[prefix.len()..] + .find(delimiter) + .map(|offset| (object.key[..prefix.len() + offset + delimiter.len()].to_owned(), true)) + }) { + Some(entry) => entry, + None => (object.key.clone(), false), + }; + if after.is_none_or(|after| key.as_str() > after) { + entries.insert(key, is_prefix); + } + } + let truncated = max > 0 && entries.len() > max; + let selected: Vec<_> = entries.into_iter().take(max).collect(); + Page { + next_key: truncated.then(|| selected.last().expect("positive truncated page has an entry").0.clone()), + objects: selected + .iter() + .filter(|(_, prefix)| !prefix) + .map(|(key, _)| key.clone()) + .collect(), + prefixes: selected + .iter() + .filter(|(_, prefix)| *prefix) + .map(|(key, _)| key.clone()) + .collect(), + truncated, + } +} + +fn marker_key(marker: &str) -> &str { + marker.split_once("[rustfs_cache:").map_or(marker, |(key, _)| key) +} + +struct Fixture { + store: Arc, + bucket: String, + objects: Vec, + dirs: Vec, +} + +impl Fixture { + async fn new(bucket: &str, objects: Vec, authoritative: bool) -> Self { + let (dirs, store) = isolated_store_over_temp_disks().await; + if authoritative { + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("corpus bucket should be created with authoritative metadata"); + } else { + for disk in store.pools[0].disk_set[0].disks.read().await.iter().flatten() { + disk.make_volume(bucket) + .await + .expect("legacy corpus volume should be created"); + } + } + let fixture = Self { + dirs, + store, + bucket: bucket.to_owned(), + objects, + }; + for object in &fixture.objects { + for disk_index in 0..fixture.dirs.len() { + fixture.write_metadata(disk_index, object).await; + } + } + if !fixture.objects.is_empty() { + observe_list_objects_mutations(&fixture.store, bucket, fixture.objects.len()).await; + } + fixture + } + + fn metadata(&self, object: &Object, disk_index: usize) -> Vec { + let mut meta = FileMeta::new(); + // Writing from oldest to newest exercises the real on-disk format's + // canonicalization rather than giving the listing a pre-sorted model. + for version in object.versions.iter().rev() { + let mut info = if version.kind == VersionKind::Delete { + FileInfo::default() + } else { + FileInfo::new(&object.key, 2, 2) + }; + info.name = object.key.clone(); + info.volume = self.bucket.clone(); + info.version_id = version.id; + info.versioned = version.id.is_some(); + info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(version.modified).expect("corpus timestamp")); + info.deleted = version.kind == VersionKind::Delete; + if !info.deleted { + info.erasure.index = disk_index + 1; + info.data_dir = Some(Uuid::from_u128(0x1234)); + info.size = 1; + info.parts = vec![ObjectPartInfo { + number: 1, + size: 1, + actual_size: 1, + ..Default::default() + }]; + info.metadata.insert("etag".to_owned(), "corpus-etag".to_owned()); + info.data = Some(Bytes::from_static(b"x")); + info.set_inline_data(); + } + info.validate_for_metadata_read() + .expect("corpus metadata should be readable by the live index verifier"); + if info.deleted { + assert!(info.is_canonical_delete_marker(), "corpus delete markers must be canonical"); + } + if version.kind == VersionKind::Free { + info.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_owned(); + info.transition_tier = "WARM".to_owned(); + info.transitioned_objname = "corpus-remote-key".to_owned(); + info.transition_version = Some("remote-version".to_owned()); + info.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact; + rustfs_utils::http::metadata_compat::insert_str( + &mut info.metadata, + rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + "00".repeat(32), + ); + meta.add_version(info.clone()) + .expect("transitioned fixture version should serialize"); + let mut delete = FileInfo { + name: object.key.clone(), + version_id: info.version_id, + ..Default::default() + }; + delete.set_tier_free_version_id(&Uuid::from_u128(0xf00).to_string()); + meta.delete_version(&delete) + .expect("fixture should leave a real free-version cleanup owner"); + } else { + meta.add_version(info).expect("fixture version should serialize"); + } + } + meta.marshal_msg().expect("fixture xl.meta should encode") + } + + async fn write_metadata(&self, disk_index: usize, object: &Object) { + let physical_key = rustfs_utils::path::encode_dir_object(&object.key); + let disk = self.store.pools[0].disk_set[0].disks.read().await[disk_index] + .clone() + .expect("fixture disk should be online"); + disk.write_all( + &self.bucket, + &format!("{physical_key}/{STORAGE_FORMAT_FILE}"), + Bytes::from(self.metadata(object, disk_index)), + ) + .await + .expect("corpus xl.meta should reach the real local disk"); + } + + async fn page( + &self, + layer: usize, + prefix: &str, + marker: Option, + delimiter: Option<&str>, + max: i32, + ) -> ListObjectsInfo { + let request = format!("corpus layer={layer}, prefix={prefix}, marker={marker:?}, delimiter={delimiter:?}, max={max}"); + let delimiter = delimiter.map(str::to_owned); + timeout(Duration::from_secs(10), async { + match layer { + 0 => { + self.store + .clone() + .list_objects_generic(&self.bucket, prefix, marker, delimiter, max, false) + .await + } + 1 => { + self.store.pools[0] + .clone() + .list_objects_generic(&self.bucket, prefix, marker, delimiter, max, false) + .await + } + _ => { + self.store.pools[0].disk_set[0] + .clone() + .list_objects_generic(&self.bucket, prefix, marker, delimiter, max, false) + .await + } + } + }) + .await + .unwrap_or_else(|error| panic!("{request}: page must finish: {error}")) + .unwrap_or_else(|error| panic!("corpus layer={layer}, prefix={prefix}, max={max}: {error:?}")) + } + + async fn assert_walk(&self, layer: usize, prefix: &str, delimiter: Option<&str>, start: Option<&str>, max: usize) { + let mut marker = start.map(str::to_owned); + let mut seen = Vec::new(); + for page_number in 0..=self.objects.len() { + let expected = reference_page(&self.objects, prefix, delimiter, marker.as_deref().map(marker_key), max); + let actual = self + .page(layer, prefix, marker.clone(), delimiter, i32::try_from(max).expect("small corpus limit")) + .await; + let observed = Page { + objects: actual.objects.into_iter().map(|object| object.name).collect(), + prefixes: actual.prefixes, + next_key: actual.next_marker.as_deref().map(marker_key).map(str::to_owned), + truncated: actual.is_truncated, + }; + assert_eq!( + observed, + expected, + "layer={layer}, prefix={prefix}, delimiter={delimiter:?}, max={max}, page={page_number}, marker={marker:?}, mode={:?}, provider={:?}", + std::env::var(ENV_API_LIST_OBJECTS_INDEX_MODE).ok(), + std::env::var(ENV_API_LIST_OBJECTS_INDEX_PROVIDER).ok(), + ); + seen.extend(observed.objects); + seen.extend(observed.prefixes); + if !actual.is_truncated { + let complete = reference_page(&self.objects, prefix, delimiter, start, if max == 0 { 0 } else { usize::MAX }); + let mut complete_keys = complete.objects; + complete_keys.extend(complete.prefixes); + complete_keys.sort(); + seen.sort(); + assert_eq!(seen, complete_keys, "complete walk must contain every identity exactly once"); + return; + } + assert!(actual.next_marker.is_some(), "truncated page must supply a cursor"); + assert_ne!(actual.next_marker, marker, "cursor must advance"); + marker = actual.next_marker; + } + panic!("corpus pagination did not converge"); + } + + async fn disk_walk(&self, options: WalkDirOptions) -> Vec { + let disk = self.store.pools[0].disk_set[0].disks.read().await[0] + .clone() + .expect("corpus walker disk should be online"); + let (reader, mut writer) = duplex(4096); + let producer = tokio::spawn(async move { disk.walk_dir(options, &mut writer).await }); + let entries = MetacacheReader::new(reader) + .read_all() + .await + .expect("real walker stream should decode"); + producer + .await + .expect("real walker producer should join") + .expect("real walker should reach EOF"); + entries + } +} + +#[tokio::test] +async fn list_objects_shared_corpus_static_namespace() { + let keys = shared_keys(); + for count in [0, 1, 3, 4, keys.len()] { + let fixture = Fixture::new(&format!("list-corpus-static-{count}"), namespace(&keys[..count]), true).await; + let entries = fixture + .disk_walk(WalkDirOptions { + bucket: fixture.bucket.clone(), + recursive: true, + ..Default::default() + }) + .await; + let names: Vec<_> = entries + .into_iter() + .filter(|entry| !entry.metadata.is_empty()) + .map(|entry| entry.name) + .collect(); + assert_eq!(names, keys[..count], "raw walker should preserve byte order and __XLDIR__ decoding"); + for layer in 0..3 { + for prefix in ["", "a", "a/", "中/", "missing/"] { + for delimiter in [None, Some("/")] { + for max in [0, 1, 3, 4] { + fixture.assert_walk(layer, prefix, delimiter, None, max).await; + } + } + } + } + } +} + +#[tokio::test] +async fn list_objects_shared_corpus_cross_subdirectory_limit() { + let keys = ["a/first", "a/nested/second", "a0", "b/final"].map(str::to_owned); + let fixture = Fixture::new("list-corpus-subdirectory-limit", namespace(&keys), true).await; + for count in [1, 2, 3, 4] { + let names: Vec<_> = fixture + .disk_walk(WalkDirOptions { + bucket: fixture.bucket.clone(), + recursive: true, + limit: i32::try_from(count).expect("small count"), + ..Default::default() + }) + .await + .into_iter() + .filter(|entry| !entry.metadata.is_empty()) + .map(|entry| entry.name) + .collect(); + assert_eq!( + names, + keys[..count], + "#7049: a recursive child that reaches the limit must stop its parent" + ); + } + for layer in 0..3 { + for max in [1, 2, 3, 4] { + fixture.assert_walk(layer, "", None, None, max).await; + } + } +} + +#[tokio::test] +async fn list_objects_shared_corpus_max_and_max_plus_one() { + let max = usize::try_from(MAX_OBJECT_LIST).expect("maximum LIST size should be positive"); + for count in [max, max + 1] { + let keys: Vec<_> = (0..count).map(|index| format!("bulk/key-{index:04}")).collect(); + let fixture = Fixture::new(&format!("list-corpus-max-{count}"), namespace(&keys), true).await; + for layer in 0..3 { + fixture.assert_walk(layer, "bulk/", None, None, max).await; + } + let index_dir = tempfile::tempdir().expect("maximum-boundary index fixture"); + let index_path = index_dir.path().join("maximum.idx"); + for mode in [ListSourceMode::IndexKeyOnly, ListSourceMode::IndexVerifiedPage] { + temp_env::async_with_vars( + [ + (ENV_API_LIST_OBJECTS_INDEX_MODE, Some(mode.cursor_value())), + (ENV_API_LIST_OBJECTS_INDEX_PROVIDER, Some(LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY)), + (ENV_API_LIST_OBJECTS_INDEX_PROVIDER_PATH, index_path.to_str()), + (ENV_API_LIST_OBJECTS_INDEX_PROVIDER_GENERATION, Some("maximum-corpus")), + ], + async { fixture.assert_walk(0, "bulk/", None, None, max).await }, + ) + .await; + } + let names: Vec<_> = fixture + .disk_walk(WalkDirOptions { + bucket: fixture.bucket.clone(), + base_dir: "bulk/".to_owned(), + recursive: true, + limit: MAX_OBJECT_LIST, + ..Default::default() + }) + .await + .into_iter() + .filter(|entry| !entry.metadata.is_empty()) + .map(|entry| entry.name) + .collect(); + assert_eq!(names, keys[..max], "walker must stop at max for both adjacent namespaces"); + } +} + +#[tokio::test] +async fn list_objects_shared_corpus_fixed_seed_marker_and_prefix() { + let fixture = Fixture::new("list-corpus-seeded", namespace(&shared_keys()), true).await; + let prefixes = ["", "a", "a/", "b", "中/"]; + let markers = [None, Some("a"), Some("a/"), Some("a/b"), Some("b"), Some("space key")]; + // Replayable query generation over one static real namespace. No cross-page + // snapshot guarantee is assumed for concurrent writers. + let mut seed = 0x00ec_5707_u64; + for _ in 0..32 { + seed = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + let prefix = prefixes[usize::try_from(seed % 5).expect("small prefix index")]; + let marker = markers[usize::try_from((seed >> 8) % 6).expect("small marker index")]; + // A start-after outside the selected prefix is outside these APIs' contract. + let marker = marker.filter(|marker| marker.starts_with(prefix)); + let delimiter = (seed & 1 == 0).then_some("/"); + let max = usize::try_from((seed >> 16) % 4 + 1).expect("small page size"); + for layer in 0..3 { + fixture.assert_walk(layer, prefix, delimiter, marker, max).await; + } + } + // A folded prefix can consume more raw candidates than the producer's + // ordinary page budget before the next visible identity appears. + let mut keys: Vec<_> = (0..12).map(|index| format!("data-{index:02}")).collect(); + keys.push("other-key".to_owned()); + let folded = Fixture::new("list-corpus-long-fold", namespace(&keys), true).await; + for layer in 0..3 { + for max in [1, 2] { + folded.assert_walk(layer, "", Some("-"), None, max).await; + } + } +} + +#[tokio::test] +async fn list_objects_shared_corpus_offline_corrupt_and_minority_stale() { + let fixture = Fixture::new("list-corpus-faults", namespace(&shared_keys()), true).await; + let set = &fixture.store.pools[0].disk_set[0]; + // Three copies are a legal write quorum. Losing one of those three leaves + // two readable copies among three online disks, the #7010 boundary. + for object in &fixture.objects { + let physical_key = rustfs_utils::path::encode_dir_object(&object.key); + tokio::fs::remove_file( + fixture.dirs[3] + .path() + .join(&fixture.bucket) + .join(physical_key) + .join(STORAGE_FORMAT_FILE), + ) + .await + .expect("fixture should retain exactly a write quorum of metadata copies"); + } + fixture.assert_walk(0, "", None, None, 3).await; + let offline_disk = set.disks.write().await[0] + .take() + .expect("a disk with a committed copy should exist before the offline fault"); + assert_eq!( + set.disks.read().await.iter().flatten().count(), + 3, + "offline fault must reach the real set" + ); + for layer in 0..3 { + fixture.assert_walk(layer, "", None, None, 3).await; + fixture.assert_walk(layer, "", Some("/"), None, 3).await; + } + set.disks.write().await[0] = Some(offline_disk); + for object in &fixture.objects { + fixture.write_metadata(3, object).await; + } + + let object = &fixture.objects[2]; + let metadata_path = fixture.dirs[3] + .path() + .join(&fixture.bucket) + .join(&object.key) + .join(STORAGE_FORMAT_FILE); + tokio::fs::write(&metadata_path, b"corrupt xl.meta") + .await + .expect("corrupt fault should reach disk bytes"); + assert!(FileMeta::load(&tokio::fs::read(&metadata_path).await.expect("fault bytes should be readable")).is_err()); + for layer in 0..3 { + fixture.assert_walk(layer, "a", None, None, 1).await; + } + fixture.write_metadata(3, object).await; + + let mut stale = object.clone(); + stale.versions[0].modified -= 10; + fixture.write_metadata(3, &stale).await; + let phantom = Object { + key: "minority-only".to_owned(), + versions: stale.versions.clone(), + }; + fixture.write_metadata(3, &phantom).await; + for layer in 0..3 { + fixture.assert_walk(layer, "", None, None, 3).await; + } +} + +#[tokio::test] +async fn list_objects_shared_corpus_versions_delete_markers_and_free_versions() { + let mut objects = vec![ + Object { + key: "a".to_owned(), + versions: vec![ + Version { + id: Some(Uuid::from_u128(3)), + modified: 1_705_312_302, + kind: VersionKind::Object, + }, + Version { + id: Some(Uuid::from_u128(2)), + modified: 1_705_312_301, + kind: VersionKind::Delete, + }, + Version { + id: Some(Uuid::from_u128(1)), + modified: 1_705_312_300, + kind: VersionKind::Object, + }, + ], + }, + Object { + key: "a/hidden".to_owned(), + versions: vec![ + Version { + id: Some(Uuid::from_u128(5)), + modified: 1_705_312_301, + kind: VersionKind::Delete, + }, + Version { + id: Some(Uuid::from_u128(4)), + modified: 1_705_312_300, + kind: VersionKind::Object, + }, + ], + }, + Object { + key: "b/tie".to_owned(), + versions: vec![ + Version { + id: Some(Uuid::from_u128(6)), + modified: 1_705_312_300, + kind: VersionKind::Object, + }, + Version { + id: Some(Uuid::from_u128(7)), + modified: 1_705_312_300, + kind: VersionKind::Delete, + }, + ], + }, + Object { + key: "folder/".to_owned(), + versions: vec![Version { + id: Some(Uuid::from_u128(9)), + modified: 1_705_312_301, + kind: VersionKind::Delete, + }], + }, + Object { + key: "folder/child".to_owned(), + versions: vec![Version { + id: Some(Uuid::from_u128(10)), + modified: 1_705_312_300, + kind: VersionKind::Object, + }], + }, + Object { + key: "free-only".to_owned(), + versions: vec![Version { + id: Some(Uuid::from_u128(8)), + modified: 1_705_312_300, + kind: VersionKind::Free, + }], + }, + Object { + key: "null".to_owned(), + versions: vec![Version { + id: None, + modified: 1_705_312_300, + kind: VersionKind::Object, + }], + }, + ]; + // More cleanup-only keys than the old raw page budget can hold must not + // hide the visible null version that follows them. + objects.splice( + 5..5, + (0..12).map(|index| Object { + key: format!("free-dense-{index:02}"), + versions: vec![Version { + id: Some(Uuid::from_u128(0x100 + index)), + modified: 1_705_312_300, + kind: VersionKind::Free, + }], + }), + ); + let fixture = Fixture::new("list-corpus-versions", objects, true).await; + let physical = fixture + .disk_walk(WalkDirOptions { + bucket: fixture.bucket.clone(), + recursive: true, + incl_deleted: true, + ..Default::default() + }) + .await; + let free = physical + .iter() + .find(|entry| entry.name == "free-only") + .expect("free-version fault must exist on disk"); + assert_eq!( + free.file_info_versions_with_free_versions(&fixture.bucket) + .expect("physical free metadata should decode") + .free_versions + .len(), + 1 + ); + assert!( + free.file_info_versions(&fixture.bucket) + .expect("public free-only projection should decode") + .versions + .is_empty(), + "the free-only key must occupy no public version slot" + ); + let (cleanup_tx, mut cleanup_rx) = mpsc::channel(1); + let cleanup_walk = fixture.store.pools[0].disk_set[0].clone().walk_internal( + CancellationToken::new(), + &fixture.bucket, + "free-only", + cleanup_tx, + WalkOptions { + include_free_versions: true, + filter: Some(FileInfo::tier_free_version), + limit: 1, + ..Default::default() + }, + ); + let cleanup_drain = async { + let mut entries = Vec::new(); + while let Some(entry) = cleanup_rx.recv().await { + entries.push(entry); + } + entries + }; + let (walk_result, mut cleanup_entries) = + timeout(Duration::from_secs(10), async { tokio::join!(cleanup_walk, cleanup_drain) }) + .await + .expect("cleanup walk and consumer must finish together"); + walk_result.expect("the actual cleanup walk must retain free versions"); + assert_eq!(cleanup_entries.len(), 1, "cleanup walk must join and emit the owner once"); + let cleanup = cleanup_entries.pop().expect("cleanup walk must emit the physical free owner"); + assert!(cleanup.err.is_none(), "cleanup owner must not become a read error"); + let cleanup = cleanup.item.expect("cleanup walk should emit an object"); + assert_eq!(cleanup.name, "free-only"); + assert!( + cleanup.transitioned_object.free_version, + "cleanup must preserve the free-version identity" + ); + + let expected: Vec<_> = fixture + .objects + .iter() + .flat_map(|object| { + object + .versions + .iter() + .filter(|version| version.kind != VersionKind::Free) + .map(|version| (object.key.clone(), version.id, version.kind == VersionKind::Delete)) + }) + .collect(); + for layer in 0..3 { + fixture.assert_walk(layer, "", None, None, 1).await; + fixture.assert_walk(layer, "", Some("/"), None, 1).await; + for max in [0, 1, expected.len() - 1, expected.len(), expected.len() + 1] { + let mut marker = None; + let mut version_marker = None; + let mut offset = 0; + loop { + let result = timeout(Duration::from_secs(10), async { + match layer { + 0 => { + fixture + .store + .clone() + .inner_list_object_versions( + &fixture.bucket, + "", + marker.clone(), + version_marker.clone(), + None, + i32::try_from(max).expect("small limit"), + ) + .await + } + 1 => { + fixture.store.pools[0] + .clone() + .inner_list_object_versions( + &fixture.bucket, + "", + marker.clone(), + version_marker.clone(), + None, + i32::try_from(max).expect("small limit"), + ) + .await + } + _ => { + fixture.store.pools[0].disk_set[0] + .clone() + .inner_list_object_versions( + &fixture.bucket, + "", + marker.clone(), + version_marker.clone(), + None, + i32::try_from(max).expect("small limit"), + ) + .await + } + } + }) + .await + .unwrap_or_else(|error| { + panic!("version layer={layer}, max={max}, offset={offset}, marker={marker:?}, version_marker={version_marker:?}: page must finish: {error}") + }) + .expect("real version page should list"); + let end = (offset + max).min(expected.len()); + let actual: Vec<_> = result + .objects + .into_iter() + .map(|object| (object.name, object.version_id.filter(|id| !id.is_nil()), object.delete_marker)) + .collect(); + assert_eq!(actual, expected[offset..end], "version layer={layer}, max={max}, offset={offset}"); + assert!(result.prefixes.is_empty()); + let more = max > 0 && end < expected.len(); + assert_eq!( + result.is_truncated, more, + "version layer={layer}, max={max}, offset={offset}, marker={marker:?}, version_marker={version_marker:?}, returned={actual:?}, next_key={:?}, next_version={:?}: page needs exactly one lookahead", + result.next_marker, result.next_version_idmarker, + ); + if !more { + assert!(result.next_marker.is_none() && result.next_version_idmarker.is_none()); + break; + } + let last = &expected[end - 1]; + assert_eq!(result.next_marker.as_deref().map(marker_key), Some(last.0.as_str())); + assert_eq!( + result.next_version_idmarker, + Some(last.1.map_or_else(|| "null".to_owned(), |id| id.to_string())) + ); + marker = result.next_marker; + version_marker = result.next_version_idmarker; + offset = end; + } + } + } + // A deleted directory-marker key cannot hide a different live child. + // Keep the child's four copies and reduce only the marker to write quorum. + tokio::fs::remove_file( + fixture.dirs[3] + .path() + .join(&fixture.bucket) + .join(rustfs_utils::path::encode_dir_object("folder/")) + .join(STORAGE_FORMAT_FILE), + ) + .await + .expect("delete-marker fixture should retain three committed copies"); + for layer in 0..3 { + fixture.assert_walk(layer, "folder/", None, None, 1).await; + fixture.assert_walk(layer, "", Some("/"), None, 1).await; + } +} + +#[tokio::test] +async fn list_objects_shared_corpus_orphan_cleanup_requires_authoritative_evidence() { + for authoritative in [false, true] { + let fixture = Fixture::new( + &format!("list-corpus-orphan-{authoritative}"), + namespace(&["legal/child".to_owned()]), + authoritative, + ) + .await; + for dir in &fixture.dirs { + tokio::fs::create_dir_all(dir.path().join(&fixture.bucket).join("ghost/nested/leaf")) + .await + .expect("orphan input must exist"); + tokio::fs::create_dir_all(dir.path().join(&fixture.bucket).join("legal/orphan/leaf")) + .await + .expect("orphan input must share an ancestor with a legal child"); + } + let result = fixture.page(0, "ghost/", None, None, 1).await; + assert!(result.objects.is_empty() && result.prefixes.is_empty() && !result.is_truncated); + for dir in &fixture.dirs { + assert_eq!( + dir.path().join(&fixture.bucket).join("ghost").exists(), + !authoritative, + "unknown metadata must fail closed" + ); + assert!( + dir.path() + .join(&fixture.bucket) + .join("legal/child") + .join(STORAGE_FORMAT_FILE) + .exists(), + "purge must preserve legal siblings" + ); + } + fixture.assert_walk(0, "legal/", None, None, 1).await; + for dir in &fixture.dirs { + assert!( + dir.path() + .join(&fixture.bucket) + .join("legal/child") + .join(STORAGE_FORMAT_FILE) + .exists(), + "an orphan in the same subtree must not cause the legitimate child to be purged" + ); + } + if authoritative { + for dir in &fixture.dirs { + tokio::fs::create_dir_all(dir.path().join(&fixture.bucket).join("timed-out/leaf")) + .await + .expect("timeout orphan input should exist"); + } + let metadata_sys = fixture + .store + .ctx + .bucket_metadata_sys() + .expect("authoritative fixture has a metadata system"); + let guard = metadata_sys.write().await; + assert!( + timeout(Duration::from_millis(100), fixture.page(0, "timed-out/", None, None, 1)) + .await + .is_err(), + "blocked authoritative evidence must time out before cleanup can classify the prefix as empty" + ); + drop(guard); + for dir in &fixture.dirs { + assert!( + dir.path().join(&fixture.bucket).join("timed-out").exists(), + "a timed-out evidence check must preserve its orphan input" + ); + } + } + } +} + +struct BackpressureWriter { + inner: DuplexStream, + blocked: Arc, +} + +impl AsyncWrite for BackpressureWriter { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, bytes: &[u8]) -> Poll> { + let result = Pin::new(&mut self.inner).poll_write(cx, bytes); + if result.is_pending() { + self.blocked.notify_one(); + } + result + } + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_flush(cx) + } + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_shutdown(cx) + } +} + +#[tokio::test] +async fn list_objects_shared_corpus_consumer_drop_joins_real_disk_producer() { + let fixture = Fixture::new("list-corpus-cancel", namespace(&shared_keys()), true).await; + let disk = fixture.store.pools[0].disk_set[0].disks.read().await[0] + .clone() + .expect("real disk should exist"); + let (reader, writer) = duplex(64); + let blocked = Arc::new(Notify::new()); + let mut writer = BackpressureWriter { + inner: writer, + blocked: blocked.clone(), + }; + let options = WalkDirOptions { + bucket: fixture.bucket.clone(), + recursive: true, + ..Default::default() + }; + let producer = tokio::spawn(async move { disk.walk_dir(options, &mut writer).await }); + let mut reader = MetacacheReader::new(reader); + let first = timeout(Duration::from_secs(2), reader.peek()) + .await + .expect("producer must emit before client cancellation") + .expect("first metacache entry should decode") + .expect("the fixture should contain a real entry"); + assert_eq!(first.name, "a", "cancellation must reach a real object stream"); + timeout(Duration::from_secs(2), blocked.notified()) + .await + .expect("producer should encounter bounded consumer backpressure"); + drop(reader); + let result = timeout(Duration::from_secs(2), producer) + .await + .expect("client drop must terminate the disk producer") + .expect("producer must not panic"); + assert!(result.is_err(), "the disconnected stream must reach the producer as a write error"); + // A second real walk plus metadata rewrite proves disk access is reusable + // before fixture/runtime teardown, instead of relying on process exit. + fixture.write_metadata(0, &fixture.objects[0]).await; + let entries = fixture + .disk_walk(WalkDirOptions { + bucket: fixture.bucket.clone(), + recursive: true, + ..Default::default() + }) + .await; + assert_eq!( + entries.into_iter().filter(|entry| !entry.metadata.is_empty()).count(), + fixture.objects.len() + ); +} + +#[tokio::test] +async fn list_objects_shared_corpus_cancelled_quorum_walk_releases_senders_and_disk_lock() { + let fixture = Fixture::new("list-corpus-quorum-cancel", namespace(&shared_keys()), true).await; + let set = fixture.store.pools[0].disk_set[0].clone(); + let cancel = CancellationToken::new(); + let (sender, mut receiver) = mpsc::channel(1); + let observer = sender.clone(); + let task_set = set.clone(); + let task_cancel = cancel.clone(); + let options = ListPathOptions { + bucket: fixture.bucket.clone(), + recursive: true, + ask_disks: "strict".to_owned(), + limit: 1000, + ..Default::default() + }; + let task = tokio::spawn(async move { task_set.list_path(task_cancel, options, sender).await }); + let first = timeout(Duration::from_secs(2), receiver.recv()) + .await + .expect("quorum producer should emit before cancellation") + .expect("real quorum walk should yield an object"); + assert_eq!(first.name, "a"); + timeout(Duration::from_secs(2), async { + while observer.capacity() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("real producer should fill the bounded channel before cancellation"); + assert!( + timeout(Duration::from_millis(50), observer.reserve()).await.is_err(), + "the real callback must be blocked by a full client channel before cancellation" + ); + cancel.cancel(); + drop(receiver); + timeout(Duration::from_secs(2), task) + .await + .expect("cancellation must join the real quorum walk and its disk producers") + .expect("quorum producer should not panic") + .expect("external cancellation is a clean listing shutdown"); + assert_eq!( + observer.strong_count(), + 1, + "all production callback/producer sender clones must be released" + ); + assert!(set.disks.try_write().is_ok(), "listing cancellation must not retain the set's disk lock"); + fixture.assert_walk(0, "", None, None, 3).await; +} + +#[tokio::test] +async fn list_objects_shared_corpus_enabled_index_contracts() { + let fixture = Fixture::new("list-corpus-index", namespace(&shared_keys()), true).await; + let index_dir = tempfile::tempdir().expect("index directory should exist"); + let path = index_dir.path().join("namespace.idx"); + let path_value = path.to_str().expect("test index path should be UTF-8"); + for provider in [ + LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY, + LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY, + ] { + for mode in [ListSourceMode::IndexKeyOnly, ListSourceMode::IndexVerifiedPage] { + temp_env::async_with_vars( + [ + (ENV_API_LIST_OBJECTS_INDEX_MODE, Some(mode.cursor_value())), + (ENV_API_LIST_OBJECTS_INDEX_PROVIDER, Some(provider)), + (ENV_API_LIST_OBJECTS_INDEX_PROVIDER_PATH, Some(path_value)), + (ENV_API_LIST_OBJECTS_INDEX_PROVIDER_GENERATION, Some("shared-corpus")), + ], + async { + for prefix in ["", "a", "a/", "missing/"] { + for delimiter in [None, Some("/")] { + for max in [0, 1, 3, 4] { + fixture.assert_walk(0, prefix, delimiter, None, max).await; + } + } + } + let result = fixture.page(0, "", None, None, 1).await; + let mut parsed = ListPathOptions { + marker: result.next_marker, + ..Default::default() + }; + parsed.parse_marker(); + assert_eq!(parsed.cursor_source, Some(mode), "enabled adapter must actually serve the index path"); + assert_eq!( + parsed.cursor_generation.as_deref(), + Some(if provider == LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY { + LIST_CURSOR_GENERATION_LIVE + } else { + "shared-corpus" + }) + ); + }, + ) + .await; + } + } + temp_env::async_with_vars( + [ + (ENV_API_LIST_OBJECTS_INDEX_MODE, Some(ListSourceMode::IndexMetadataFast.cursor_value())), + (ENV_API_LIST_OBJECTS_INDEX_PROVIDER, Some(LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY)), + (ENV_API_LIST_OBJECTS_INDEX_PROVIDER_PATH, Some(path_value)), + (ENV_API_LIST_OBJECTS_INDEX_PROVIDER_GENERATION, Some("shared-corpus")), + (ENV_API_LIST_OBJECTS_METADATA_FAST_ENABLED, Some("true")), + (ENV_API_LIST_OBJECTS_METADATA_FAST_STALENESS_MS, Some("5000")), + ], + async { + fixture.assert_walk(0, "", None, None, 3).await; + let first = fixture.page(0, "", None, None, 1).await; + let mut cursor = ListPathOptions { + marker: first.next_marker.clone(), + ..Default::default() + }; + cursor.parse_marker(); + assert_eq!(cursor.cursor_source, Some(ListSourceMode::IndexMetadataFast)); + assert_eq!(cursor.cursor_generation.as_deref(), Some("shared-corpus")); + assert!( + !ListSourceMode::IndexMetadataFast.can_satisfy_strong_listing(), + "snapshot mode carries an eventual-consistency contract" + ); + + let state = ListObjectsIndexProviderState::persistent_key_only(Some(path.clone()), Some("shared-corpus".to_owned())); + let mut expired = ListPathOptions { + bucket: fixture.bucket.clone(), + limit: 2, + marker: first.next_marker, + ..Default::default() + }; + expired.parse_marker(); + expired.cursor_generation = Some("expired-generation".to_owned()); + assert!( + fixture + .store + .clone() + .list_objects_from_metadata_fast_provider(&expired, &state, "persistent_key_only", 1, false) + .await + .expect("expired cursor must be handled") + .is_none(), + "expired generation must fall back to walker" + ); + }, + ) + .await; + temp_env::async_with_vars( + [ + (ENV_API_LIST_OBJECTS_INDEX_MODE, Some(ListSourceMode::IndexKeyOnly.cursor_value())), + (ENV_API_LIST_OBJECTS_INDEX_PROVIDER, Some(LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY)), + (ENV_API_LIST_OBJECTS_INDEX_PROVIDER_PATH, index_dir.path().to_str()), + (ENV_API_LIST_OBJECTS_INDEX_PROVIDER_GENERATION, Some("shared-corpus")), + ], + async { + let options = ListPathOptions { + bucket: fixture.bucket.clone(), + limit: 2, + ..Default::default() + }; + assert!( + fixture + .store + .clone() + .list_objects_from_opt_in_key_only_provider(&options, ListSourceMode::IndexKeyOnly, 1, false) + .await + .expect("unreadable provider must return a fallback decision") + .is_none(), + "an index path that is a directory must fail the enabled provider" + ); + fixture.assert_walk(0, "", None, None, 3).await; + let result = fixture.page(0, "", None, None, 1).await; + let mut cursor = ListPathOptions { + marker: result.next_marker, + ..Default::default() + }; + cursor.parse_marker(); + assert_ne!( + cursor.cursor_source, + Some(ListSourceMode::IndexKeyOnly), + "degraded provider must not advertise an index cursor" + ); + }, + ) + .await; +} diff --git a/crates/ecstore/tests/fixtures/list_namespace_keys.json b/crates/ecstore/tests/fixtures/list_namespace_keys.json new file mode 100644 index 000000000..a0f917e5b --- /dev/null +++ b/crates/ecstore/tests/fixtures/list_namespace_keys.json @@ -0,0 +1,14 @@ +[ + "a", + "a/", + "a/b", + "a/b/child", + "a0", + "b", + "b/leaf", + "quote\"&<", + "space key", + "z", + "é", + "中/文" +] diff --git a/crates/filemeta/src/metacache.rs b/crates/filemeta/src/metacache.rs index 2c02a7bee..61a2e4d8e 100644 --- a/crates/filemeta/src/metacache.rs +++ b/crates/filemeta/src/metacache.rs @@ -618,6 +618,7 @@ impl MetaCacheEntries { } let mut dir_exists = 0; + let mut directory = None; let mut selected = None; params.candidates.clear(); @@ -633,7 +634,7 @@ impl MetaCacheEntries { } if entry.is_dir() { dir_exists += 1; - selected = Some(entry.clone()); + directory = Some(entry.clone()); debug!(entry = %entry.name, "metacache resolve observed directory candidate"); continue; } @@ -664,6 +665,10 @@ impl MetaCacheEntries { } } + // A prefix and an explicit directory-marker object can share a name. + // Keep directory evidence separate so its arrival cannot displace an + // object that meets the metadata quorum. + let directory = directory.filter(|_| dir_exists >= params.dir_quorum); let Some(selected) = selected else { debug!( bucket = %params.bucket, @@ -671,19 +676,9 @@ impl MetaCacheEntries { obj_quorum = params.obj_quorum, "metacache resolve could not select any candidate entry" ); - return None; + return directory; }; - if selected.is_dir() && dir_exists >= params.dir_quorum { - debug!( - entry = %selected.name, - dir_exists, - dir_quorum = params.dir_quorum, - "metacache resolve selected directory candidate after quorum reconciliation" - ); - return Some(selected); - } - // If we would never be able to reach the required object quorum. if objs_valid < params.obj_quorum { debug!( @@ -691,7 +686,7 @@ impl MetaCacheEntries { obj_quorum = params.obj_quorum, "metacache resolve did not have enough valid object candidates to satisfy quorum" ); - return None; + return directory; } if objs_agree == objs_valid { @@ -724,7 +719,7 @@ impl MetaCacheEntries { let Some(cached) = selected.cached else { debug!(selected = %selected.name, "metacache resolve could not merge because selected candidate had no cached metadata"); - return None; + return directory; }; let versions = if enforce_write_quorum { @@ -744,7 +739,7 @@ impl MetaCacheEntries { requested_versions = params.requested_versions, "metacache resolve produced no merged versions after reconciliation" ); - return None; + return directory; } let merged_cached = FileMeta { @@ -761,7 +756,7 @@ impl MetaCacheEntries { error = ?e, "metacache resolve failed to marshal merged metadata entry" ); - return None; + return directory; } }; @@ -2534,6 +2529,57 @@ mod tests { assert!(resolved.is_none()); } + #[test] + fn resolve_directory_marker_and_prefix_preserve_quorum_in_both_orders() { + let modified = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); + let mut object = metacache_entry_with_erasure(modified, "marker-etag", 2, 2); + object.name = "folder/".to_string(); + let directory = metacache_dir_entry("folder/"); + for enforce_write_quorum in [false, true] { + for (object_copies, directory_copies) in [(3, 1), (3, 3), (2, 3), (0, 3), (2, 2)] { + for reverse in [false, true] { + let mut entries = vec![Some(object.clone()); object_copies]; + entries.extend(vec![Some(directory.clone()); directory_copies]); + if reverse { + entries.reverse(); + } + let entries = MetaCacheEntries(entries); + let params = MetadataResolutionParams { + dir_quorum: 3, + obj_quorum: 3, + requested_versions: 1, + bucket: "bucket".to_string(), + strict: true, + ..Default::default() + }; + let resolved = if enforce_write_quorum { + entries.resolve_with_write_quorum(params) + } else { + entries.resolve(params) + }; + let context = format!( + "objects={object_copies}, directories={directory_copies}, reverse={reverse}, write={enforce_write_quorum}" + ); + if object_copies >= 3 { + let resolved = resolved.expect("metadata at quorum must shadow a same-named prefix"); + assert!(resolved.is_object(), "{context}"); + assert_eq!( + resolved + .to_fileinfo("bucket") + .expect("marker metadata should decode") + .mod_time, + Some(modified) + ); + } else if directory_copies >= 3 { + assert!(resolved.expect("prefix at quorum must remain a fallback").is_dir(), "{context}"); + } else { + assert!(resolved.is_none(), "{context}"); + } + } + } + } + } + fn build_hashmap_cache(update_size: usize) -> Arc>> { let generation = Arc::new(AtomicUsize::new(0)); Arc::new(Cache::new( diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index e1d0cb5f5..423ca0bd5 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -11,6 +11,8 @@ ## Open Items +- `backlog-2253-metadata-observation-adapter` metadata observations: preserve aligned metadata/error slices for existing layout, shard and exact-version PUT, MPU and delete consumers while read/heal scheduling uses typed disk observations. Pending slots acquire empty placeholders only in this adapter. Remove after those remaining consumers use typed observations; delete the adapter and old facade in a separate cleanup. + - `connect-894` pending Connect heartbeats: replay the exact preceding producer-capability list, with or without jobs, when upgrading to the memory-service capability. Preserve request ID, sequence, and persisted body rather than inserting the new capability into a retry. Remove after upgrades from the pre-service-memory capability set are unsupported. - `backlog-2539` administrator erasure-set scope: decode historical pending intents whose empty bucket list omitted the all-buckets marker. Normalize that representation to the explicit scope before replay or checkpoint comparison. Remove after all pre-marker administrator ErasureSet intents are retired; deployment verification must confirm that no such pending records remain on coordinator disks. diff --git a/docs/architecture/ecstore-module-split-plan.md b/docs/architecture/ecstore-module-split-plan.md index eb26b73f4..dd9cd8d84 100644 --- a/docs/architecture/ecstore-module-split-plan.md +++ b/docs/architecture/ecstore-module-split-plan.md @@ -83,3 +83,39 @@ A candidate is ready for code movement only when all of these hold: - old facade names have compatibility tests or explicit deprecation coverage; - focused tests cover the changed owner path before any full gate is attempted; - rollback preserves object IO, quorum, lifecycle/replication queues, scanner repair, notification/audit events, and metadata compatibility. + +## Metadata read ownership and identity + +`set_disk/core/metadata_read.rs` owns disk/RPC scheduling, coalescing, timeouts, +hedging and cancellation. Its result retains each disk slot as pending, a +successful `FileInfo`, or a typed `DiskError`. Pending includes both an +unscheduled disk and a response not yet observed; it contributes no vote and +does not establish absence. Task IDs retain the disk slot when a task panics. +The existing full-wait and early-stop gates, coalescer admission, cancellation +drain and fallback policy remain at this scheduling boundary. + +`set_disk/core/metadata_quorum.rs` consumes those observations without IO. +`SetDisks` remains the sole owner of topology, locks and caches. Read and heal +callers use the typed result; `MetadataReadResult::into_legacy` is the explicit +adapter for existing quorum/layout and shard-reader helpers. It moves successful +metadata without cloning and creates empty placeholders only at that adapter. +Old full-wait facade callers retain the same aligned metadata/error slices. + +These identities have different purposes and must not become one generic hash: + +| Decision | Equal fields | Deliberately different or separately checked | +| --- | --- | --- | +| Quorum grouping (`file_info_quorum_hash`) | Size, deletion/restore/transition state, timestamps, version and data-directory UUIDs, checksums, mode/writer version, normalized metadata, target delete-marker versions and part identities. Payloads also include data/parity counts and distribution. | Object name and volume come from the request; per-disk erasure index and shard bytes are not an object identity. Canonical delete markers and zero-size objects do not add payload geometry. | +| Early-stop strict match (`metadata_early_stop_candidate_matches`) | Request name/volume, version/latest/deletion state, transition fields, size/time, mode/writer version, complete metadata/replication state/parts/checksum, version-history fields, data directory, algorithm/block size/data/parity/distribution. | Per-disk erasure index and data bytes differ. A non-inline read reserve additionally proves distinct indexes mapped to their disk slots; inline reads verify shard identity and bitrot/body content before stopping. | +| Late materialization (`LateMetadataIdentity`) | Request name/volume, algorithm, block size, legacy-checksum mode and quorum hash. | Each late shard must match the original distribution at its disk slot. A changed identity or insufficient matching shards fails read quorum; it cannot mix generations into a body. | + +`metadata_observation_all_four_slot_states_and_arrival_orders_preserve_reduction` +checks all 625 four-disk success/missing/corrupt/offline/pending combinations and +24 arrival orders with early-stop on and off. The observation regressions also +cover a newer pending response, same-time different directories, duplicated +shard indexes, null versions, canonical delete markers and invalid success +payloads. Existing bounded-fanout, legacy inline bitrot, cache, late-metadata and +GET/Range regressions continue to exercise the real scheduling and reader paths; +the pure matrix does not replace those checks. Their call counters bound actual +disk reads, including canceled tails, without adding a scheduling policy or a +per-request metadata clone. diff --git a/docs/architecture/unified-object-generation.md b/docs/architecture/unified-object-generation.md index 0fe5d3c70..7d235bad0 100644 --- a/docs/architecture/unified-object-generation.md +++ b/docs/architecture/unified-object-generation.md @@ -82,7 +82,7 @@ A single filesystem rename does not atomically commit a sidecar plus `xl.meta`. A laggard need not contain the predecessor. Recovery fetches the chosen decision and its verified metadata, reconstructs or validates its shards under [erasure-coding.md](erasure-coding.md), and installs that state. An empty replacement uses a fresh disk incarnation and cannot reuse old preparation receipts. A disk whose durable state claims a later decision than the supplied one rejects the operation; a conflicting same-revision digest is quarantined. Never erase a divergent disk merely because it is in the minority. -`rollback_committed_rename_std` and `rollback_inline_metadata_commit_std` in `crates/ecstore/src/disk/local/commit.rs`, and `restore_metadata_backup` in `crates/ecstore/src/disk/local.rs`, must become decision-aware before strict mode includes them. The permitted rollback is limited to a transaction's unchosen private preparation, or restoration proven by recovery to be necessary before any newer local effect. A chosen operation is repaired forward. Neither a client timeout nor a rename-tail error permits reverting an acknowledged decision. +`RenameCommitContext::rollback_non_inline_std` and `RenameCommitContext::rollback_inline_std` in `crates/ecstore/src/disk/local/commit.rs`, and `restore_metadata_backup` in `crates/ecstore/src/disk/local.rs`, must become decision-aware before strict mode includes them. The permitted rollback is limited to a transaction's unchosen private preparation, or restoration proven by recovery to be necessary before any newer local effect. A chosen operation is repaired forward. Neither a client timeout nor a rename-tail error permits reverting an acknowledged decision. `RenameConvergence` remains a post-publication repair signal. `PartialCommit`, `SignatureDivergent`, and `Unknown` do not decide which transaction won. Keep their diagnostics and quorum accounting; resolve authority first. Early ACK may still precede minority-tail completion after the two quorum conditions hold. Tests that inspect all disks must synchronize the tail or assert the permitted minority residue separately. diff --git a/docs/testing/ecstore-validation-suite-design.md b/docs/testing/ecstore-validation-suite-design.md index de717d1b9..ba8536db3 100644 --- a/docs/testing/ecstore-validation-suite-design.md +++ b/docs/testing/ecstore-validation-suite-design.md @@ -58,7 +58,7 @@ Fault injection is explicit and deterministic: local disk mocks for unit tests, `ObjectOptions::default()` uses `WriteCompletion::Quorum`: a namespace-lock-owning PUT may acknowledge write quorum while its rename tail retains the lock. A fixture that immediately inspects every disk or primes a metadata generation must set `write_completion: WriteCompletion::TailDrained` and keep normal locking. TailDrained waits for the existing rename fan-out; it does not require every disk to succeed or change fsync policy. Codec-only `no_lock` fixtures do not cover namespace locking. -The object tests reuse `rename_fanout_barrier::arm(object, disk_slot, phase)` and `observe_tasks(object)`. Wait for the barrier with a deadline, observe actual metadata quorum with `wait_for_paused_tail_metadata_quorum`, then release or cancel. The metadata check distinguishes a real quorum from disk tasks that have not started. Assert zero remaining rename tasks after the owned coordinator releases its lock; cancellation tests also wait for staging cleanup. +The object tests reuse `rename_fanout_barrier::arm(object, disk_slot, phase)` and `observe_tasks(object)`. The fixture uses typed rename, rollback, cleanup, read-version and hedge-timer phases; multiple disk/phase checkpoints can be armed for one object. A duplicate live checkpoint is rejected instead of replacing its owner. `wait_until_paused_before(deadline)` reports the exact unreached checkpoint on timeout. Wait for the barrier with a deadline, observe actual metadata quorum with `wait_for_paused_tail_metadata_quorum`, then release or cancel. The metadata check distinguishes a real quorum from disk tasks that have not started. Assert zero remaining rename tasks after the owned coordinator releases its lock; cancellation tests also wait for staging cleanup. | Fixture | Completion boundary | |---|---| diff --git a/rustfs/src/on_demand_migration/list_through.rs b/rustfs/src/on_demand_migration/list_through.rs index 3613eca03..fb7ab99a5 100644 --- a/rustfs/src/on_demand_migration/list_through.rs +++ b/rustfs/src/on_demand_migration/list_through.rs @@ -1113,22 +1113,11 @@ mod tests { #[test] fn list_through_static_namespace_boundary_matrix() { - let corpus = [ - "a", - "a/", - "a/b", - "a/b/child", - "a0", - "b", - "b/leaf", - "quote\"&<", - "space key", - "z", - "é", - "中/文", - ]; + let corpus: Vec = + serde_json::from_str(include_str!("../../../crates/ecstore/tests/fixtures/list_namespace_keys.json")) + .expect("shared LIST namespace corpus should decode"); for count in [0, 1, 3, 4, corpus.len()] { - let keys: Vec = corpus[..count].iter().map(|key| (*key).to_string()).collect(); + let keys = corpus[..count].to_vec(); for placement in 0..3 { let (local, source): (Vec<_>, Vec<_>) = keys.iter()