mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-29 03:45:46 +08:00
fix(test): wait for EC materialization in relocated-pool resume fixture (#6718)
fix(test): wait for EC write materialization before staging relocated-pool fixture An erasure-coded write returns once write-quorum disks commit, so a lagging disk can legally still be missing its xl.meta when the relocated-pool resume test starts staging its fixture by iterating every disk of the owning pool. Under CI load this raced into a NotFound panic in the staging loop. Add a bounded readiness poll that waits for xl.meta on every pool disk before the normalization and staging steps. Fixes #6703
This commit is contained in:
@@ -7023,6 +7023,29 @@ mod tests {
|
||||
body
|
||||
}
|
||||
|
||||
// An erasure-coded write returns once write-quorum disks commit, so a
|
||||
// lagging disk can legally still be missing its xl.meta when the write
|
||||
// call returns. Fixtures that iterate every disk of the owning pool must
|
||||
// wait for full materialization first, or they race the trailing disk
|
||||
// writes under CI load (issue #6703). Bounded so a genuinely failed disk
|
||||
// write still surfaces as a test failure instead of a hang.
|
||||
async fn wait_for_object_on_every_disk(disk_paths: &[std::path::PathBuf], bucket: &str, object: &str) {
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);
|
||||
loop {
|
||||
if disk_paths
|
||||
.iter()
|
||||
.all(|path| path.join(bucket).join(object).join("xl.meta").is_file())
|
||||
{
|
||||
return;
|
||||
}
|
||||
assert!(
|
||||
std::time::Instant::now() < deadline,
|
||||
"object {bucket}/{object} must materialize xl.meta on every pool disk within the readiness window"
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
|
||||
}
|
||||
}
|
||||
|
||||
// Deletes the given part files from every version data dir present on the
|
||||
// disks, simulating rebalance removing the object data while xl.meta stays
|
||||
// readable. Returns the number of version dirs visited and files removed.
|
||||
@@ -7172,6 +7195,7 @@ mod tests {
|
||||
.any(|path| path.join(&bucket).join(object).join("xl.meta").is_file())
|
||||
})
|
||||
.expect("multipart object must be placed in one source pool");
|
||||
wait_for_object_on_every_disk(&pool_disk_paths[upload_pool], &bucket, object).await;
|
||||
if upload_pool != 0 {
|
||||
let mut normalized_disks = 0;
|
||||
for (source_disk, target_disk) in pool_disk_paths[upload_pool].iter().zip(&pool_disk_paths[0]) {
|
||||
|
||||
Reference in New Issue
Block a user