diff --git a/.config/e2e-protocols-selection.txt b/.config/e2e-protocols-selection.txt index b0c07f708..705c834b8 100644 --- a/.config/e2e-protocols-selection.txt +++ b/.config/e2e-protocols-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=55534a97fbd376f64c8f6c341d319017d11ff77cad6da8629a1a7f6a874e0315 -sha256-linux=c06fb8c19aed6f388b9dc61cb8251b7a44f8561a9bf764ad2b9e635598f8dc17 +sha256-darwin=a2cb67d91eb0d093928371d6e84a84b2dd41b2ba975a28094e81879bd65838ad +sha256-linux=82db0561cf9b111f11782d647d0859c9d2a44c5c68481b5e39c48505828bd392 diff --git a/.config/ecstore-required-tests.json b/.config/ecstore-required-tests.json index 24b723e39..4a95d9790 100644 --- a/.config/ecstore-required-tests.json +++ b/.config/ecstore-required-tests.json @@ -55,6 +55,261 @@ "invariant": "odm-source-contract-gcs", "suite": "rustfs", "name": "on_demand_migration::gcs::tests::gcs_native_backend_satisfies_the_shared_backend_contract" + }, + { + "invariant": "rollback-inspection", + "suite": "rustfs-ecstore", + "name": "set_disk::core::io_primitives::tests::rename_rollback_incomplete_preserves_overwrite_data_dirs_and_staging" + }, + { + "invariant": "rollback-closed-inspection-channel", + "suite": "rustfs-ecstore", + "name": "set_disk::core::io_primitives::tests::rename_rollback_closed_heal_receiver_preserves_recovery_material" + }, + { + "invariant": "rollback-cancellation", + "suite": "rustfs-ecstore", + "name": "set_disk::core::io_primitives::tests::rename_rollback_incomplete_cancelled_task_is_not_success" + }, + { + "invariant": "rollback-completion-equivalence", + "suite": "rustfs-ecstore", + "name": "set_disk::core::io_primitives::tests::rename_rollback_incomplete_matches_early_ack_and_full_wait_after_reopen" + }, + { + "invariant": "rollback-physical-owner", + "suite": "rustfs-ecstore", + "name": "set_disk::core::io_primitives::tests::rename_rollback_children_keep_namespace_ownership_after_coordinator_panic" + }, + { + "invariant": "late-ack-retains-commit", + "suite": "rustfs-ecstore", + "name": "set_disk::core::io_primitives::tests::rename_data_early_ack_post_mutation_tail_error_never_rolls_back_commit" + }, + { + "invariant": "tail-drained-cas", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::put_object_tmp_cleanup_tests::tail_drained_put_waits_for_tail_and_allows_immediate_cas" + }, + { + "invariant": "seeded-put-tail-pressure", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::put_object_tmp_cleanup_tests::seeded_put_completion_pressure_drains_every_real_tail" + }, + { + "invariant": "borrowed-lock-tail", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::put_object_tmp_cleanup_tests::no_lock_put_waits_for_rename_tail_under_outer_guard" + }, + { + "invariant": "cancelled-writer-owner", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::put_object_tmp_cleanup_tests::tail_drained_put_owned_commit_survives_waiter_cancellation" + }, + { + "invariant": "get-body-reconstruction", + "suite": "rustfs-ecstore", + "name": "ecstore_validation_blackbox::blackbox_get_restores_body_after_one_shard_file_is_removed" + }, + { + "invariant": "range-reconstruction", + "suite": "rustfs-ecstore", + "name": "ecstore_validation_blackbox::blackbox_range_read_restores_exact_slice_with_one_offline_disk" + }, + { + "invariant": "multipart-abort-linearization", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::multipart::tests::abort_and_complete_linearize_for_plain_sse_and_legacy_layouts" + }, + { + "invariant": "multipart-write-quorum", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::multipart::tests::abort_enforces_delete_write_quorum_boundary" + }, + { + "invariant": "multipart-physical-owner", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::multipart::tests::complete_multipart_timeout_keeps_namespace_owner_until_physical_publication" + }, + { + "invariant": "restore-abort-cleanup", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::transition_commit_failure_tests::multipart_restore_aborts_every_post_create_failure" + }, + { + "invariant": "legacy-metadata", + "suite": "rustfs-filemeta", + "name": "filemeta::test::test_issue_2288_legacy_xlmeta_compatibility" + }, + { + "invariant": "legacy-nil-version", + "suite": "rustfs-filemeta", + "name": "filemeta::test::test_into_fileinfo_reads_legacy_nil_uuid_inline_key" + }, + { + "invariant": "commit-phase-schedule", + "suite": "rustfs-ecstore", + "name": "set_disk::core::io_primitives::tests::commit_fault_schedule_keeps_multiple_disk_phases_independent" + }, + { + "invariant": "commit-phase-ownership", + "suite": "rustfs-ecstore", + "name": "set_disk::core::io_primitives::tests::commit_fault_schedule_rejects_duplicate_checkpoint_without_disarming_owner" + }, + { + "invariant": "metadata-observation-all-four-slot-states-and-arrival-orders-preserve-reduction", + "suite": "rustfs-ecstore", + "name": "set_disk::core::metadata_quorum::tests::metadata_observation_all_four_slot_states_and_arrival_orders_preserve_reduction" + }, + { + "invariant": "metadata-observation-pending-newer-version-and-same-time-different-directory-force-full-wait", + "suite": "rustfs-ecstore", + "name": "set_disk::core::metadata_quorum::tests::metadata_observation_pending_newer_version_and_same_time_different_directory_force_full_wait" + }, + { + "invariant": "metadata-observation-duplicate-shard-cannot-supply-a-read-reserve", + "suite": "rustfs-ecstore", + "name": "set_disk::core::metadata_quorum::tests::metadata_observation_duplicate_shard_cannot_supply_a_read_reserve" + }, + { + "invariant": "metadata-observation-null-versions-markers-and-invalid-success-keep-their-meaning", + "suite": "rustfs-ecstore", + "name": "set_disk::core::metadata_quorum::tests::metadata_observation_null_versions_markers_and_invalid_success_keep_their_meaning" + }, + { + "invariant": "list-objects-shared-corpus-static-namespace", + "suite": "rustfs-ecstore", + "name": "store::list_objects::differential_tests::list_objects_shared_corpus_static_namespace" + }, + { + "invariant": "list-objects-shared-corpus-cross-subdirectory-limit", + "suite": "rustfs-ecstore", + "name": "store::list_objects::differential_tests::list_objects_shared_corpus_cross_subdirectory_limit" + }, + { + "invariant": "list-objects-shared-corpus-max-and-max-plus-one", + "suite": "rustfs-ecstore", + "name": "store::list_objects::differential_tests::list_objects_shared_corpus_max_and_max_plus_one" + }, + { + "invariant": "list-objects-shared-corpus-fixed-seed-marker-and-prefix", + "suite": "rustfs-ecstore", + "name": "store::list_objects::differential_tests::list_objects_shared_corpus_fixed_seed_marker_and_prefix" + }, + { + "invariant": "list-objects-shared-corpus-offline-corrupt-and-minority-stale", + "suite": "rustfs-ecstore", + "name": "store::list_objects::differential_tests::list_objects_shared_corpus_offline_corrupt_and_minority_stale" + }, + { + "invariant": "list-objects-shared-corpus-versions-delete-markers-and-free-versions", + "suite": "rustfs-ecstore", + "name": "store::list_objects::differential_tests::list_objects_shared_corpus_versions_delete_markers_and_free_versions" + }, + { + "invariant": "list-objects-shared-corpus-orphan-cleanup-requires-authoritative-evidence", + "suite": "rustfs-ecstore", + "name": "store::list_objects::differential_tests::list_objects_shared_corpus_orphan_cleanup_requires_authoritative_evidence" + }, + { + "invariant": "list-objects-shared-corpus-consumer-drop-joins-real-disk-producer", + "suite": "rustfs-ecstore", + "name": "store::list_objects::differential_tests::list_objects_shared_corpus_consumer_drop_joins_real_disk_producer" + }, + { + "invariant": "list-objects-shared-corpus-cancelled-quorum-walk-releases-senders-and-disk-lock", + "suite": "rustfs-ecstore", + "name": "store::list_objects::differential_tests::list_objects_shared_corpus_cancelled_quorum_walk_releases_senders_and_disk_lock" + }, + { + "invariant": "list-objects-shared-corpus-enabled-index-contracts", + "suite": "rustfs-ecstore", + "name": "store::list_objects::differential_tests::list_objects_shared_corpus_enabled_index_contracts" + }, + { + "invariant": "borrowed-context-boundary", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::put_object_tmp_cleanup_tests::borrowed_write_context_rejects_wrong_namespace_and_cache_flag" + }, + { + "invariant": "borrowed-put-cancellation", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::put_object_tmp_cleanup_tests::borrowed_tail_drained_put_keeps_owner_after_caller_cancel" + }, + { + "invariant": "borrowed-mpu-cancellation", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::multipart::tests::borrowed_complete_keeps_namespace_owner_after_caller_cancel" + }, + { + "invariant": "pool-cas-owner", + "suite": "rustfs-ecstore", + "name": "core::pools::pools_tests::pool_meta_cas_context_retains_actual_owner_for_exact_target" + }, + { + "invariant": "pool-cas-released-owner", + "suite": "rustfs-ecstore", + "name": "core::pools::pools_tests::pool_meta_cas_rejects_released_owner" + }, + { + "invariant": "copy-tail-drained", + "suite": "rustfs-ecstore", + "name": "store::object::tests::borrowed_copy_tail_drained_waits_for_rename_tail" + }, + { + "invariant": "catalog-borrowed-publication", + "suite": "rustfs", + "name": "admin::handlers::table_catalog::tests::commit_backend_borrowed_put_preserves_owner_authorization_and_precondition" + }, + { + "invariant": "directory-marker-quorum", + "suite": "rustfs-filemeta", + "name": "metacache::tests::resolve_directory_marker_and_prefix_preserve_quorum_in_both_orders" + }, + { + "invariant": "logical-prefix-limit", + "suite": "rustfs-ecstore", + "name": "store::list_objects::test::list_path_gather_results_counts_common_prefixes_before_page_limit" + }, + { + "invariant": "quota-same-name-owner-and-disk-authority", + "suite": "rustfs-ecstore", + "name": "bucket::metadata_sys::tests::quota_snapshot_reuses_same_name_owner_and_rereads_disk_authority" + }, + { + "invariant": "quota-snapshot-scope-and-owner", + "suite": "rustfs-ecstore", + "name": "bucket::metadata_sys::tests::quota_snapshot_rejects_wrong_scope_incarnation_and_transaction_guard" + }, + { + "invariant": "quota-snapshot-lifecycle-and-incarnation", + "suite": "rustfs-ecstore", + "name": "bucket::metadata_sys::tests::quota_snapshot_rejects_lifecycle_loss_during_read_and_changed_disk_incarnation" + }, + { + "invariant": "marker-purge-receipt-reopen", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::marker_purge_receipt_tests::quorum_marker_purge_receipt_survives_reopen_with_stale_member" + }, + { + "invariant": "external-minio-shards", + "suite": "rustfs-ecstore::pinned_erasure_fixtures_test", + "name": "pinned_minio_shards_decode_and_reconstruct_plaintext" + }, + { + "invariant": "external-legacy-shards", + "suite": "rustfs-ecstore::pinned_erasure_fixtures_test", + "name": "pinned_legacy_shards_decode_and_reconstruct_plaintext" + }, + { + "invariant": "external-shard-bitrot", + "suite": "rustfs-ecstore::pinned_erasure_fixtures_test", + "name": "pinned_erasure_shards_reject_bitrot_corruption" + }, + { + "invariant": "external-shard-quorum-codec", + "suite": "rustfs-ecstore::pinned_erasure_fixtures_test", + "name": "pinned_erasure_shards_reject_insufficient_quorum_and_wrong_codec" } ], "fixtures": [ @@ -82,6 +337,36 @@ "path": "crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex", "sha256": "3b6de589519c08a1614c8bd409bb8199c17d42043861b07bce513075e6fbfc12", "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + }, + { + "path": "crates/filemeta/tests/fixtures/issue_2288_legacy_xlmeta.hex", + "sha256": "b3e8d32f9950a38b5dc71f679519cc830a2b2a807e8b0b39f78840b238d8eacf", + "source": "Legacy xl.meta captured for rustfs/backlog#2288; filemeta::test::test_issue_2288_legacy_xlmeta_compatibility" + }, + { + "path": "crates/filemeta/tests/fixtures/issue_2265_legacy_meta_v2_object.hex", + "sha256": "89dd7ec92d16e54b5d31e0c49c9edffff8a7aa58890aaeff59bc2ff84da0a62f", + "source": "Legacy metadata v2 object captured for rustfs/backlog#2265; crates/filemeta/src/test_data.rs" + }, + { + "path": "crates/ecstore/tests/fixtures/list_namespace_keys.json", + "sha256": "5afddb60565deba0938e617470255563f0e15a15fab699645c59bb7ccc84898e", + "source": "Shared static LIST namespace corpus; ECStore differential tests and ODM list_through_static_namespace_boundary_matrix" + }, + { + "path": "crates/ecstore/tests/fixtures/erasure-shards/minio/manifest.json", + "sha256": "c398f8dcb76903f6cf8eee07931c76592895ced201ff9d59ed4ff9b0ead2b13c", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/erasure-shards/README.md" + }, + { + "path": "crates/ecstore/tests/fixtures/erasure-shards/legacy/manifest.json", + "sha256": "dce397d5b2f42748eb5091b4355bb9d133a50cbba003626fafac28095dbe540b", + "source": "RustFS 1.0.0-alpha.37; crates/ecstore/tests/fixtures/erasure-shards/README.md" + }, + { + "path": "crates/ecstore/tests/fixtures/erasure-shards/plaintext.bin", + "sha256": "b5a83332327961a31b65a8731c64559fd04ff8ac9f6124022201dbbdba5ffa01", + "source": "Independent fixed SHA256 counter-stream oracle; crates/ecstore/tests/fixtures/erasure-shards/README.md" } ] } diff --git a/.config/nextest.toml b/.config/nextest.toml index 6134ac90b..80b2d8398 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -170,10 +170,11 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost test-group = 'embedded-test-ports' # Registration, inventory delivery, and real drive/object probes include -# durable filesystem IO in short deadlines. Reserve capacity so sibling test +# filesystem or HTTP IO in short deadlines. Reserve capacity so sibling test # processes cannot exhaust those budgets. Keep deadlines and assertions intact. +# The short HTTP fixtures are tracked in rustfs/backlog#2714. [[profile.default.overrides]] -filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)))' +filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))' threads-required = "num-test-threads" # This cancellation probe hashes the serving executable before acquiring its @@ -428,7 +429,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost test-group = 'embedded-test-ports' [[profile.ci.overrides]] -filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)))' +filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))' threads-required = "num-test-threads" # Preserve the cancellation probe's startup budget and assertions in CI too. diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml index 83677c3be..ea42bcd2b 100644 --- a/.github/workflows/rustfs-pool-expand-test.yml +++ b/.github/workflows/rustfs-pool-expand-test.yml @@ -266,7 +266,10 @@ jobs: # the previous 45m step cap killed four consecutive runs mid-rebalance # with steps 1-7 green. 240m covers the observed pace (rebalance 1/4 # in ~21m) plus one decommission pass; override per dispatch. - timeout-minutes: ${{ inputs.pool_timeout_minutes || 240 }} + # fromJSON coerces the string dispatch input to the number that + # timeout-minutes requires; the bare || form yielded the string '240' + # and the runner rejected the template (Unexpected value '240'). + timeout-minutes: ${{ fromJSON(inputs.pool_timeout_minutes || '240') }} # Case failures keep the run green: the report and the backlog # issue manager carry the product signal. continue-on-error: true @@ -583,25 +586,6 @@ jobs: echo "pool test script reported a failure (step FAIL verdict or die() abort)" >&2 failed=1 fi - # A runner-level kill (step timeout) truncates the suite without a - # FAIL verdict or an [ERROR] line, so require every expected - # [POOL-STEP] marker to be present with PASS - missing markers mean - # the run was truncated, not green. This also closes the gap for - # standalone dispatches, where the chain driver's evidence - # validation does not run. - last_step=6 - if [ "${{ inputs.pools || '3' }}" = "3" ]; then - last_step=8 - if [ "${{ inputs.run_decommission != 'false' }}" = "true" ]; then - last_step=9 - fi - fi - for n in $(seq 1 "${last_step}"); do - grep -q "^\[POOL-STEP\] ${n} .* PASS" "${POOL_ARTIFACT_DIR}/pool-test.log" 2>/dev/null || { - echo "pool test log is missing [POOL-STEP] ${n} PASS (expected through step ${last_step}); the run was truncated" >&2 - failed=1 - } - done [ "${failed}" -eq 0 ] || exit 1 - name: Upload functional report to dashboard diff --git a/crates/e2e_test/src/cluster_multidrive_pool_test.rs b/crates/e2e_test/src/cluster_multidrive_pool_test.rs index 411336754..535f4d0f4 100644 --- a/crates/e2e_test/src/cluster_multidrive_pool_test.rs +++ b/crates/e2e_test/src/cluster_multidrive_pool_test.rs @@ -33,6 +33,10 @@ //! tracked separately. use crate::common::{ClusterTopology, RustFSTestClusterEnvironment}; +use crate::fault_proxy::FaultMode; +use aws_sdk_s3::Client; +use std::time::Duration; +use tokio::time::{Instant, timeout}; type TestResult = Result<(), Box>; @@ -144,7 +148,99 @@ async fn cluster_volume_fault_proxy_pass_smoke() -> TestResult { cluster.start().await?; cluster.create_test_bucket(BUCKET).await?; let payload = vec![0x6Du8; 256 * 1024]; - put_get_roundtrip(&cluster, "volume-proxy/object", &payload).await + put_get_roundtrip(&cluster, "volume-proxy/object", &payload).await?; + + // Node 1 owns only two disks; a successful three-vote write must + // authenticate at least one disk mutation through node 0's proxy. + let client = cluster.create_s3_client(1)?; + let client = Client::from_conf( + client + .config() + .to_builder() + .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1)) + .build(), + ); + let forwarded_before = proxy.forwarded_bytes(); + client + .put_object() + .bucket(BUCKET) + .key("volume-proxy/authenticated") + .body(bytes::Bytes::copy_from_slice(&payload).into()) + .send() + .await?; + assert!( + proxy.forwarded_bytes() > forwarded_before, + "successful peer-disk publication must traverse the proxy" + ); + + let dropped_before = proxy.dropped_bytes(); + proxy.set_mode(FaultMode::Blackhole); + assert_eq!(proxy.mode(), FaultMode::Blackhole); + let error = timeout( + Duration::from_secs(30), + client + .put_object() + .bucket(BUCKET) + .key("volume-proxy/rejected") + .body(bytes::Bytes::copy_from_slice(&payload).into()) + .send(), + ) + .await? + .expect_err("two local disks cannot commit a write while the peer is blackholed"); + assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503)); + assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable")); + assert!( + proxy.dropped_bytes() > dropped_before, + "the injected blackhole must discard actual peer traffic" + ); + + proxy.set_mode(FaultMode::Pass); + assert_eq!(proxy.mode(), FaultMode::Pass); + let http = reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(3)) + .build()?; + let deadline = Instant::now() + Duration::from_secs(30); + loop { + let response = http.get(format!("{}/health/ready", cluster.nodes[1].url)).send().await?; + let status = response.status(); + let body: serde_json::Value = response.json().await?; + // Reachable peers can still have Returning disk handles that reject + // writes. Wait for this node's live writer inventory to be Online. + if status.as_u16() == 200 + && body["details"]["storage"]["source"] == "local_runtime" + && body["details"]["storage"]["writeQuorum"] == true + && body["details"]["storage"]["unavailableDrives"] + .as_array() + .is_some_and(Vec::is_empty) + && body["details"]["lock"]["ready"] == true + { + break; + } + assert!(Instant::now() < deadline, "peer write quorum did not recover: HTTP {status}, {body}"); + tokio::time::sleep(Duration::from_millis(200)).await; + } + client + .put_object() + .bucket(BUCKET) + .key("volume-proxy/recovered") + .body(bytes::Bytes::copy_from_slice(&payload).into()) + .send() + .await?; + for key in ["volume-proxy/object", "volume-proxy/authenticated", "volume-proxy/recovered"] { + let got = client.get_object().bucket(BUCKET).key(key).send().await?; + let body = got.body.collect().await?.into_bytes(); + assert_eq!(body.as_ref(), payload, "proxy recovery must preserve the full object body for {key}"); + } + let rejected = client + .head_object() + .bucket(BUCKET) + .key("volume-proxy/rejected") + .send() + .await + .expect_err("a rejected proxied write must never become visible after recovery"); + assert_eq!(rejected.raw_response().map(|response| response.status().as_u16()), Some(404)); + Ok(()) } .await; diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index 5b8e4bba5..622fe9015 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -35,7 +35,7 @@ use s3s::Body; use serde_json; use std::fs as stdfs; use std::io::ErrorKind; -use std::net::SocketAddr; +use std::net::{Ipv6Addr, SocketAddr}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; use std::sync::Once; @@ -1442,7 +1442,11 @@ impl RustFSTestClusterEnvironment { return Err(format!("a volume proxy is already configured for node {node_idx}").into()); } let target = self.nodes[node_idx].address.parse::()?; - let proxy = crate::fault_proxy::FaultProxy::start(target).await?; + // Endpoint locality and RPC audience both use the advertised authority. + // Keep the node's port while the second loopback IP separates the proxy + // listener from the direct IPv4 server and S3 client address. + let listen = SocketAddr::from((Ipv6Addr::LOCALHOST, target.port())); + let proxy = crate::fault_proxy::FaultProxy::start_on(listen, target).await?; self.volume_proxy_addresses[node_idx] = Some(proxy.local_addr()); Ok(proxy) } @@ -2467,6 +2471,20 @@ mod tests { assert!(volumes.contains(&proxied), "volumes must use the proxy address: {volumes}"); assert!(!volumes.contains(&direct), "volumes must not retain the direct address: {volumes}"); + assert_eq!(proxy.local_addr().port(), proxy.target_addr().port()); + assert_ne!(proxy.local_addr().ip(), proxy.target_addr().ip()); + assert_eq!(env.nodes[0].address, direct, "S3 clients must retain the direct address"); + let mut endpoint = + crate::storage_api::Endpoint::try_from(volumes.as_str()).expect("proxied volume should be a valid endpoint"); + endpoint + .update_is_local(proxy.target_addr().port()) + .expect("proxied endpoint locality should resolve"); + assert!(endpoint.is_local, "the proxied node must keep its local disk and first-disk authority"); + assert_eq!( + endpoint.host_port(), + proxied, + "the local RPC audience must be the advertised proxy authority" + ); proxy.shutdown().await; } diff --git a/crates/e2e_test/src/fault_proxy.rs b/crates/e2e_test/src/fault_proxy.rs index 1879532ca..92002a255 100644 --- a/crates/e2e_test/src/fault_proxy.rs +++ b/crates/e2e_test/src/fault_proxy.rs @@ -61,6 +61,8 @@ use std::io; use std::net::{Ipv4Addr, SocketAddr}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -108,19 +110,33 @@ pub struct FaultProxy { mode_tx: watch::Sender, shutdown_tx: watch::Sender, accept_task: JoinHandle<()>, + traffic: Arc, +} + +#[derive(Default)] +struct ProxyTraffic { + forwarded_bytes: AtomicUsize, + dropped_bytes: AtomicUsize, } impl FaultProxy { /// Bind a listener on `127.0.0.1:0` and start forwarding accepted /// connections to `target`. Starts in [`FaultMode::Pass`]. pub async fn start(target: SocketAddr) -> io::Result { - let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + Self::start_on(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)), target).await + } + + /// Bind a specific address so advertised volume endpoints can retain the + /// server's port and authenticated RPC identity on another loopback IP. + pub async fn start_on(listen: SocketAddr, target: SocketAddr) -> io::Result { + let listener = TcpListener::bind(listen).await?; let listen_addr = listener.local_addr()?; let (mode_tx, mode_rx) = watch::channel(FaultMode::Pass); let (shutdown_tx, shutdown_rx) = watch::channel(false); - let accept_task = tokio::spawn(accept_loop(listener, target, mode_rx, shutdown_rx)); + let traffic = Arc::new(ProxyTraffic::default()); + let accept_task = tokio::spawn(accept_loop(listener, target, mode_rx, shutdown_rx, traffic.clone())); Ok(Self { listen_addr, @@ -128,9 +144,20 @@ impl FaultProxy { mode_tx, shutdown_tx, accept_task, + traffic, }) } + /// Bytes actually forwarded in either direction, excluding dropped traffic. + pub fn forwarded_bytes(&self) -> usize { + self.traffic.forwarded_bytes.load(Ordering::Relaxed) + } + + /// Bytes actually discarded by a blackhole or one-way partition. + pub fn dropped_bytes(&self) -> usize { + self.traffic.dropped_bytes.load(Ordering::Relaxed) + } + /// Local address the proxy is listening on. Point clients here instead of /// the real target to route their traffic through the proxy. pub fn local_addr(&self) -> SocketAddr { @@ -171,6 +198,7 @@ async fn accept_loop( target: SocketAddr, mode_rx: watch::Receiver, mut shutdown_rx: watch::Receiver, + traffic: Arc, ) { loop { tokio::select! { @@ -188,6 +216,7 @@ async fn accept_loop( target, mode_rx.clone(), shutdown_rx.clone(), + traffic.clone(), )); } // A transient accept error should not tear the proxy down. @@ -205,6 +234,7 @@ async fn handle_connection( target: SocketAddr, mode_rx: watch::Receiver, shutdown_rx: watch::Receiver, + traffic: Arc, ) { let server = match TcpStream::connect(target).await { Ok(s) => s, @@ -221,8 +251,9 @@ async fn handle_connection( server_wr, mode_rx.clone(), shutdown_rx.clone(), + traffic.clone(), )); - let down = tokio::spawn(pump(Direction::ServerToClient, server_rd, client_wr, mode_rx, shutdown_rx)); + let down = tokio::spawn(pump(Direction::ServerToClient, server_rd, client_wr, mode_rx, shutdown_rx, traffic)); let _ = up.await; let _ = down.await; @@ -237,6 +268,7 @@ async fn pump( mut to: W, mut mode_rx: watch::Receiver, mut shutdown_rx: watch::Receiver, + traffic: Arc, ) where R: AsyncReadExt + Unpin, W: AsyncWriteExt + Unpin, @@ -257,15 +289,22 @@ async fn pump( let mode = *mode_rx.borrow_and_update(); match mode { // Whole connection is a black hole: drain and drop. - FaultMode::Blackhole => continue, + FaultMode::Blackhole => { + traffic.dropped_bytes.fetch_add(n, Ordering::Relaxed); + continue; + } // This direction is partitioned off: drop its bytes; the peer keeps // reading nothing while the other direction may still flow. - FaultMode::Partition(blocked) if blocked == dir => continue, + FaultMode::Partition(blocked) if blocked == dir => { + traffic.dropped_bytes.fetch_add(n, Ordering::Relaxed); + continue; + } FaultMode::Latency(delay) => { tokio::time::sleep(delay).await; if to.write_all(&buf[..n]).await.is_err() { break; } + traffic.forwarded_bytes.fetch_add(n, Ordering::Relaxed); if to.flush().await.is_err() { break; } @@ -274,6 +313,7 @@ async fn pump( if to.write_all(&buf[..n]).await.is_err() { break; } + traffic.forwarded_bytes.fetch_add(n, Ordering::Relaxed); if to.flush().await.is_err() { break; } @@ -285,7 +325,6 @@ async fn pump( #[cfg(test)] mod tests { use super::*; - use std::sync::Arc; use std::time::Instant; use tokio::sync::Mutex; @@ -375,6 +414,8 @@ mod tests { assert_eq!(got, payload, "pass mode must forward bytes unchanged"); assert_eq!(echo.received().await, payload, "server must have received the bytes"); + assert_eq!(proxy.forwarded_bytes(), payload.len() * 2); + assert_eq!(proxy.dropped_bytes(), 0); proxy.shutdown().await; } @@ -414,6 +455,8 @@ mod tests { assert!(got.is_empty(), "blackhole mode must not return any bytes, got {got:?}"); assert!(echo.received().await.is_empty(), "blackhole must not forward to the server"); + assert_eq!(proxy.forwarded_bytes(), 0); + assert_eq!(proxy.dropped_bytes(), b"into-the-void".len()); // Proxy is still alive and steerable after a blackholed connection. proxy.set_mode(FaultMode::Pass); diff --git a/crates/e2e_test/src/namespace_lock_quorum_test.rs b/crates/e2e_test/src/namespace_lock_quorum_test.rs index 14e3fe49e..46a59a943 100644 --- a/crates/e2e_test/src/namespace_lock_quorum_test.rs +++ b/crates/e2e_test/src/namespace_lock_quorum_test.rs @@ -300,17 +300,28 @@ async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResul } cluster.stop_node(read_quorum - 1)?; + let lock_read_quorum = node_count - node_count / 2; + // EC:2 on six nodes still has three shared lock votes after losing + // its fourth data shard. Preserve the distinct data-quorum error. + let expected_code = if read_quorum - 1 < lock_read_quorum { + "ServiceUnavailable" + } else { + "SlowDownRead" + }; for client in clients.iter().take(read_quorum - 1) { match client.get_object().bucket(&bucket).key("below-quorum").send().await { Ok(response) => assert!( response.body.collect().await.is_err(), "fewer than {read_quorum} valid fragments must not reconstruct an uncached object" ), - Err(error) => assert_eq!( - error.as_service_error().and_then(|error| error.meta().code()), - Some("ServiceUnavailable"), - "a quorum loss must not be mistaken for a missing object" - ), + Err(error) => { + assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503)); + assert_eq!( + error.as_service_error().and_then(|error| error.meta().code()), + Some(expected_code), + "a quorum loss must not be mistaken for a missing object" + ); + } } } @@ -425,6 +436,60 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum() Ok(()) } +fn unavailable_readiness_drive_indices(payload: &serde_json::Value) -> Vec { + let mut indices = payload["details"]["storage"]["unavailableDrives"] + .as_array() + .expect("node readiness must report unavailable drives") + .iter() + .map(|drive| { + usize::try_from(drive["diskIndex"].as_u64().expect("drive index must be unsigned")) + .expect("drive index must fit usize") + }) + .collect::>(); + indices.sort_unstable(); + indices +} + +fn assert_node_readiness_contract(payload: &serde_json::Value, survivors: usize) { + let read_ready = survivors >= 2; + let write_ready = survivors >= 3; + let storage = &payload["details"]["storage"]; + assert_eq!(payload["ready"], read_ready); + assert_eq!(storage["ready"], read_ready); + assert_eq!(storage["readQuorum"], read_ready); + assert_eq!(storage["writeQuorum"], write_ready); + assert_eq!(storage["readinessScope"], "read_quorum"); + assert_eq!(storage["source"], "local_runtime"); + assert_eq!(storage["status"], if read_ready { "connected" } else { "disconnected" }); + assert_eq!(payload["details"]["poolMetadata"]["ready"], true); + assert_eq!(payload["details"]["iam"]["ready"], true); + assert_eq!(payload["details"]["lock"]["ready"], read_ready); + assert_eq!(unavailable_readiness_drive_indices(payload), (survivors..4).collect::>()); + for drive in storage["unavailableDrives"].as_array().expect("unavailable drives") { + assert_eq!(drive["poolIndex"], 0); + assert_eq!(drive["setIndex"], 0); + let state = drive["runtimeState"].as_str().expect("runtime state must be reported"); + let host_online = drive["hostOnline"].as_bool().expect("host reachability must be reported"); + assert!( + matches!(state, "online" | "suspect" | "offline" | "returning"), + "unknown runtime state: {state}" + ); + assert!( + state != "online" || !host_online, + "a reachable online drive cannot be excluded from quorum" + ); + } + if !read_ready { + assert!( + payload["degradedReasons"] + .as_array() + .expect("degraded reasons") + .iter() + .any(|reason| reason == "storage_and_lock_unavailable") + ); + } +} + async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvironment) -> TestResult { let clients: Vec<_> = cluster .create_all_clients()? @@ -483,27 +548,14 @@ async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvi && payload["details"]["poolMetadata"]["ready"] == true && payload["details"]["iam"]["ready"] == true && payload["details"]["lock"]["ready"] == read_quorum + && unavailable_readiness_drive_indices(&payload) == (survivors..4).collect::>() { break payload; } assert!(Instant::now() < deadline, "node {idx}, survivors={survivors}: HTTP {status}, {payload}"); tokio::time::sleep(Duration::from_millis(200)).await; }; - assert_eq!(payload["details"]["storage"]["readinessScope"], "read_quorum"); - assert_eq!(payload["details"]["storage"]["source"], "local_runtime"); - assert_eq!( - payload["details"]["storage"]["status"], - if node_ready { "connected" } else { "disconnected" } - ); - if !node_ready { - assert!( - payload["degradedReasons"] - .as_array() - .expect("degraded reasons") - .iter() - .any(|reason| reason == "storage_and_lock_unavailable") - ); - } + assert_node_readiness_contract(&payload, survivors); for path in ["/health/ready", "/minio/health/ready"] { let head = http.head(format!("{url}{path}")).send().await?; @@ -512,7 +564,9 @@ async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvi let response = http.get(format!("{url}{path}")).send().await?; assert_eq!(response.status().as_u16(), node_status); let body: serde_json::Value = response.json().await?; - assert_eq!(body["details"]["storage"], payload["details"]["storage"]); + // Each response samples live diagnostics; suspect -> offline + // must not change the quorum contract or excluded topology. + assert_node_readiness_contract(&body, survivors); assert_eq!(body["details"]["poolMetadata"], payload["details"]["poolMetadata"]); } let live = http.get(format!("{url}/health/live")).send().await?; diff --git a/crates/e2e_test/src/storage_api.rs b/crates/e2e_test/src/storage_api.rs index 8a3523fb2..98f1200ab 100644 --- a/crates/e2e_test/src/storage_api.rs +++ b/crates/e2e_test/src/storage_api.rs @@ -15,7 +15,7 @@ #[cfg(test)] pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys; #[cfg(test)] -pub(crate) use rustfs_ecstore::api::disk::{RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions}; +pub(crate) use rustfs_ecstore::api::disk::{Endpoint, RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions}; #[cfg(test)] pub(crate) use rustfs_ecstore::api::rpc::PeerRestClient; pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth}; diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 4a3d259ce..fb8ef876d 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -176,7 +176,7 @@ regex = { workspace = true } path-absolutize = { workspace = true } rmp.workspace = true rmp-serde.workspace = true -tokio-util = { workspace = true, features = ["io", "compat"] } +tokio-util = { workspace = true, features = ["io", "compat", "rt"] } tokio-stream = { workspace = true, features = ["sync"] } hmac = { workspace = true } sha2 = { workspace = true } diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 6324406f3..327149682 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -519,7 +519,7 @@ pub mod object { ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError, - ScannerPublicationCommitState, ShardIntegrityWriteMode, StreamConsumer, WriteCompletion, + ScannerPublicationCommitState, ShardIntegrityWriteMode, StreamConsumer, WriteCommitGuard, WriteCompletion, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook, }; diff --git a/crates/ecstore/src/bucket/lifecycle/recovery_disposition.rs b/crates/ecstore/src/bucket/lifecycle/recovery_disposition.rs index 229ec3eb3..418c1764d 100644 --- a/crates/ecstore/src/bucket/lifecycle/recovery_disposition.rs +++ b/crates/ecstore/src/bucket/lifecycle/recovery_disposition.rs @@ -1122,9 +1122,11 @@ pub(crate) async fn resume_recovery_disposition( // Lock order is control record, then legacy source. Disposition revisions // use ETag CAS and never acquire either namespace lock internally. let control_lock = api.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?; - let control_guard = control_lock - .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) - .await?; + let control_guard = Arc::new( + control_lock + .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?, + ); let source_lock = api .new_ns_lock(RUSTFS_META_BUCKET, &initial.disposition.identity.canonical_source_path) .await?; @@ -1504,7 +1506,7 @@ fn control_is_abandoned_successor(disposition: &IlmRecoveryDisposition, control: async fn abandon_recovery_control_fenced( api: Arc, disposition: &IlmRecoveryDisposition, - control_guard: &rustfs_lock::NamespaceLockGuard, + control_guard: &Arc, source_guard: &rustfs_lock::NamespaceLockGuard, proof: &IlmRecoveryExportFleetProofToken, ) -> EcstoreResult<()> { @@ -1557,7 +1559,7 @@ async fn abandon_recovery_control_fenced( }), ..Default::default() }; - options.add_namespace_lock_guard(control_guard); + options.add_owned_write_lock(Arc::clone(control_guard), RUSTFS_META_BUCKET, &object); options.add_namespace_lock_guard(source_guard); let write_result = config_boundary::save_config_with_opts(api.clone(), &object, encoded, &options).await; let observed = match load_recovery_control_no_lock(api.clone(), disposition).await { @@ -1630,7 +1632,7 @@ async fn delete_exact_local_recovery_copy( api: Arc, copy: &IlmRecoverySourceCopy, source_schema: &str, - control_guard: &rustfs_lock::NamespaceLockGuard, + control_guard: &Arc, source_guard: &rustfs_lock::NamespaceLockGuard, ) -> EcstoreResult<()> { let set = recovery_copy_set(&api, ©.authority)?; @@ -1704,7 +1706,7 @@ async fn save_recovery_disposition_step_fenced( api: Arc, current: &ObservedIlmRecoveryDisposition, next: &IlmRecoveryDisposition, - control_guard: &rustfs_lock::NamespaceLockGuard, + control_guard: &Arc, source_guard: &rustfs_lock::NamespaceLockGuard, proof: &IlmRecoveryExportFleetProofToken, ) -> EcstoreResult { diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 91478e0e4..a0d039975 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -1338,10 +1338,54 @@ pub(crate) async fn get_quota_config_and_incarnation_from_disk_in( let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?; let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone(); - match bucket_meta_sys - .read_authoritative_metadata_from_disk_under_transaction_lock(bucket) - .await? + quota_config_and_incarnation_from_authority( + bucket, + bucket_meta_sys + .read_authoritative_metadata_from_disk_under_transaction_lock(bucket) + .await?, + ) +} + +pub(crate) async fn get_quota_config_and_incarnation_from_disk_for_options_in( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, + opts: &ObjectOptions, + transaction_guard: &Arc, +) -> Result<(Option, Uuid, OffsetDateTime)> { + let Some(snapshot) = opts.object_lock_config_snapshot.as_ref() else { + return get_quota_config_and_incarnation_from_disk_in(ctx, bucket).await; + }; + let sys = bucket_metadata_sys_of(ctx)?.read().await.clone(); + let store = sys.object_store(); + let snapshot_guard = snapshot + .metadata_transaction_guard_for(store.id, bucket, opts.expected_bucket_incarnation_id) + .filter(|guard| Arc::ptr_eq(guard, transaction_guard)) + .ok_or_else(|| Error::other("quota snapshot does not hold this bucket's metadata transaction fence"))?; + if snapshot_guard.is_released() { + return Err(Error::other("quota snapshot metadata transaction fence was released")); + } + + // The snapshot carries the caller's lifecycle fence, excluding deletion. + // Its transaction guard excludes config writes, so a same-name object + // owner need not recursively acquire the bucket namespace here. + let authority = sys + .read_authoritative_metadata_from_disk_under_guard(bucket, transaction_guard) + .await?; + if snapshot_guard.is_lock_lost() + || snapshot_guard.is_released() + || matches!(&authority, BucketMetadataAuthority::Authoritative(metadata) + if !snapshot.is_valid_for_destructive_put(store.id, bucket, metadata.bucket_incarnation_id)) { + return Err(Error::other("quota snapshot fence or bucket incarnation changed during the disk read")); + } + quota_config_and_incarnation_from_authority(bucket, authority) +} + +fn quota_config_and_incarnation_from_authority( + bucket: &str, + authority: BucketMetadataAuthority, +) -> Result<(Option, Uuid, OffsetDateTime)> { + match authority { BucketMetadataAuthority::Authoritative(metadata) if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() => { @@ -2722,29 +2766,30 @@ impl BucketMetadataSys { async fn read_authoritative_metadata_from_disk_under_transaction_lock( &self, bucket: &str, + ) -> Result { + let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?; + let namespace_guard = namespace_lock + .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + self.read_authoritative_metadata_from_disk_under_guard(bucket, &namespace_guard) + .await + } + + async fn read_authoritative_metadata_from_disk_under_guard( + &self, + bucket: &str, + guard: &rustfs_lock::NamespaceLockGuard, ) -> Result { #[cfg(test)] if self.object_lock_disk_read_errors.write().await.remove(bucket) { return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}"))); } - let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?; - let namespace_guard = namespace_lock - .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) - .await?; - match await_bucket_namespace_operation( - Some(&namespace_guard), - bucket, - "bucket metadata snapshot existence check", - async { - self.object_store() - .get_bucket_info_from_sets_at_read_quorum( - bucket, - &crate::storage_api_contracts::bucket::BucketOptions::default(), - ) - .await - }, - ) + match await_bucket_namespace_operation(Some(guard), bucket, "bucket metadata snapshot existence check", async { + self.object_store() + .get_bucket_info_from_sets_at_read_quorum(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default()) + .await + }) .await { Ok(_) => {} @@ -2753,7 +2798,7 @@ impl BucketMetadataSys { } let (metadata, persisted) = await_bucket_namespace_operation( - Some(&namespace_guard), + Some(guard), bucket, "bucket metadata authoritative snapshot", load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true), @@ -4776,6 +4821,231 @@ mod tests { .expect("DeleteBucket should succeed"); } + #[tokio::test] + #[serial] + async fn quota_snapshot_reuses_same_name_owner_and_rereads_disk_authority() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "quota-same-name-owner"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("quota fixture bucket"); + let snapshot = store + .object_lock_config_snapshot(bucket) + .await + .expect("guarded Object Lock snapshot"); + let incarnation = store.bucket_incarnation_id(bucket).await.expect("fixture incarnation"); + let transaction_guard = snapshot + .metadata_transaction_guard_for(store.id, bucket, Some(incarnation)) + .expect("snapshot's actual transaction guard"); + let expected = get_quota_config_and_incarnation_from_disk_for_options_in( + &store.ctx, + bucket, + &ObjectOptions::default(), + &transaction_guard, + ) + .await + .expect("a reader without a snapshot must retain the ordinary namespace path"); + + let sys = bucket_metadata_sys_of(&store.ctx).expect("fixture metadata system"); + let sys = sys.read().await.clone(); + let mut stale = (*sys.get(bucket).await.expect("cached fixture metadata")).clone(); + let metadata_path = stale.save_file_path(); + stale.quota_config = Some(BucketQuota { + quota: Some(123), + ..Default::default() + }); + sys.set(bucket.to_string(), Arc::new(stale)).await; + let namespace = store.new_ns_lock(bucket, bucket).await.expect("same-name object namespace"); + let _owner = namespace + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await + .expect("same-name object write owner"); + let opts = ObjectOptions { + object_lock_config_snapshot: Some(snapshot), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + assert_eq!( + get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard) + .await + .expect("quota disk read must reuse the held lifecycle and transaction fences"), + expected, + "the cached quota must not replace disk authority" + ); + + crate::config::com::save_config_with_opts( + store.clone(), + &metadata_path, + b"corrupt bucket metadata".to_vec(), + &ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("inject committed metadata corruption"); + assert!( + get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard) + .await + .is_err(), + "a valid snapshot must not hide corrupt disk metadata" + ); + crate::config::com::delete_config(store.clone(), &metadata_path) + .await + .expect("remove the persisted metadata while retaining its sidecar"); + assert!( + get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard) + .await + .is_err(), + "a valid snapshot must not promote fabricated metadata to authority" + ); + } + + #[tokio::test] + #[serial] + async fn quota_snapshot_rejects_wrong_scope_incarnation_and_transaction_guard() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "quota-snapshot-scope"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("quota fixture bucket"); + let incarnation = store.bucket_incarnation_id(bucket).await.expect("fixture incarnation"); + let snapshot = store + .object_lock_config_snapshot(bucket) + .await + .expect("guarded Object Lock snapshot"); + let transaction_guard = snapshot + .metadata_transaction_guard_for(store.id, bucket, Some(incarnation)) + .expect("snapshot's actual transaction guard"); + let mut opts = ObjectOptions { + object_lock_config_snapshot: Some(snapshot), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + assert!( + get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, "another-bucket", &opts, &transaction_guard) + .await + .is_err() + ); + opts.expected_bucket_incarnation_id = Some(Uuid::new_v4()); + assert!( + get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard) + .await + .is_err() + ); + opts.expected_bucket_incarnation_id = Some(incarnation); + let other_guard = Arc::new( + acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, bucket) + .await + .expect("distinct real transaction read guard"), + ); + assert!( + get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &other_guard) + .await + .is_err(), + "the same namespace key does not prove that this is the snapshot's actual guard" + ); + let (_other_dirs, other_store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(other_store.clone(), Vec::new()).await; + assert!( + get_quota_config_and_incarnation_from_disk_for_options_in(&other_store.ctx, bucket, &opts, &transaction_guard) + .await + .is_err() + ); + opts.object_lock_config_snapshot = Some(Arc::new(crate::object_api::ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))); + assert!( + get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard) + .await + .is_err(), + "an unguarded snapshot must not authorize namespace reuse" + ); + } + + #[tokio::test] + #[serial] + async fn quota_snapshot_rejects_lifecycle_loss_during_read_and_changed_disk_incarnation() { + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = "quota-snapshot-loss"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("quota fixture bucket"); + let sys = bucket_metadata_sys_of(&store.ctx).expect("fixture metadata system"); + let sys = sys.read().await.clone(); + let mut metadata = (*sys.get(bucket).await.expect("fixture metadata")).clone(); + let incarnation = metadata.bucket_incarnation_id; + let _lifecycle_guard = store + .acquire_bucket_lifecycle_read_lock(bucket) + .await + .expect("real lifecycle owner"); + let transaction_guard = Arc::new( + acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, bucket) + .await + .expect("real metadata transaction owner"), + ); + let (lifecycle_fence, lost) = crate::object_api::NamespaceLockFence::loss_handle_for_test(); + let snapshot = crate::object_api::ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence( + store.id, + bucket, + incarnation, + metadata.object_lock_config_updated_at, + ObjectLockConfigState::ConfirmedAbsent, + lifecycle_fence, + Arc::clone(&transaction_guard), + ); + let opts = ObjectOptions { + object_lock_config_snapshot: Some(Arc::new(snapshot)), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + let mut read = Box::pin(get_quota_config_and_incarnation_from_disk_for_options_in( + &store.ctx, + bucket, + &opts, + &transaction_guard, + )); + assert!( + futures::poll!(read.as_mut()).is_pending(), + "the authoritative disk read must be in flight" + ); + lost.store(true, std::sync::atomic::Ordering::Release); + assert!(read.await.is_err(), "a lifecycle fence lost during the disk read must reject its result"); + + let snapshot = store + .object_lock_config_snapshot(bucket) + .await + .expect("fresh guarded snapshot"); + let transaction_guard = snapshot + .metadata_transaction_guard_for(store.id, bucket, Some(incarnation)) + .expect("snapshot's actual transaction guard"); + let opts = ObjectOptions { + object_lock_config_snapshot: Some(snapshot), + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }; + metadata.bucket_incarnation_id = Uuid::new_v4(); + save_bucket_incarnation(store.clone(), bucket, metadata.bucket_incarnation_id) + .await + .expect("inject a new disk incarnation"); + metadata + .save_with_store_committed(store.clone()) + .await + .expect("persist its matching metadata blob"); + assert!( + get_quota_config_and_incarnation_from_disk_for_options_in(&store.ctx, bucket, &opts, &transaction_guard) + .await + .is_err(), + "a matching sidecar and blob from another incarnation must not reuse the old snapshot" + ); + } + /// Writers on different nodes updating *different* config files of one /// bucket must both survive. Each rewrites the whole metadata blob, so /// without a lock spanning the read-modify-write the later save carries diff --git a/crates/ecstore/src/bucket/migration.rs b/crates/ecstore/src/bucket/migration.rs index 6724a408a..723b75c69 100644 --- a/crates/ecstore/src/bucket/migration.rs +++ b/crates/ecstore/src/bucket/migration.rs @@ -280,11 +280,11 @@ where debug!("Found {} migrating bucket metadata, migrating...", buckets.len()); - let opts = ObjectOptions { + let mut opts = ObjectOptions { max_parity: true, - no_lock: true, ..Default::default() }; + opts.use_offline_recovery_write(); let h = HeaderMap::new(); for bucket in buckets { @@ -397,11 +397,11 @@ where return Ok(()); } - let opts = ObjectOptions { + let mut opts = ObjectOptions { max_parity: true, - no_lock: true, ..Default::default() }; + opts.use_offline_recovery_write(); let h = HeaderMap::new(); let prefix = format!("{IAM_CONFIG_PREFIX}/"); let mut continuation: Option = None; diff --git a/crates/ecstore/src/bucket/quota/reservation.rs b/crates/ecstore/src/bucket/quota/reservation.rs index 2294dc0b4..1760a926a 100644 --- a/crates/ecstore/src/bucket/quota/reservation.rs +++ b/crates/ecstore/src/bucket/quota/reservation.rs @@ -276,7 +276,7 @@ impl QuotaContext { reap_stale_reservations(Arc::clone(&store), &ledger_data.bucket, &ledger_data.ledger_object).await?; let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_data.ledger_object).await?; - let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + let ledger_guard = Arc::new(ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?); fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_data.ledger_object, None).await?; let mut ledger = load_current_ledger_locked( Arc::clone(&store), @@ -563,7 +563,7 @@ pub(crate) async fn begin( } let config_read_started = rustfs_io_metrics::put_stage_timer(); let (quota, bucket_incarnation, quota_revision) = - metadata_sys::get_quota_config_and_incarnation_from_disk_in(ctx, bucket).await?; + metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(ctx, bucket, opts, &metadata_guard).await?; if let Some(cached) = cached_quota_snapshot.as_ref() { let reason = quota_cache_reason(quota.as_ref()); if cached.quota == quota && cached.bucket_incarnation == bucket_incarnation && cached.quota_revision == quota_revision { @@ -758,7 +758,7 @@ async fn mark_commit_started(data: &LedgerReservationData) -> Result<()> { let data = data.clone(); tokio::spawn(async move { let ledger_lock = data.store.new_ns_lock(RUSTFS_META_BUCKET, &data.ledger_object).await?; - let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + let ledger_guard = Arc::new(ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?); fence_namespace_mutations(&data.store, RUSTFS_META_BUCKET, &data.ledger_object, None).await?; let mut ledger = load_ledger_locked(Arc::clone(&data.store), &data.ledger_object).await?; ledger.mark_commit_started(data.operation_id, &data.reservation)?; @@ -779,7 +779,7 @@ async fn settle(data: &LedgerReservationData, committed: bool) -> Result<()> { // object lock is released and would otherwise revoke a later write's // newly acquired fence for the same object. let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?; - let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + let ledger_guard = Arc::new(ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?); fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_object, None).await?; let mut ledger = load_ledger_locked(Arc::clone(&store), &ledger_object).await?; if committed { @@ -862,7 +862,7 @@ async fn reap_stale_reservations(store: Arc, bucket: &str, ledger_objec } let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, ledger_object).await?; - let ledger_guard = ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?; + let ledger_guard = Arc::new(ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?); fence_namespace_mutations(&store, RUSTFS_META_BUCKET, ledger_object, None).await?; let mut ledger = load_ledger_locked(Arc::clone(&store), ledger_object).await?; let cursor_changed = next_cursor.is_some() && ledger.reap_cursor != next_cursor; @@ -1022,7 +1022,7 @@ async fn save_ledger_locked( store: Arc, ledger_object: &str, ledger: &QuotaLedger, - ledger_guard: &NamespaceLockGuard, + ledger_guard: &Arc, ) -> Result<()> { if ledger_guard.is_lock_lost() { return Err(StorageError::NamespaceLockQuorumUnavailable { @@ -1043,7 +1043,8 @@ async fn save_ledger_locked( ..Default::default() }; let _ = opts.set_quota_admission(0, u64::MAX); - opts.add_namespace_lock_guard(ledger_guard); + opts.add_owned_write_lock(Arc::clone(ledger_guard), RUSTFS_META_BUCKET, ledger_object); + opts.write_completion = crate::object_api::WriteCompletion::TailDrained; save_config_with_opts(store, ledger_object, serde_json::to_vec(ledger)?, &opts).await } diff --git a/crates/ecstore/src/bucket/replication/replication_config_store.rs b/crates/ecstore/src/bucket/replication/replication_config_store.rs index 5ac6a48af..9f30eb6f3 100644 --- a/crates/ecstore/src/bucket/replication/replication_config_store.rs +++ b/crates/ecstore/src/bucket/replication/replication_config_store.rs @@ -13,7 +13,9 @@ // limitations under the License. use super::replication_error_boundary::Result; -use super::replication_storage_boundary::{HTTPPreconditions, ObjectInfo, ObjectOptions, ReplicationObjectIO}; +use super::replication_storage_boundary::{ + HTTPPreconditions, ObjectInfo, ObjectOptions, ReplicationObjectIO, WriteCommitGuard, WriteCompletion, +}; use crate::config::{com, storageclass}; use std::sync::Arc; @@ -77,13 +79,6 @@ impl ReplicationConfigStore { com::save_config(api, file, data).await } - pub(crate) async fn save_no_lock(api: Arc, file: &str, data: Vec) -> Result<()> - where - S: ReplicationObjectIO, - { - com::save_config_no_lock(api, file, data).await - } - pub(crate) async fn save_conditional( api: Arc, file: &str, @@ -111,21 +106,18 @@ impl ReplicationConfigStore { file: &str, data: Vec, http_preconditions: HTTPPreconditions, + guard: &WriteCommitGuard, ) -> Result<()> where S: ReplicationObjectIO, { - com::save_config_with_opts_quiet( - api, - file, - data, - &ObjectOptions { - max_parity: true, - no_lock: true, - http_preconditions: Some(http_preconditions), - ..Default::default() - }, - ) - .await + let mut opts = ObjectOptions { + max_parity: true, + http_preconditions: Some(http_preconditions), + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }; + opts.add_write_commit_guard(guard); + com::save_config_with_opts_quiet(api, file, data, &opts).await } } diff --git a/crates/ecstore/src/bucket/replication/replication_pool.rs b/crates/ecstore/src/bucket/replication/replication_pool.rs index c1f5ae916..f0eb97e2d 100644 --- a/crates/ecstore/src/bucket/replication/replication_pool.rs +++ b/crates/ecstore/src/bucket/replication/replication_pool.rs @@ -44,7 +44,7 @@ use super::replication_resyncer::{ use super::replication_state::ReplicationStats; use super::replication_storage_boundary::{ HTTPPreconditions, ObjectInfo, ObjectOptions, ObjectToDelete, ReplicationDeletedObject, ReplicationObjectIO, - ReplicationStorage, + ReplicationStorage, WriteCommitGuard, }; use super::replication_target_boundary::{BucketTargetError, ReplicationTargetStore, replication_object_is_ssec_encrypted}; use super::replication_versioning_boundary::ReplicationVersioningStore; @@ -513,7 +513,7 @@ where // Lock order is transaction namespace lock -> force-delete journal object lock. // Keep the transaction guard alive through the conditional write so legacy // writers cannot interleave a read-modify-write transition within this process. - let guard = lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await?; + let guard = WriteCommitGuard::acquire(&lock, ReplicationLockTiming::acquire_timeout()).await?; let (mut entries, preconditions, exists) = read_force_delete_intents(storage.clone(), file).await?; if !update(&mut entries, exists)? { return Ok(()); @@ -565,7 +565,7 @@ async fn read_force_delete_intents( async fn save_force_delete_intents( storage: Arc, file: &str, - guard: &rustfs_lock::NamespaceLockGuard, + guard: &WriteCommitGuard, entries: Vec, preconditions: HTTPPreconditions, ) -> Result<(), EcstoreError> { @@ -628,7 +628,7 @@ async fn acknowledge_mrf_recovery( let lock = storage .new_ns_lock(ReplicationMetadataStore::rustfs_meta_bucket(), file) .await?; - let guard = lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await?; + let guard = WriteCommitGuard::acquire(&lock, ReplicationLockTiming::acquire_timeout()).await?; let current = ReplicationConfigStore::read_no_lock_with_metadata_preserve_empty(storage.clone(), file).await; let (current_data, current_etag, current_exists) = match current { Ok((data, object_info)) => (data, object_info.etag, true), @@ -658,7 +658,7 @@ async fn acknowledge_mrf_recovery( } else { encode_mrf_file(&retained)? }; - match ReplicationConfigStore::save_conditional_no_lock(storage.clone(), file, data, preconditions).await { + match ReplicationConfigStore::save_conditional_no_lock(storage.clone(), file, data, preconditions, &guard).await { Ok(()) => return Ok(retained), Err(EcstoreError::PreconditionFailed) => continue, Err(error) => return Err(error), @@ -2522,7 +2522,7 @@ async fn quarantine_mrf_file(storage: &Arc, data: &[u8 continue; } }; - let guard = match lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await { + let guard = match WriteCommitGuard::acquire(&lock, ReplicationLockTiming::acquire_timeout()).await { Ok(guard) => guard, Err(error) => { warn!( @@ -2573,6 +2573,7 @@ async fn quarantine_mrf_file(storage: &Arc, data: &[u8 ReplicationMetadataStore::MRF_REPLICATION_FILE, Vec::new(), preconditions, + &guard, ) .await { @@ -2627,13 +2628,12 @@ async fn recover_corrupt_mrf_generation( entries: &[MrfReplicateEntry], preconditions: Option, storage: &Arc, - guard: &rustfs_lock::NamespaceLockGuard, + guard: &WriteCommitGuard, pending_payload: &mut Option, started: Instant, ) -> Option { let quarantine_file = format!("{MRF_CORRUPT_FILE_PREFIX}.{}.bin", OffsetDateTime::now_utc().unix_timestamp_nanos()); - if let Err(error) = ReplicationConfigStore::save_no_lock(storage.clone(), &quarantine_file, corrupt_generation.to_vec()).await - { + if let Err(error) = ReplicationConfigStore::save(storage.clone(), &quarantine_file, corrupt_generation.to_vec()).await { warn!( component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_REPLICATION, @@ -2688,6 +2688,7 @@ async fn recover_corrupt_mrf_generation( ReplicationMetadataStore::MRF_REPLICATION_FILE, data, preconditions, + guard, ) .await { @@ -2735,7 +2736,7 @@ async fn append_mrf_entries_to_disk( return None; } }; - let guard = match lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await { + let guard = match WriteCommitGuard::acquire(&lock, ReplicationLockTiming::acquire_timeout()).await { Ok(guard) => guard, Err(error) => { warn!( @@ -2862,6 +2863,7 @@ async fn append_mrf_entries_to_disk( ReplicationMetadataStore::MRF_REPLICATION_FILE, data, preconditions, + &guard, ) .await { diff --git a/crates/ecstore/src/bucket/replication/replication_storage_boundary.rs b/crates/ecstore/src/bucket/replication/replication_storage_boundary.rs index 1e39819bf..0a5cbd756 100644 --- a/crates/ecstore/src/bucket/replication/replication_storage_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_storage_boundary.rs @@ -19,7 +19,8 @@ use super::replication_error_boundary::Error; use super::replication_filemeta_boundary::{replication_state_from_filemeta, version_purge_status_from_filemeta}; pub(crate) type ReplicationObjectStore = crate::store::ECStore; pub(crate) use crate::object_api::{ - GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode, + GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ReplicationStatusWritebackCondition, + ReplicationStatusWritebackMode, WriteCommitGuard, WriteCompletion, }; #[cfg(test)] pub(crate) use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence}; diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 2521729a3..b35e776da 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -74,7 +74,7 @@ fn config_task_join_error(operation: &'static str, error: tokio::task::JoinError /// The operation must use the corresponding no-lock read/save functions. pub async fn with_server_config_write_lock(store: Arc, operation: F) -> Result where - F: FnOnce() -> Fut + Send + 'static, + F: FnOnce(crate::object_api::WriteCommitGuard) -> Fut + Send + 'static, Fut: std::future::Future + Send + 'static, T: Send + 'static, { @@ -85,8 +85,8 @@ where let transaction_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?; let _transaction_guard = transaction_lock.get_write_lock(get_lock_acquire_timeout()).await?; let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?; - let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?; - Ok(operation().await) + let write_guard = crate::object_api::WriteCommitGuard::acquire(&namespace_lock, get_lock_acquire_timeout()).await?; + Ok(operation(write_guard.clone()).await) }) .await .map_err(|error| config_task_join_error("server config write", error))? @@ -119,14 +119,14 @@ where /// one metadata config object. The operation must use no-lock object I/O. pub async fn with_config_object_write_lock(store: Arc, object: String, operation: F) -> Result where - F: FnOnce() -> Fut + Send + 'static, + F: FnOnce(crate::object_api::WriteCommitGuard) -> Fut + Send + 'static, Fut: std::future::Future + Send + 'static, T: Send + 'static, { tokio::spawn(async move { let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?; - let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?; - Ok(operation().await) + let write_guard = crate::object_api::WriteCommitGuard::acquire(&namespace_lock, get_lock_acquire_timeout()).await?; + Ok(operation(write_guard.clone()).await) }) .await .map_err(|error| config_task_join_error("config object write", error))? @@ -638,7 +638,12 @@ where .await } -pub async fn save_config_no_lock(api: Arc, file: &str, data: Vec) -> Result<()> +pub async fn save_config_no_lock( + api: Arc, + file: &str, + data: Vec, + guard: &crate::object_api::WriteCommitGuard, +) -> Result<()> where S: ObjectIO< Error = Error, @@ -650,17 +655,13 @@ where PutObjectReader = PutObjReader, >, { - save_config_with_opts( - api, - file, - data, - &ObjectOptions { - max_parity: true, - no_lock: true, - ..Default::default() - }, - ) - .await + let mut opts = ObjectOptions { + max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }; + opts.add_write_commit_guard(guard); + save_config_with_opts(api, file, data, &opts).await } /// `delete_config` with `no_lock` set — for callers already holding the @@ -846,12 +847,12 @@ where Ok(cfg) } -async fn new_and_save_server_config_no_lock(api: Arc) -> Result +async fn new_and_save_server_config_no_lock(api: Arc, guard: &crate::object_api::WriteCommitGuard) -> Result where S: EcstoreObjectIO + StorageAdminApi, { let cfg = new_server_config(); - save_server_config_no_lock(api, &cfg).await?; + save_server_config_no_lock(api, &cfg, guard).await?; Ok(cfg) } @@ -2138,14 +2139,18 @@ where } /// Handle the situation where the configuration file does not exist, create and save a new configuration -async fn handle_missing_config(api: Arc, context: &str, namespace_lock_held: bool) -> Result +async fn handle_missing_config( + api: Arc, + context: &str, + write_guard: Option<&crate::object_api::WriteCommitGuard>, +) -> Result where S: EcstoreObjectIO + StorageAdminApi + NamespaceLocking, { warn!("Configuration not found ({}): Start initializing new configuration", context); let cfg = if runtime_sources::first_cluster_node_is_local().await { - if namespace_lock_held { - new_and_save_server_config_no_lock(api.clone()).await? + if let Some(guard) = write_guard { + new_and_save_server_config_no_lock(api.clone(), guard).await? } else { new_and_save_server_config(api.clone()).await? } @@ -2175,13 +2180,13 @@ where > + StorageAdminApi, S: NamespaceLocking, { - read_config_without_migrate_inner(api, false).await + read_config_without_migrate_inner(api, None).await } /// Reads the server config while an upper layer holds the namespace write lock /// for [`SERVER_CONFIG_OBJECT`]. Missing-config initialization uses the matching /// no-lock save path and therefore cannot recursively acquire the same lock. -pub async fn read_config_without_migrate_no_lock(api: Arc) -> Result +pub async fn read_config_without_migrate_no_lock(api: Arc, guard: &crate::object_api::WriteCommitGuard) -> Result where S: ObjectIO< Error = Error, @@ -2194,7 +2199,7 @@ where > + StorageAdminApi + NamespaceLocking, { - read_config_without_migrate_inner(api, true).await + read_config_without_migrate_inner(api, Some(guard)).await } /// Reads an already-initialized server config while a caller owns a namespace @@ -2204,7 +2209,10 @@ pub async fn read_existing_server_config_no_lock(api: Arc) -> Result(api: Arc, namespace_lock_held: bool) -> Result +async fn read_config_without_migrate_inner( + api: Arc, + write_guard: Option<&crate::object_api::WriteCommitGuard>, +) -> Result where S: ObjectIO< Error = Error, @@ -2220,19 +2228,23 @@ where let config_file = server_config_path(); // Try to read the configuration file. - let data = if namespace_lock_held { + let data = if write_guard.is_some() { read_config_no_lock(api.clone(), &config_file).await } else { read_config(api.clone(), &config_file).await }; match data { - Ok(data) => read_server_config(api, &data, namespace_lock_held).await, - Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration", namespace_lock_held).await, + Ok(data) => read_server_config(api, &data, write_guard).await, + Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration", write_guard).await, Err(err) => handle_config_read_error(err, &config_file), } } -async fn read_server_config(api: Arc, data: &[u8], namespace_lock_held: bool) -> Result +async fn read_server_config( + api: Arc, + data: &[u8], + write_guard: Option<&crate::object_api::WriteCommitGuard>, +) -> Result where S: EcstoreObjectIO + StorageAdminApi + NamespaceLocking, { @@ -2242,7 +2254,7 @@ where warn!("Received empty configuration data, try to reread from '{}'", config_file); // Try to read the configuration again - let data = if namespace_lock_held { + let data = if write_guard.is_some() { read_config_no_lock(api.clone(), &config_file).await } else { read_config(api.clone(), &config_file).await @@ -2253,7 +2265,7 @@ where return Ok(cfg.merge()); } Err(Error::ConfigNotFound) => { - return handle_missing_config(api, "Read alternate configuration", namespace_lock_held).await; + return handle_missing_config(api, "Read alternate configuration", write_guard).await; } Err(err) => return handle_config_read_error(err, &config_file), } @@ -2687,7 +2699,7 @@ where /// Saves the server config while an upper layer holds the namespace write /// lock for [`SERVER_CONFIG_OBJECT`]. -pub async fn save_server_config_no_lock(api: Arc, cfg: &Config) -> Result<()> +pub async fn save_server_config_no_lock(api: Arc, cfg: &Config, guard: &crate::object_api::WriteCommitGuard) -> Result<()> where S: ObjectIO< Error = Error, @@ -2699,10 +2711,10 @@ where PutObjectReader = PutObjReader, >, { - save_server_config_inner(api, cfg, true).await + save_server_config_inner(api, cfg, guard).await } -async fn save_server_config_inner(api: Arc, cfg: &Config, no_lock: bool) -> Result<()> +async fn save_server_config_inner(api: Arc, cfg: &Config, guard: &crate::object_api::WriteCommitGuard) -> Result<()> where S: ObjectIO< Error = Error, @@ -2715,11 +2727,7 @@ where >, { let config_file = get_config_file(); - let existing = match if no_lock { - read_config_no_lock(api.clone(), &config_file).await - } else { - read_config(api.clone(), &config_file).await - } { + let existing = match read_config_no_lock(api.clone(), &config_file).await { Ok(v) => Some(v), Err(Error::ConfigNotFound) => None, Err(err) => { @@ -2743,21 +2751,7 @@ where return Ok(()); } - if no_lock { - save_config_with_opts( - api, - &config_file, - data, - &ObjectOptions { - max_parity: true, - no_lock: true, - ..Default::default() - }, - ) - .await - } else { - save_config(api, &config_file, data).await - } + save_config_no_lock(api, &config_file, data, guard).await } pub async fn lookup_configs(cfg: &mut Config, api: Arc) -> Result<()> diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 795fe950f..f807eb603 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -3692,8 +3692,8 @@ fn activation_pool_meta_lock_error(err: rustfs_lock::LockError) -> Error { } pub(crate) struct PoolRebalanceActivationFence { - pool_meta_guard: rustfs_lock::NamespaceLockGuard, - rebalance_meta_guard: rustfs_lock::NamespaceLockGuard, + pool_meta_guard: Arc, + rebalance_meta_guard: Arc, fleet_proof: Option, decommission_target_fence_proof: Option, #[cfg(test)] @@ -3720,7 +3720,12 @@ impl PoolRebalanceActivationFence { let forced_lost = self.forced_lost.load(Ordering::Acquire); #[cfg(not(test))] let forced_lost = false; - if forced_lost || self.pool_meta_guard.is_lock_lost() || self.rebalance_meta_guard.is_lock_lost() { + if forced_lost + || self.pool_meta_guard.is_lock_lost() + || self.pool_meta_guard.is_released() + || self.rebalance_meta_guard.is_lock_lost() + || self.rebalance_meta_guard.is_released() + { return Err(Error::other("activation lock lost before metadata commit or worker admission")); } if self @@ -3742,8 +3747,8 @@ impl PoolRebalanceActivationFence { } pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) { - opts.add_namespace_lock_guard(&self.pool_meta_guard); - opts.add_namespace_lock_guard(&self.rebalance_meta_guard); + opts.add_owned_write_lock(Arc::clone(&self.pool_meta_guard), RUSTFS_META_BUCKET, POOL_META_NAME); + opts.add_owned_write_lock(Arc::clone(&self.rebalance_meta_guard), RUSTFS_META_BUCKET, REBAL_META_NAME); } #[cfg(test)] @@ -3784,8 +3789,8 @@ where .map_err(activation_pool_meta_lock_error)?; Ok(PoolRebalanceActivationFence { - pool_meta_guard, - rebalance_meta_guard, + pool_meta_guard: Arc::new(pool_meta_guard), + rebalance_meta_guard: Arc::new(rebalance_meta_guard), fleet_proof, decommission_target_fence_proof: None, #[cfg(test)] @@ -4067,7 +4072,7 @@ fn rollback_start_decommission_pool_meta(pool_meta: &mut PoolMeta, previous_pool } fn ensure_pool_meta_write_fence(guard: &rustfs_lock::NamespaceLockGuard, operation: &str) -> Result<()> { - if guard.is_lock_lost() { + if guard.is_lock_lost() || guard.is_released() { return Err(Error::other(format!("{operation}: pool metadata distributed fence was lost"))); } Ok(()) @@ -6339,6 +6344,9 @@ fn encode_pool_meta_v3_envelope( } enum PoolMetaPersistenceFence<'a> { + Owned(Arc), + OfflineRecovery, + #[cfg(test)] Distributed(Option>), Activation(&'a PoolRebalanceActivationFence), } @@ -6346,6 +6354,10 @@ enum PoolMetaPersistenceFence<'a> { impl PoolMetaPersistenceFence<'_> { fn ensure_held(&self) -> Result<()> { match self { + Self::Owned(guard) if guard.is_lock_lost() || guard.is_released() => { + Err(Error::other("pool metadata distributed fence was lost before a replica write")) + } + #[cfg(test)] Self::Distributed(Some(signal)) if signal.is_lost() => { Err(Error::other("pool metadata distributed fence was lost before a replica write")) } @@ -6354,14 +6366,30 @@ impl PoolMetaPersistenceFence<'_> { } } - fn add_to_options(&self, opts: &mut ObjectOptions) { + fn add_to_options(&self, opts: &mut ObjectOptions, object: &str) { match self { + Self::Owned(guard) => opts.add_owned_write_lock(Arc::clone(guard), RUSTFS_META_BUCKET, object), + Self::OfflineRecovery => opts.use_offline_recovery_write(), + #[cfg(test)] Self::Distributed(Some(signal)) => opts.add_namespace_lock_lost_signal(Arc::clone(signal)), - Self::Activation(fence) => fence.add_namespace_lock_fence(opts), + Self::Activation(fence) => { + fence.add_namespace_lock_fence(opts); + opts.add_owned_write_lock(Arc::clone(&fence.pool_meta_guard), RUSTFS_META_BUCKET, object); + } + #[cfg(test)] Self::Distributed(None) => {} } } + fn allows_concurrent_phases(&self) -> bool { + match self { + Self::Owned(guard) => guard.lock_lost_signal().is_some(), + #[cfg(test)] + Self::Distributed(signal) => signal.is_some(), + _ => false, + } + } + fn is_activation(&self) -> bool { matches!(self, Self::Activation(_)) } @@ -6662,7 +6690,7 @@ fn pool_meta_cas_options(object: &str, token: &PoolMetaCasToken, fence: &PoolMet http_preconditions: Some(pool_meta_cas_preconditions(token, object)?), ..Default::default() }; - fence.add_to_options(&mut opts); + fence.add_to_options(&mut opts, object); Ok(opts) } @@ -7049,7 +7077,7 @@ where write_state, initialized, PoolMetaIdentityWriteScope::All, - &PoolMetaPersistenceFence::Distributed(None), + &PoolMetaPersistenceFence::OfflineRecovery, &mut transaction_arm, ) .await?; @@ -7076,7 +7104,7 @@ where write_state, false, PoolMetaIdentityWriteScope::Pools(pool_indices), - &PoolMetaPersistenceFence::Distributed(None), + &PoolMetaPersistenceFence::OfflineRecovery, &mut transaction_arm, ) .await?; @@ -8154,11 +8182,16 @@ impl PoolMeta { .cloned() .ok_or_else(|| Error::other("pool metadata save failed: no storage pools available"))?; let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?; - let pool_meta_guard = pool_meta_lock.get_write_lock(get_lock_acquire_timeout()).await?; + let pool_meta_guard = Arc::new(pool_meta_lock.get_write_lock(get_lock_acquire_timeout()).await?); let mut write_state = PoolMetaWriteState::default(); let indices = (0..self.pools.len()).collect::>(); let outcome = self - .save_no_lock_armed_scoped(pools, &mut write_state, pool_meta_guard.lock_lost_signal(), Some(&indices)) + .save_no_lock_armed_scoped( + pools, + &mut write_state, + PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)), + Some(&indices), + ) .await?; outcome.disarm(); Ok(()) @@ -8182,14 +8215,16 @@ impl PoolMeta { where S: EcstoreObjectIO, { - let outcome = self.save_no_lock_armed_scoped(pools, write_state, None, None).await?; + let outcome = self + .save_no_lock_armed_scoped(pools, write_state, PoolMetaPersistenceFence::OfflineRecovery, None) + .await?; Ok(outcome.into_committed()) } async fn save_no_lock_with_fence( &self, pools: Vec>, - lock_lost: Option>, + fence: PoolMetaPersistenceFence<'_>, indices: &[usize], ) -> Result<()> where @@ -8197,7 +8232,7 @@ impl PoolMeta { { let mut write_state = PoolMetaWriteState::default(); let outcome = self - .save_no_lock_armed_scoped(pools, &mut write_state, lock_lost, Some(indices)) + .save_no_lock_armed_scoped(pools, &mut write_state, fence, Some(indices)) .await?; outcome.disarm(); Ok(()) @@ -8233,21 +8268,20 @@ impl PoolMeta { &self, pools: Vec>, write_state: &mut PoolMetaWriteState, - lock_lost: Option>, + fence: PoolMetaPersistenceFence<'_>, indices: &[usize], ) -> Result where S: EcstoreObjectIO, { - self.save_no_lock_armed_scoped(pools, write_state, lock_lost, Some(indices)) - .await + self.save_no_lock_armed_scoped(pools, write_state, fence, Some(indices)).await } async fn save_no_lock_armed_scoped( &self, pools: Vec>, write_state: &mut PoolMetaWriteState, - lock_lost: Option>, + fence: PoolMetaPersistenceFence<'_>, indices: Option<&[usize]>, ) -> Result where @@ -8258,7 +8292,6 @@ impl PoolMeta { // returned outcome is dropped before publication, Drop latches the // sticky recovery gate. let mut transaction_arm = write_state.arm_transaction(); - let fence = PoolMetaPersistenceFence::Distributed(lock_lost); let committed = self .save_no_lock_transaction(pools, write_state, &fence, indices, &mut transaction_arm) .await @@ -8489,7 +8522,7 @@ impl PoolMeta { && selection.revision.is_generation_protocol() && !selection.replica_state.needs_repair && !bootstrap_generation_required - && matches!(fence, PoolMetaPersistenceFence::Distributed(Some(_))); + && fence.allows_concurrent_phases(); #[cfg(feature = "e2e-test-hooks")] let phase_barrier = if concurrent_phases && pools.len() == 4 { PoolMetaPhaseBarrier::bind(&previous, &committed, revision, &durable).await? @@ -8635,7 +8668,9 @@ impl PoolMeta { S: EcstoreObjectIO, { let indices = (0..self.pools.len()).collect::>(); - let outcome = self.save_no_lock_armed(pools, write_state, None, &indices).await?; + let outcome = self + .save_no_lock_armed(pools, write_state, PoolMetaPersistenceFence::OfflineRecovery, &indices) + .await?; outcome.disarm(); Ok(()) } @@ -10831,7 +10866,7 @@ impl ECStore { &self, write_state: &mut PoolMetaWriteState, operation: &str, - ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> { + ) -> Result<(Arc, PoolMeta)> { self.acquire_pool_meta_write_guard_with_lock_error(write_state, operation, activation_pool_meta_lock_error) .await } @@ -10841,7 +10876,7 @@ impl ECStore { write_state: &mut PoolMetaWriteState, operation: &str, map_lock_error: F, - ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> + ) -> Result<(Arc, PoolMeta)> where F: FnOnce(rustfs_lock::LockError) -> Error, { @@ -10862,7 +10897,7 @@ impl ECStore { let selection = load_pool_meta_replicas_observing(self.pools.clone(), true, write_state).await?; write_state.observe_replicas(selection.replica_state); write_state.ensure_write_safe(operation)?; - Ok((pool_meta_guard, selection.meta)) + Ok((Arc::new(pool_meta_guard), selection.meta)) } async fn acquire_pool_meta_read_guard( @@ -11132,7 +11167,7 @@ impl ECStore { pub(crate) async fn acquire_pool_meta_object_heal_fence( &self, target_pool_indices: &[usize], - ) -> Result<(rustfs_lock::NamespaceLockGuard, Vec>)> { + ) -> Result<(Arc, Vec>)> { let mut save_guard = self.pool_meta_save_gate.lock().await; let (pool_meta_guard, snapshot) = self .acquire_pool_meta_write_guard_with_lock_error(&mut save_guard, "pool metadata heal admission failed", Error::from) @@ -11361,7 +11396,12 @@ impl ECStore { ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?; let outcome = snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &source_pool_indices) + .save_no_lock_armed( + self.pools.clone(), + &mut save_guard, + PoolMetaPersistenceFence::Owned(Arc::clone(&write_guard)), + &source_pool_indices, + ) .await?; ensure_pool_meta_write_fence(&write_guard, "exact delete capacity reconciliation save failed")?; for (target_pool_index, target_guard) in &target_guards { @@ -11470,7 +11510,7 @@ impl ECStore { .save_no_lock_armed( self.pools.clone(), &mut save_guard, - pool_meta_guard.lock_lost_signal(), + PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)), &[source_pool_index], ) .await?; @@ -11659,7 +11699,7 @@ impl ECStore { .save_no_lock_armed( self.pools.clone(), &mut save_guard, - pool_meta_guard.lock_lost_signal(), + PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)), &[source_pool_index], ) .await?; @@ -11965,7 +12005,12 @@ impl ECStore { }; if reservation_grew || pending_added > 0 { let outcome = snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &[source_pool_index]) + .save_no_lock_armed( + self.pools.clone(), + &mut save_guard, + PoolMetaPersistenceFence::Owned(Arc::clone(&write_guard)), + &[source_pool_index], + ) .await?; ensure_pool_meta_write_fence(&write_guard, "decommission target capacity intent save failed")?; snapshot = outcome.committed.clone(); @@ -12137,7 +12182,12 @@ impl ECStore { return result; } let outcome = snapshot - .save_no_lock_armed(self.pools.clone(), save_guard, write_guard.lock_lost_signal(), &[source_pool_index]) + .save_no_lock_armed( + self.pools.clone(), + save_guard, + PoolMetaPersistenceFence::Owned(Arc::clone(write_guard)), + &[source_pool_index], + ) .await?; ensure_pool_meta_write_fence(write_guard, "decommission target capacity progress save failed")?; if let Some(target_guard) = target_guard.as_ref() { @@ -12272,8 +12322,8 @@ impl ECStore { .first() .ok_or_else(|| Error::other("pool metadata recovery has no storage pools"))?; let lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?; - let guard = lock.get_write_lock(get_lock_acquire_timeout()).await?; - let fence = PoolMetaPersistenceFence::Distributed(guard.lock_lost_signal()); + let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?); + let fence = PoolMetaPersistenceFence::Owned(Arc::clone(&guard)); let mut candidate = PoolMetaWriteState { aborted_transaction: Arc::new(AtomicBool::new(false)), transaction_failure: Arc::default(), @@ -12449,7 +12499,12 @@ impl ECStore { merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, indices, "pool metadata save failed")?; } let outcome = snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), indices) + .save_no_lock_armed( + self.pools.clone(), + &mut save_guard, + PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)), + indices, + ) .await?; let mut pool_meta = self.pool_meta.write().await; ensure_pool_meta_write_fence(&pool_meta_guard, "pool metadata save failed")?; @@ -12544,7 +12599,12 @@ impl ECStore { }; let outcome = match snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx]) + .save_no_lock_armed( + self.pools.clone(), + &mut save_guard, + PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)), + &[idx], + ) .await { Ok(outcome) => outcome, @@ -12610,7 +12670,12 @@ impl ECStore { #[cfg(test)] decommission_test_wrap_result("bucket_completion_before_save", &bucket.name, &bucket.prefix, 0, Ok(()))?; let outcome = snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx]) + .save_no_lock_armed( + self.pools.clone(), + &mut save_guard, + PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)), + &[idx], + ) .await?; #[cfg(test)] decommission_test_wrap_result("bucket_completion_after_save", &bucket.name, &bucket.prefix, 0, Ok(()))?; @@ -12744,7 +12809,12 @@ impl ECStore { .await?; rollback_start_decommission_pool_meta(&mut snapshot, previous_pool_meta, indices); let outcome = snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), indices) + .save_no_lock_armed( + self.pools.clone(), + &mut save_guard, + PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)), + indices, + ) .await?; let mut pool_meta = self.pool_meta.write().await; ensure_pool_meta_write_fence(&pool_meta_guard, "decommission start rollback failed")?; @@ -13044,7 +13114,7 @@ impl ECStore { save_pool_meta: Save, ) -> Result<()> where - Save: FnOnce(PoolMeta, Option>) -> SaveFuture + Send + 'static, + Save: FnOnce(PoolMeta, PoolMetaPersistenceFence<'static>) -> SaveFuture + Send + 'static, SaveFuture: Future> + Send + 'static, { let store = self.clone(); @@ -13064,7 +13134,7 @@ impl ECStore { save_pool_meta: Save, ) -> Result<()> where - Save: FnOnce(PoolMeta, Option>) -> SaveFuture, + Save: FnOnce(PoolMeta, PoolMetaPersistenceFence<'static>) -> SaveFuture, SaveFuture: Future>, { let owner = owner.as_ref(); @@ -13113,9 +13183,17 @@ impl ECStore { )); } } - let pool_meta_fence = _pool_meta_guard - .as_ref() - .and_then(rustfs_lock::NamespaceLockGuard::lock_lost_signal); + let pool_meta_fence = match _pool_meta_guard.as_ref() { + Some(guard) => PoolMetaPersistenceFence::Owned(Arc::clone(guard)), + None => { + #[cfg(test)] + { + PoolMetaPersistenceFence::OfflineRecovery + } + #[cfg(not(test))] + return Err(Error::other("runtime pool metadata cancellation requires its write-lock owner")); + } + }; // Lock order: start gate, target gates, save gate, distributed pool // metadata fence, rebalance_meta, decommission_cancelers, then @@ -13554,7 +13632,12 @@ impl ECStore { let (save_outcome, save_error) = if changed { ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?; match snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &changed_indices) + .save_no_lock_armed( + self.pools.clone(), + &mut save_guard, + PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)), + &changed_indices, + ) .await { Ok(outcome) => (Some(outcome), None), @@ -13646,7 +13729,12 @@ impl ECStore { .await?; snapshot.mark_decommission_capacity_blocked(idx, err.to_string(), OffsetDateTime::now_utc())?; let outcome = snapshot - .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx]) + .save_no_lock_armed( + self.pools.clone(), + &mut save_guard, + PoolMetaPersistenceFence::Owned(Arc::clone(&pool_meta_guard)), + &[idx], + ) .await?; ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?; { @@ -22023,7 +22111,7 @@ mod tests { let mut state = store.pool_meta_save_gate.lock().await; let (fence, _) = store.acquire_pool_meta_write_guard(&mut state, "test save").await.unwrap(); let outcome = requested - .save_no_lock_armed(store.pools.clone(), &mut state, fence.lock_lost_signal(), &[0]) + .save_no_lock_armed(store.pools.clone(), &mut state, PoolMetaPersistenceFence::Owned(Arc::clone(&fence)), &[0]) .await .unwrap(); assert_eq!(outcome.committed.pools[0].last_update, requested.pools[0].last_update); @@ -24825,11 +24913,12 @@ mod pools_tests { config[3][0].gate = Some(gate.clone()); let fixture = Fixture::new(config); let mut state = fixture.state(); - let mut save = Box::pin( - fixture - .desired - .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]), - ); + let mut save = Box::pin(fixture.desired.save_no_lock_armed( + fixture.pools.clone(), + &mut state, + PoolMetaPersistenceFence::Distributed(signal()), + &[0], + )); tokio::select! { _ = fixture.trace.wait_for(4, 3) => {}, _ = &mut save => panic!("commit crossed the last prepare") } assert_eq!(fixture.trace.commit_count(), 0); gate.add_permits(1); @@ -24849,7 +24938,7 @@ mod pools_tests { let mut state = fixture.state(); let err = fixture .desired - .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]) + .save_no_lock_armed(fixture.pools.clone(), &mut state, PoolMetaPersistenceFence::Distributed(signal()), &[0]) .await .expect_err("a missing conditional-write revision must fail"); assert!(err.to_string().contains("without a conditional-write revision")); @@ -24866,11 +24955,12 @@ mod pools_tests { config[3][0].gate = Some(gate.clone()); let fixture = Fixture::new(config); let mut state = fixture.state(); - let mut save = Box::pin( - fixture - .desired - .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]), - ); + let mut save = Box::pin(fixture.desired.save_no_lock_armed( + fixture.pools.clone(), + &mut state, + PoolMetaPersistenceFence::Distributed(signal()), + &[0], + )); tokio::select! { _ = fixture.trace.wait_for(4, 3) => {}, _ = &mut save => panic!("phase returned with a live replica") } assert_eq!(fixture.trace.commit_count(), 0); gate.add_permits(1); @@ -25045,11 +25135,12 @@ mod pools_tests { } let fixture = Fixture::new(config); let mut state = fixture.state(); - let mut save = Box::pin( - fixture - .desired - .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]), - ); + let mut save = Box::pin(fixture.desired.save_no_lock_armed( + fixture.pools.clone(), + &mut state, + PoolMetaPersistenceFence::Distributed(signal()), + &[0], + )); tokio::select! { _ = fixture.trace.wait_for(4 * (phase + 1), 4 * phase) => {}, _ = &mut save => panic!("blocked phase finished") } drop(save); assert!(state.ensure_write_safe("cancelled phase").is_err()); @@ -25060,7 +25151,7 @@ mod pools_tests { let mut state = fixture.state(); let outcome = fixture .desired - .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]) + .save_no_lock_armed(fixture.pools.clone(), &mut state, PoolMetaPersistenceFence::Distributed(signal()), &[0]) .await .expect("durable save before publication"); drop(outcome); @@ -25088,6 +25179,7 @@ mod pools_tests { .await .expect("lease request") .expect("lease acquired"); + let guard = Arc::new(guard); let gate = Arc::new(tokio::sync::Semaphore::new(0)); let mut config = steps(6); for step in &mut config { @@ -25096,7 +25188,7 @@ mod pools_tests { let fixture = Fixture::new(config); let state = fixture.state(); let mut arm = state.arm_transaction(); - let fence = PoolMetaPersistenceFence::Distributed(guard.lock_lost_signal()); + let fence = PoolMetaPersistenceFence::Owned(Arc::clone(&guard)); let mut pending = Box::pin(prepare(&fixture, &fence, &mut arm)); tokio::select! { _ = fixture.trace.wait_for(4, 0) => {}, _ = &mut pending => panic!("gate ignored") } tokio::time::timeout(StdDuration::from_secs(3), guard.lock_lost_notified()) @@ -25122,7 +25214,7 @@ mod pools_tests { let mut state = fixture.state(); let result = fixture .desired - .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]) + .save_no_lock_armed(fixture.pools.clone(), &mut state, PoolMetaPersistenceFence::Distributed(signal()), &[0]) .await; assert_eq!(fixture.trace.commit_count(), 4); if matches!(fault, WriteFault::TimeoutAfterWrite) { @@ -25172,7 +25264,7 @@ mod pools_tests { let mut save = Box::pin(fixture.desired.save_no_lock_armed( fixture.pools.clone(), &mut state, - if legacy || needs_repair { signal() } else { None }, + PoolMetaPersistenceFence::Distributed(if legacy || needs_repair { signal() } else { None }), &[0], )); tokio::select! { _ = fixture.trace.wait_for(1, 0) => {}, _ = &mut save => panic!("serial write gate ignored") } @@ -25732,7 +25824,7 @@ mod pools_tests { let save_task = tokio::spawn(async move { let mut save_guard = save_store.pool_meta_save_gate.lock().await; let outcome = snapshot - .save_no_lock_armed(vec![save_committed], &mut save_guard, None, &[0]) + .save_no_lock_armed(vec![save_committed], &mut save_guard, PoolMetaPersistenceFence::OfflineRecovery, &[0]) .await?; task_publish_started.notify_one(); task_publish_release.notified().await; @@ -25759,6 +25851,73 @@ mod pools_tests { .expect_err("cancellation after replica save but before publication must keep writes blocked"); } + #[tokio::test] + async fn pool_meta_cas_context_retains_actual_owner_for_exact_target() { + use crate::object_api::write_commit_context::WriteCommitContext; + + let lock = NamespaceLock::with_local_manager("pool-meta-cas-owner".to_string(), Arc::new(GlobalLockManager::new())); + let request = LockRequest::new( + ObjectKey::new(super::RUSTFS_META_BUCKET, POOL_META_NAME), + LockType::Exclusive, + "cas-writer", + ) + .with_acquire_timeout(StdDuration::from_secs(1)); + let owner = Arc::new( + lock.acquire_guard(&request) + .await + .expect("pool metadata lock acquisition should succeed") + .expect("pool metadata lock should be available"), + ); + let weak_owner = Arc::downgrade(&owner); + let fence = PoolMetaPersistenceFence::Owned(Arc::clone(&owner)); + let mut contexts = Vec::new(); + for object in [POOL_META_NAME, POOL_META_IDENTITY_NAME] { + let options = super::pool_meta_cas_options(object, &PoolMetaCasToken::Missing, &fence) + .expect("a held pool metadata lock should authorize its CAS target"); + let cloned_options = options.clone(); + let context = WriteCommitContext::from_options(&cloned_options, super::RUSTFS_META_BUCKET, object, false) + .expect("the actual CAS target should retain its borrowed namespace owner"); + assert!(context.has_borrowed_owner(), "CAS must borrow the real protocol lock owner"); + assert!( + WriteCommitContext::from_options(&cloned_options, super::RUSTFS_META_BUCKET, "unrelated-object", false).is_err(), + "a pool metadata CAS capability must not authorize an unrelated object" + ); + drop(options); + drop(cloned_options); + contexts.push(context); + } + drop(fence); + drop(owner); + assert!( + weak_owner.upgrade().is_some(), + "queued commit contexts must keep the actual lock guard alive" + ); + drop(contexts); + assert!(weak_owner.upgrade().is_none(), "the last commit context must release its lock owner"); + } + + #[tokio::test] + async fn pool_meta_cas_rejects_released_owner() { + let lock = NamespaceLock::with_local_manager("pool-meta-released-owner".to_string(), Arc::new(GlobalLockManager::new())); + let request = LockRequest::new( + ObjectKey::new(super::RUSTFS_META_BUCKET, POOL_META_NAME), + LockType::Exclusive, + "released-cas-writer", + ) + .with_acquire_timeout(StdDuration::from_secs(1)); + let mut owner = lock + .acquire_guard(&request) + .await + .expect("pool metadata lock acquisition should succeed") + .expect("pool metadata lock should be available"); + assert!(owner.release(), "the test must release a real namespace owner"); + let fence = PoolMetaPersistenceFence::Owned(Arc::new(owner)); + assert!( + super::pool_meta_cas_options(POOL_META_NAME, &PoolMetaCasToken::Missing, &fence).is_err(), + "a released namespace owner must not dispatch another CAS" + ); + } + #[tokio::test] async fn test_lost_pool_meta_fence_rejects_replica_write() { let client = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))); @@ -25776,6 +25935,7 @@ mod pools_tests { .await .expect("pool metadata fence acquisition should not error") .expect("the stale writer should acquire the pool metadata fence"); + let guard = Arc::new(guard); tokio::time::timeout(StdDuration::from_secs(2), guard.lock_lost_notified()) .await .expect("the stale writer lease should expire"); @@ -25797,7 +25957,12 @@ mod pools_tests { }; let mut write_state = super::PoolMetaWriteState::default(); let err = snapshot - .save_no_lock_armed(vec![storage.clone()], &mut write_state, guard.lock_lost_signal(), &[0]) + .save_no_lock_armed( + vec![storage.clone()], + &mut write_state, + PoolMetaPersistenceFence::Owned(Arc::clone(&guard)), + &[0], + ) .await .expect_err("a writer must not persist after losing the distributed pool metadata fence"); @@ -25825,6 +25990,7 @@ mod pools_tests { .await .expect("pool metadata fence acquisition should not error") .expect("the publishing writer should acquire the pool metadata fence"); + let guard = Arc::new(guard); let storage = Arc::new(PartialPoolMetaWriteStorage { fail_write: false, fail_after_first_write: false, @@ -25845,7 +26011,7 @@ mod pools_tests { current.pools[0].last_update = OffsetDateTime::UNIX_EPOCH; let mut write_state = super::PoolMetaWriteState::for_test_bootstrap(); let outcome = saved - .save_no_lock_armed(vec![storage], &mut write_state, guard.lock_lost_signal(), &[0]) + .save_no_lock_armed(vec![storage], &mut write_state, PoolMetaPersistenceFence::Owned(Arc::clone(&guard)), &[0]) .await .expect("the replica save should finish while the fence is valid"); diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 81ff74d14..7f8e971ed 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -165,12 +165,13 @@ fn is_delete_transaction_marker(entry: &str, prefix: &str) -> bool { .is_some_and(|transaction| Uuid::parse_str(transaction).is_ok_and(|uuid| !uuid.is_nil())) } -/// Whether a `list_dir` entry inside a UUID data dir is erasure data or a -/// delete-transaction marker. Anything else (a subdirectory, an `xl.meta`, an -/// unknown file) means the directory is not plain delete residue. +/// Whether a `list_dir` entry inside a UUID data dir is erasure data, a rollback +/// metadata backup, or a delete-transaction marker. Anything else (a subdirectory, +/// an `xl.meta`, an unknown file) means the directory is not plain delete residue. fn is_metadata_less_data_dir_entry(entry: &str) -> bool { !entry.ends_with(SLASH_SEPARATOR) && (metadata_less_part_file(entry) + || entry == STORAGE_FORMAT_FILE_BACKUP || is_delete_transaction_marker(entry, DELETE_DATA_DIR_MARKER_PREFIX) || is_delete_transaction_marker(entry, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX)) } @@ -8601,9 +8602,9 @@ impl LocalDisk { /// Whether the metadata-less directory `dir_name` holds nothing that a /// listing could show: every leaf under it is the data dir of a deleted - /// version (a non-nil UUID directory of `part.N` files and - /// delete-transaction markers) or an empty directory. That is what an - /// interrupted or deferred version delete leaves behind once the `xl.meta` + /// version (a non-nil UUID directory of `part.N` files, rollback metadata + /// backups, and delete-transaction markers) or an empty directory. That is + /// what an interrupted or deferred version delete leaves behind once the `xl.meta` /// is gone, and neither the object directory nor the date-style ancestors /// above it may surface as prefixes (#6898). Real objects are directories /// carrying their own `xl.meta`, so the first file met outside a UUID data @@ -19796,13 +19797,42 @@ mod test { .await .expect("delete marker should be written"); + // Inline deletes can leave only a rollback backup; non-inline deletes + // can leave the backup alongside parts. Neither has listable metadata. + for (prefix, has_parts) in [("backup-only", false), ("backup-parts", true)] { + let backup = bucket_dir + .join(prefix) + .join("2026/08/28/23/object.parquet") + .join(Uuid::new_v4().to_string()); + fs::create_dir_all(&backup).await.expect("backup residue should be created"); + fs::write(backup.join(STORAGE_FORMAT_FILE_BACKUP), b"rollback metadata") + .await + .expect("rollback backup should be written"); + if has_parts { + fs::write(backup.join("part.1"), b"stale") + .await + .expect("stale part beside backup should be written"); + } + } + // A user prefix made of UUID-named directories holding real objects. let upload = Uuid::new_v4().to_string(); write_object(&bucket_dir.join("uploads").join(&upload).join("file"), &format!("uploads/{upload}/file")).await; + write_object( + &bucket_dir.join("uploads").join(&upload).join(STORAGE_FORMAT_FILE_BACKUP), + &format!("uploads/{upload}/{STORAGE_FORMAT_FILE_BACKUP}"), + ) + .await; // An object whose key is itself a UUID. let named = Uuid::new_v4().to_string(); write_object(&bucket_dir.join("named").join(&named), &format!("named/{named}")).await; + fs::write( + bucket_dir.join("named").join(&named).join(STORAGE_FORMAT_FILE_BACKUP), + b"rollback metadata", + ) + .await + .expect("backup beside live metadata should be written"); // Residue next to a live child object. let mixed_residue = bucket_dir.join("mixed").join(Uuid::new_v4().to_string()); @@ -19859,10 +19889,25 @@ mod test { assert_eq!(scan_names(&disk, bucket, "residue/2026/").await, Vec::::new()); assert_eq!(scan_names(&disk, bucket, "residue/").await, Vec::::new()); assert_eq!(scan_names(&disk, bucket, "committed/").await, Vec::::new()); + for prefix in ["backup-only", "backup-parts"] { + assert_eq!( + scan_names(&disk, bucket, &format!("{prefix}/2026/08/28/23/")).await, + Vec::::new(), + "rollback backups must not surface as object prefixes" + ); + assert_eq!(scan_names(&disk, bucket, &format!("{prefix}/")).await, Vec::::new()); + } // UUID-named directories holding real objects, an object keyed by a // UUID, and residue beside a live child all remain visible. assert_eq!(scan_names(&disk, bucket, "uploads/").await, vec![format!("uploads/{upload}/")]); + assert_eq!( + scan_names(&disk, bucket, &format!("uploads/{upload}/")).await, + vec![ + format!("uploads/{upload}/file"), + format!("uploads/{upload}/{STORAGE_FORMAT_FILE_BACKUP}") + ] + ); assert_eq!(scan_names(&disk, bucket, "named/").await, vec![format!("named/{named}")]); assert_eq!(scan_names(&disk, bucket, "mixed/").await, vec!["mixed/child".to_owned()]); assert_eq!( @@ -19952,6 +19997,9 @@ mod test { fs::write(bucket_dir.join("stray/2026/08/notes.txt"), b"?") .await .expect("stray file should be written"); + fs::write(bucket_dir.join("stray/2026/08").join(STORAGE_FORMAT_FILE_BACKUP), b"?") + .await + .expect("backup outside a UUID data directory should be written"); // A UUID directory holding a subdirectory is not a data dir shape the // probe understands, so it surfaces even when the subdirectory is empty. diff --git a/crates/ecstore/src/disk/local/commit.rs b/crates/ecstore/src/disk/local/commit.rs index f011a0511..343a54796 100644 --- a/crates/ecstore/src/disk/local/commit.rs +++ b/crates/ecstore/src/disk/local/commit.rs @@ -234,7 +234,6 @@ pub(super) fn create_local_inline_rollback_backup( Ok(backup_path) } -#[cfg(test)] pub(super) async fn lock_rename_commit_directories( source_parent: &Path, destination_parent: &Path, @@ -273,41 +272,32 @@ pub(super) async fn lock_rename_commit_directories( } async fn prepare_rename_destination_metadata( - source_parent: &Path, file_path: &Path, - base_dir: &Path, - publication_root: &os::PublicationRoot, + rename_commit_guard: &os::RenameCommitGuard, mutation_lease: Arc, -) -> Result<(os::RenameCommitGuard, Option)> { - let source_parent = source_parent.to_path_buf(); - let file_path = file_path.to_path_buf(); - let base_dir = base_dir.to_path_buf(); - let publication_root = publication_root.clone(); - os::run_blocking_namespace_operation(mutation_lease, move || { - let parent = file_path - .parent() - .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "missing metadata parent"))?; - let guard = os::prepare_rename_commit_guard(&source_parent, parent, &base_dir, &publication_root).map_err(|err| { - match std::fs::symlink_metadata(&base_dir) { - Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err, - _ => err, - } - })?; - #[cfg(all(test, windows))] - run_destination_commit_directory_preparation(parent); - #[cfg(any(windows, target_os = "linux"))] - let data = os::read_destination_file_with_commit_guard(&file_path, &guard)?; - #[cfg(not(any(windows, target_os = "linux")))] - let data = match std::fs::read(&file_path) { - Ok(data) => Some(data), - Err(err) if err.kind() == ErrorKind::NotFound => None, - Err(err) => return Err(err), - }; - Ok((guard, data.map(Bytes::from))) - }) - .await - .map_err(to_file_error) - .map_err(DiskError::from) +) -> Result> { + #[cfg(windows)] + let result = { + let file_path = file_path.to_path_buf(); + let rename_commit_guard = rename_commit_guard.clone(); + os::run_blocking_namespace_operation(mutation_lease, move || { + os::read_destination_file_with_commit_guard(&file_path, &rename_commit_guard) + }) + .await + }; + #[cfg(not(windows))] + let _ = (rename_commit_guard, mutation_lease); + #[cfg(not(windows))] + let result = match fs::read(file_path).await { + Ok(data) => Ok(Some(data)), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), + Err(err) => Err(err), + }; + + result + .map(|data| data.map(Bytes::from)) + .map_err(to_file_error) + .map_err(DiskError::from) } /// Proof produced only when the local rename returns at an existing access @@ -489,11 +479,11 @@ impl LocalDisk { if !no_inline { fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?; } - // Open the commit directories and read the old metadata in one blocking - // operation. The returned guard owns the same identities through publish. - let (rename_commit_guard, has_dst_buf) = prepare_rename_destination_metadata( + // Acquire the common trees before reading destination metadata. On + // Windows this pins the object directory identity across publication. + let rename_commit_guard = lock_rename_commit_directories( src_file_parent, - &dst_file_path, + dst_file_parent, &dst_volume_dir, &self.publication_root, mutation_lease.clone(), @@ -508,6 +498,12 @@ impl LocalDisk { source_volume_dir: &src_volume_dir, data_paths: has_data_dir_path.as_ref(), }; + let has_dst_buf = prepare_rename_destination_metadata( + commit.destination_metadata, + &commit.directory_guard, + commit.mutation_lease.clone(), + ) + .await?; if no_inline { // Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta diff --git a/crates/ecstore/src/ecstore_validation_blackbox.rs b/crates/ecstore/src/ecstore_validation_blackbox.rs index 7806fe098..6fccc53b5 100644 --- a/crates/ecstore/src/ecstore_validation_blackbox.rs +++ b/crates/ecstore/src/ecstore_validation_blackbox.rs @@ -259,7 +259,7 @@ async fn blackbox_put_unknown_actual_size_restores_body_and_records_written_size .map(|idx| ((idx * 17) % 251) as u8) .collect::>(); let opts = ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, ..Default::default() }; @@ -300,7 +300,7 @@ async fn blackbox_get_restores_body_after_one_shard_file_is_removed() { .map(|idx| ((idx * 19) % 251) as u8) .collect::>(); let opts = ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, ..Default::default() }; @@ -406,7 +406,7 @@ async fn blackbox_heal_requests_preserve_repair_scope() { put_object, &mut put_reader, &ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, versioned: true, expected_bucket_incarnation_id: Some(put_incarnation), ..Default::default() @@ -469,7 +469,7 @@ async fn blackbox_heal_requests_preserve_repair_scope() { put_object, &mut healthy_reader, &ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, ..Default::default() }, ) @@ -509,7 +509,7 @@ async fn blackbox_heal_requests_preserve_repair_scope() { .map(|idx| ((idx * 29) % 251) as u8) .collect::>(); let opts = ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, ..Default::default() }; @@ -567,7 +567,7 @@ async fn blackbox_heal_requests_preserve_repair_scope() { let suspended_object = "suspended.bin"; let payload = vec![0x5a; 1 << 20]; let mpu_opts = ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, versioned: true, ..Default::default() }; @@ -675,7 +675,7 @@ async fn blackbox_heal_requests_preserve_repair_scope() { .expect("fourth disk should be online before suspended completion") }; let suspended_opts = ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, version_suspended: true, ..Default::default() }; @@ -712,7 +712,7 @@ async fn blackbox_range_read_restores_exact_slice_with_one_offline_disk() { let range_start = 513usize; let range_len = 8192usize; let opts = ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, ..Default::default() }; @@ -756,7 +756,7 @@ async fn blackbox_delete_marker_hides_object_body_without_erasing_prior_version_ let bucket = "bb-delete-marker-read-negative"; let object = "object.bin"; let opts = ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, version_suspended: true, object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))), ..Default::default() @@ -853,7 +853,7 @@ async fn blackbox_issue3031_diag_covers_put_success_cleanup_and_error_summary() let first_payload = b"first diagnostic body".to_vec(); let second_payload = b"second diagnostic body that replaces the first".to_vec(); let opts = ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, ..Default::default() }; @@ -959,7 +959,7 @@ mod old_current_size_backfill { .await .expect("bucket should be created"); let opts = ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, ..Default::default() }; @@ -993,7 +993,7 @@ mod old_current_size_backfill { // latest's size, not the incoming version's. let versioned = "versioned.bin"; let first_version_opts = ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, versioned: true, version_id: Some(uuid::Uuid::new_v4().to_string()), ..Default::default() @@ -1005,7 +1005,7 @@ mod old_current_size_backfill { let expected = prelookup_expectation(&set_disks, bucket, versioned).await; assert_eq!(expected, OldCurrentSize::Present(111)); let second_version_opts = ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, versioned: true, version_id: Some(uuid::Uuid::new_v4().to_string()), ..Default::default() @@ -1037,7 +1037,7 @@ mod old_current_size_backfill { let expected = prelookup_expectation(&set_disks, bucket, versioned).await; assert_eq!(expected, OldCurrentSize::Present(0)); let third_version_opts = ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, versioned: true, version_id: Some(uuid::Uuid::new_v4().to_string()), ..Default::default() @@ -1061,7 +1061,7 @@ mod old_current_size_backfill { .await .expect("bucket should be created"); let opts = ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, ..Default::default() }; diff --git a/crates/ecstore/src/object_api/mod.rs b/crates/ecstore/src/object_api/mod.rs index 60f7e6776..008005547 100644 --- a/crates/ecstore/src/object_api/mod.rs +++ b/crates/ecstore/src/object_api/mod.rs @@ -104,6 +104,7 @@ mod hook_slot; mod object_mutation_hook; mod readers; mod types; +pub(crate) mod write_commit_context; #[cfg(test)] mod persisted_metadata_keys_tests; @@ -125,3 +126,4 @@ pub(crate) use object_mutation_hook::notify_object_mutation; pub use object_mutation_hook::{ObjectMutationHook, register_object_mutation_hook, unregister_object_mutation_hook}; pub use readers::*; pub use types::*; +pub use write_commit_context::{WriteCommitGuard, WriteLockContext}; diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 0fda80ac5..7666ef2b7 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -935,6 +935,9 @@ pub struct ObjectOptions { /// Persisted bucket incarnation observed before authorization. pub expected_bucket_incarnation_id: Option, pub no_lock: bool, + /// A genuine target write guard; loss fences alone do not authorize a borrowed write. + #[doc(hidden)] + pub write_lock_context: Option, /// Internal read-only inspection must not enqueue metadata or payload repairs. pub suppress_read_repair: bool, /// Control-plane writers that immediately read or CAS the same namespace diff --git a/crates/ecstore/src/object_api/write_commit_context.rs b/crates/ecstore/src/object_api/write_commit_context.rs new file mode 100644 index 000000000..571664265 --- /dev/null +++ b/crates/ecstore/src/object_api/write_commit_context.rs @@ -0,0 +1,186 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{ObjectOptions, WriteCompletion}; +use crate::error::{Error, Result}; +use std::sync::Arc; +use std::time::Duration; + +/// A real write guard shared with an inner commit, never a lock-loss signal alone. +/// +/// Callers cannot fabricate a guard without acquiring its namespace lock. +/// +/// ```compile_fail +/// use rustfs_ecstore::api::object::WriteCommitGuard; +/// use std::sync::Arc; +/// +/// let guard = WriteCommitGuard { +/// bucket: "bucket".into(), +/// object: "object".into(), +/// guards: Arc::new(Vec::new()), +/// }; +/// ``` +#[doc(hidden)] +#[derive(Clone, Debug)] +pub struct WriteCommitGuard { + bucket: Arc, + object: Arc, + guards: Arc>>, +} + +impl WriteCommitGuard { + /// Acquire the key recorded by the namespace wrapper and retain its owner. + pub async fn acquire( + lock: &rustfs_lock::NamespaceLockWrapper, + timeout: Duration, + ) -> std::result::Result { + let guard = Arc::new(lock.get_write_lock(timeout).await?); + let key = lock.resource(); + Ok(Self::from_guard(guard, &key.bucket, &key.object)) + } + + pub(crate) fn from_guard(guard: Arc, bucket: &str, object: &str) -> Self { + Self { + bucket: bucket.into(), + object: object.into(), + guards: Arc::new(vec![guard]), + } + } + + pub fn is_lock_lost(&self) -> bool { + self.guards.iter().any(|guard| guard.is_lock_lost() || guard.is_released()) + } + + fn matches(&self, bucket: &str, object: &str) -> bool { + self.bucket.as_ref() == bucket && self.object.as_ref() == object + } +} + +/// Opaque boundary capability; only real acquisitions or offline startup construct it. +/// +/// ```compile_fail +/// use rustfs_ecstore::object_api::WriteLockContext; +/// +/// let context = WriteLockContext {}; +/// ``` +#[doc(hidden)] +#[derive(Clone, Debug)] +pub struct WriteLockContext { + mode: WriteLockMode, +} + +#[derive(Clone, Debug)] +enum WriteLockMode { + Borrowed(Vec), + OfflineRecovery, +} + +pub(crate) enum WriteCommitContext { + Owned, + Borrowed(Vec), + Publication, + OfflineRecovery, + #[cfg(test)] + UnlockedFixture, +} + +impl WriteCommitContext { + pub(crate) fn from_options(opts: &ObjectOptions, bucket: &str, object: &str, publication: bool) -> Result { + if publication { + if let Some(WriteLockContext { + mode: WriteLockMode::Borrowed(guards), + }) = opts.write_lock_context.as_ref() + && !guards.iter().any(|guard| guard.matches(bucket, object)) + { + return Err(Error::InvalidArgument( + bucket.to_owned(), + object.to_owned(), + "publication carries a different borrowed namespace".to_owned(), + )); + } + return Ok(Self::Publication); + } + match (opts.no_lock, opts.write_lock_context.as_ref().map(|context| &context.mode)) { + (false, None | Some(WriteLockMode::Borrowed(_))) => Ok(Self::Owned), + (true, Some(WriteLockMode::Borrowed(guards))) if guards.iter().any(|guard| guard.matches(bucket, object)) => { + Ok(Self::Borrowed(guards.clone())) + } + (true, Some(WriteLockMode::OfflineRecovery)) => Ok(Self::OfflineRecovery), + #[cfg(test)] + (true, None) if !opts.metadata_cache_safe => Ok(Self::UnlockedFixture), + _ => Err(Error::InvalidArgument( + bucket.to_owned(), + object.to_owned(), + "write lock context does not prove the target namespace".to_owned(), + )), + } + } + + pub(crate) fn acquires_namespace(&self) -> bool { + matches!(self, Self::Owned) + } + + pub(crate) fn has_borrowed_owner(&self) -> bool { + matches!(self, Self::Borrowed(_)) + } + + pub(crate) fn is_lock_lost(&self) -> bool { + matches!(self, Self::Borrowed(guards) if guards.iter().any(WriteCommitGuard::is_lock_lost)) + } + + pub(crate) fn allows_early_ack(&self, completion: WriteCompletion) -> bool { + completion == WriteCompletion::Quorum && matches!(self, Self::Owned | Self::Borrowed(_) | Self::Publication) + } +} + +impl ObjectOptions { + pub fn add_write_commit_guard(&mut self, guard: &WriteCommitGuard) { + for owner in guard.guards.iter() { + self.add_namespace_lock_guard(owner); + } + let mut guards = match self.write_lock_context.as_ref() { + Some(WriteLockContext { + mode: WriteLockMode::Borrowed(existing), + }) => existing.clone(), + _ => Vec::new(), + }; + if let Some(existing) = guards.iter_mut().find(|held| held.matches(&guard.bucket, &guard.object)) { + let owners = Arc::make_mut(&mut existing.guards); + for owner in guard.guards.iter() { + if !owners.iter().any(|held| Arc::ptr_eq(held, owner)) { + owners.push(Arc::clone(owner)); + } + } + } else { + guards.push(guard.clone()); + } + self.no_lock = true; + self.write_lock_context = Some(WriteLockContext { + mode: WriteLockMode::Borrowed(guards), + }); + } + + pub(crate) fn add_owned_write_lock(&mut self, guard: Arc, bucket: &str, object: &str) { + self.add_write_commit_guard(&WriteCommitGuard::from_guard(guard, bucket, object)); + } + + /// Only startup migration may publish without a running namespace owner. + pub(crate) fn use_offline_recovery_write(&mut self) { + self.no_lock = true; + self.write_lock_context = Some(WriteLockContext { + mode: WriteLockMode::OfflineRecovery, + }); + self.write_completion = WriteCompletion::TailDrained; + } +} diff --git a/crates/ecstore/src/services/rebalance/control.rs b/crates/ecstore/src/services/rebalance/control.rs index a0b2a2cb9..a8fef5fd8 100644 --- a/crates/ecstore/src/services/rebalance/control.rs +++ b/crates/ecstore/src/services/rebalance/control.rs @@ -521,11 +521,12 @@ impl ECStore { .map_err(|err| rebalance_meta_lock_error(err, "write")) }) .await?; + let guard = Arc::new(guard); let mut opts = ObjectOptions { - no_lock: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, ..Default::default() }; - opts.add_namespace_lock_guard(&guard); + opts.add_owned_write_lock(Arc::clone(&guard), crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME); merge_and_save_rebalance_meta_no_lock(pool, local_snapshot, stage, opts, None, expected_id).await?; if guard.is_lock_lost() { diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 94f5374ae..4b8ebdb32 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -10659,10 +10659,68 @@ mod tests { .await; } + const CLOSED_HEAL_RECEIVER_CHILD: &str = "RUSTFS_TEST_ROLLBACK_CLOSED_HEAL_RECEIVER_CHILD"; + + #[test] + fn rename_rollback_closed_heal_receiver_preserves_recovery_material() { + // The shared test binary has one heal receiver owner. Use a fresh process + // to close the real receiver without replacing its process-wide sender. + let output = std::process::Command::new(std::env::current_exe().expect("test executable")) + .args([ + "--exact", + "set_disk::core::io_primitives::tests::rename_rollback_incomplete_preserves_overwrite_data_dirs_and_staging", + "--nocapture", + ]) + .env(CLOSED_HEAL_RECEIVER_CHILD, "1") + .output() + .expect("isolated closed-receiver test should run"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "closed-receiver child failed: {stdout} {stderr}"); + assert!(stdout.contains("1 passed"), "child must execute the selected regression: {stdout}"); + assert_eq!( + stderr + .lines() + .filter(|line| { + line.contains(EVENT_SET_DISK_RENAME_ROLLBACK) + && line.contains("stage=\"inspection_admission\"") + && line.contains("outcome=Failed(\"Failed to send heal command:") + }) + .count(), + 10, + "every actual rollback inspection must encounter the closed-channel send error: {stderr}" + ); + assert_eq!( + stderr.matches("rollback inspection sender=closed,").count(), + 10, + "all rollback fault and early-ACK combinations must run with the closed sender: {stderr}" + ); + eprint!("{stderr}"); + } + #[tokio::test(flavor = "current_thread")] #[serial_test::serial] #[serial_test::serial(capacity_dirty_scope)] async fn rename_rollback_incomplete_preserves_overwrite_data_dirs_and_staging() { + let _closed_receiver_logs = if std::env::var_os(CLOSED_HEAL_RECEIVER_CHILD).is_some() { + assert!( + rustfs_heal_contracts::heal_channel::get_heal_channel_sender().is_none(), + "the isolated child must own a fresh heal channel" + ); + let receiver = rustfs_heal_contracts::heal_channel::init_heal_channel().expect("isolated heal receiver"); + drop(receiver); + let sender = rustfs_heal_contracts::heal_channel::get_heal_channel_sender().expect("initialized heal sender"); + assert!(sender.is_closed(), "the initialized heal receiver must be closed"); + Some(tracing::subscriber::set_default( + tracing_subscriber::fmt() + .with_ansi(false) + .with_max_level(tracing::Level::WARN) + .with_writer(std::io::stderr) + .finish(), + )) + } else { + None + }; let heal_sender_state = match rustfs_heal_contracts::heal_channel::get_heal_channel_sender() { None => "not_initialized", Some(sender) => { diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 5a65e6627..1fa371c86 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -972,7 +972,7 @@ static OBJECT_LOCK_DIAG_ENABLED: OnceLock = OnceLock::new(); mod core; #[cfg(test)] pub(crate) use core::io_primitives::disk_call_counters; -#[cfg(all(test, feature = "test-util"))] +#[cfg(test)] pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier}; mod ctx; mod metadata; @@ -4097,7 +4097,6 @@ impl SetDisks { b"rustfs-marker-purge-receipt-v1".to_vec(), &ObjectOptions { max_parity: true, - no_lock: true, ..Default::default() }, ) diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 6531f895f..3961fca44 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -21,6 +21,7 @@ //! unchanged; method bodies are moved verbatim and runtime behavior is the same. use crate::core::pools::DecommissionCapacityAdmission; +use crate::object_api::write_commit_context::WriteCommitContext; #[cfg(test)] use super::super::GetObjectMetadataCacheKey; @@ -2446,17 +2447,19 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { opts: &ObjectOptions, ) -> Result { crate::hp_guard!("SetDisks::complete_multipart_upload"); + let write_context = WriteCommitContext::from_options(opts, bucket, object, false)?; self.invalidate_get_object_metadata_cache(bucket, object).await; let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement); - let range_seek_rollout_enabled = crate::object_api::legacy_encrypted_range_seek_enabled() && !opts.no_lock; + let range_seek_rollout_enabled = + crate::object_api::legacy_encrypted_range_seek_enabled() && write_context.acquires_namespace(); let mut object_lock_guard = None; let mut decommission_object_lock_guard = None; let mut decommission_target_lock_covered = false; let mut decommission_capacity_guard = None; if opts.http_preconditions.is_some() { - if !opts.no_lock { + if write_context.acquires_namespace() { object_lock_guard = Some( self.acquire_write_lock_diag("complete_multipart_upload_precondition", bucket, object) .await?, @@ -2479,7 +2482,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { self.pool_index, bucket, object, - opts.no_lock || object_lock_guard.is_some(), + !write_context.acquires_namespace() || object_lock_guard.is_some(), DecommissionCapacityAdmission::ExistingMultipart, ) .await?; @@ -2487,7 +2490,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { decommission_target_lock_covered = target_lock_covered; decommission_capacity_guard = capacity_guard; } - if !opts.no_lock && object_lock_guard.is_none() && !decommission_target_lock_covered { + if write_context.acquires_namespace() && object_lock_guard.is_none() && !decommission_target_lock_covered { object_lock_guard = Some( self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object) .await?, @@ -3362,9 +3365,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { // usable as a CopyObject source. Do not return on rename quorum while // a tail owner may still hold the object guard and finish shard moves. let commit_allows_early_ack = false; - let detach_commit_owner = commit_allows_early_ack || upload_guard.is_some() || quota_mutation_fence; + let detach_commit_owner = + write_context.has_borrowed_owner() || commit_allows_early_ack || upload_guard.is_some() || quota_mutation_fence; let commit = async move { let mut metadata_cache_mutation_guard = metadata_cache_mutation_guard; + let write_context = write_context; let mut _object_lock_guard = commit_object_lock_guard; let mut _decommission_object_lock_guard = commit_decommission_object_lock_guard; let mut _upload_guard = upload_guard; @@ -3386,6 +3391,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeQuotaRename).await; if quota_reservation.is_lock_lost() || !quota_reservation.capability_proof_matches() + || write_context.is_lock_lost() || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) || _decommission_object_lock_guard .as_ref() @@ -3440,6 +3446,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { } if quota_reservation.is_lock_lost() || !quota_reservation.capability_proof_matches() + || write_context.is_lock_lost() || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) || _decommission_object_lock_guard .as_ref() @@ -3777,6 +3784,7 @@ mod tests { use crate::layout::endpoints::SetupType; use crate::services::notification_sys::install_remote_version_state_fleet_proof_for_test; use crate::set_disk::core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier}; + use crate::set_disk::get_lock_acquire_timeout; // No-locker helpers resolve to the isolated-context variants (see // `hermetic_set_disks_isolated`); the guard-based tests build through // `hermetic_set_disks_with_lockers`, which stays on the bootstrap context @@ -7138,6 +7146,54 @@ mod tests { .await; } + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn borrowed_complete_keeps_namespace_owner_after_caller_cancel() { + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "multipart-borrowed-cancel"; + let object = "object"; + make_bucket_on_all(&disks, bucket).await; + let (upload_id, parts) = + stage_upload_with_create_opts(&set, bucket, object, &[0x53; 4096], &ObjectOptions::default()).await; + let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename); + let writer = Arc::clone(&set); + let complete = tokio::spawn(async move { + let outer_guard = crate::object_api::WriteCommitGuard::acquire( + &writer.new_ns_lock(bucket, object).await.expect("namespace wrapper"), + get_lock_acquire_timeout(), + ) + .await + .expect("outer MPU namespace guard"); + let mut opts = ObjectOptions::default(); + opts.add_write_commit_guard(&outer_guard); + writer + .complete_multipart_upload(bucket, object, &upload_id, parts, &opts) + .await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("borrowed completion reaches committed cleanup"); + complete.abort(); + assert!(complete.await.expect_err("completion caller canceled").is_cancelled()); + let mut probe = Box::pin(set.acquire_write_lock_diag("borrowed_complete_probe", bucket, object)); + assert!( + futures::poll!(probe.as_mut()).is_pending(), + "borrowed completion retains the real outer namespace owner" + ); + barrier.release(); + drop( + tokio::time::timeout(Duration::from_secs(30), probe) + .await + .expect("completion owner finishes cleanup") + .expect("next writer obtains namespace"), + ); + let info = set + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("borrowed MPU result remains readable"); + assert_eq!(info.size, 4096); + } + #[tokio::test(flavor = "multi_thread")] #[serial] async fn cancelled_complete_keeps_upload_lock_through_tail_cleanup() { diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 2e9c99c16..5e8692461 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -668,7 +668,7 @@ use super::super::GetObjectMetadataCacheEntry; use super::super::GetObjectMetadataCacheKey; #[cfg(test)] use super::super::capacity_scope_from_disks; -#[cfg(all(test, feature = "test-util"))] +#[cfg(test)] use super::super::get_lock_acquire_timeout; use crate::bucket::lifecycle::{ tier_delete_journal::{TierDeleteDispatchAuthorization, record_tier_delete_journal_backend_identity}, @@ -707,10 +707,13 @@ use crate::disk::format::FormatV3; use crate::disk::new_disk; use crate::disk::{DataDirDeleteStatus, OldCurrentSize}; use crate::error::is_err_invalid_upload_id; +#[cfg(test)] +use crate::object_api::WriteCompletion; +use crate::object_api::write_commit_context::WriteCommitContext; use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed}; use crate::object_api::{ NamespaceLockFence, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode, - SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, WriteCompletion, + SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, }; use crate::services::notification_sys::RemoteVersionStateFleetProofToken; use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata}; @@ -3981,6 +3984,7 @@ impl SetDisks { opts: &ObjectOptions, mut publication_fence: Option, ) -> Result<(ObjectInfo, Option)> { + let write_context = WriteCommitContext::from_options(opts, bucket, object, publication_fence.is_some())?; let protect_write = opts.shard_integrity_write_enabled(); let source_bucket_incarnation_id = match opts.expected_bucket_incarnation_id { Some(incarnation_id) => Some(incarnation_id), @@ -4531,7 +4535,7 @@ impl SetDisks { self.pool_index, bucket, object, - opts.no_lock || object_lock_guard.is_some(), + !write_context.acquires_namespace() || object_lock_guard.is_some(), DecommissionCapacityAdmission::Mutation, ) .await?; @@ -4539,7 +4543,8 @@ impl SetDisks { decommission_target_lock_covered = target_lock_covered; decommission_capacity_guard = capacity_guard; } - if publication_fence.is_some() || (!opts.no_lock && object_lock_guard.is_none() && !decommission_target_lock_covered) + if publication_fence.is_some() + || (write_context.acquires_namespace() && object_lock_guard.is_none() && !decommission_target_lock_covered) { #[cfg(any(test, feature = "test-util"))] pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await; @@ -4956,10 +4961,11 @@ impl SetDisks { // complete rename fan-out drains. Keep this path synchronous so // its terminal state is known before the coordinator releases // remote leases. - let commit_owns_namespace_guard = commit_object_lock_guard.is_some() + let commit_owns_namespace_guard = write_context.has_borrowed_owner() + || commit_object_lock_guard.is_some() || commit_decommission_object_lock_guard.is_some() || commit_publication_guard.is_some(); - let commit_allows_early_ack = opts.write_completion == WriteCompletion::Quorum + let commit_allows_early_ack = write_context.allows_early_ack(opts.write_completion) && !(opts.data_movement && opts.has_decommission_capacity_reservation()) && commit_owns_namespace_guard && commit_scanner_publication_scope.is_none(); @@ -4996,6 +5002,7 @@ impl SetDisks { let commit = move |commit_submit_to_closure_enter_started: Option, cancellation: Option| async move { + let write_context = write_context; let mut metadata_cache_mutation_guard = metadata_cache_mutation_guard; let commit_closure_total_started = rustfs_io_metrics::put_stage_timer(); rustfs_io_metrics::record_put_object_stage_duration_from( @@ -5026,6 +5033,7 @@ impl SetDisks { let first_fence_check_started = rustfs_io_metrics::put_stage_timer(); if quota_reservation.is_lock_lost() || !quota_reservation.capability_proof_matches() + || write_context.is_lock_lost() || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) || _decommission_object_lock_guard .as_ref() @@ -5100,6 +5108,7 @@ impl SetDisks { let second_fence_check_started = rustfs_io_metrics::put_stage_timer(); if quota_reservation.is_lock_lost() || !quota_reservation.capability_proof_matches() + || write_context.is_lock_lost() || _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) || _decommission_object_lock_guard .as_ref() @@ -5354,6 +5363,7 @@ impl SetDisks { guard_release_rx, ( object_lock_guard, + write_context, publication_guard, bucket_lifecycle_guard, decommission_object_lock_guard, @@ -5374,6 +5384,7 @@ impl SetDisks { }, move |( object_lock_guard, + write_context, publication_guard, bucket_lifecycle_guard, decommission_object_lock_guard, @@ -5381,6 +5392,7 @@ impl SetDisks { ), targets| { drop(object_lock_guard); + drop(write_context); drop(publication_guard); drop(bucket_lifecycle_guard); cleanup_runtime::run(cleanup_reservation, async move { @@ -20702,6 +20714,282 @@ mod put_object_tmp_cleanup_tests { .expect("three disks must publish metadata while the fourth rename remains paused"); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(capacity_dirty_scope)] + async fn seeded_put_completion_pressure_drains_every_real_tail() { + use super::hermetic_set_disks_support::hermetic_set_disks_isolated; + use crate::disk::os; + use rand::{Rng, SeedableRng, rngs::StdRng}; + use tokio::time::{Instant, timeout_at}; + + const SEED: u64 = 0x2248_0009; + let mut rng = StdRng::seed_from_u64(SEED); + for (round, (completion, cancel)) in [ + (WriteCompletion::Quorum, false), + (WriteCompletion::Quorum, true), + (WriteCompletion::TailDrained, false), + (WriteCompletion::TailDrained, true), + ] + .into_iter() + .enumerate() + { + let (dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "seeded-put-pressure"; + make_completion_test_bucket(&disks, bucket).await; + let deadline = Instant::now() + Duration::from_secs(30); + let objects: Vec<_> = (0..3).map(|slot| format!("pressure-{round}-{slot}")).collect(); + let mut bodies = Vec::with_capacity(3); + let mut barriers = Vec::with_capacity(3); + let mut trackers = Vec::with_capacity(3); + let mut puts = Vec::with_capacity(3); + let mut paused_disks = Vec::with_capacity(3); + let quorum_barrier = (completion == WriteCompletion::Quorum && cancel) + .then(|| PutObjectCommitBarrier::install(bucket, &objects[0], PutObjectCommitPause::AfterRenameQuorum)); + let handoff_barrier = (completion == WriteCompletion::Quorum && cancel) + .then(|| PutObjectCommitBarrier::install(bucket, &objects[0], PutObjectCommitPause::AfterRenameHandoff)); + for (slot, object) in objects.iter().enumerate() { + let mut body = vec![0; TEST_OBJECT_SIZE + 17 + 16 * slot]; + rng.fill_bytes(&mut body); + let disk = usize::try_from(rng.next_u32() % 4).expect("four-disk schedule slot"); + paused_disks.push(disk); + trackers.push(rename_fanout_barrier::observe_tasks(object)); + barriers.push(Some(rename_fanout_barrier::arm(object, disk, rename_fanout_barrier::PHASE_RENAME))); + bodies.push(body.clone()); + let writer = Arc::clone(&set); + let object = object.clone(); + let write_completion = if slot == 0 { completion } else { WriteCompletion::TailDrained }; + puts.push(tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(body); + writer + .put_object( + bucket, + &object, + &mut reader, + &ObjectOptions { + write_completion, + ..Default::default() + }, + ) + .await + })); + } + for (slot, object) in objects.iter().enumerate() { + barriers[slot] + .as_ref() + .expect("armed real-disk checkpoint") + .wait_until_paused_before(deadline) + .await; + timeout_at(deadline, wait_for_paused_tail_metadata_quorum(&disks, bucket, object)) + .await + .expect("target and both pressure writes must publish three real metadata records"); + assert!(trackers[slot].running() >= 1, "round {round} key {slot}: parked rename remains owned"); + } + assert!( + set.instance_ctx().namespace_commits_pending(), + "parked real mutations must remain pending" + ); + assert!( + !puts[1].is_finished() && !puts[2].is_finished(), + "pressure full-tail callers remain parked" + ); + let mut target = Some(puts.remove(0)); + let mut written = None; + if let Some(barrier) = quorum_barrier.as_ref() { + timeout_at(deadline, barrier.wait_until_paused()) + .await + .expect("cancelled Quorum write reaches its actual quorum boundary"); + } + if cancel { + let caller = target.take().expect("target ACK waiter"); + assert!(!caller.is_finished(), "cancellation must reach an owned, still-pending ACK waiter"); + caller.abort(); + assert!(caller.await.expect_err("cancel target ACK waiter").is_cancelled()); + } else if completion == WriteCompletion::Quorum { + written = Some( + timeout_at(deadline, target.take().expect("Quorum ACK waiter")) + .await + .expect("Quorum ACK must precede the parked tail") + .expect("Quorum caller joins") + .expect("three real disk publications acknowledge the target"), + ); + } else { + assert!( + !target.as_ref().expect("full-tail ACK waiter").is_finished(), + "full-tail ACK waits past metadata quorum" + ); + } + if let Some(barrier) = quorum_barrier { + barrier.release(); + drop(barrier); + timeout_at( + deadline, + handoff_barrier + .as_ref() + .expect("Quorum handoff checkpoint") + .wait_until_paused(), + ) + .await + .expect("cancelled Quorum continuation installs its tail owner"); + } + let mut probe = Box::pin(set.acquire_write_lock_diag("seeded_pressure_owner_probe", bucket, &objects[0])); + assert!( + futures::poll!(probe.as_mut()).is_pending(), + "round {round}: target owner survives ACK or cancellation" + ); + assert!(trackers[0].running() >= 1, "target tail survives its caller"); + if let Some(barrier) = handoff_barrier { + barrier.release(); + drop(barrier); + } + + // One pressure rename races the target; the other stays parked to + // distinguish per-object full-tail completion from global quiescence. + let first_pressure = 1 + usize::try_from(rng.next_u32() % 2).expect("two pressure keys"); + let held_pressure = 3 - first_pressure; + let release_order = if rng.next_u32() & 1 == 0 { + [first_pressure, 0] + } else { + [0, first_pressure] + }; + for slot in release_order { + let barrier = barriers[slot].take().expect("release reached checkpoint once"); + barrier.release(); + drop(barrier); + tokio::task::yield_now().await; + } + if let Some(caller) = target.take() { + written = Some( + timeout_at(deadline, caller) + .await + .expect("full-tail ACK follows target rename drain") + .expect("full-tail caller joins") + .expect("target full-tail write commits"), + ); + assert_eq!(trackers[0].running(), 0, "full-tail ACK follows every target rename task"); + } + drop( + timeout_at(deadline, probe) + .await + .expect("target tail releases its actual namespace owner") + .expect("next target writer acquires the namespace"), + ); + assert_eq!(trackers[0].running(), 0, "target rename owner has drained"); + assert!( + barriers[held_pressure] + .as_ref() + .expect("held pressure checkpoint") + .is_paused() + && trackers[held_pressure].running() >= 1, + "target must hand off while the other real pressure mutation remains parked" + ); + { + let mut read = timeout_at( + deadline, + set.get_object_reader(bucket, &objects[0], None, HeaderMap::new(), &ObjectOptions::default()), + ) + .await + .expect("target reader acquires within the round deadline") + .expect("ACKed or cancelled-owned target remains readable"); + let mut actual = Vec::new(); + timeout_at(deadline, read.stream.read_to_end(&mut actual)) + .await + .expect("target body drains before the round deadline") + .expect("target body streams"); + assert_eq!(actual, bodies[0], "round {round}: exact target bytes survive the handoff"); + } + if let Some(written) = written { + let etag = written.etag.expect("successful target ACK must supply the CAS ETag"); + assert!(!etag.is_empty(), "same-key CAS must carry an actual If-Match condition"); + let mut successor = vec![0; TEST_OBJECT_SIZE + 65]; + rng.fill_bytes(&mut successor); + let mut reader = PutObjReader::from_vec(successor.clone()); + timeout_at( + deadline, + set.put_object( + bucket, + &objects[0], + &mut reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + http_preconditions: Some(HTTPPreconditions { + if_match: Some(etag), + ..Default::default() + }), + ..Default::default() + }, + ), + ) + .await + .expect("same-key CAS acquires the handed-off owner under pressure") + .expect("same-key CAS accepts the exact target ETag"); + bodies[0] = successor; + } + let barrier = barriers[held_pressure].take().expect("final pressure checkpoint"); + barrier.release(); + drop(barrier); + for caller in puts { + timeout_at(deadline, caller) + .await + .expect("pressure writer drains within the finite round") + .expect("pressure caller joins") + .expect("pressure full-tail write commits"); + } + for (slot, object) in objects.iter().enumerate() { + let mut read = timeout_at( + deadline, + set.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()), + ) + .await + .expect("every reader acquires within the finite round") + .expect("every pressure and target object is readable"); + let mut actual = Vec::new(); + timeout_at(deadline, read.stream.read_to_end(&mut actual)) + .await + .expect("complete body drains within the round") + .expect("complete body streams"); + assert_eq!(actual, bodies[slot], "round {round} key {slot}: exact production GET oracle"); + for disk in &disks { + let fi = timeout_at(deadline, disk.read_version("", bucket, object, "", &ReadOptions::default())) + .await + .expect("physical metadata read remains bounded") + .expect("every healthy disk publishes the object"); + assert!(!fi.inline_data(), "pressure must write actual external shards"); + assert_eq!(fi.parts[0].size, bodies[slot].len()); + let root = disk.path(); + let path = root.join(bucket).join(object); + let part = path + .join(fi.data_dir.expect("non-inline data directory").to_string()) + .join("part.1"); + assert!( + timeout_at(deadline, tokio::fs::metadata(part)) + .await + .expect("external shard inspection remains bounded") + .expect("actual external shard file") + .len() + > 32 + ); + drop( + timeout_at(deadline, os::acquire_rename_data_mutation_lease(&root, bucket, &path)) + .await + .expect("physical mutation owner must release every real disk and key"), + ); + } + assert_eq!(trackers[slot].running(), 0, "round {round} key {slot}: no fan-out task remains"); + } + timeout_at(deadline, async { + while set.instance_ctx().namespace_commits_pending() || !non_trash_tmp_entries(&dirs).await.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("all real mutation owners and staging entries must drain"); + assert!(!set.instance_ctx().namespace_commits_pending()); + eprintln!( + "seed={SEED:#x} round={round} completion={completion:?} cancel={cancel} paused_disks={paused_disks:?} drained" + ); + } + } + #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] async fn tail_drained_put_waits_for_tail_and_allows_immediate_cas() { @@ -20832,6 +21120,16 @@ mod put_object_tmp_cleanup_tests { .await .expect("committed object must remain readable despite the failed tail"); assert_eq!(info.size, TEST_OBJECT_SIZE as i64); + let mut read = set + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("committed body must remain readable despite the failed tail"); + let mut body = Vec::new(); + read.stream + .read_to_end(&mut body) + .await + .expect("committed body must stream completely after the failed tail"); + assert_eq!(body, vec![b'1'; TEST_OBJECT_SIZE]); } #[tokio::test] @@ -21017,6 +21315,93 @@ mod put_object_tmp_cleanup_tests { wait_for_tmp_workspace_to_drain(&dirs, "cancelled full-tail commit should release staging ownership").await; } + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn borrowed_write_context_rejects_wrong_namespace_and_cache_flag() { + let (_dirs, _disks, set) = hermetic_set_disks(4).await; + let bucket = "borrowed-write-target"; + let object = "owned-object"; + let guard = crate::object_api::WriteCommitGuard::acquire( + &set.new_ns_lock(bucket, object).await.expect("namespace wrapper"), + get_lock_acquire_timeout(), + ) + .await + .expect("outer write lock"); + let mut opts = ObjectOptions::default(); + opts.add_write_commit_guard(&guard); + assert!(matches!( + WriteCommitContext::from_options(&opts, bucket, "other-object", false), + Err(Error::InvalidArgument(..)) + )); + assert!(matches!( + WriteCommitContext::from_options(&opts, bucket, "other-object", true), + Err(Error::InvalidArgument(..)) + )); + let fake = ObjectOptions { + no_lock: true, + metadata_cache_safe: true, + ..Default::default() + }; + assert!(matches!( + WriteCommitContext::from_options(&fake, bucket, object, false), + Err(Error::InvalidArgument(..)) + )); + } + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn borrowed_tail_drained_put_keeps_owner_after_caller_cancel() { + let (_dirs, disks, set) = hermetic_set_disks(4).await; + let bucket = "borrowed-put-cancel"; + let object = "borrowed-put-cancel-object"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume"); + } + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let tasks = rename_fanout_barrier::observe_tasks(object); + let writer = Arc::clone(&set); + let put = tokio::spawn(async move { + let guard = crate::object_api::WriteCommitGuard::acquire( + &writer.new_ns_lock(bucket, object).await.expect("namespace wrapper"), + get_lock_acquire_timeout(), + ) + .await + .expect("outer write guard"); + let mut opts = ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }; + opts.add_write_commit_guard(&guard); + let mut reader = PutObjReader::from_vec(vec![b'b'; TEST_OBJECT_SIZE]); + writer.put_object(bucket, object, &mut reader, &opts).await + }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("borrowed commit reaches rename phase"); + wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await; + assert!(!put.is_finished(), "full-tail borrowed write remains pending"); + put.abort(); + assert!(put.await.expect_err("caller was canceled").is_cancelled()); + let mut probe = Box::pin(set.acquire_write_lock_diag("borrowed_cancel_probe", bucket, object)); + assert!( + futures::poll!(probe.as_mut()).is_pending(), + "commit must retain the real borrowed namespace owner" + ); + barrier.release(); + drop( + tokio::time::timeout(Duration::from_secs(30), probe) + .await + .expect("borrowed owner drains") + .expect("next writer acquires"), + ); + assert_eq!(tasks.running(), 0, "all borrowed rename tasks are reaped before releasing the owner"); + let info = set + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("committed borrowed write remains readable"); + assert_eq!(info.size, TEST_OBJECT_SIZE as i64); + } + #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] async fn no_lock_put_waits_for_rename_tail_under_outer_guard() { @@ -21028,27 +21413,23 @@ mod put_object_tmp_cleanup_tests { disk.make_volume(bucket).await.expect("bucket volume should be created"); } - let outer_guard = set_disks - .acquire_write_lock_diag("outer_no_lock_put", bucket, object) - .await - .expect("the outer caller should hold the namespace guard"); + let outer_guard = crate::object_api::WriteCommitGuard::acquire( + &set_disks.new_ns_lock(bucket, object).await.expect("outer namespace wrapper"), + get_lock_acquire_timeout(), + ) + .await + .expect("the outer caller should hold the namespace guard"); let rename_tasks = rename_fanout_barrier::observe_tasks(object); let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let mut opts = ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }; + opts.add_write_commit_guard(&outer_guard); let put_store = Arc::clone(&set_disks); let put = tokio::spawn(async move { let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]); - put_store - .put_object( - bucket, - object, - &mut reader, - &ObjectOptions { - no_lock: true, - write_completion: WriteCompletion::TailDrained, - ..Default::default() - }, - ) - .await + put_store.put_object(bucket, object, &mut reader, &opts).await }); tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused()) @@ -23442,6 +23823,119 @@ mod body_cache_hook_e2e_tests { } } +#[cfg(test)] +mod marker_purge_receipt_tests { + use super::hermetic_set_disks_support::hermetic_set_disks_isolated; + use super::*; + use crate::disk::ReadOptions; + + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn quorum_marker_purge_receipt_survives_reopen_with_stale_member() { + let (_dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "marker-purge-receipt"; + let object = "explicit-marker-version"; + let version = Uuid::new_v4(); + let incarnation = Uuid::new_v4(); + for disk in &disks { + disk.make_volume(bucket).await.expect("fixture bucket"); + } + let mut marker = FileInfo { + name: object.to_string(), + version_id: Some(version), + deleted: true, + mark_deleted: true, + mod_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }; + marker.set_delete_marker_incarnation(incarnation); + let owner = set + .acquire_write_lock_diag("marker_purge_receipt_test", bucket, object) + .await + .expect("own the deleted object namespace"); + set.delete_object_version_with_purge(bucket, object, &marker, true, None, Some(incarnation)) + .await + .expect("seed the real marker on every disk"); + let marker = disks[0] + .read_version("", bucket, object, &version.to_string(), &ReadOptions::default()) + .await + .expect("read the persisted marker identity"); + let purge = delete_marker_purge_candidate(&marker, Some(incarnation)) + .expect("the canonical marker must authorize its exact purge"); + let request = FileInfo { + name: object.to_string(), + version_id: Some(version), + ..Default::default() + }; + + // A member that misses the explicit DELETE retains its original + // marker. The other three disks still satisfy configured write quorum. + set.disks.write().await[0] = None; + tokio::time::timeout( + Duration::from_secs(10), + set.delete_object_version_with_purge(bucket, object, &request, false, Some(purge.clone()), Some(incarnation)), + ) + .await + .expect("the receipt must not reacquire the deleted object's owner") + .expect("the explicit marker purge must reach write quorum"); + drop(owner); + for disk in &disks[1..] { + assert!( + matches!( + disk.read_version("", bucket, object, &version.to_string(), &ReadOptions::default()) + .await, + Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) + ), + "acknowledging members must remove the exact marker" + ); + } + let stale = disks[0] + .read_version("", bucket, object, &version.to_string(), &ReadOptions::default()) + .await + .expect("the offline member must retain its marker"); + assert!(stale.deleted); + assert_eq!(stale.version_id, Some(version)); + + let ctx = Arc::new(InstanceContext::new()); + ctx.update_erasure_type(crate::layout::endpoints::SetupType::Erasure).await; + let reopened = SetDisks::new_with_instance_ctx( + set.locker_owner.clone(), + Arc::new(tokio::sync::RwLock::new(disks.iter().cloned().map(Some).collect())), + set.set_drive_count, + set.default_parity_count, + set.set_index, + set.pool_index, + set.set_endpoints.clone(), + set.format.clone(), + Vec::new(), + ctx, + ) + .await; + assert!( + reopened.has_marker_purge_receipt(bucket, object, version, &purge).await, + "a successful explicit DELETE must leave durable proof for stale-marker cleanup after reopen" + ); + let receipt = crate::set_disk::marker_purge_receipt_path(bucket, object, version, &purge); + assert_eq!( + crate::config::com::read_config_limited_preserve_empty(reopened.clone(), &receipt, 128) + .await + .expect("read the physical quorum receipt"), + b"rustfs-marker-purge-receipt-v1" + ); + assert!( + !reopened + .has_marker_purge_receipt(bucket, object, Uuid::new_v4(), &purge) + .await, + "the receipt must not authorize another marker version" + ); + reopened.consume_marker_purge_receipt(bucket, object, version, &purge).await; + assert!( + !reopened.has_marker_purge_receipt(bucket, object, version, &purge).await, + "cleanup must be able to consume the persisted receipt" + ); + } +} + #[cfg(test)] mod single_delete_namespace_owner_tests { use super::hermetic_set_disks_support::hermetic_set_disks_isolated; diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 6ecabdad2..ec7a3aac4 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -69,11 +69,11 @@ use std::sync::{ }; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::io::duplex; -use tokio::sync::broadcast::{self}; use tokio::sync::mpsc::{self, Receiver, Sender}; use tokio::sync::{OnceCell, RwLock}; -use tokio::task::{JoinHandle, JoinSet}; +use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; +use tokio_util::task::AbortOnDropHandle; use tracing::{Instrument, debug, error, info, warn}; use uuid::Uuid; @@ -4334,96 +4334,15 @@ impl ECStore { // cancel channel let cancel = CancellationToken::new(); - let _cancel_guard = cancel.clone().drop_guard(); ensure_producer_limit_state(&mut o); - let (err_tx, mut err_rx) = broadcast::channel::>(1); - let (sender, recv) = mpsc::channel(o.limit as usize); - let store = self.clone(); - let opts = o.clone(); - let cancel_rx1 = cancel.clone(); - let cancel_rx1_for_err = cancel_rx1.clone(); - let err_tx1 = err_tx.clone(); - let job1_context = log_context.clone(); - let job1 = tokio::spawn( - async move { - let mut opts = opts; - opts.stop_disk_at_limit = true; - if let Err(err) = store.list_merged(cancel_rx1, opts, sender).await - && !cancel_rx1_for_err.is_cancelled() - { - log_list_path_worker_error("store", "list_merged", &job1_context, &err); - let _ = err_tx1.send(Arc::new(err)); - } - } - .instrument(tracing::Span::current()), - ); - - let cancel_rx2 = cancel.clone(); - - let (result_tx, mut result_rx) = mpsc::channel(1); - let err_tx2 = err_tx.clone(); - let opts = o.clone(); - let job2_context = log_context.clone(); - let job2 = tokio::spawn( - async move { - match gather_results(cancel_rx2, opts, recv, result_tx).await { - Ok(GatherResultsState::LimitReached) => cancel.cancel(), - Ok(GatherResultsState::InputClosed) => {} - // Consumer disconnect (e.g. client cancelled the request) - // is a benign completion: no error log, no err_tx send. - // The explicit cancel is idempotent and avoids relying on - // the wrapper's drop-guard ordering to stop the producer. - Ok(GatherResultsState::ConsumerGone) => cancel.cancel(), - // Invariant (rustfs/backlog#1306): gather_results maps a - // consumer disconnect to Ok(ConsumerGone) and has no - // fallible pre-send path, so this arm is currently - // unreachable. It is kept as a guard: it stays correct only - // while a *real* gather_results error would still surface as - // an error here. Real producer/listing errors take the - // separate job1 -> err_tx -> err_rx path below, unaffected. - Err(err) => { - log_list_path_worker_error("store", "gather_results", &job2_context, &err); - let _ = err_tx2.send(Arc::new(err)); - cancel.cancel(); - } - } - } - .instrument(tracing::Span::current()), - ); - - let mut result = { - // receiver result - tokio::select! { - res = err_rx.recv() =>{ - - match res{ - Ok(err) => { - log_list_path_worker_error("store", "worker_error", &log_context, err.as_ref()); - MetaCacheEntriesSortedResult{ entries: None, err: Some(to_filemeta_err(err.as_ref().clone())) } - }, - Err(err) => { - log_list_path_worker_error("store", "error_channel_closed", &log_context, &err); - - MetaCacheEntriesSortedResult{ entries: None, err: Some(rustfs_filemeta::Error::other(err)) } - }, - } - }, - Some(result) = result_rx.recv()=>{ - result - } - } - }; - - // wait spawns exit - join_all(vec![job1, job2]).await; - - if let Ok(err) = err_rx.try_recv() { - log_list_path_worker_error("store", "trailing_worker_error", &log_context, err.as_ref()); - result.err = Some(to_filemeta_err(err.as_ref().clone())); - } + let mut opts = o.clone(); + opts.stop_disk_at_limit = true; + let producer_cancel = cancel.clone(); + let producer = async move { store.list_merged(producer_cancel, opts, sender).await.map(|_| ()) }; + let mut result = collect_list_path_results("store", &log_context, cancel, o.clone(), recv, producer).await; if result.err.is_some() { log_list_path_finished("store", &log_context, list_path_started.elapsed().as_secs_f64() * 1000.0, 0, true); @@ -4969,6 +4888,68 @@ impl ECStore { } } +/// Own every page worker until completion, including when the request is dropped. +async fn collect_list_path_results

( + component: &'static str, + context: &ListPathLogContext, + cancel: CancellationToken, + opts: ListPathOptions, + recv: Receiver, + producer: P, +) -> MetaCacheEntriesSortedResult +where + P: Future> + Send + 'static, +{ + let _cancel_guard = cancel.clone().drop_guard(); + let (result_tx, mut result_rx) = mpsc::channel(1); + let mut workers = JoinSet::new(); + workers.spawn(async move { producer.await.map(|()| None) }.instrument(tracing::Span::current())); + let gather_cancel = cancel.clone(); + workers.spawn( + async move { gather_results(gather_cancel, opts, recv, result_tx).await.map(Some) }.instrument(tracing::Span::current()), + ); + + let mut failure = None; + let mut stopping = false; + while let Some(completion) = workers.join_next().await { + let error = match completion { + Ok(Ok(Some(GatherResultsState::LimitReached | GatherResultsState::ConsumerGone))) => { + stopping = true; + None + } + // EOF is only authoritative after the producer also succeeds. + Ok(Ok(None | Some(GatherResultsState::InputClosed))) => None, + Ok(Err(err)) => Some(err), + Err(err) if stopping && err.is_cancelled() => None, + // Do not expose a panic payload through the S3 error or logs. + Err(err) if err.is_panic() => Some(Error::other("listing worker panicked")), + Err(_) => Some(Error::other("listing worker unexpectedly cancelled")), + }; + if let Some(error) = error { + failure.get_or_insert(error); + stopping = true; + } + if stopping { + cancel.cancel(); + workers.abort_all(); + } + // Drain even after abort: a completed error/panic must not be hidden + // by a page limit becoming ready at the same time. + } + + // gather_results sends at most one page into a capacity-1 channel, so it + // never needs this receiver to make progress. Read only after both joins. + let mut result = result_rx.try_recv().unwrap_or_else(|_| MetaCacheEntriesSortedResult { + entries: None, + err: Some(rustfs_filemeta::Error::other("listing worker exited without a page")), + }); + if let Some(error) = failure { + log_list_path_worker_error(component, "worker_error", context, &error); + result.err = Some(to_filemeta_err(error)); + } + result +} + async fn gather_results( rx: CancellationToken, opts: ListPathOptions, @@ -5366,8 +5347,8 @@ fn spawn_listing_merge( rx: CancellationToken, inputs: Vec>, sender: Sender, -) -> JoinHandle> { - tokio::spawn( +) -> AbortOnDropHandle> { + AbortOnDropHandle::new(tokio::spawn( async move { let result = merge_entry_channels(rx.clone(), inputs, sender, 1).await; if result.is_err() { @@ -5376,7 +5357,7 @@ fn spawn_listing_merge( result } .instrument(tracing::Span::current()), - ) + )) } async fn merge_entry_channels( @@ -5776,85 +5757,14 @@ impl Sets { let log_context = ListPathLogContext::from_options(&o); let cancel = CancellationToken::new(); - let _cancel_guard = cancel.clone().drop_guard(); - let (err_tx, mut err_rx) = broadcast::channel::>(1); - let (sender, recv) = mpsc::channel(o.limit as usize); - ensure_producer_limit_state(&mut o); + let (sender, recv) = mpsc::channel(o.limit as usize); let sets = self.clone(); - let opts = o.clone(); - let cancel_rx1 = cancel.clone(); - let cancel_rx1_for_err = cancel_rx1.clone(); - let err_tx1 = err_tx.clone(); - let job1_context = log_context.clone(); - let job1 = tokio::spawn( - async move { - let mut opts = opts; - opts.stop_disk_at_limit = true; - if let Err(err) = sets.list_merged(cancel_rx1, opts, sender).await - && !cancel_rx1_for_err.is_cancelled() - { - log_list_path_worker_error("sets", "list_merged", &job1_context, &err); - let _ = err_tx1.send(Arc::new(err)); - } - } - .instrument(tracing::Span::current()), - ); - - let cancel_rx2 = cancel.clone(); - let (result_tx, mut result_rx) = mpsc::channel(1); - let err_tx2 = err_tx.clone(); - let opts = o.clone(); - let job2_context = log_context.clone(); - let job2 = tokio::spawn( - async move { - match gather_results(cancel_rx2, opts, recv, result_tx).await { - Ok(GatherResultsState::LimitReached) => cancel.cancel(), - Ok(GatherResultsState::InputClosed) => {} - // Consumer disconnect (e.g. client cancelled the request) - // is a benign completion: no error log, no err_tx send. - // The explicit cancel is idempotent and avoids relying on - // the wrapper's drop-guard ordering to stop the producer. - Ok(GatherResultsState::ConsumerGone) => cancel.cancel(), - // Invariant (rustfs/backlog#1306): gather_results maps a - // consumer disconnect to Ok(ConsumerGone) and has no - // fallible pre-send path, so this arm is currently - // unreachable. It is kept as a guard: it stays correct only - // while a *real* gather_results error would still surface as - // an error here. Real producer/listing errors take the - // separate job1 -> err_tx -> err_rx path below, unaffected. - Err(err) => { - log_list_path_worker_error("sets", "gather_results", &job2_context, &err); - let _ = err_tx2.send(Arc::new(err)); - cancel.cancel(); - } - } - } - .instrument(tracing::Span::current()), - ); - - let mut result = tokio::select! { - res = err_rx.recv() => { - match res { - Ok(err) => { - log_list_path_worker_error("sets", "worker_error", &log_context, err.as_ref()); - MetaCacheEntriesSortedResult { entries: None, err: Some(to_filemeta_err(err.as_ref().clone())) } - }, - Err(err) => { - log_list_path_worker_error("sets", "error_channel_closed", &log_context, &err); - MetaCacheEntriesSortedResult { entries: None, err: Some(rustfs_filemeta::Error::other(err)) } - }, - } - } - Some(result) = result_rx.recv() => result, - }; - - join_all(vec![job1, job2]).await; - - if let Ok(err) = err_rx.try_recv() { - log_list_path_worker_error("sets", "trailing_worker_error", &log_context, err.as_ref()); - result.err = Some(to_filemeta_err(err.as_ref().clone())); - } + let mut opts = o.clone(); + opts.stop_disk_at_limit = true; + let producer_cancel = cancel.clone(); + let producer = async move { sets.list_merged(producer_cancel, opts, sender).await.map(|_| ()) }; + let mut result = collect_list_path_results("sets", &log_context, cancel, o.clone(), recv, producer).await; if result.err.is_some() { log_list_path_finished("sets", &log_context, list_path_started.elapsed().as_secs_f64() * 1000.0, 0, true); @@ -6793,85 +6703,14 @@ impl SetDisks { let log_context = ListPathLogContext::from_options(&o); let cancel = CancellationToken::new(); - let _cancel_guard = cancel.clone().drop_guard(); - let (err_tx, mut err_rx) = broadcast::channel::>(1); - let (sender, recv) = mpsc::channel(o.limit as usize); - ensure_producer_limit_state(&mut o); + let (sender, recv) = mpsc::channel(o.limit as usize); let set = self.clone(); - let opts = o.clone(); - let cancel_rx1 = cancel.clone(); - let cancel_rx1_for_err = cancel_rx1.clone(); - let err_tx1 = err_tx.clone(); - let job1_context = log_context.clone(); - let job1 = tokio::spawn( - async move { - let mut opts = opts; - opts.stop_disk_at_limit = true; - if let Err(err) = set.list_path(cancel_rx1, opts, sender).await - && !cancel_rx1_for_err.is_cancelled() - { - log_list_path_worker_error("set_disks", "list_path", &job1_context, &err); - let _ = err_tx1.send(Arc::new(err)); - } - } - .instrument(tracing::Span::current()), - ); - - let cancel_rx2 = cancel.clone(); - let (result_tx, mut result_rx) = mpsc::channel(1); - let err_tx2 = err_tx.clone(); - let opts = o.clone(); - let job2_context = log_context.clone(); - let job2 = tokio::spawn( - async move { - match gather_results(cancel_rx2, opts, recv, result_tx).await { - Ok(GatherResultsState::LimitReached) => cancel.cancel(), - Ok(GatherResultsState::InputClosed) => {} - // Consumer disconnect (e.g. client cancelled the request) - // is a benign completion: no error log, no err_tx send. - // The explicit cancel is idempotent and avoids relying on - // the wrapper's drop-guard ordering to stop the producer. - Ok(GatherResultsState::ConsumerGone) => cancel.cancel(), - // Invariant (rustfs/backlog#1306): gather_results maps a - // consumer disconnect to Ok(ConsumerGone) and has no - // fallible pre-send path, so this arm is currently - // unreachable. It is kept as a guard: it stays correct only - // while a *real* gather_results error would still surface as - // an error here. Real producer/listing errors take the - // separate job1 -> err_tx -> err_rx path below, unaffected. - Err(err) => { - log_list_path_worker_error("set_disks", "gather_results", &job2_context, &err); - let _ = err_tx2.send(Arc::new(err)); - cancel.cancel(); - } - } - } - .instrument(tracing::Span::current()), - ); - - let mut result = tokio::select! { - res = err_rx.recv() => { - match res { - Ok(err) => { - log_list_path_worker_error("set_disks", "worker_error", &log_context, err.as_ref()); - MetaCacheEntriesSortedResult { entries: None, err: Some(to_filemeta_err(err.as_ref().clone())) } - }, - Err(err) => { - log_list_path_worker_error("set_disks", "error_channel_closed", &log_context, &err); - MetaCacheEntriesSortedResult { entries: None, err: Some(rustfs_filemeta::Error::other(err)) } - }, - } - } - Some(result) = result_rx.recv() => result, - }; - - join_all(vec![job1, job2]).await; - - if let Ok(err) = err_rx.try_recv() { - log_list_path_worker_error("set_disks", "trailing_worker_error", &log_context, err.as_ref()); - result.err = Some(to_filemeta_err(err.as_ref().clone())); - } + let mut opts = o.clone(); + opts.stop_disk_at_limit = true; + let producer_cancel = cancel.clone(); + let producer = async move { set.list_path(producer_cancel, opts, sender).await.map(|_| ()) }; + let mut result = collect_list_path_results("set_disks", &log_context, cancel, o.clone(), recv, producer).await; if result.err.is_some() { log_list_path_finished("set_disks", &log_context, list_path_started.elapsed().as_secs_f64() * 1000.0, 0, true); @@ -10125,6 +9964,7 @@ mod test { .expect("bucket should be created with authoritative metadata"); let committed_dir = uuid::Uuid::new_v4(); let unmarked_dir = uuid::Uuid::new_v4(); + let backup_dir = uuid::Uuid::new_v4(); let transaction = uuid::Uuid::new_v4(); for dir in &dirs { let committed = dir @@ -10135,6 +9975,9 @@ mod test { tokio::fs::create_dir_all(&committed) .await .expect("committed delete residue should be created"); + tokio::fs::write(committed.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP), b"rollback metadata") + .await + .expect("committed rollback backup should be written"); tokio::fs::write(committed.join("part.1"), b"stale") .await .expect("stale part should be written"); @@ -10158,6 +10001,18 @@ mod test { tokio::fs::write(unmarked.join("part.1"), b"stale") .await .expect("stale part should be written"); + + let backup = dir + .path() + .join(bucket) + .join("metrics/statefulset/2026/08/26/18/object.parquet") + .join(backup_dir.to_string()); + tokio::fs::create_dir_all(&backup) + .await + .expect("rollback backup directory should be created"); + tokio::fs::write(backup.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP), b"rollback metadata") + .await + .expect("rollback backup should be written"); } let result = store @@ -10166,7 +10021,7 @@ mod test { .await .expect("delimiter listing should succeed"); assert!(result.objects.is_empty()); - assert!(result.prefixes.is_empty(), "neither residue subtree may surface as a prefix"); + assert!(result.prefixes.is_empty(), "residue subtrees may not surface as prefixes"); for dir in &dirs { let metrics = dir.path().join(bucket).join("metrics"); assert!( @@ -10181,6 +10036,17 @@ mod test { .exists(), "residue without a committed marker must survive the purge" ); + assert_eq!( + tokio::fs::read( + metrics + .join("statefulset/2026/08/26/18/object.parquet") + .join(backup_dir.to_string()) + .join(crate::disk::STORAGE_FORMAT_FILE_BACKUP) + ) + .await + .expect("uncommitted rollback backup must survive the listing"), + b"rollback metadata" + ); } } @@ -11996,6 +11862,166 @@ mod test { assert!(!versions.versions[0].deleted); } + async fn collect_list_path_results

( + cancel: CancellationToken, + opts: ListPathOptions, + recv: mpsc::Receiver, + producer: P, + ) -> rustfs_filemeta::MetaCacheEntriesSortedResult + where + P: std::future::Future> + Send + 'static, + { + let context = super::ListPathLogContext::from_options(&opts); + super::collect_list_path_results("test", &context, cancel, opts, recv, producer).await + } + + #[tokio::test] + async fn list_path_workers_report_producer_panic_instead_of_empty_eof() { + let (sender, recv) = mpsc::channel(1); + let result = timeout( + Duration::from_secs(1), + collect_list_path_results(CancellationToken::new(), ListPathOptions::default(), recv, async move { + let _sender = sender; + panic!("test listing producer panic"); + }), + ) + .await + .expect("producer panic must not strand the collector"); + assert!( + result + .err + .expect("panic must be reported") + .to_string() + .contains("listing worker panicked") + ); + } + + #[tokio::test] + async fn list_path_workers_preserve_error_after_input_closes() { + let (sender, recv) = mpsc::channel(1); + let result = collect_list_path_results(CancellationToken::new(), ListPathOptions::default(), recv, async move { + drop(sender); + tokio::task::yield_now().await; + Err(StorageError::FileCorrupt) + }) + .await; + assert_eq!(result.err, Some(rustfs_filemeta::Error::FileCorrupt)); + } + + #[tokio::test] + async fn list_path_workers_drain_buffer_before_successful_eof() { + let (sender, recv) = mpsc::channel(3); + let result = collect_list_path_results( + CancellationToken::new(), + ListPathOptions { + limit: 4, + incl_deleted: true, + ..Default::default() + }, + recv, + async move { + for name in ["a", "b", "c"] { + sender.try_send(test_meta_entry(name)).expect("queue page"); + } + Ok(()) + }, + ) + .await; + assert_eq!(result.err, Some(rustfs_filemeta::Error::Unexpected)); + let entries = result.entries.expect("EOF retains entries"); + assert_eq!( + entries.entries().iter().map(|entry| entry.name.as_str()).collect::>(), + ["a", "b", "c"] + ); + } + + #[tokio::test] + async fn list_path_workers_abort_stalled_producer_at_page_limit() { + let (sender, recv) = mpsc::channel(1); + let dropped = CancellationToken::new(); + let producer_dropped = dropped.clone(); + let result = timeout( + Duration::from_secs(1), + collect_list_path_results( + CancellationToken::new(), + ListPathOptions { + limit: 1, + incl_deleted: true, + ..Default::default() + }, + recv, + async move { + let _drop_guard = producer_dropped.drop_guard(); + sender.send(test_meta_entry("a")).await.expect("send page"); + std::future::pending::>().await + }, + ), + ) + .await + .expect("page limit must not wait for a producer that ignores cancellation"); + assert!(result.err.is_none()); + assert_eq!(result.entries.expect("page").entries().len(), 1); + assert!(dropped.is_cancelled(), "producer must be dropped before the page is returned"); + } + + #[tokio::test] + async fn list_path_workers_parent_abort_drops_both_children() { + let (sender, recv) = mpsc::channel(1); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let cancel = CancellationToken::new(); + let dropped = CancellationToken::new(); + let producer_dropped = dropped.clone(); + let parent = tokio::spawn(collect_list_path_results(cancel.clone(), ListPathOptions::default(), recv, async move { + let _drop_guard = producer_dropped.drop_guard(); + started_tx.send(()).expect("notify parent"); + std::future::pending::>().await + })); + started_rx.await.expect("producer started"); + parent.abort(); + assert!(parent.await.expect_err("parent aborted").is_cancelled()); + timeout(Duration::from_secs(1), dropped.cancelled()) + .await + .expect("producer dropped"); + timeout(Duration::from_secs(1), sender.closed()) + .await + .expect("collector dropped"); + assert!(cancel.is_cancelled()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn list_path_workers_preserve_ready_error_at_page_limit() { + for _ in 0..128 { + let (sender, recv) = mpsc::channel(1); + let result = collect_list_path_results( + CancellationToken::new(), + ListPathOptions { + limit: 1, + incl_deleted: true, + ..Default::default() + }, + recv, + async move { + sender.try_send(test_meta_entry("a")).expect("queue page"); + Err(StorageError::FileCorrupt) + }, + ) + .await; + assert_eq!(result.err, Some(rustfs_filemeta::Error::FileCorrupt)); + } + } + + #[tokio::test] + async fn listing_merge_drop_stops_blocked_inputs() { + let (input_tx, input_rx) = mpsc::channel(1); + let (output_tx, _output_rx) = mpsc::channel(1); + let task = super::spawn_listing_merge(CancellationToken::new(), vec![input_rx], output_tx); + drop(task); + + timeout(Duration::from_secs(1), input_tx.closed()) + .await + .expect("dropping the merge owner must release an idle producer"); + } + #[tokio::test] async fn listing_merge_preserves_error_after_partial_output_without_cancelling_request() { let time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 52fdad5ce..444d00ab8 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -1038,8 +1038,23 @@ impl fmt::Display for ObjectLockDiagMode { } } +enum ObjectLockOwner { + Read(rustfs_lock::NamespaceLockGuard), + Write(Arc), +} + +impl ObjectLockOwner { + fn as_ref(&self) -> &rustfs_lock::NamespaceLockGuard { + match self { + Self::Read(guard) => guard, + Self::Write(guard) => guard, + } + } +} + pub(crate) struct ObjectLockDiagGuard { - guard: rustfs_lock::NamespaceLockGuard, + guard: ObjectLockOwner, + write_key: Option<(String, String)>, #[cfg(test)] test_namespace_lock_fence: Option, enabled: bool, @@ -1063,7 +1078,11 @@ impl ObjectLockDiagGuard { mode: ObjectLockDiagMode, ) -> Self { Self { - guard, + guard: match mode { + ObjectLockDiagMode::Read => ObjectLockOwner::Read(guard), + ObjectLockDiagMode::Write => ObjectLockOwner::Write(Arc::new(guard)), + }, + write_key: None, #[cfg(test)] test_namespace_lock_fence: None, enabled, @@ -1082,15 +1101,20 @@ impl ObjectLockDiagGuard { self } + fn with_write_key(mut self, bucket: &str, object: &str) -> Self { + self.write_key = Some((bucket.to_owned(), object.to_owned())); + self + } + pub(crate) fn lock_lost_signal(&self) -> Option> { - match &self.guard { + match self.guard.as_ref() { rustfs_lock::NamespaceLockGuard::Standard(guard) => Some(guard.lock_lost()), rustfs_lock::NamespaceLockGuard::Fast(_) => None, } } pub(crate) fn is_lock_lost(&self) -> bool { - self.guard.is_lock_lost() || { + self.guard.as_ref().is_lock_lost() || { #[cfg(test)] { self.test_namespace_lock_fence @@ -1105,6 +1129,13 @@ impl ObjectLockDiagGuard { } pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) { + if let Some((bucket, object)) = self.write_key.as_ref() + && let ObjectLockOwner::Write(guard) = &self.guard + { + let no_lock = opts.no_lock; + opts.add_owned_write_lock(Arc::clone(guard), bucket, object); + opts.no_lock = no_lock; + } opts.ensure_namespace_lock_fence(); if let Some(signal) = self.lock_lost_signal() { opts.add_namespace_lock_lost_signal(signal); @@ -1301,7 +1332,7 @@ impl RestoreAcceptGuard { /// refresh lost quorum). Callers must check this before committing the /// restore-status write the guard exists to serialize. pub fn is_lock_lost(&self) -> bool { - self.0.guard.is_lock_lost() + self.0.guard.as_ref().is_lock_lost() } pub fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) { @@ -1322,7 +1353,7 @@ impl RestoreWorkerGuard { impl Drop for ObjectLockDiagGuard { fn drop(&mut self) { - if !self.enabled || self.guard.is_released() { + if !self.enabled || self.guard.as_ref().is_released() { return; } @@ -3029,7 +3060,8 @@ impl ECStore { owner, ObjectLockDiagMode::Write, ) - .with_attempt(attempt)) + .with_attempt(attempt) + .with_write_key(bucket, object)) } async fn acquire_object_write_lock_if_needed( @@ -3044,11 +3076,9 @@ impl ECStore { } let guard = self.acquire_object_write_lock(op, bucket, object).await?; - if let Some(signal) = guard.lock_lost_signal() { - opts.add_namespace_lock_lost_signal(signal); - } - opts.ensure_namespace_lock_fence(); + guard.add_namespace_lock_fence(opts); opts.no_lock = true; + opts.write_completion = crate::object_api::WriteCompletion::TailDrained; Ok(Some(guard)) } @@ -3187,7 +3217,10 @@ impl ECStore { let store = Arc::clone(self); let write = async move { let object = "buckets/.scanner-pause-backlog.json"; - let mut opts = ObjectOptions::default(); + let mut opts = ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }; // Match migration: fixed object namespace -> durable pool metadata -> // actual replica namespace. The replica need not be the hash-routed set. let object_guard = if store.single_pool() { @@ -3727,15 +3760,18 @@ impl ECStore { acquire_start.elapsed(), diag_enabled, ); - guards.push(ObjectLockDiagGuard::new( - guard, - diag_enabled, - op, - diag_enabled.then(|| bucket.to_string()), - diag_enabled.then(|| object.to_string()), - owner, - ObjectLockDiagMode::Write, - )); + guards.push( + ObjectLockDiagGuard::new( + guard, + diag_enabled, + op, + diag_enabled.then(|| bucket.to_string()), + diag_enabled.then(|| object.to_string()), + owner, + ObjectLockDiagMode::Write, + ) + .with_write_key(bucket, object), + ); locked_sets.push(Arc::clone(set)); } } @@ -3826,7 +3862,8 @@ impl ECStore { diag_enabled.then(|| object.to_string()), owner, ObjectLockDiagMode::Write, - ); + ) + .with_write_key(bucket, object); #[cfg(test)] let guard = { let mut guard = guard; @@ -4571,11 +4608,13 @@ impl ECStore { shard_integrity_write_mode: dst_opts.shard_integrity_write_mode, version_id: dst_opts.version_id.clone(), no_lock: dst_opts.no_lock, + write_completion: dst_opts.write_completion, mod_time: dst_opts.mod_time, http_preconditions: dst_opts.http_preconditions.clone(), expected_current_version_id: dst_opts.expected_current_version_id.clone(), expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id, namespace_lock_fence: dst_opts.namespace_lock_fence.clone(), + write_lock_context: dst_opts.write_lock_context.clone(), bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(), object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(), quota_admission: dst_opts.quota_admission, @@ -4614,11 +4653,13 @@ impl ECStore { shard_integrity_write_mode: dst_opts.shard_integrity_write_mode, version_id: dst_opts.version_id.clone(), no_lock: dst_opts.no_lock, + write_completion: dst_opts.write_completion, mod_time: dst_opts.mod_time, http_preconditions: dst_opts.http_preconditions.clone(), expected_current_version_id: dst_opts.expected_current_version_id.clone(), expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id, namespace_lock_fence: dst_opts.namespace_lock_fence.clone(), + write_lock_context: dst_opts.write_lock_context.clone(), bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(), object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(), quota_admission: dst_opts.quota_admission, @@ -4667,11 +4708,13 @@ impl ECStore { shard_integrity_write_mode: dst_opts.shard_integrity_write_mode, version_id: dst_opts.version_id.clone(), no_lock: dst_opts.no_lock, + write_completion: dst_opts.write_completion, mod_time: dst_opts.mod_time, http_preconditions: dst_opts.http_preconditions.clone(), expected_current_version_id: dst_opts.expected_current_version_id.clone(), expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id, namespace_lock_fence: dst_opts.namespace_lock_fence.clone(), + write_lock_context: dst_opts.write_lock_context.clone(), bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(), object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(), quota_admission: dst_opts.quota_admission, @@ -8721,6 +8764,108 @@ mod tests { } } + #[tokio::test] + #[serial_test::serial] + async fn borrowed_copy_tail_drained_waits_for_rename_tail() { + use crate::object_api::{WriteCommitGuard, WriteCompletion}; + use crate::set_disk::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier}; + + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], ctx).await); + let bucket = RUSTFS_META_BUCKET; + let source = "copy-tail-drained-source"; + let target = "copy-tail-drained-target"; + let payload = vec![0x64; 4097]; + let set = store.pools[0].get_disks_by_key(target); + let mut source_info = set + .put_object( + bucket, + source, + &mut PutObjReader::from_vec(payload.clone()), + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("seed copy source"); + source_info.put_object_reader = Some(PutObjReader::from_vec(payload.clone())); + + let barrier = rename_fanout_barrier::arm(target, 0, rename_fanout_barrier::PHASE_RENAME); + let rename_tasks = rename_fanout_barrier::observe_tasks(target); + let copy_store = Arc::clone(&store); + let mut copy = tokio::spawn(async move { + let lock = copy_store + .new_ns_lock(bucket, target) + .await + .expect("target namespace wrapper"); + let guard = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout()) + .await + .expect("real Copy target namespace owner"); + let mut options = ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }; + options.add_write_commit_guard(&guard); + copy_store + .handle_copy_object(bucket, source, bucket, target, &mut source_info, &ObjectOptions::default(), &options) + .await + }); + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + tokio::time::timeout_at(deadline, barrier.wait_until_paused()) + .await + .expect("copy reaches the paused rename checkpoint"); + tokio::time::timeout_at(deadline, async { + loop { + if let Ok(info) = set + .get_object_info( + bucket, + target, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + && info.size == 4097 + && rename_tasks.running() == 1 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("copy commits a metadata quorum while one rename remains paused"); + assert!( + tokio::time::timeout(Duration::from_secs(1), &mut copy).await.is_err(), + "Copy TailDrained must remain pending after metadata quorum" + ); + barrier.release(); + let result = tokio::time::timeout_at(deadline, copy) + .await + .expect("copy drains its rename tail") + .expect("copy task completes") + .expect("copy succeeds"); + assert_eq!(result.size, 4097); + assert_eq!(rename_tasks.running(), 0); + let mut reader = store + .get_object_reader(bucket, target, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read copied target"); + let mut actual = Vec::new(); + reader + .stream + .read_to_end(&mut actual) + .await + .expect("read complete copied bytes"); + assert_eq!(actual, payload); + }) + .await; + } + #[tokio::test] #[serial_test::serial] async fn object_lock_diagnostics_report_historical_get_blocked_by_put_commit() { diff --git a/crates/ecstore/tests/fixtures/erasure-shards/README.md b/crates/ecstore/tests/fixtures/erasure-shards/README.md new file mode 100644 index 000000000..2c6be11be --- /dev/null +++ b/crates/ecstore/tests/fixtures/erasure-shards/README.md @@ -0,0 +1,90 @@ +# Pinned external erasure shards + +These are backend object files captured from two checksum-verified released +processes, then copied after each process stopped: + +| Corpus | Producer | Source commit | Read layout | +| --- | --- | --- | --- | +| `minio/` | MinIO `RELEASE.2025-07-23T15-54-02Z` | `7ced9663e6a791fef9dc6be798ff24cda9c730ac` | GF8 Vandermonde, MinIO HighwayHash key | +| `legacy/` | RustFS `1.0.0-alpha.37` | `371119f7336b74fdbcee37a8fcccb9d91c276f47` | Historical GF16, even-padded shards, `[3,4,2,1]` HighwayHash key | + +Each manifest pins the official release URL, archive and executable SHA256, +producer version output, capture geometry, and every `xl.meta` and external +`part.1` file. The MinIO executable was downloaded directly from its official +GitHub release. The historical RustFS executable came from its official release +archive. Both hashes were checked before execution. No current RustFS encoder +generated these shards. + +The legacy sample proves reading a historical released RustFS format. It is not +evidence from an independently implemented GF16 codec. The MinIO sample is from +an external producer. Neither sample proves distributed failure tolerance, +power-loss durability, transitioned objects, or reverse migration into a live +MinIO drive set. + +## Payload and capture + +Both processes used twelve disposable local directories with `EC:6`, producing +six data and six parity shards. The object is the smallest odd size above six +128-KiB inline thresholds: 786433 bytes. Its actual external shard payload is +131073 bytes for MinIO and 131074 bytes for the historical even-padded layout. +All twelve shard file hashes are distinct in each corpus. + +`plaintext.bin` is the complete independent oracle. It is a nonperiodic SHA256 +counter stream, not an encoder roundtrip: + +```python +import hashlib +size = 6 * 128 * 1024 + 1 +seed = b"RustFS backlog #2248 E10 pinned fixture v1" +payload = b"".join( + hashlib.sha256(seed + counter.to_bytes(8, "little")).digest() + for counter in range((size + 31) // 32) +)[:size] +``` + +The pinned plaintext SHA256 is +`b5a83332327961a31b65a8731c64559fd04ff8ac9f6124022201dbbdba5ffa01`. +Each producer's S3 GET was compared with the complete oracle before stopping +the process and copying its object tree. Credentials were temporary environment +values and are absent from this corpus. + +The capture commands and nonsecret settings are in the manifests. Both servers +bound to a disposable loopback port. MinIO used `MINIO_CI_CD=1` to allow multiple +directories on the host's root filesystem; this is a local format capture, not +a production drive deployment. RustFS used `RUSTFS_CONSOLE_ENABLE=false` and a +temporary log directory. + +MinIO exited on SIGTERM. The historical RustFS process did not exit within the +20-second shutdown budget, so it was killed and reaped before copying. That +shutdown is recorded in its manifest; it is not a crash-durability assertion. + +To reproduce the upload after starting the pinned producer, use the existing +`crates/rio-v2/tests/minio_fixture_lab/lab.py` S3 client with the temporary +endpoint and credentials: create `erasure-fixtures`, PUT the oracle as +`non-inline.bin`, then GET it and compare every byte. Stop the producer before +copying `backend/disk1` through `backend/disk12`'s object directories. Preserve +their data-directory UUIDs and metadata without re-encoding. A fresh capture +has different UUIDs and timestamps, so replacing this corpus also requires +reviewing and updating all per-file hashes. + +## Required test + +```bash +cargo nextest run --locked -p rustfs-ecstore \ + --test pinned_erasure_fixtures_test --profile ci --retries 0 +``` + +CI must select and pass all four tests without skips or retries. + +The positive tests import the captured files into isolated twelve-drive stores +and use the production `get_object_reader` metadata and codec dispatch. A data +slot's actual part file is removed before the first GET in each recovery case; +no current encoder writes the imported payload. Each schedule has a 30-second +budget. Low-level decode is used only for the opposite-codec and insufficient +quorum negative checks. + +The test validates the manifest and every file before decoding. Missing or +modified files fail; no fixture-dependent successful return or ignore flag is +provided. It checks exact full bodies, odd Range boundaries and tails, recovery +after removing a real data shard, bitrot rejection before exposing bytes, +insufficient read quorum, and rejection of an opposite-codec reconstruction. diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk1/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk1/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 new file mode 100644 index 000000000..7f5fc71f7 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk1/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk1/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk1/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..c4fe71d65 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk1/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk10/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk10/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 new file mode 100644 index 000000000..3f6f82ee7 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk10/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk10/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk10/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..83b402d1e Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk10/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk11/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk11/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 new file mode 100644 index 000000000..ef35fa48d Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk11/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk11/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk11/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..35398c813 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk11/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk12/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk12/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 new file mode 100644 index 000000000..12f5bc4da Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk12/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk12/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk12/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..cc8be17af Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk12/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk2/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk2/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 new file mode 100644 index 000000000..0e6e6040c Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk2/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk2/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk2/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..2acbe1f95 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk2/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk3/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk3/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 new file mode 100644 index 000000000..f8ed8ea8c Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk3/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk3/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk3/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..51f3068e3 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk3/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk4/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk4/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 new file mode 100644 index 000000000..4151b09ad Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk4/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk4/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk4/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..077c1f9fe Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk4/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk5/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk5/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 new file mode 100644 index 000000000..ff917b60d Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk5/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk5/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk5/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..e13447592 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk5/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk6/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk6/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 new file mode 100644 index 000000000..dac41d26d Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk6/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk6/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk6/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..ef0781038 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk6/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk7/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk7/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 new file mode 100644 index 000000000..14b8b2591 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk7/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk7/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk7/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..98f89adf4 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk7/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk8/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk8/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 new file mode 100644 index 000000000..cd65325e6 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk8/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk8/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk8/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..56eef9bb6 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk8/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk9/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk9/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 new file mode 100644 index 000000000..808e1b21a Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk9/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk9/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk9/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..ac53ef26e Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/legacy/disk9/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/legacy/manifest.json b/crates/ecstore/tests/fixtures/erasure-shards/legacy/manifest.json new file mode 100644 index 000000000..b4046a75c --- /dev/null +++ b/crates/ecstore/tests/fixtures/erasure-shards/legacy/manifest.json @@ -0,0 +1,66 @@ +{ + "block_size": 1048576, + "bucket": "erasure-fixtures", + "capture": { + "command": "rustfs --address 127.0.0.1:$PORT backend/disk{1..12}", + "disk_count": 12, + "environment": { + "RUSTFS_CONSOLE_ENABLE": "false", + "RUSTFS_STORAGE_CLASS_STANDARD": "EC:6" + }, + "inline_threshold_per_shard": 131072, + "payload": "SHA256(b'RustFS backlog #2248 E10 pinned fixture v1' + counter.to_bytes(8, 'little')), concatenated and truncated to 786433 bytes", + "producer_get_equals_oracle": true, + "producer_stopped_before_copy": true, + "shutdown": { + "forced_kill_after_seconds": 20, + "initial_signal": "SIGTERM", + "process_reaped_before_copy": true, + "returncode": -9 + }, + "storage_class_standard": "EC:6" + }, + "data_shards": 6, + "files": { + "disk1/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1": "125f8204f39211fed4a5a7b927de9c7c4d7d39544cfc5a93602daf86a8298225", + "disk1/erasure-fixtures/non-inline.bin/xl.meta": "acf42a683c1e81ec8925189c693574d74e6b5682a5091884974722bd3edd5f4f", + "disk10/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1": "39d226f036a22a0099dd3cf3f0d6c5eb8902e7a9560642528d46e002ab01b3db", + "disk10/erasure-fixtures/non-inline.bin/xl.meta": "73e1f9a2f86f7214d3e76bb906a4c657e375e992d178ee23f967be289f464f81", + "disk11/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1": "d56d32d1eb32bdb32b77e6458205f8c42bd4b59d692e084330344deab7e48810", + "disk11/erasure-fixtures/non-inline.bin/xl.meta": "67392bea6c1c9d8e6a13afa3540633c6ff9bfa0caa6a49b019bfc9e707adbb62", + "disk12/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1": "c64d7c5eb33aef912913dbc85692ae291c61cb67fcacda9abe87de8059c369b5", + "disk12/erasure-fixtures/non-inline.bin/xl.meta": "9d327a0e6d101f11ad8db0fa693eb81ce8e893a200c44aafd436b4b6e0e38f2a", + "disk2/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1": "ae4209ab1e4d7ce8e26da414e4d0e21490c81b39e475e768a664a4e000b0fe16", + "disk2/erasure-fixtures/non-inline.bin/xl.meta": "6570482acc478529dce1af0525f05e4bf617c57e7f775ced8b4b0b80dea20a40", + "disk3/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1": "e0abbab5f232c0b4fe1744b9e78d94ce477514766fd475dbefe2e712b1b4d87b", + "disk3/erasure-fixtures/non-inline.bin/xl.meta": "dfb141d97bb6b43a8f4b96b43d7f214ceea4b447bec1fa12a3fdd2f36f76a299", + "disk4/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1": "870a31918d708e83583f0dbc5fad3941af603d602b134672aad84c018b1506e2", + "disk4/erasure-fixtures/non-inline.bin/xl.meta": "63e9d7462e12930f003f416954514108e231cde4813f5cf7b804b3f7546c1ab1", + "disk5/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1": "07b5c7b23fd43fb7f8bbba22f32fdb430895929ef702a2453ee3c26ef9e3d28d", + "disk5/erasure-fixtures/non-inline.bin/xl.meta": "b10bc26f2f92ca59a50f25315f9ba5ce614b1de627c6edb366109367f3b905cd", + "disk6/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1": "623f6131ab6c6b8d5cb5962f7c88a8b289da08c65fbd91a99c591a27b16298da", + "disk6/erasure-fixtures/non-inline.bin/xl.meta": "dc351b240b90baf19bd3684709157b02aa330bcb0c42bd404ef3938c9a1b0782", + "disk7/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1": "2795b0d98bd5026ad765e276a8f22b1d601b8f699258d9b3a1663186c7a300ce", + "disk7/erasure-fixtures/non-inline.bin/xl.meta": "0add42e3251e057c10637786e9e3060bacb95e0b084b21ad7acc369e1f6c89aa", + "disk8/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1": "254ef640d6a193bb23673a19116861280649e6255f63800baf4595c8b4815a7b", + "disk8/erasure-fixtures/non-inline.bin/xl.meta": "8ed8e05d9a9013b1cfd8ba5c816727f5ca6bfa888f3af76a97ca1127a73e7c9a", + "disk9/erasure-fixtures/non-inline.bin/8a24d3f4-dbc3-4c77-8e6d-10ceecbd5a71/part.1": "ac754f505a62ccbf29046324ed9fd4172b5cdd44bdb1d78f0e3150322acc45cc", + "disk9/erasure-fixtures/non-inline.bin/xl.meta": "47d021868d9f5580934adfd3321e9c618d1357758890eaa81f5d27af0c324ad5" + }, + "object": "non-inline.bin", + "parity_shards": 6, + "plaintext_sha256": "b5a83332327961a31b65a8731c64559fd04ff8ac9f6124022201dbbdba5ffa01", + "plaintext_size": 786433, + "schema_version": 1, + "source": { + "archive_sha256": "c7d1d5b060fad53ea3c9eac9deb6d1ea20a22d441ec3494cd5a18fe6f7bbb215", + "archive_url": "https://github.com/rustfs/rustfs/releases/download/1.0.0-alpha.37/rustfs-macos-aarch64-v1.0.0-alpha.37.zip", + "binary_sha256": "158b5e0a96c6caf201a72e8c20e417124c190f295160e7c8610bf54228b5292a", + "binary_version": "rustfs 1.0.0-alpha.37\nbuild time : 2025-07-24 09:05:37 +00:00\nbuild profile: release\nbuild os : macos-aarch64\nrust version : rustc 1.88.0 (6b00bc388 2025-06-23)\nrust channel : stable-aarch64-apple-darwin\ngit branch : \ngit commit : 371119f7336b74fdbcee37a8fcccb9d91c276f47\ngit tag : 1.0.0-alpha.37\ngit status :", + "codec": "gf16", + "producer": "RustFS", + "release": "1.0.0-alpha.37", + "source_commit": "371119f7336b74fdbcee37a8fcccb9d91c276f47", + "uses_legacy": true + } +} diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk1/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk1/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 new file mode 100644 index 000000000..d55f7e7ec Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk1/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk1/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk1/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..e8f9c6a78 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk1/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk10/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk10/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 new file mode 100644 index 000000000..45c03c4d2 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk10/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk10/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk10/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..791a32148 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk10/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk11/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk11/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 new file mode 100644 index 000000000..920f33cdd Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk11/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk11/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk11/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..d4d436492 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk11/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk12/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk12/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 new file mode 100644 index 000000000..308ca8106 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk12/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk12/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk12/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..8738db2ee Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk12/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk2/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk2/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 new file mode 100644 index 000000000..fbc91b943 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk2/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk2/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk2/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..b9216a273 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk2/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk3/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk3/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 new file mode 100644 index 000000000..00d64cd45 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk3/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk3/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk3/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..f02c3b0ef Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk3/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk4/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk4/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 new file mode 100644 index 000000000..6327af4c6 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk4/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk4/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk4/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..e805d3a21 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk4/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk5/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk5/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 new file mode 100644 index 000000000..012eb6492 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk5/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk5/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk5/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..46639d675 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk5/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk6/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk6/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 new file mode 100644 index 000000000..9fbe2f49f Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk6/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk6/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk6/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..0143036a9 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk6/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk7/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk7/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 new file mode 100644 index 000000000..91513607e Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk7/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk7/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk7/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..b5c1179fa Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk7/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk8/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk8/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 new file mode 100644 index 000000000..f24d32528 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk8/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk8/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk8/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..60a0b7f2b Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk8/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk9/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk9/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 new file mode 100644 index 000000000..17244c5d1 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk9/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1 differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/disk9/erasure-fixtures/non-inline.bin/xl.meta b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk9/erasure-fixtures/non-inline.bin/xl.meta new file mode 100644 index 000000000..eec896664 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/minio/disk9/erasure-fixtures/non-inline.bin/xl.meta differ diff --git a/crates/ecstore/tests/fixtures/erasure-shards/minio/manifest.json b/crates/ecstore/tests/fixtures/erasure-shards/minio/manifest.json new file mode 100644 index 000000000..1b3ffc279 --- /dev/null +++ b/crates/ecstore/tests/fixtures/erasure-shards/minio/manifest.json @@ -0,0 +1,66 @@ +{ + "block_size": 1048576, + "bucket": "erasure-fixtures", + "capture": { + "command": "minio server --address 127.0.0.1:$PORT --console-address 127.0.0.1:0 backend/disk{1..12}", + "disk_count": 12, + "environment": { + "MINIO_CI_CD": "1", + "MINIO_STORAGE_CLASS_STANDARD": "EC:6" + }, + "inline_threshold_per_shard": 131072, + "payload": "SHA256(b'RustFS backlog #2248 E10 pinned fixture v1' + counter.to_bytes(8, 'little')), concatenated and truncated to 786433 bytes", + "producer_get_equals_oracle": true, + "producer_stopped_before_copy": true, + "shutdown": { + "forced_kill_after_seconds": null, + "initial_signal": "SIGTERM", + "process_reaped_before_copy": true, + "returncode": 0 + }, + "storage_class_standard": "EC:6" + }, + "data_shards": 6, + "files": { + "disk1/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1": "8e9d24c2762b5388f44f1f055b2c7abaef4b4ed5dcc8d7f7ef0c276110c3a245", + "disk1/erasure-fixtures/non-inline.bin/xl.meta": "f70f0c6372fa0948fa01d8e67660efc8dd6b0aef044900f11e3880edbacc8f17", + "disk10/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1": "e5aa98934500c67cfb400df62e0559f9e616b9840b1f1f32dc01b7e37d97269b", + "disk10/erasure-fixtures/non-inline.bin/xl.meta": "ea32b1b2b2bd97d4392e3ddef22b39e615e686658bdee62c40a451866da0d0e9", + "disk11/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1": "3a4b00c571f9ab20b883f0f49d4660d0b3ab627a849426a24ae7caf05066128f", + "disk11/erasure-fixtures/non-inline.bin/xl.meta": "935c2c5099259771b74dd0ee50703215c586a819aca56288f0401f7c98ca4643", + "disk12/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1": "1471c1ab6f7e23ba356586b0316d16d43d3a36c6a9044830af8e0120d734d57a", + "disk12/erasure-fixtures/non-inline.bin/xl.meta": "fab8f55f7f963ffd21d450e38cad7c308428a7d8a80796e4f5857c724354e7fc", + "disk2/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1": "bdbb60059c63d689b8ff4d5f2b455fe41c609adacfc85ba8ead7bda775899547", + "disk2/erasure-fixtures/non-inline.bin/xl.meta": "d5b663daaf6792fb2b7fc8371a2dce3b8a36c2afe0d2decdcc7e5a1a272846ca", + "disk3/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1": "cc52eca6c0a5224984dfdad0ef44a228aec021d353248599e9f422e7ed1d23de", + "disk3/erasure-fixtures/non-inline.bin/xl.meta": "88b5d4abab47448edb434ce4929be1854d1803b5b103b10a96f90d7ac66c7ff1", + "disk4/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1": "e557cdf771923e0ff8ac4af951aa2a002b963647cefa547b5a1bf3ab646718fe", + "disk4/erasure-fixtures/non-inline.bin/xl.meta": "f81ca7009b274dcdb58e094175e7ec12e7e2a0937a6b8082e20a8f9dadfaaf48", + "disk5/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1": "be63b337c86fdb99e53496fc3e9bc3657ba1274175f12ea319e03cdde5d0f38d", + "disk5/erasure-fixtures/non-inline.bin/xl.meta": "25865e2195f190af28f50f555b14f66dc05efcf8826e13994ef9830ec6dcae82", + "disk6/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1": "c7ad2137d8d886d5b83139acdb359d5916bb49541bad4febfa555a888f1e1e2d", + "disk6/erasure-fixtures/non-inline.bin/xl.meta": "2df65a8b9bddcac4d136ac2ef96189eecf13e8974c8f76e7cb2f95abd063a7a1", + "disk7/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1": "58ea9a83ede34f0dafe9f1d163dea88b6262b65c31b1da2bcf4c88cfa4e86912", + "disk7/erasure-fixtures/non-inline.bin/xl.meta": "210c11e3f59ff7b3f6cf6e31e59e247dee9b68eed9d8fe7c7bb058f2385b69b8", + "disk8/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1": "1dec3b901b2add4bc143288b8f1a706e1b6b5cad1b5fb0082da280c2aedb829c", + "disk8/erasure-fixtures/non-inline.bin/xl.meta": "c57b57c9d0845c610d31a5d109744d0f2020ac63dfb8209accf38e4db45f3cb0", + "disk9/erasure-fixtures/non-inline.bin/6acc300d-4557-4eb3-8d50-c70923243c2a/part.1": "c261ddd5ec1c2aa77771185b6de547c9bd23af10311096c5f41c17085509c42c", + "disk9/erasure-fixtures/non-inline.bin/xl.meta": "b03bbec380fc0444975466624e239a0be0dbc178b8a0f613cf73f7c935068f9e" + }, + "object": "non-inline.bin", + "parity_shards": 6, + "plaintext_sha256": "b5a83332327961a31b65a8731c64559fd04ff8ac9f6124022201dbbdba5ffa01", + "plaintext_size": 786433, + "schema_version": 1, + "source": { + "archive_sha256": "0939ce5553ce9e6451b69e049fbf399794368276b61da8166f84cbd8c7f2d641", + "archive_url": "https://github.com/minio/minio/releases/download/RELEASE.2025-07-23T15-54-02Z/minio.darwin-arm64.RELEASE.2025-07-23T15-54-02Z", + "binary_sha256": "0939ce5553ce9e6451b69e049fbf399794368276b61da8166f84cbd8c7f2d641", + "binary_version": "minio version RELEASE.2025-07-23T15-54-02Z (commit-id=7ced9663e6a791fef9dc6be798ff24cda9c730ac)\nRuntime: go1.24.5 darwin/arm64\nLicense: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html\nCopyright: 2015-2025 MinIO, Inc.", + "codec": "gf8", + "producer": "MinIO", + "release": "RELEASE.2025-07-23T15-54-02Z", + "source_commit": "7ced9663e6a791fef9dc6be798ff24cda9c730ac", + "uses_legacy": false + } +} diff --git a/crates/ecstore/tests/fixtures/erasure-shards/plaintext.bin b/crates/ecstore/tests/fixtures/erasure-shards/plaintext.bin new file mode 100644 index 000000000..e712350f4 Binary files /dev/null and b/crates/ecstore/tests/fixtures/erasure-shards/plaintext.bin differ diff --git a/crates/ecstore/tests/pinned_erasure_fixtures_test.rs b/crates/ecstore/tests/pinned_erasure_fixtures_test.rs new file mode 100644 index 000000000..cadfc0c26 --- /dev/null +++ b/crates/ecstore/tests/pinned_erasure_fixtures_test.rs @@ -0,0 +1,349 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod storage_api; + +use rustfs_filemeta::{FileInfoOpts, get_file_info}; +use rustfs_utils::HashAlgorithm; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use std::{ + collections::{BTreeMap, HashSet}, + fs, + io::{Cursor, ErrorKind}, + path::PathBuf, + sync::Arc, + time::Duration, +}; +use storage_api::pinned_erasure_fixtures::{ + BitrotReader, BucketOperations, ECStore, Endpoint, EndpointServerPools, Endpoints, Erasure, Error, HTTPRangeSpec, + InstanceContext, MakeBucketOptions, ObjectIO, ObjectOptions, PoolEndpoints, init_bucket_metadata_sys, + init_local_disks_with_instance_ctx, +}; +use tokio::io::AsyncReadExt; +use tokio::time::timeout; +use tokio_util::sync::CancellationToken; + +const PLAINTEXT_SHA256: &str = "b5a83332327961a31b65a8731c64559fd04ff8ac9f6124022201dbbdba5ffa01"; + +#[derive(Deserialize)] +struct Source { + release: String, + source_commit: String, + archive_sha256: String, + binary_sha256: String, + uses_legacy: bool, +} + +#[derive(Deserialize)] +struct Manifest { + schema_version: usize, + source: Source, + bucket: String, + object: String, + data_shards: usize, + parity_shards: usize, + block_size: usize, + plaintext_size: usize, + plaintext_sha256: String, + files: BTreeMap, +} + +struct Fixture { + case: PathBuf, + manifest: Manifest, + slot_paths: Vec, + plaintext: Vec, + shards: Vec>, + uses_legacy: bool, + hash: HashAlgorithm, +} + +fn sha256(bytes: &[u8]) -> String { + rustfs_utils::crypto::hex(Sha256::digest(bytes)) +} + +fn load_fixture(kind: &str) -> Fixture { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/erasure-shards"); + let case = root.join(kind); + let manifest: Manifest = serde_json::from_slice(&fs::read(case.join("manifest.json")).expect("required fixture manifest")) + .expect("parse pinned fixture manifest"); + assert_eq!(manifest.schema_version, 1); + assert_eq!((manifest.data_shards, manifest.parity_shards, manifest.block_size), (6, 6, 1_048_576)); + assert_eq!(manifest.files.len(), 24, "all twelve metadata and external shard files are required"); + let (release, commit, archive, binary, uses_legacy) = match kind { + "minio" => ( + "RELEASE.2025-07-23T15-54-02Z", + "7ced9663e6a791fef9dc6be798ff24cda9c730ac", + "0939ce5553ce9e6451b69e049fbf399794368276b61da8166f84cbd8c7f2d641", + "0939ce5553ce9e6451b69e049fbf399794368276b61da8166f84cbd8c7f2d641", + false, + ), + "legacy" => ( + "1.0.0-alpha.37", + "371119f7336b74fdbcee37a8fcccb9d91c276f47", + "c7d1d5b060fad53ea3c9eac9deb6d1ea20a22d441ec3494cd5a18fe6f7bbb215", + "158b5e0a96c6caf201a72e8c20e417124c190f295160e7c8610bf54228b5292a", + true, + ), + _ => panic!("unknown fixture source"), + }; + assert_eq!(manifest.source.release, release); + assert_eq!(manifest.source.source_commit, commit); + assert_eq!(manifest.source.archive_sha256, archive); + assert_eq!(manifest.source.binary_sha256, binary); + assert_eq!(manifest.source.uses_legacy, uses_legacy); + + let plaintext = fs::read(root.join("plaintext.bin")).expect("required independent plaintext oracle"); + assert_eq!(plaintext.len(), 6 * 128 * 1024 + 1); + assert_eq!(manifest.plaintext_size, plaintext.len()); + assert_eq!(manifest.plaintext_sha256, PLAINTEXT_SHA256); + assert_eq!(sha256(&plaintext), PLAINTEXT_SHA256); + let read_file = |relative: &str| { + let expected_hash = manifest.files.get(relative).expect("file must be pinned in the manifest"); + let bytes = fs::read(case.join(relative)).unwrap_or_else(|error| panic!("required fixture {relative}: {error}")); + assert_eq!(sha256(&bytes), *expected_hash, "fixture SHA256 mismatch: {relative}"); + bytes + }; + let mut shards = vec![None; 12]; + let mut slot_paths = vec![String::new(); 12]; + let mut selected_legacy = None; + for disk in 1..=12 { + let object_path = format!("disk{disk}/{}/{}", manifest.bucket, manifest.object); + let metadata = read_file(&format!("{object_path}/xl.meta")); + let fi = get_file_info( + &metadata, + &manifest.bucket, + &manifest.object, + "", + FileInfoOpts { + data: true, + include_free_versions: false, + include_part_checksums: true, + }, + ) + .expect("decode actual producer xl.meta"); + fi.validate_for_metadata_read() + .expect("producer metadata must describe a valid payload"); + assert!(!fi.inline_data(), "capture must exercise external part files"); + assert!(!fi.is_remote(), "transitioned layout is outside this corpus"); + assert_eq!(fi.uses_legacy_checksum, uses_legacy); + selected_legacy = Some(fi.uses_legacy_checksum); + assert_eq!( + (fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size), + (6, 6, 1_048_576) + ); + assert_eq!(fi.parts.len(), 1); + assert_eq!((fi.parts[0].number, fi.parts[0].size), (1, plaintext.len())); + assert_eq!(fi.erasure.get_checksum_info(1).algorithm, HashAlgorithm::HighwayHash256S); + let data_dir = fi.data_dir.expect("external part data directory"); + let relative = format!("{object_path}/{data_dir}/part.1"); + let part = read_file(&relative); + assert_eq!(part.len(), if uses_legacy { 131_074 + 32 } else { 131_073 + 32 }); + let slot = fi.erasure.index.checked_sub(1).expect("one-based erasure index"); + let target = shards.get_mut(slot).expect("erasure index is within the captured set"); + assert!(target.replace(part).is_none(), "duplicate captured erasure slot"); + slot_paths[slot] = relative; + } + let shards: Vec<_> = shards + .into_iter() + .map(|part| part.expect("all erasure slots are present")) + .collect(); + assert_eq!( + shards.iter().map(|part| sha256(part)).collect::>().len(), + 12, + "shards must be distinguishable" + ); + Fixture { + case, + manifest, + slot_paths, + plaintext, + shards, + uses_legacy: selected_legacy.expect("format selected from real xl.meta"), + hash: if selected_legacy.expect("checksum variant selected from real xl.meta") { + HashAlgorithm::HighwayHash256SLegacy + } else { + HashAlgorithm::HighwayHash256S + }, + } +} + +async fn decode( + fixture: &Fixture, + missing: &[usize], + uses_legacy: bool, + offset: usize, + length: usize, +) -> (Vec, Option) { + let erasure = Erasure::try_new_with_options(6, 6, 1_048_576, uses_legacy).expect("supported captured geometry"); + let readers = fixture + .shards + .iter() + .enumerate() + .map(|(slot, part)| { + (!missing.contains(&slot)) + .then(|| BitrotReader::new(Cursor::new(part.clone()), erasure.shard_size(), fixture.hash.clone(), false)) + }) + .collect(); + let mut plaintext = Vec::new(); + let (written, error) = erasure + .decode(&mut plaintext, readers, offset, length, fixture.plaintext.len()) + .await; + assert_eq!(written, plaintext.len()); + (plaintext, error) +} + +async fn assert_complete_and_reconstructed(kind: &str) { + let fixture = load_fixture(kind); + let erasure = Erasure::try_new_with_options(6, 6, 1_048_576, fixture.uses_legacy).expect("captured codec"); + assert_eq!(erasure.shard_size(), if fixture.uses_legacy { 174_764 } else { 174_763 }); + for missing_data in [false, true] { + let mut dirs = Vec::with_capacity(12); + let mut endpoints = Vec::with_capacity(12); + for disk in 0..12 { + let dir = tempfile::tempdir().expect("isolated fixture disk"); + let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path UTF-8")).expect("local endpoint"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(disk); + dirs.push(dir); + endpoints.push(endpoint); + } + let endpoint_pools = EndpointServerPools(vec![PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 12, + endpoints: Endpoints::from(endpoints), + cmd_line: "pinned-erasure-fixture-test".to_string(), + platform: "test".to_string(), + }]); + let instance = Arc::new(InstanceContext::new()); + init_local_disks_with_instance_ctx(&instance, endpoint_pools.clone()) + .await + .expect("initialize fixture disks"); + let cancel = CancellationToken::new(); + let _cancel_on_drop = cancel.clone().drop_guard(); + let store = ECStore::new_with_instance_ctx( + "127.0.0.1:0".parse().expect("local address"), + endpoint_pools, + cancel.clone(), + instance, + ) + .await + .expect("initialize isolated twelve-drive store"); + init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await; + store + .make_bucket(&fixture.manifest.bucket, &MakeBucketOptions::default()) + .await + .expect("create fixture namespace"); + for relative in fixture.manifest.files.keys() { + let (disk, path) = relative.split_once('/').expect("captured disk path"); + let disk: usize = disk.strip_prefix("disk").expect("disk prefix").parse().expect("disk number"); + let target = dirs[disk - 1].path().join(path); + fs::create_dir_all(target.parent().expect("object directory")).expect("create captured object directory"); + fs::copy(fixture.case.join(relative), target).expect("copy actual producer file without re-encoding"); + } + if missing_data { + let (disk, path) = fixture.slot_paths[0].split_once('/').expect("data shard disk path"); + let disk: usize = disk.strip_prefix("disk").expect("disk prefix").parse().expect("disk number"); + let target = dirs[disk - 1].path().join(path); + fs::remove_file(&target).expect("remove actual data slot zero before the first GET"); + assert!(!target.exists()); + } + for (offset, length) in [ + (0, fixture.plaintext.len()), + (0, 17), + (131_070, 37), + (fixture.plaintext.len() - 47, 47), + ] { + let range = (length != fixture.plaintext.len()).then(|| HTTPRangeSpec { + is_suffix_length: false, + start: offset.try_into().expect("range start"), + end: (offset + length - 1).try_into().expect("range end"), + }); + let mut reader = store + .get_object_reader( + &fixture.manifest.bucket, + &fixture.manifest.object, + range, + Default::default(), + &ObjectOptions::default(), + ) + .await + .unwrap_or_else(|error| { + panic!("production GET dispatch: {kind} missing_data={missing_data} range={offset}+{length}: {error}") + }); + let mut actual = Vec::new(); + reader.read_to_end(&mut actual).await.unwrap_or_else(|error| { + panic!("production GET body: {kind} missing_data={missing_data} range={offset}+{length}: {error}") + }); + assert_eq!( + actual, + fixture.plaintext[offset..offset + length], + "{kind} missing_data={missing_data} range={offset}+{length}" + ); + } + cancel.cancel(); + } +} + +#[tokio::test] +async fn pinned_minio_shards_decode_and_reconstruct_plaintext() { + timeout(Duration::from_secs(30), assert_complete_and_reconstructed("minio")) + .await + .expect("pinned MinIO GET and reconstruction schedule must finish"); +} + +#[tokio::test] +async fn pinned_legacy_shards_decode_and_reconstruct_plaintext() { + timeout(Duration::from_secs(30), assert_complete_and_reconstructed("legacy")) + .await + .expect("pinned legacy GET and reconstruction schedule must finish"); +} + +#[tokio::test] +async fn pinned_erasure_shards_reject_bitrot_corruption() { + for kind in ["minio", "legacy"] { + let fixture = load_fixture(kind); + let erasure = Erasure::try_new_with_options(6, 6, 1_048_576, fixture.uses_legacy).expect("captured codec"); + let mut corrupt = fixture.shards[0].clone(); + corrupt[32] ^= 0x80; + let mut reader = BitrotReader::new(Cursor::new(corrupt), erasure.shard_size(), fixture.hash, false); + let mut output = vec![0xa5; fixture.shards[0].len() - 32]; + let error = reader + .read(&mut output) + .await + .expect_err("real captured shard corruption must fail bitrot verification"); + assert_eq!(error.kind(), ErrorKind::InvalidData, "{kind}: {error}"); + assert!(output.iter().all(|byte| *byte == 0xa5), "unverified bytes must not reach the caller"); + } +} + +#[tokio::test] +async fn pinned_erasure_shards_reject_insufficient_quorum_and_wrong_codec() { + for kind in ["minio", "legacy"] { + let fixture = load_fixture(kind); + let missing = [0, 1, 2, 3, 4, 5, 6]; + let (actual, error) = decode(&fixture, &missing, fixture.uses_legacy, 0, fixture.plaintext.len()).await; + let error = error.expect("five remaining shards cannot satisfy six-data-shard quorum"); + assert_eq!(error.to_string(), Error::ErasureReadQuorum.to_string(), "{kind}"); + assert!(actual.len() < fixture.plaintext.len()); + let (actual, error) = decode(&fixture, &[0], !fixture.uses_legacy, 0, fixture.plaintext.len()).await; + assert!( + error.is_some() || actual != fixture.plaintext, + "the opposite codec cannot reconstruct the pinned plaintext: {kind}" + ); + } +} diff --git a/crates/ecstore/tests/storage_api.rs b/crates/ecstore/tests/storage_api.rs index 531b543d2..b283c5400 100644 --- a/crates/ecstore/tests/storage_api.rs +++ b/crates/ecstore/tests/storage_api.rs @@ -73,3 +73,15 @@ pub(crate) mod fd_relative { pub(crate) use super::{DiskAPI, DiskOption, DiskStore, Endpoint, new_disk}; pub(crate) use rustfs_ecstore::api::disk::{DiskError, LocalPublicationPause, LocalPublicationStage, ReadOptions}; } + +pub(crate) mod pinned_erasure_fixtures { + pub(crate) use super::storage_contracts::{BucketOperations, HTTPRangeSpec, MakeBucketOptions, ObjectIO}; + pub(crate) use super::{ECStore, Endpoint, Erasure, Error, ObjectOptions}; + pub(crate) use rustfs_ecstore::api::{ + bucket::metadata_sys::init_bucket_metadata_sys, + erasure::BitrotReader, + layout::{EndpointServerPools, Endpoints, PoolEndpoints}, + runtime::InstanceContext, + storage::init_local_disks_with_instance_ctx, + }; +} diff --git a/crates/heal/tests/shard_identity_receipt_test.rs b/crates/heal/tests/shard_identity_receipt_test.rs index 4811c0d81..aa0f59a8d 100644 --- a/crates/heal/tests/shard_identity_receipt_test.rs +++ b/crates/heal/tests/shard_identity_receipt_test.rs @@ -21,11 +21,13 @@ use rustfs_heal::heal::{ use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode}; use rustfs_test_utils::TestECStoreEnv; use serial_test::serial; +use std::time::Duration; use tokio::io::AsyncReadExt as _; mod storage_api; use storage_api::integration::{ - DiskAPI, ObjectIO, ObjectOptions, PutObjReader, ReadOptions, ShardIntegrityWriteMode, WriteCompletion, + DiskAPI, NamespaceLocking as _, ObjectIO, ObjectOptions, PutObjReader, ReadOptions, ShardIntegrityWriteMode, + WriteCommitGuard, WriteCompletion, }; #[tokio::test] @@ -132,17 +134,18 @@ async fn receipt_requires_independent_deep_verification() { let object = format!("target-{corrupt_count}"); let donor = format!("donor-{corrupt_count}"); for (name, byte) in [(&object, 0x3c), (&donor, 0xa9)] { + let lock = env.ecstore.new_ns_lock(bucket, name).await.expect("fixture namespace lock"); + let guard = WriteCommitGuard::acquire(&lock, Duration::from_secs(30)) + .await + .expect("fixture write owner"); + let mut opts = ObjectOptions { + shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected), + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }; + opts.add_write_commit_guard(&guard); env.ecstore - .put_object( - bucket, - name, - &mut PutObjReader::from_vec(vec![byte; 1024 * 1024 + 123]), - &ObjectOptions { - shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected), - no_lock: true, - ..Default::default() - }, - ) + .put_object(bucket, name, &mut PutObjReader::from_vec(vec![byte; 1024 * 1024 + 123]), &opts) .await .expect("commit all fixture shards"); } diff --git a/crates/heal/tests/storage_api.rs b/crates/heal/tests/storage_api.rs index 4edf01007..b47737203 100644 --- a/crates/heal/tests/storage_api.rs +++ b/crates/heal/tests/storage_api.rs @@ -24,7 +24,9 @@ pub(crate) mod integration { DiskAPI, DiskError, DiskOption, DiskStore, Endpoint, RUSTFS_META_BUCKET, ReadOptions, new_disk, }; pub(crate) use rustfs_ecstore::api::error::StorageError as EcstoreStorageError; - pub(crate) use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCompletion}; + pub(crate) use rustfs_ecstore::api::object::{ + ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCommitGuard, WriteCompletion, + }; pub(crate) use rustfs_ecstore::api::storage::ECStore; pub(crate) use rustfs_storage_api::BucketOperations; pub(crate) use rustfs_storage_api::MakeBucketOptions; diff --git a/crates/lock/src/namespace/mod.rs b/crates/lock/src/namespace/mod.rs index ec90c5ccc..64238295b 100644 --- a/crates/lock/src/namespace/mod.rs +++ b/crates/lock/src/namespace/mod.rs @@ -57,6 +57,11 @@ impl NamespaceLockWrapper { &self.owner } + /// The exact namespace key used by this acquisition wrapper. + pub fn resource(&self) -> &ObjectKey { + &self.resource + } + /// Logical namespace label for diagnostics; local namespaces may share a manager. pub fn namespace(&self) -> &str { self.lock.namespace() diff --git a/crates/protocols/Cargo.toml b/crates/protocols/Cargo.toml index 24cd74d7f..d75b76297 100644 --- a/crates/protocols/Cargo.toml +++ b/crates/protocols/Cargo.toml @@ -85,7 +85,6 @@ swift = [ "dep:rustfs-ecstore", "dep:rustfs-rio", "dep:axum", - "dep:http", "dep:tower", "dep:regex", "dep:percent-encoding", @@ -106,7 +105,7 @@ swift = [ "dep:base64-simd", "dep:async-compression", ] -webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http", "dep:http-body-util", "dep:tokio-rustls", "dep:base64-simd", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime", "dep:subtle"] +webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http-body-util", "dep:tokio-rustls", "dep:base64-simd", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime", "dep:subtle"] sftp = ["dep:russh", "dep:russh-sftp", "dep:uuid", "dep:subtle", "dep:tokio-util", "dep:socket2"] [dependencies] @@ -132,6 +131,7 @@ thiserror = { workspace = true } serde_json = { workspace = true, features = ["raw_value"] } # Utilities +http = { workspace = true } async-trait = { workspace = true } time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] } bytes = { workspace = true, features = ["serde"] } @@ -149,7 +149,6 @@ rustfs-keystone = { workspace = true, optional = true } rustfs-ecstore = { workspace = true, optional = true } rustfs-rio = { workspace = true, optional = true } axum = { workspace = true, optional = true } -http = { workspace = true, optional = true } tower = { workspace = true, optional = true, features = ["timeout"] } regex = { workspace = true, optional = true } percent-encoding = { workspace = true, optional = true } @@ -183,7 +182,6 @@ subtle = { workspace = true, optional = true } socket2 = { workspace = true, optional = true, features = ["all"] } [dev-dependencies] -http = { workspace = true } tempfile = { workspace = true } proptest = "1" rustfs-test-utils = { workspace = true } diff --git a/crates/protocols/src/common/client/s3.rs b/crates/protocols/src/common/client/s3.rs index 6dda40806..de140ab72 100644 --- a/crates/protocols/src/common/client/s3.rs +++ b/crates/protocols/src/common/client/s3.rs @@ -16,7 +16,7 @@ use async_trait::async_trait; use rustfs_credentials::Credentials; use s3s::dto::*; -#[cfg(feature = "webdav")] +use crate::common::gateway::{AuthorizationError, S3Action}; use crate::common::session::SessionContext; /// Per-bucket capacity inputs for WebDAV quota reporting. @@ -51,6 +51,30 @@ pub struct SessionCapacityView { pub trait StorageBackend: Send + Sync { /// Error type for this storage backend type Error: std::error::Error + Send + Sync + 'static; + /// Authorize a protocol operation against the backend's policy domain. + /// Backends with resource policies must override this identity-policy fallback. + /// Use `authorize_list_objects` for ListObjectsV2 requests. + async fn authorize_operation( + &self, + session: &SessionContext, + action: &S3Action, + bucket: &str, + object: Option<&str>, + ) -> Result<(), AuthorizationError> { + crate::common::gateway::authorize_operation(session, action, bucket, object).await + } + + /// Authorize listing with the exact parameters passed to `list_objects_v2`. + /// Backends with resource or request-condition policies must override this fallback. + async fn authorize_list_objects( + &self, + session: &SessionContext, + input: &ListObjectsV2Input, + ) -> Result<(), AuthorizationError> { + self.authorize_operation(session, &S3Action::ListBucket, &input.bucket, input.prefix.as_deref()) + .await + } + /// Get object content and metadata async fn get_object( &self, diff --git a/crates/protocols/src/common/dummy_storage.rs b/crates/protocols/src/common/dummy_storage.rs index 92707d090..129cd31ba 100644 --- a/crates/protocols/src/common/dummy_storage.rs +++ b/crates/protocols/src/common/dummy_storage.rs @@ -161,6 +161,8 @@ struct Inner { upload_part_copy: VecDeque>, // Observation logs. + list_authorizations: Vec, + list_objects_calls: Vec, abort_multipart_calls: Vec, put_object_calls: Vec, create_multipart_calls: Vec, @@ -218,6 +220,8 @@ impl Inner { complete_multipart_upload: VecDeque::new(), abort_multipart_upload: VecDeque::new(), upload_part_copy: VecDeque::new(), + list_authorizations: Vec::new(), + list_objects_calls: Vec::new(), abort_multipart_calls: Vec::new(), put_object_calls: Vec::new(), create_multipart_calls: Vec::new(), @@ -249,6 +253,7 @@ impl Inner { #[derive(Clone)] pub struct DummyBackend { inner: Arc>, + deny_authorization: bool, } // Drivers whose trait bounds require `Debug` (for example FtpsDriver) cannot be @@ -272,9 +277,23 @@ impl DummyBackend { pub fn new() -> Self { Self { inner: Arc::new(Mutex::new(Inner::new())), + deny_authorization: false, } } + pub fn list_authorizations(&self) -> Vec { + self.inner.lock().expect("list authorization log").list_authorizations.clone() + } + + pub fn list_objects_calls(&self) -> Vec { + self.inner.lock().expect("list call log").list_objects_calls.clone() + } + + pub fn deny_authorization(mut self) -> Self { + self.deny_authorization = true; + self + } + /// Opt in to consuming upload bodies for byte-for-byte protocol tests. pub fn capture_upload_bodies(&self) { self.inner.lock().expect("lock").capture_upload_bodies = true; @@ -302,6 +321,14 @@ impl DummyBackend { // Queue-configuration helpers. Each test stages the responses it // expects in order. The method pops in FIFO order. + pub fn queue_head_bucket_ok(&self) { + self.inner + .lock() + .expect("queue bucket metadata") + .head_bucket + .push_back(Ok(HeadBucketOutput::default())); + } + /// Queue a head_object Ok response with the given size and mtime. pub fn queue_head_object_ok(&self, size: u64, mtime: Option) { let out = HeadObjectOutput { @@ -525,6 +552,15 @@ impl DummyBackend { self.inner.lock().expect("lock").get_object_range.push_back(Err(err)); } + pub fn queue_get_object_bytes(&self, payload: Vec) { + let output = GetObjectOutput { + content_length: Some(i64::try_from(payload.len()).expect("test payload length")), + body: Some(StreamingBlob::from_bytes(Bytes::from(payload))), + ..Default::default() + }; + self.inner.lock().expect("lock").get_object.push_back(Ok(output)); + } + /// Queue a get_object_range Ok response carrying the given bytes as /// the streaming body. content_length is set to bytes.len(). pub fn queue_get_object_range_bytes(&self, payload: Vec) { @@ -649,6 +685,38 @@ impl DummyBackend { impl StorageBackend for DummyBackend { type Error = DummyError; + async fn authorize_operation( + &self, + session: &crate::common::session::SessionContext, + action: &crate::common::gateway::S3Action, + bucket: &str, + object: Option<&str>, + ) -> Result<(), crate::common::gateway::AuthorizationError> { + if self.deny_authorization { + return Err(crate::common::gateway::AuthorizationError::AccessDenied); + } + crate::common::gateway::authorize_operation(session, action, bucket, object).await + } + + async fn authorize_list_objects( + &self, + session: &crate::common::session::SessionContext, + input: &ListObjectsV2Input, + ) -> Result<(), crate::common::gateway::AuthorizationError> { + self.inner + .lock() + .expect("record list authorization") + .list_authorizations + .push(input.clone()); + self.authorize_operation( + session, + &crate::common::gateway::S3Action::ListBucket, + &input.bucket, + input.prefix.as_deref(), + ) + .await + } + async fn get_object( &self, bucket: &str, @@ -746,7 +814,7 @@ impl StorageBackend for DummyBackend { async fn list_objects_v2( &self, - _input: ListObjectsV2Input, + input: ListObjectsV2Input, _credentials: &Credentials, ) -> Result { // Decide control flow while holding the lock. Release before @@ -754,6 +822,7 @@ impl StorageBackend for DummyBackend { // an await point. let (stall, entered, popped) = { let mut inner = self.inner.lock().expect("lock"); + inner.list_objects_calls.push(input); let stall = inner.stall_list_objects_v2; let entered = inner.list_objects_v2_entered.clone(); let popped = if stall { None } else { inner.list_objects_v2.pop_front() }; diff --git a/crates/protocols/src/common/session.rs b/crates/protocols/src/common/session.rs index d1acba1ce..f05959b37 100644 --- a/crates/protocols/src/common/session.rs +++ b/crates/protocols/src/common/session.rs @@ -68,6 +68,10 @@ pub struct SessionContext { pub protocol: Protocol, /// The source IP address pub source_ip: IpAddr, + /// Policy-relevant request headers, excluding authentication secrets. + pub request_headers: http::HeaderMap, + /// Whether the client transport protects both authentication and data. + pub secure_transport: bool, } impl SessionContext { @@ -77,6 +81,8 @@ impl SessionContext { principal, protocol, source_ip, + request_headers: http::HeaderMap::new(), + secure_transport: protocol == Protocol::Sftp, } } diff --git a/crates/protocols/src/ftps/driver.rs b/crates/protocols/src/ftps/driver.rs index f3edc0498..da6e70007 100644 --- a/crates/protocols/src/ftps/driver.rs +++ b/crates/protocols/src/ftps/driver.rs @@ -14,7 +14,6 @@ use crate::common::client::s3::StorageBackend as S3StorageBackend; use crate::common::gateway::S3Action; -use crate::common::gateway::authorize_operation; use async_trait::async_trait; use rustfs_utils::MaskedAccessKey; use rustfs_utils::path; @@ -123,7 +122,11 @@ where session_context: &crate::common::session::SessionContext, ) -> Result as unftp_core::storage::StorageBackend>::Metadata>>> { - match authorize_operation(session_context, &S3Action::ListBuckets, "", None).await { + match self + .storage + .authorize_operation(session_context, &S3Action::ListBuckets, "", None) + .await + { Ok(_) => {} Err(_e) => { return Err(Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied")); @@ -168,10 +171,6 @@ where // SECURITY: s3:DeleteBucket does not imply the right to destroy the // bucket contents. Enumerating and deleting each object are separate // authorization boundaries and must be cleared on their own. - authorize_operation(session_context, &S3Action::ListBucket, bucket, None) - .await - .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?; - // First, delete all objects in the bucket (with pagination) let mut continuation_token = None; loop { @@ -185,12 +184,18 @@ where Error::new(ErrorKind::PermanentFileNotAvailable, format!("Failed to build ListObjectsV2Input: {}", e)) })?; + self.storage + .authorize_list_objects(session_context, &list_input) + .await + .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?; + if let Ok(output) = self.storage.list_objects_v2(list_input, session_context.credentials()).await { // Delete all objects in this page if let Some(objects) = output.contents { for obj in objects { if let Some(obj_key) = obj.key { - authorize_operation(session_context, &S3Action::DeleteObject, bucket, Some(&obj_key)) + self.storage + .authorize_operation(session_context, &S3Action::DeleteObject, bucket, Some(&obj_key)) .await .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?; @@ -247,7 +252,8 @@ where if let Some(key) = key { // Authorize HeadObject - authorize_operation(session_context, &S3Action::HeadObject, &bucket, Some(&key)) + self.storage + .authorize_operation(session_context, &S3Action::HeadObject, &bucket, Some(&key)) .await .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?; @@ -283,7 +289,8 @@ where } else { // Directory metadata - use HeadBucket // Authorize HeadBucket - authorize_operation(session_context, &S3Action::HeadBucket, &bucket, None) + self.storage + .authorize_operation(session_context, &S3Action::HeadBucket, &bucket, None) .await .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?; @@ -331,11 +338,6 @@ where let (bucket, prefix) = parse_s3_path(&path_str) .map_err(|e| Error::new(ErrorKind::PermanentFileNotAvailable, format!("{}: {}", "Invalid path", e)))?; - // Authorize the operation - authorize_operation(session_context, &S3Action::ListBucket, &bucket, prefix.as_deref()) - .await - .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?; - let prefix_with_slash = prefix.clone().map(|p| if p.ends_with('/') { p } else { format!("{}/", p) }); let list_input = ListObjectsV2Input::builder() @@ -347,6 +349,11 @@ where Error::new(ErrorKind::PermanentFileNotAvailable, format!("Failed to build ListObjectsV2Input: {}", e)) })?; + self.storage + .authorize_list_objects(session_context, &list_input) + .await + .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?; + match self.storage.list_objects_v2(list_input, session_context.credentials()).await { Ok(output) => { let mut fileinfos = Vec::new(); @@ -455,7 +462,8 @@ where let key = key.ok_or_else(|| Error::new(ErrorKind::PermanentFileNotAvailable, "Cannot get directory"))?; // Authorize GetObject - authorize_operation(session_context, &S3Action::GetObject, &bucket, Some(&key)) + self.storage + .authorize_operation(session_context, &S3Action::GetObject, &bucket, Some(&key)) .await .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?; @@ -542,7 +550,8 @@ where } // Authorize the operation - authorize_operation(session_context, &S3Action::PutObject, &bucket, Some(&key)) + self.storage + .authorize_operation(session_context, &S3Action::PutObject, &bucket, Some(&key)) .await .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?; @@ -579,7 +588,8 @@ where if let Some(key) = key { // Authorize delete object - authorize_operation(session_context, &S3Action::DeleteObject, &bucket, Some(&key)) + self.storage + .authorize_operation(session_context, &S3Action::DeleteObject, &bucket, Some(&key)) .await .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?; @@ -607,7 +617,8 @@ where // If path ends with '/', treat it as bucket deletion request if path_str.ends_with('/') { // Authorize delete bucket - authorize_operation(session_context, &S3Action::DeleteBucket, &bucket, None) + self.storage + .authorize_operation(session_context, &S3Action::DeleteBucket, &bucket, None) .await .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?; @@ -637,7 +648,8 @@ where // MKD creates a bucket, so it has to clear the same authorization boundary as // an S3 CreateBucket call. - authorize_operation(session_context, &S3Action::CreateBucket, &bucket, None) + self.storage + .authorize_operation(session_context, &S3Action::CreateBucket, &bucket, None) .await .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?; @@ -681,7 +693,8 @@ where .map_err(|e| Error::new(ErrorKind::PermanentFileNotAvailable, format!("{}: {}", "Invalid path", e)))?; // Authorize delete bucket - authorize_operation(session_context, &S3Action::DeleteBucket, &bucket, None) + self.storage + .authorize_operation(session_context, &S3Action::DeleteBucket, &bucket, None) .await .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?; @@ -737,7 +750,8 @@ where .map_err(|e| Error::new(ErrorKind::PermanentFileNotAvailable, format!("{}: {}", "Invalid path", e)))?; // Authorize HeadBucket (CWD probes bucket existence) - authorize_operation(session_context, &S3Action::HeadBucket, &bucket, None) + self.storage + .authorize_operation(session_context, &S3Action::HeadBucket, &bucket, None) .await .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?; @@ -785,6 +799,40 @@ mod tests { use super::parse_s3_path; use rustfs_utils::path; + #[tokio::test] + async fn listing_authorizes_normalized_prefix_and_delimiter() { + use crate::common::dummy_storage::DummyBackend; + use crate::common::gateway::with_test_auth_override; + use crate::common::session::{Protocol, test_session}; + use unftp_core::storage::StorageBackend as _; + let backend = DummyBackend::new(); + let driver = super::FtpsDriver::new(backend.clone()); + let user = super::super::server::FtpsUser { + username: "protocol-user".into(), + name: None, + session_context: test_session(Protocol::Ftps), + }; + with_test_auth_override( + |_, bucket, prefix| bucket == "bucket" && prefix == Some("private/"), + driver.list(&user, "/bucket/private"), + ) + .await + .expect("normalized prefix satisfies conditional allow"); + let authorized = backend.list_authorizations(); + let executed = backend.list_objects_calls(); + assert_eq!(authorized.len(), 1); + assert_eq!(executed.len(), 1); + assert_eq!(authorized[0].prefix.as_deref(), Some("private/")); + assert_eq!(authorized[0].delimiter.as_deref(), Some("/")); + assert_eq!(authorized[0].prefix, executed[0].prefix); + assert_eq!(authorized[0].delimiter, executed[0].delimiter); + assert_eq!(authorized[0].max_keys, executed[0].max_keys); + let denied = + with_test_auth_override(|_, _, prefix| prefix != Some("private/"), driver.list(&user, "/bucket/private")).await; + assert!(denied.is_err(), "matching prefix deny blocks LIST"); + assert_eq!(backend.list_objects_calls().len(), 1, "denied LIST cannot reach storage"); + } + /// GHSA-g3vq-vv42-f647: MKD creates a bucket, so it must clear the /// `s3:CreateBucket` authorization boundary before touching the backend. /// The queued success is what makes this a real guard — an unqueued @@ -816,6 +864,27 @@ mod tests { ); } + #[tokio::test] + async fn backend_policy_denial_blocks_ftp_read_and_bucket_creation() { + use crate::common::dummy_storage::DummyBackend; + use crate::common::gateway::with_test_auth_override; + use crate::common::session::{Protocol, test_session}; + use unftp_core::storage::StorageBackend as _; + let backend = DummyBackend::new().deny_authorization(); + backend.queue_create_bucket_ok(); + backend.queue_get_object_bytes(b"secret content".to_vec()); + let driver = super::FtpsDriver::new(backend); + let user = super::super::server::FtpsUser { + username: "protocol-user".into(), + name: None, + session_context: test_session(Protocol::Ftps), + }; + let result = with_test_auth_override(|_, _, _| true, driver.mkd(&user, "/bucket")).await; + assert!(result.is_err(), "backend policy deny must override identity allow"); + let result = with_test_auth_override(|_, _, _| true, driver.get(&user, "/bucket/secret.txt", 0)).await; + assert!(result.is_err(), "backend policy denial must prevent returning the queued object body"); + } + /// RMD deletes every object in the bucket, so `s3:DeleteBucket` alone must /// not be enough: each object needs its own `s3:DeleteObject` boundary. The /// backend is primed so that the whole recursive delete would succeed if the @@ -894,11 +963,7 @@ mod tests { } mod upload { - use crate::common::{ - client::s3::StorageBackend, - gateway::{S3Action, authorize_operation}, - session::SessionContext, - }; + use crate::common::{client::s3::StorageBackend, gateway::S3Action, session::SessionContext}; use bytes::Bytes; use s3s::dto::*; use std::{ @@ -924,8 +989,15 @@ mod upload { Error::new(ErrorKind::TransientFileNotAvailable, error.to_string()) } - async fn authorize(session: &SessionContext, action: S3Action, bucket: &str, key: &str) -> Result<()> { - authorize_operation(session, &action, bucket, Some(key)) + async fn authorize( + storage: &S, + session: &SessionContext, + action: S3Action, + bucket: &str, + key: &str, + ) -> Result<()> { + storage + .authorize_operation(session, &action, bucket, Some(key)) .await .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied")) } @@ -974,7 +1046,7 @@ mod upload { async fn abort_upload(storage: &S, session: &SessionContext, input: AbortMultipartUploadInput) { let cleanup = async { - authorize(session, S3Action::AbortMultipartUpload, &input.bucket, &input.key).await?; + authorize(storage, session, S3Action::AbortMultipartUpload, &input.bucket, &input.key).await?; storage .abort_multipart_upload(input, session.credentials()) .await @@ -1028,7 +1100,7 @@ mod upload { let mut part = read_part(&mut reader).await?; if part.len() < PART_SIZE { let size = u64::try_from(part.len()).map_err(upload_error)?; - authorize(session, S3Action::PutObject, bucket, key).await?; + authorize(storage.as_ref(), session, S3Action::PutObject, bucket, key).await?; let input = PutObjectInput::builder() .bucket(bucket.to_owned()) .key(key.to_owned()) @@ -1040,7 +1112,7 @@ mod upload { return Ok(size); } - authorize(session, S3Action::CreateMultipartUpload, bucket, key).await?; + authorize(storage.as_ref(), session, S3Action::CreateMultipartUpload, bucket, key).await?; let input = CreateMultipartUploadInput::builder() .bucket(bucket.to_owned()) .key(key.to_owned()) @@ -1075,7 +1147,7 @@ mod upload { total = total .checked_add(u64::try_from(part.len()).map_err(upload_error)?) .ok_or_else(|| upload_error("FTPS upload size overflow"))?; - authorize(session, S3Action::UploadPart, bucket, key).await?; + authorize(storage.as_ref(), session, S3Action::UploadPart, bucket, key).await?; let input = UploadPartInput::builder() .bucket(bucket.to_owned()) .key(key.to_owned()) @@ -1097,7 +1169,7 @@ mod upload { }); part = read_part(&mut reader).await?; } - authorize(session, S3Action::CompleteMultipartUpload, bucket, key).await?; + authorize(storage.as_ref(), session, S3Action::CompleteMultipartUpload, bucket, key).await?; let input = CompleteMultipartUploadInput::builder() .bucket(bucket.to_owned()) .key(key.to_owned()) diff --git a/crates/protocols/src/ftps/server.rs b/crates/protocols/src/ftps/server.rs index c2c00750c..ec1cfe710 100644 --- a/crates/protocols/src/ftps/server.rs +++ b/crates/protocols/src/ftps/server.rs @@ -115,9 +115,11 @@ where let storage_clone = self.storage.clone(); let mut server_builder = libunftp::ServerBuilder::with_user_detail_provider( Box::new(move || FtpsDriver::new(storage_clone.clone())), - Arc::new(FtpsUserDetailProvider), + Arc::new(ConnectionUserDetailProvider), ) - .authenticator(Arc::new(FtpsAuthenticator::new())); + .authenticator(Arc::new(ConnectionAuthenticator { + require_data_tls: self.config.tls_enabled && self.config.ftps_required, + })); // Configure passive ports for data connections if let Some(passive_ports) = &self.config.passive_ports { @@ -379,6 +381,44 @@ impl UserDetailProvider for FtpsUserDetailProvider { } } +// libunftp passes only Principal between authentication and user construction. +// Encode the verified connection attributes in that private handoff so concurrent +// logins for the same account cannot overwrite each other's source address. +#[derive(Debug)] +struct ConnectionAuthenticator { + require_data_tls: bool, +} + +#[async_trait::async_trait] +impl Authenticator for ConnectionAuthenticator { + async fn authenticate(&self, username: &str, creds: &Credentials) -> Result { + let principal = FtpsAuthenticator::new().authenticate(username, creds).await?; + let secure = + self.require_data_tls && matches!(creds.command_channel_security, unftp_core::auth::ChannelEncryptionState::Tls); + let username = serde_json::to_string(&(principal.username, creds.source_ip, secure)).map_err(|error| { + AuthenticationError::ImplPropagated("Connection context unavailable".to_owned(), Some(Box::new(error))) + })?; + Ok(Principal { username }) + } +} + +#[derive(Debug)] +struct ConnectionUserDetailProvider; + +#[async_trait::async_trait] +impl UserDetailProvider for ConnectionUserDetailProvider { + type User = FtpsUser; + + async fn provide_user_detail(&self, principal: &Principal) -> Result { + let (username, source_ip, secure_transport): (String, IpAddr, bool) = serde_json::from_str(&principal.username) + .map_err(|error| UserDetailError::with_source("Invalid connection context", error))?; + let mut user = FtpsUserDetailProvider.provide_user_detail(&Principal { username }).await?; + user.session_context.source_ip = source_ip; + user.session_context.secure_transport = secure_transport; + Ok(user) + } +} + /// FTPS authenticator implementation #[derive(Debug, Default)] pub struct FtpsAuthenticator; diff --git a/crates/protocols/src/sftp/attrs.rs b/crates/protocols/src/sftp/attrs.rs index aac6089f3..330b3a6b9 100644 --- a/crates/protocols/src/sftp/attrs.rs +++ b/crates/protocols/src/sftp/attrs.rs @@ -170,7 +170,6 @@ impl SftpDriver { // No object at this key. Check whether it is a // directory by listing with the key as a prefix. let prefix = format!("{object_key}/"); - self.authorize(&S3Action::ListBucket, &bucket, Some(prefix.as_str())).await?; let input = ListObjectsV2Input::builder() .bucket(bucket.clone()) .prefix(Some(prefix)) @@ -178,6 +177,7 @@ impl SftpDriver { .max_keys(Some(1)) .build() .map_err(|e| s3_error_to_sftp("build_list_objects", e))?; + self.authorize_list_objects(&input).await?; let out = self .run_backend("list_objects_v2", self.storage.list_objects_v2(input, self.credentials())) .await?; diff --git a/crates/protocols/src/sftp/dir.rs b/crates/protocols/src/sftp/dir.rs index 36d137b1d..06ab0d982 100644 --- a/crates/protocols/src/sftp/dir.rs +++ b/crates/protocols/src/sftp/dir.rs @@ -56,6 +56,22 @@ pub(super) fn dot_entries() -> Vec { ] } +fn directory_listing_input( + bucket: &str, + prefix: &str, + continuation: &ListingContinuation, +) -> Result { + let mut builder = ListObjectsV2Input::builder() + .bucket(bucket.to_owned()) + .prefix(Some(prefix.to_owned())) + .delimiter(Some("/".to_owned())) + .max_keys(Some(READDIR_PAGE_MAX_KEYS)); + if let ListingContinuation::Next(token) = continuation { + builder = builder.continuation_token(Some(token.clone())); + } + builder.build().map_err(|e| s3_error_to_sftp("build_list_objects", e)) +} + impl SftpDriver { /// Fetch one S3 ListObjectsV2 page for a Listing cursor, convert it to /// File entries (subdirectories from common_prefixes, objects from @@ -89,17 +105,8 @@ impl SftpDriver { // list_objects_v2 wire call. Matching that means a policy // revoked mid-iteration takes effect on the next page rather // than at session end. - self.authorize(&S3Action::ListBucket, bucket, None).await?; - - let mut builder = ListObjectsV2Input::builder() - .bucket(bucket.clone()) - .prefix(Some(prefix.clone())) - .delimiter(Some("/".to_string())) - .max_keys(Some(READDIR_PAGE_MAX_KEYS)); - if let ListingContinuation::Next(token) = continuation { - builder = builder.continuation_token(Some(token.clone())); - } - let input = builder.build().map_err(|e| s3_error_to_sftp("build_list_objects", e))?; + let input = directory_listing_input(bucket, prefix, continuation)?; + self.authorize_list_objects(&input).await?; let out = self .run_backend("list_objects_v2", self.storage.list_objects_v2(input, self.credentials())) @@ -169,10 +176,6 @@ impl SftpDriver { /// must not fall through to a destructive call when this returns /// Err. pub(super) async fn validate_directory_empty(&self, bucket: &str, prefix: &str) -> Result<(), SftpError> { - let prefix_for_authorization = if prefix.is_empty() { None } else { Some(prefix) }; - self.authorize(&S3Action::ListBucket, bucket, prefix_for_authorization) - .await?; - // For sub-directory prefixes, max_keys=2 because the backend // may return the directory's own __XLDIR__ marker (decoded to // the prefix itself, e.g. "subdir/") as a content entry. @@ -190,6 +193,8 @@ impl SftpDriver { } let input = builder.build().map_err(|e| s3_error_to_sftp("build_list_objects", e))?; + self.authorize_list_objects(&input).await?; + // Issue list_objects_v2. On Err the destructive caller never // runs because validate_directory_empty returns the Err. let out = self @@ -382,12 +387,8 @@ impl SftpDriver { Some(k) if k.ends_with('/') => k.clone(), Some(k) => format!("{k}/"), }; - self.authorize( - &S3Action::ListBucket, - &bucket, - if prefix.is_empty() { None } else { Some(prefix.as_str()) }, - ) - .await?; + let input = directory_listing_input(&bucket, &prefix, &ListingContinuation::Initial)?; + self.authorize_list_objects(&input).await?; self.run_backend("head_bucket", self.storage.head_bucket(&bucket, self.credentials())) .await?; DirCursor::Listing { @@ -452,6 +453,57 @@ mod tests { use tokio::sync::Notify; use tracing::Level; + #[tokio::test] + async fn opendir_and_readdir_authorize_the_same_listing_parameters() { + let backend = Arc::new(DummyBackend::new()); + backend.queue_head_bucket_ok(); + let mut driver = build_driver(backend.clone(), TEST_PART_SIZE); + let handle = with_test_auth_override( + |_, bucket, prefix| bucket == "bucket" && prefix == Some("private/"), + driver.opendir_inner(1, "/bucket/private"), + ) + .await + .expect("OPENDIR conditional allow"); + let mut cursor = DirCursor::Listing { + bucket: "bucket".into(), + prefix: "private/".into(), + continuation: ListingContinuation::Initial, + dots_emitted: true, + }; + with_test_auth_override(|_, _, prefix| prefix == Some("private/"), driver.next_listing_page(&mut cursor)) + .await + .expect("READDIR conditional allow"); + cursor = DirCursor::Listing { + bucket: "bucket".into(), + prefix: "private/".into(), + continuation: ListingContinuation::Next("next-page".into()), + dots_emitted: true, + }; + with_test_auth_override(|_, _, prefix| prefix == Some("private/"), driver.next_listing_page(&mut cursor)) + .await + .expect("continued READDIR conditional allow"); + let authorized = backend.list_authorizations(); + let executed = backend.list_objects_calls(); + assert_eq!(authorized.len(), 3); + assert_eq!(executed.len(), 2); + for input in &authorized { + assert_eq!(input.prefix.as_deref(), Some("private/")); + assert_eq!(input.delimiter.as_deref(), Some("/")); + assert_eq!(input.max_keys, Some(super::READDIR_PAGE_MAX_KEYS)); + } + for (auth, call) in authorized[1..].iter().zip(&executed) { + assert_eq!(auth.prefix, call.prefix); + assert_eq!(auth.delimiter, call.delimiter); + assert_eq!(auth.max_keys, call.max_keys); + assert_eq!(auth.continuation_token, call.continuation_token); + } + assert_eq!(authorized[2].continuation_token.as_deref(), Some("next-page")); + let denied = + with_test_auth_override(|_, _, prefix| prefix != Some("private/"), driver.readdir_inner(2, handle.handle)).await; + assert!(denied.is_err(), "policy change must deny the next READDIR"); + assert_eq!(backend.list_objects_calls().len(), 2, "denied READDIR cannot reach storage"); + } + #[tokio::test] async fn validate_directory_empty_propagates_list_error() { // Safety contract: when the empty-check list_objects_v2 itself diff --git a/crates/protocols/src/sftp/driver.rs b/crates/protocols/src/sftp/driver.rs index dcd5191a9..e6ed150c1 100644 --- a/crates/protocols/src/sftp/driver.rs +++ b/crates/protocols/src/sftp/driver.rs @@ -31,12 +31,12 @@ use super::paths::{parse_s3_path, sanitise_control_bytes}; use super::state::{HandleState, WritePhase}; use super::write::{build_write_tombstone, fstat_reported_size, rejects_excl_or_trunc_without_create, should_abort_on_drop}; use crate::common::client::s3::StorageBackend; -use crate::common::gateway::{AuthorizationError, S3Action, authorize_operation}; +use crate::common::gateway::{AuthorizationError, S3Action}; use crate::common::session::SessionContext; use russh_sftp::protocol::{Attrs, Data, File, FileAttributes, Handle, Name, OpenFlags, Packet, Status, StatusCode, Version}; use rustfs_credentials::Credentials; use rustfs_utils::MaskedAccessKey; -use s3s::dto::{AbortMultipartUploadInput, CopyObjectInput, CopySource}; +use s3s::dto::{AbortMultipartUploadInput, CopyObjectInput, CopySource, ListObjectsV2Input}; use std::collections::HashMap; use std::sync::atomic::AtomicU64; use std::sync::{Arc, LazyLock}; @@ -297,7 +297,23 @@ impl SftpDriver { /// the SSH keepalive closed the transport (~45 s). The deadline /// closes that gap and returns IamUnavailable to the client. pub(super) async fn authorize(&self, action: &S3Action, bucket: &str, key: Option<&str>) -> Result<(), SftpError> { - let auth_fut = authorize_operation(&self.session_context, action, bucket, key); + let auth_fut = self.storage.authorize_operation(&self.session_context, action, bucket, key); + self.await_authorization(action, bucket, key, auth_fut).await + } + + pub(super) async fn authorize_list_objects(&self, input: &ListObjectsV2Input) -> Result<(), SftpError> { + let auth_fut = self.storage.authorize_list_objects(&self.session_context, input); + self.await_authorization(&S3Action::ListBucket, &input.bucket, input.prefix.as_deref(), auth_fut) + .await + } + + async fn await_authorization( + &self, + action: &S3Action, + bucket: &str, + key: Option<&str>, + auth_fut: impl std::future::Future>, + ) -> Result<(), SftpError> { let outcome = match tokio::time::timeout(std::time::Duration::from_secs(self.backend_op_timeout_secs), auth_fut).await { Ok(inner) => inner, Err(_elapsed) => { diff --git a/crates/protocols/src/sftp/read.rs b/crates/protocols/src/sftp/read.rs index 5d366b68d..5ea8e7401 100644 --- a/crates/protocols/src/sftp/read.rs +++ b/crates/protocols/src/sftp/read.rs @@ -111,8 +111,10 @@ impl SftpDriver { let remaining = size - offset; let actual_len = (capped_len as u64).min(remaining); + self.authorize(&S3Action::GetObject, &bucket, Some(&key)).await?; + // Cache-hit fast path. Probe the cache while only borrowing - // the handle table. No backend call, no auth call, no await, + // the handle table. No backend call or await, // so cancellation cannot fire between the probe and the // return. let cached = self.with_handle_ref(&handle, |state| match state { @@ -131,7 +133,6 @@ impl SftpDriver { // smaller than actual_len, or when read_cache_window is the // READ_CACHE_DISABLED sentinel (0), the backend call still // returns the bytes the client requested. - self.authorize(&S3Action::GetObject, &bucket, Some(&key)).await?; let fetch_len = self.read_cache_window.max(actual_len).min(remaining); let window_bytes = self.fetch_object_range(&bucket, &key, offset, fetch_len).await?; @@ -240,6 +241,20 @@ mod tests { use std::time::{Duration, Instant}; use tracing::Level; + #[tokio::test] + async fn cached_read_checks_backend_policy() { + let backend = Arc::new(DummyBackend::new().deny_authorization()); + let mut driver = build_driver_with_read_cache(backend, TEST_PART_SIZE, 4096, 1024 * 1024); + let handle = driver + .allocate_handle(file_handle("bucket", "secret.txt", 4096, FileAttributes::default())) + .expect("allocate read handle"); + driver.try_populate_read_cache(&handle, 0, vec![42; 4096]); + let error = with_test_auth_override(|_, _, _| true, driver.read(1, handle, 0, 1024)) + .await + .expect_err("backend policy denial must prevent cached content disclosure"); + assert!(matches!(StatusCode::from(error), StatusCode::PermissionDenied)); + } + #[tokio::test] async fn read_with_len_zero_returns_bad_message_before_backend_call() { let backend = Arc::new(DummyBackend::new()); diff --git a/crates/protocols/src/webdav/driver.rs b/crates/protocols/src/webdav/driver.rs index 6b682f20a..3ba88ec16 100644 --- a/crates/protocols/src/webdav/driver.rs +++ b/crates/protocols/src/webdav/driver.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::common::client::s3::{SessionCapacityView, StorageBackend as S3StorageBackend}; -use crate::common::gateway::{AuthorizationError, S3Action, authorize_operation}; +use crate::common::gateway::{AuthorizationError, S3Action}; use crate::common::session::SessionContext; use bytes::Bytes; use dav_server::davpath::DavPath; @@ -497,6 +497,9 @@ where /// Attach the request context used by IAM policy conditions. pub fn with_request_context(mut self, request_headers: http::HeaderMap, secure_transport: bool) -> Self { + let session = Arc::make_mut(&mut self.session_context); + session.request_headers = request_headers.clone(); + session.secure_transport = secure_transport; self.request_headers = Some(request_headers); self.secure_transport = secure_transport; self @@ -523,6 +526,11 @@ where .build() .map_err(|_| FsError::GeneralFailure)?; + self.storage + .authorize_list_objects(&self.session_context, &list_input) + .await + .map_err(|_| FsError::Forbidden)?; + let output = self.storage.list_objects_v2(list_input, credentials).await.map_err(|e| { error!( event = EVENT_WEBDAV_LIST_FAILED, @@ -655,7 +663,9 @@ where async fn probe_head_object(&self, bucket: &str, key: &str) -> FsResult { let credentials = self.credentials(); - if authorize_operation(&self.session_context, &S3Action::HeadObject, bucket, Some(key)) + if self + .storage + .authorize_operation(&self.session_context, &S3Action::HeadObject, bucket, Some(key)) .await .is_err() { @@ -700,12 +710,16 @@ where }); } - if size == 0 - && authorize_operation(&self.session_context, &S3Action::ListBucket, bucket, Some(&prefix)) - .await - .is_ok() - && self.prefix_has_entries(bucket, &prefix).await? - { + let has_entries = if size == 0 { + match self.prefix_has_entries(bucket, &prefix).await { + Ok(has_entries) => has_entries, + Err(FsError::Forbidden) => false, + Err(error) => return Err(error), + } + } else { + false + }; + if has_entries { return Ok(ResolvedPath::Directory { prefix, metadata: Some(output), @@ -733,14 +747,11 @@ where HeadObjectProbe::Forbidden => {} } - if authorize_operation(&self.session_context, &S3Action::ListBucket, bucket, Some(&prefix)) - .await - .is_ok() - { - had_visibility = true; - if self.prefix_has_entries(bucket, &prefix).await? { - return Ok(ResolvedPath::Directory { prefix, metadata: None }); - } + match self.prefix_has_entries(bucket, &prefix).await { + Ok(true) => return Ok(ResolvedPath::Directory { prefix, metadata: None }), + Ok(false) => had_visibility = true, + Err(FsError::Forbidden) => {} + Err(error) => return Err(error), } if had_visibility { @@ -784,7 +795,11 @@ where /// List all buckets (for root path) async fn list_buckets(&self) -> FsResult> { - match authorize_operation(&self.session_context, &S3Action::ListBuckets, "", None).await { + match self + .storage + .authorize_operation(&self.session_context, &S3Action::ListBuckets, "", None) + .await + { Ok(()) => { let credentials = self.credentials(); return match self.storage.list_buckets(credentials).await { @@ -864,11 +879,6 @@ where /// List objects in a bucket async fn list_objects(&self, bucket: &str, prefix: Option<&str>) -> FsResult> { - // Authorize the operation - authorize_operation(&self.session_context, &S3Action::ListBucket, bucket, prefix) - .await - .map_err(|_| FsError::Forbidden)?; - let prefix_with_slash = prefix.map(|p| if p.ends_with('/') { p.to_string() } else { format!("{}/", p) }); let list_input = ListObjectsV2Input::builder() @@ -878,6 +888,11 @@ where .build() .map_err(|_| FsError::GeneralFailure)?; + self.storage + .authorize_list_objects(&self.session_context, &list_input) + .await + .map_err(|_| FsError::Forbidden)?; + match self.storage.list_objects_v2(list_input, self.credentials()).await { Ok(output) => { let mut entries = Vec::new(); @@ -1001,10 +1016,6 @@ where // SECURITY: s3:DeleteBucket does not imply the right to destroy the // bucket contents. Enumerating and deleting each object are separate // authorization boundaries and must be cleared on their own. - authorize_operation(&self.session_context, &S3Action::ListBucket, bucket, None) - .await - .map_err(|_| FsError::Forbidden)?; - // First, delete all objects in the bucket (with pagination) let mut continuation_token = None; loop { @@ -1016,12 +1027,18 @@ where let list_input = list_input.build().map_err(|_| FsError::GeneralFailure)?; + self.storage + .authorize_list_objects(&self.session_context, &list_input) + .await + .map_err(|_| FsError::Forbidden)?; + if let Ok(output) = self.storage.list_objects_v2(list_input, self.credentials()).await { // Delete all objects in this page if let Some(objects) = output.contents { for obj in objects { if let Some(obj_key) = obj.key { - authorize_operation(&self.session_context, &S3Action::DeleteObject, bucket, Some(&obj_key)) + self.storage + .authorize_operation(&self.session_context, &S3Action::DeleteObject, bucket, Some(&obj_key)) .await .map_err(|_| FsError::Forbidden)?; @@ -1104,11 +1121,13 @@ where // Check authorization based on operation type if options.write || options.create || options.create_new || options.append { - authorize_operation(&session_context, &S3Action::PutObject, &bucket, Some(&key)) + self.storage + .authorize_operation(&session_context, &S3Action::PutObject, &bucket, Some(&key)) .await .map_err(|_| FsError::Forbidden)?; } else { - authorize_operation(&session_context, &S3Action::GetObject, &bucket, Some(&key)) + self.storage + .authorize_operation(&session_context, &S3Action::GetObject, &bucket, Some(&key)) .await .map_err(|_| FsError::Forbidden)?; } @@ -1209,7 +1228,8 @@ where }; } else { // Get bucket metadata - authorize_operation(&self.session_context, &S3Action::HeadBucket, &bucket, None) + self.storage + .authorize_operation(&self.session_context, &S3Action::HeadBucket, &bucket, None) .await .map_err(|_| FsError::Forbidden)?; @@ -1256,7 +1276,8 @@ where format!("{}/", key_str) }; - authorize_operation(&self.session_context, &S3Action::PutObject, &bucket, Some(&dir_key)) + self.storage + .authorize_operation(&self.session_context, &S3Action::PutObject, &bucket, Some(&dir_key)) .await .map_err(|_| FsError::Forbidden)?; @@ -1303,7 +1324,8 @@ where } // Create bucket - authorize_operation(&self.session_context, &S3Action::CreateBucket, &bucket, None) + self.storage + .authorize_operation(&self.session_context, &S3Action::CreateBucket, &bucket, None) .await .map_err(|_| FsError::Forbidden)?; @@ -1352,7 +1374,8 @@ where format!("{}/", prefix) }; - authorize_operation(&self.session_context, &S3Action::DeleteObject, &bucket, Some(&prefix_with_slash)) + self.storage + .authorize_operation(&self.session_context, &S3Action::DeleteObject, &bucket, Some(&prefix_with_slash)) .await .map_err(|_| FsError::Forbidden)?; @@ -1360,10 +1383,6 @@ where // says nothing about the children stored under it. Enumerating the // prefix and deleting each child are separate authorization // boundaries and must be cleared on their own. - authorize_operation(&self.session_context, &S3Action::ListBucket, &bucket, Some(&prefix_with_slash)) - .await - .map_err(|_| FsError::Forbidden)?; - // List and delete all objects with this prefix let mut continuation_token = None; loop { @@ -1377,11 +1396,22 @@ where let list_input = list_input.build().map_err(|_| FsError::GeneralFailure)?; + self.storage + .authorize_list_objects(&self.session_context, &list_input) + .await + .map_err(|_| FsError::Forbidden)?; + if let Ok(output) = self.storage.list_objects_v2(list_input, self.credentials()).await { if let Some(objects) = output.contents { for obj in objects { if let Some(obj_key) = obj.key { - authorize_operation(&self.session_context, &S3Action::DeleteObject, &bucket, Some(&obj_key)) + self.storage + .authorize_operation( + &self.session_context, + &S3Action::DeleteObject, + &bucket, + Some(&obj_key), + ) .await .map_err(|_| FsError::Forbidden)?; @@ -1409,7 +1439,8 @@ where } // Delete bucket - authorize_operation(&self.session_context, &S3Action::DeleteBucket, &bucket, None) + self.storage + .authorize_operation(&self.session_context, &S3Action::DeleteBucket, &bucket, None) .await .map_err(|_| FsError::Forbidden)?; @@ -1429,7 +1460,8 @@ where let key = key.ok_or(FsError::Forbidden)?; // Authorize delete object - authorize_operation(&self.session_context, &S3Action::DeleteObject, &bucket, Some(&key)) + self.storage + .authorize_operation(&self.session_context, &S3Action::DeleteObject, &bucket, Some(&key)) .await .map_err(|_| FsError::Forbidden)?; @@ -1479,13 +1511,16 @@ where let resolved_src = self.resolve_path(&src_bucket, &src_key).await?; let (src_prefix, include_src_marker) = match resolved_src { ResolvedPath::File(_) => { - authorize_operation(&self.session_context, &S3Action::GetObject, &src_bucket, Some(&src_key)) + self.storage + .authorize_operation(&self.session_context, &S3Action::GetObject, &src_bucket, Some(&src_key)) .await .map_err(|_| FsError::Forbidden)?; - authorize_operation(&self.session_context, &S3Action::PutObject, &dst_bucket, Some(&dst_key)) + self.storage + .authorize_operation(&self.session_context, &S3Action::PutObject, &dst_bucket, Some(&dst_key)) .await .map_err(|_| FsError::Forbidden)?; - authorize_operation(&self.session_context, &S3Action::DeleteObject, &src_bucket, Some(&src_key)) + self.storage + .authorize_operation(&self.session_context, &S3Action::DeleteObject, &src_bucket, Some(&src_key)) .await .map_err(|_| FsError::Forbidden)?; @@ -1532,21 +1567,28 @@ where }; let dst_prefix = format!("{}/", dst_key); - authorize_operation(&self.session_context, &S3Action::ListBucket, &src_bucket, Some(&src_prefix)) + let mut list_input = ListObjectsV2Input::builder() + .bucket(src_bucket.clone()) + .prefix(Some(src_prefix.clone())) + .build() + .map_err(|_| FsError::GeneralFailure)?; + self.storage + .authorize_list_objects(&self.session_context, &list_input) .await .map_err(|_| FsError::Forbidden)?; - - let mut continuation_token: Option = None; let mut renamed_any = false; if include_src_marker { - authorize_operation(&self.session_context, &S3Action::GetObject, &src_bucket, Some(&src_key)) + self.storage + .authorize_operation(&self.session_context, &S3Action::GetObject, &src_bucket, Some(&src_key)) .await .map_err(|_| FsError::Forbidden)?; - authorize_operation(&self.session_context, &S3Action::PutObject, &dst_bucket, Some(&dst_key)) + self.storage + .authorize_operation(&self.session_context, &S3Action::PutObject, &dst_bucket, Some(&dst_key)) .await .map_err(|_| FsError::Forbidden)?; - authorize_operation(&self.session_context, &S3Action::DeleteObject, &src_bucket, Some(&src_key)) + self.storage + .authorize_operation(&self.session_context, &S3Action::DeleteObject, &src_bucket, Some(&src_key)) .await .map_err(|_| FsError::Forbidden)?; @@ -1556,30 +1598,30 @@ where } loop { - let mut list_builder = ListObjectsV2Input::builder() - .bucket(src_bucket.clone()) - .prefix(Some(src_prefix.clone())); + self.storage + .authorize_list_objects(&self.session_context, &list_input) + .await + .map_err(|_| FsError::Forbidden)?; - if let Some(ref token) = continuation_token { - list_builder = list_builder.continuation_token(Some(token.clone())); - } - - let list_input = list_builder.build().map_err(|_| FsError::GeneralFailure)?; - let output = self.storage.list_objects_v2(list_input, credentials).await.map_err(|e| { - error!( - event = EVENT_WEBDAV_RENAME_STATE, - component = LOG_COMPONENT_PROTOCOLS, - subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER, - state = "directory_list_failed", - src_bucket = %src_bucket, - src_prefix = %src_prefix, - dst_bucket = %dst_bucket, - dst_prefix = %dst_prefix, - error = %e, - "WebDAV rename directory listing failed" - ); - FsError::GeneralFailure - })?; + let output = self + .storage + .list_objects_v2(list_input.clone(), credentials) + .await + .map_err(|e| { + error!( + event = EVENT_WEBDAV_RENAME_STATE, + component = LOG_COMPONENT_PROTOCOLS, + subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER, + state = "directory_list_failed", + src_bucket = %src_bucket, + src_prefix = %src_prefix, + dst_bucket = %dst_bucket, + dst_prefix = %dst_prefix, + error = %e, + "WebDAV rename directory listing failed" + ); + FsError::GeneralFailure + })?; let mut page_pairs: Vec<(String, String)> = Vec::new(); if let Some(objects) = output.contents { @@ -1593,13 +1635,16 @@ where if !page_pairs.is_empty() { for (src_obj_key, dst_obj_key) in &page_pairs { - authorize_operation(&self.session_context, &S3Action::GetObject, &src_bucket, Some(src_obj_key)) + self.storage + .authorize_operation(&self.session_context, &S3Action::GetObject, &src_bucket, Some(src_obj_key)) .await .map_err(|_| FsError::Forbidden)?; - authorize_operation(&self.session_context, &S3Action::PutObject, &dst_bucket, Some(dst_obj_key)) + self.storage + .authorize_operation(&self.session_context, &S3Action::PutObject, &dst_bucket, Some(dst_obj_key)) .await .map_err(|_| FsError::Forbidden)?; - authorize_operation(&self.session_context, &S3Action::DeleteObject, &src_bucket, Some(src_obj_key)) + self.storage + .authorize_operation(&self.session_context, &S3Action::DeleteObject, &src_bucket, Some(src_obj_key)) .await .map_err(|_| FsError::Forbidden)?; } @@ -1612,7 +1657,7 @@ where if !output.is_truncated.unwrap_or(false) { break; } - continuation_token = output.next_continuation_token; + list_input.continuation_token = output.next_continuation_token; } if !renamed_any { @@ -1700,6 +1745,45 @@ mod tests { use std::net::{IpAddr, Ipv4Addr}; use std::sync::{Arc, Mutex}; + #[tokio::test] + async fn listing_authorizes_normalized_prefix_and_delimiter() { + let backend = DummyBackend::new(); + let driver = WebDavDriver::new(backend.clone(), Arc::new(test_session(Protocol::WebDav))); + with_test_auth_override( + |_, bucket, prefix| bucket == "bucket" && prefix == Some("private/"), + driver.list_objects("bucket", Some("private")), + ) + .await + .expect("normalized prefix satisfies conditional allow"); + let authorized = backend.list_authorizations(); + let executed = backend.list_objects_calls(); + assert_eq!(authorized.len(), 1); + assert_eq!(executed.len(), 1); + assert_eq!(authorized[0].prefix.as_deref(), Some("private/")); + assert_eq!(authorized[0].delimiter.as_deref(), Some("/")); + assert_eq!(authorized[0].prefix, executed[0].prefix); + assert_eq!(authorized[0].delimiter, executed[0].delimiter); + assert_eq!(authorized[0].max_keys, executed[0].max_keys); + let denied = + with_test_auth_override(|_, _, prefix| prefix != Some("private/"), driver.list_objects("bucket", Some("private"))) + .await; + assert!(matches!(denied, Err(FsError::Forbidden))); + assert_eq!(backend.list_objects_calls().len(), 1, "denied listing cannot reach storage"); + } + + #[tokio::test] + async fn backend_policy_denial_blocks_webdav_delete() { + let backend = DummyBackend::new().deny_authorization(); + backend.queue_delete_object_ok(); + let driver = WebDavDriver::new(backend, Arc::new(test_session(Protocol::WebDav))); + let path = DavPath::new("/bucket/secret.txt").expect("path"); + let result = with_test_auth_override(|_, _, _| true, driver.remove_file(&path)).await; + assert!( + matches!(result, Err(FsError::Forbidden)), + "backend policy deny must override identity allow" + ); + } + #[derive(Clone)] struct DummyStorage; diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index c52a890c8..f94f1c960 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -15,7 +15,7 @@ use super::heal_info::{classify_background_heal_read_error, decode_background_heal_info}; use super::*; use crate::EcstoreResult; -use crate::storage_api::owner::ecstore_hold_namespace_commit; +use crate::storage_api::owner::{WriteCommitGuard, WriteCompletion, ecstore_hold_namespace_commit}; use crate::storage_api::scan::{BucketOperations as _, ObjectIO as _}; use crate::{ DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_CACHE_KEY_FORMAT, DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, @@ -1437,18 +1437,21 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin .await .expect("fixture bucket should be created"); let mut reader = PutObjReader::from_vec(b"first".to_vec()); - store.pools[0].disk_set[0] - .put_object( - &bucket, - "object", - &mut reader, - &ObjectOptions { - no_lock: true, - ..Default::default() - }, - ) - .await - .expect("fixture object should finish its rename fanout"); + { + let set = &store.pools[0].disk_set[0]; + let lock = set.new_ns_lock(&bucket, "object").await.expect("fixture namespace lock"); + let guard = WriteCommitGuard::acquire(&lock, Duration::from_secs(30)) + .await + .expect("fixture write owner"); + let mut opts = ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }; + opts.add_write_commit_guard(&guard); + set.put_object(&bucket, "object", &mut reader, &opts) + .await + .expect("fixture object should finish its rename fanout"); + } crate::scanner_io::record_dirty_usage_bucket(&bucket); let dirty_before = crate::scanner_io::dirty_usage_buckets_for_tests(); let baseline = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) @@ -1509,18 +1512,21 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin let committed_body = b"committed-after-walk"; let mut reader = PutObjReader::from_vec(committed_body.to_vec()); - store.pools[0].disk_set[0] - .put_object( - &bucket, - "object", - &mut reader, - &ObjectOptions { - no_lock: true, - ..Default::default() - }, - ) - .await - .expect("the pending tail must change the physical object before it drains"); + { + let set = &store.pools[0].disk_set[0]; + let lock = set.new_ns_lock(&bucket, "object").await.expect("fixture namespace lock"); + let guard = WriteCommitGuard::acquire(&lock, Duration::from_secs(30)) + .await + .expect("fixture write owner"); + let mut opts = ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }; + opts.add_write_commit_guard(&guard); + set.put_object(&bucket, "object", &mut reader, &opts) + .await + .expect("the pending tail must change the physical object before it drains"); + } assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty_before); drop(pending); let retry_budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default()); diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index 552957fd8..05f51fbe3 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -23,7 +23,7 @@ use crate::scanner_folder::ScannerItem; use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot; use crate::storage_api::owner::{ EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats, - ecstore_hold_namespace_commit, + WriteCommitGuard, WriteCompletion, ecstore_hold_namespace_commit, }; use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _}; use crate::{ @@ -1424,16 +1424,17 @@ async fn pending_put_commit_keeps_scanner_walk_live_without_authoritative_usage( .enumerate() { let mut reader = ScannerPutObjReader::from_vec(body.to_vec()); - store.pools[pool_index].disk_set[0] - .put_object( - &bucket, - object, - &mut reader, - &ScannerObjectOptions { - no_lock: true, - ..Default::default() - }, - ) + let set = &store.pools[pool_index].disk_set[0]; + let lock = set.new_ns_lock(&bucket, object).await.expect("fixture namespace lock"); + let guard = WriteCommitGuard::acquire(&lock, Duration::from_secs(30)) + .await + .expect("fixture write owner"); + let mut opts = ScannerObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }; + opts.add_write_commit_guard(&guard); + set.put_object(&bucket, object, &mut reader, &opts) .await .expect("fixture objects must finish their rename fanouts before scanning"); } diff --git a/crates/scanner/src/scanner_io/tests/service_cohort.rs b/crates/scanner/src/scanner_io/tests/service_cohort.rs index 00ea5324c..deb1da547 100644 --- a/crates/scanner/src/scanner_io/tests/service_cohort.rs +++ b/crates/scanner/src/scanner_io/tests/service_cohort.rs @@ -26,18 +26,20 @@ async fn create_cohort_bucket_objects(store: &ECStore, bucket: &str, objects: us .expect("fixture bucket"); for set in store.all_set_disks() { for index in 0..objects { + let object = format!("object-{index:04}"); let mut reader = ScannerPutObjReader::from_vec(b"cohort".to_vec()); - set.put_object( - bucket, - &format!("object-{index:04}"), - &mut reader, - &ScannerObjectOptions { - no_lock: true, - ..Default::default() - }, - ) - .await - .expect("fixture object and all rename tails should persist"); + let lock = set.new_ns_lock(bucket, &object).await.expect("fixture namespace lock"); + let guard = WriteCommitGuard::acquire(&lock, Duration::from_secs(30)) + .await + .expect("fixture write owner"); + let mut opts = ScannerObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }; + opts.add_write_commit_guard(&guard); + set.put_object(bucket, &object, &mut reader, &opts) + .await + .expect("fixture object and all rename tails should persist"); } } } diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index d1be792b7..cf715f089 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -149,6 +149,8 @@ pub(crate) mod owner { #[cfg(test)] pub(crate) use super::{NativeScannerPauseBacklogWriteFault, SourceCleanupDeleteBarrier}; + #[cfg(test)] + pub(crate) use rustfs_ecstore::api::object::{WriteCommitGuard, WriteCompletion}; #[cfg(test)] pub(crate) use rustfs_ecstore::api::set_disk::get_lock_acquire_timeout as ecstore_get_lock_acquire_timeout; #[cfg(test)] diff --git a/docs/operations/ftps.md b/docs/operations/ftps.md index 38263302c..26a3482e1 100644 --- a/docs/operations/ftps.md +++ b/docs/operations/ftps.md @@ -23,3 +23,23 @@ for denied/failed cleanup, process crashes, or losing the upload ID while upload initiation is in flight. Before completion, received parts do not replace an existing completed object. A lost or failed completion response may have an ambiguous outcome; clients should verify the destination before retrying. + +## Authorization + +FTP and FTPS operations use RustFS's S3 IAM and bucket-policy evaluator. An +applicable bucket-policy Deny blocks an operation even when an identity policy +allows it. Source-IP conditions use the FTP control connection's peer address; +concurrent sessions using the same access key retain their own addresses. + +`aws:SecureTransport` is true when the control connection uses TLS and the +listener requires TLS for data connections. With optional data-channel TLS, +the authorization context conservatively reports false because the driver's +operation interface does not expose the data connection's negotiated state. +HTTP-only attributes such as `UserAgent` and `Referer` are absent and retain +normal policy missing-key semantics. + +Directory listing conditions (`s3:prefix`, `s3:delimiter`, and `s3:max-keys`) use the actual `ListObjectsV2` parameters. Directory prefixes include their trailing `/`. Each listing page checks the current policies. + +Upgrades enforce restrictions that older protocol listeners could bypass. +Previously accepted operations can therefore return permission denied. No +object format or policy migration is required. diff --git a/docs/operations/sftp.md b/docs/operations/sftp.md index ac82bdf36..78a8a225a 100644 --- a/docs/operations/sftp.md +++ b/docs/operations/sftp.md @@ -111,7 +111,11 @@ A session ended by any cancellation path logs `SFTP session cancelled (watchdog Authentication is by password only, verified against RustFS IAM users. Public-key authentication is rejected. Anonymous access is not available. Failed logins are rejected without delay and there is no lockout, so apply rate limiting externally when the listener is exposed to untrusted networks. Accepted logins log `SFTP auth accepted` at info level, rejections log `SFTP auth rejected` at warn level. -Every SFTP operation is authorized against IAM policy before it reaches storage. Policy condition keys (for example `aws:SourceIp`) are not evaluated on the SFTP path; only unconditional Allow and Deny statements take effect. +RustFS authorizes SFTP operations through the same IAM and bucket-policy evaluator used by S3. Applicable bucket-policy Deny statements constrain object operations, including root access. Request conditions include the SSH peer address as `aws:SourceIp` and the encrypted SSH connection as `aws:SecureTransport`. HTTP-only attributes such as `UserAgent` and `Referer` are absent and follow the policy engine's normal missing-key rules. Cached file reads also check authorization. + +Directory listing conditions (`s3:prefix`, `s3:delimiter`, and `s3:max-keys`) use the actual `ListObjectsV2` parameters. Directory prefixes include their trailing `/`. Each listing page checks the current policies. + +Upgrading from versions that checked only IAM with an empty condition context can cause previously accepted operations to return permission denied. Review the applicable identity and bucket policies when diagnosing this change. Object formats, bucket policies, and existing stored data require no migration. | SFTP activity | Required S3 actions | | ------------- | ------------------- | diff --git a/docs/testing/README.md b/docs/testing/README.md index 7495d7f85..208919040 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -21,6 +21,8 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh Every script named above is indexed with status and wiring in [`scripts/README.md`](../../scripts/README.md). Fixed GHSA advisories map to named regression tests in [security-regressions.md](security-regressions.md). +The [ECStore core invariant gates](ecstore-core-invariants.md) map required native commit, rollback, lock, read, multipart and metadata regressions to the existing CI run and distinguish them from process and network fault evidence. + The [scanner checkpoint fixture](scanner-checkpoint-fixture.md) diagnoses retained subtree coverage across budget interruption, persistence, reload, and plan invalidation. The [scanner cache cost profile](scanner-cache-cost.md) separates clone, subtree copy, encoding, and counted save costs without changing production cache behavior. diff --git a/docs/testing/ecstore-core-invariants.md b/docs/testing/ecstore-core-invariants.md new file mode 100644 index 000000000..e4bc865b3 --- /dev/null +++ b/docs/testing/ecstore-core-invariants.md @@ -0,0 +1,160 @@ +# ECStore core regression gates + +The required native tests and pinned fixture hashes live in +`.config/ecstore-required-tests.json`. `Test and Lint` checks that each test +was selected by the existing `ci` run, is not ignored, and has a fresh nonempty +JUnit report. The check does not start another workspace test run. + +## Native invariant coverage + +The names below identify tests in `rustfs-ecstore` unless a different crate is +shown. The manifest contains their complete module paths. + +| Boundary | Required test | Oracle | +|---|---|---| +| Write quorum | `inline_put_direct_commit_accepts_exact_quorum_and_rejects_quorum_minus_one` | W commits; W-1 fails without exposing a fresh object. | +| Metadata rollback | `write_unique_file_info_reverts_metadata_when_write_quorum_fails` | A failed write preserves the previous metadata. | +| Rollback inspection | `rename_rollback_incomplete_preserves_overwrite_data_dirs_and_staging` | Real failed undo records rejected inspection admission and retains incomplete receipts, old shards, backups, and staging across reopen. | +| Closed inspection channel | `rename_rollback_closed_heal_receiver_preserves_recovery_material` | A fresh child closes the real heal receiver; all five rollback faults under both completion policies retain recovery material and record the actual failed send. | +| Cancelled undo | `rename_rollback_incomplete_cancelled_task_is_not_success` | A cancelled undo remains incomplete. | +| Fault schedule | `commit_fault_schedule_keeps_multiple_disk_phases_independent` | Two armed phases remain independently paused and are released by their own handles. | +| Fault checkpoint owner | `commit_fault_schedule_rejects_duplicate_checkpoint_without_disarming_owner` | A duplicate checkpoint cannot replace the current handle. | +| Completion equivalence | `rename_rollback_incomplete_matches_early_ack_and_full_wait_after_reopen` | Both completion policies retain the same recovery residue. | +| Physical undo ownership | `rename_rollback_children_keep_namespace_ownership_after_coordinator_panic` | A coordinator panic cannot retire a running physical undo. | +| Late tail failure | `rename_data_early_ack_post_mutation_tail_error_never_rolls_back_commit` | A late error cannot reverse an acknowledged quorum commit. | +| Control-plane completion | `tail_drained_put_waits_for_tail_and_allows_immediate_cas` | TailDrained waits for the tail before a same-key CAS. | +| Seeded completion pressure | `seeded_put_completion_pressure_drains_every_real_tail` | Four rounds with seed `0x2248_0009` retain target ownership under two real pressure PUTs, check complete bodies and same-key CAS for normal callers, and drain all tails and staging after normal or cancelled callers. | +| Borrowed lock | `no_lock_put_waits_for_rename_tail_under_outer_guard` | A borrowed write does not reacquire the same key and waits for its tail. | +| Borrowed capability | `borrowed_write_context_rejects_wrong_namespace_and_cache_flag` | A different namespace or cache flag cannot manufacture a write owner. | +| Quota read ownership | `quota_snapshot_reuses_same_name_owner_and_rereads_disk_authority` | A same-named write reuses its actual owner without recursive acquisition; stale cache and corrupt disk metadata cannot replace disk authority. | +| Quota snapshot scope | `quota_snapshot_rejects_wrong_scope_incarnation_and_transaction_guard` | A snapshot for another store, bucket, incarnation or guard cannot authorize namespace reuse. | +| Quota lifecycle | `quota_snapshot_rejects_lifecycle_loss_during_read_and_changed_disk_incarnation` | Losing the lifecycle fence during the read or changing the disk incarnation rejects reuse. | +| Marker purge receipt | `quorum_marker_purge_receipt_survives_reopen_with_stale_member` | A purge receipt survives fresh-store reopen with a stale member, remains version-scoped and can be consumed. | +| Borrowed cancellation | `borrowed_tail_drained_put_keeps_owner_after_caller_cancel` and `borrowed_complete_keeps_namespace_owner_after_caller_cancel` | The actual namespace owner survives caller cancellation until physical publication drains. | +| Copy completion | `borrowed_copy_tail_drained_waits_for_rename_tail` | Copy preserves an explicit TailDrained policy after a metadata quorum commits. | +| Pool metadata owner | `pool_meta_cas_context_retains_actual_owner_for_exact_target` and `pool_meta_cas_rejects_released_owner` | CAS retains its real owner and rejects a released guard. | +| Catalog publication | `rustfs`: `commit_backend_borrowed_put_preserves_owner_authorization_and_precondition` | An authorized write under the actual outer owner avoids recursive acquisition, preserves IfAbsent, and leaves the original bytes after conflict. | +| Stale writer | `put_object_no_lock_aborts_after_outer_namespace_lock_loss` | A lost outer fence rejects publication. | +| Cancelled writer | `tail_drained_put_owned_commit_survives_waiter_cancellation` | Cancelling the waiter does not cancel the commit owner. | +| GET reconstruction | `blackbox_get_restores_body_after_one_shard_file_is_removed` | The complete returned body equals the original bytes. | +| Range reconstruction | `blackbox_range_read_restores_exact_slice_with_one_offline_disk` | The requested byte slice is reconstructed exactly. | +| Transitioned Range | `transitioned_compressed_object_range_get_returns_plaintext_slice` | A transitioned compressed object returns plaintext Range bytes. | +| MPU cancellation | `cancelled_complete_keeps_upload_lock_through_tail_cleanup` | Upload lock ownership survives caller cancellation and tail cleanup. | +| MPU abort/complete | `abort_and_complete_linearize_for_plain_sse_and_legacy_layouts` | Abort and complete have one outcome across the supported layouts. | +| MPU delete quorum | `abort_enforces_delete_write_quorum_boundary` | Abort requires the existing delete write quorum. | +| MPU physical publication | `complete_multipart_timeout_keeps_namespace_owner_until_physical_publication` | Timeout cannot retire an active physical publication. | +| Restore failures | `multipart_restore_aborts_every_post_create_failure` | Every injected post-create failure aborts the remote upload. | +| Partial LIST metadata | `rustfs-filemeta`: `resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge` | A partial newer version remains hidden. | +| Metadata observations | `metadata_observation_all_four_slot_states_and_arrival_orders_preserve_reduction` and its companion tests | Pending, absent, corrupt, offline and successful disk slots retain the existing quorum decisions across all four-slot arrival orders. | +| LIST backend parity | `list_objects_shared_corpus_*` | One independent namespace oracle checks actual disk, set, pool, store and index paths, including version visibility, page boundaries, faults and cancellation. | +| LIST logical budget | `list_path_gather_results_counts_common_prefixes_before_page_limit` | Repeated common prefixes consume one page slot while max+1 retains a continuation. | +| Directory-marker metadata | `rustfs-filemeta`: `resolve_directory_marker_and_prefix_preserve_quorum_in_both_orders` | Quorum object metadata survives same-named prefix candidates; a fallback directory still requires its own quorum. | +| MinIO metadata | `rustfs-filemeta`: `parses_real_minio_object_xlmeta` | Independently captured MinIO metadata is readable. | +| Legacy metadata | `rustfs-filemeta`: `test_issue_2288_legacy_xlmeta_compatibility` | The fixed legacy fixture decodes with its expected object fields. | +| Legacy null version | `rustfs-filemeta`: `test_into_fileinfo_reads_legacy_nil_uuid_inline_key` | Legacy nil UUID inline data retains null-version semantics. | +| Corrupt part arrays | `rustfs-filemeta`: `crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics` | CRC-valid inconsistent arrays return an error rather than panic. | +| External erasure shards | `pinned_erasure_fixtures_test`: `pinned_minio_shards_decode_and_reconstruct_plaintext` and `pinned_legacy_shards_decode_and_reconstruct_plaintext` | Public GET dispatch returns the complete independent body and exact Range bytes, including after a real data shard is removed before the first GET. | +| External shard bitrot | `pinned_erasure_fixtures_test`: `pinned_erasure_shards_reject_bitrot_corruption` | Corrupt captured shards fail verification before changing the caller's buffer. | +| External shard quorum and codec | `pinned_erasure_fixtures_test`: `pinned_erasure_shards_reject_insufficient_quorum_and_wrong_codec` | Five shards cannot satisfy the six-shard read quorum; the opposite codec cannot reproduce the oracle. | + +The same manifest retains the S3, Azure and GCS source-contract tests for ODM. +The static LIST keys in `crates/ecstore/tests/fixtures/list_namespace_keys.json` +are also consumed by ODM's `list_through_static_namespace_boundary_matrix`. +The ECStore query replay uses seed `0xec5707`; the ODM matrix retains seed +`0xec5706`. +Fixture provenance is recorded in +[`crates/ecstore/tests/fixtures/minio/README.md`](../../crates/ecstore/tests/fixtures/minio/README.md). +These metadata fixtures do not establish compatibility for an external legacy +erasure-shard corpus. + +The separate [external shard corpus](../../crates/ecstore/tests/fixtures/erasure-shards/README.md) +contains non-inline object files from released MinIO GF8 and historical RustFS +GF16 processes. The manifest and plaintext hashes are required by the same +gate; each integration test validates all captured metadata and shard hashes. +Missing or changed files fail rather than skip. Historical RustFS is upgrade +compatibility evidence, not an independently implemented GF16 codec. These +reads do not establish live MinIO drive-set migration or crash durability. + +## Historical LIST regressions + +Both cases below run in `Test and Lint` through the existing `ci` profile. +`reference_page` consumes the input namespace graph without reading FileMeta +or calling production pagination. Every store, pool and set page is compared +for object keys, common prefixes, continuation and truncation; the complete +walk must contain each visible identity exactly once. + +| Report | Physical input | Required test | +|---|---|---| +| [#7049](https://github.com/rustfs/rustfs/pull/7049) | `a/first`, `a/nested/second`, `a0`, `b/final`; raw and paged limits 1 through 4 exercise a child reaching its limit before a later sibling. | `list_objects_shared_corpus_cross_subdirectory_limit` | +| [#7010](https://github.com/rustfs/rustfs/pull/7010) | Four disks first retain three committed metadata copies, then a different disk goes offline. Two readable copies remain among three online disks; plain and slash-delimited pages use the same visible-namespace oracle. | `list_objects_shared_corpus_offline_corrupt_and_minority_stale` | + +Native negative controls use the unchanged corpus at main `68ad1a1d7bb`. +Removing only the two old #7049 limit guards still passes because the recursive +child's stop result provides another protection. Removing those guards and +ignoring that result in the same stack-draining loop makes raw limit 1 return +`a/first` and `a0` instead of only `a/first`. This tests the historical invariant, +not a literal revert of #7049. Setting the current offline write-quorum +allowance to zero makes the #7010 corpus return `InsufficientReadQuorum` where +the independent oracle requires visible objects. Restoring both production +files passes both tests. These local controls do not establish a concurrent +cross-page snapshot or process-crash safety. + +## Write owner and completion routes + +These routes explain the ownership exercised by the required tests. A raw +`no_lock` flag does not establish a borrowed owner. + +| Caller | Commit owner | Completion | +|---|---|---| +| Ordinary PUT | The writer acquires and retains its namespace guard. | Quorum by default; explicit TailDrained is preserved. | +| CompleteMultipartUpload | Owned by default, or a validated target-key WriteCommitGuard; the upload guard remains retained. | Full tail completion. | +| S3 CopyObject destination | The use case supplies its actual destination WriteCommitGuard to the inner PUT. | Explicit TailDrained. | +| Direct store Copy | Its actual destination owner is propagated to the inner writer. | Preserves the destination options. | +| Restore copy-back | The inner PUT or MPU acquires its own object writer; remote-tuple and bucket fences are separate. | PUT preserves its policy; MPU waits for the full tail. | +| Scanner publication | An exact-target publication capability and scanner scope. | Scanner scope disables early ACK. | +| Decommission target | Its target publication capability and capacity reservation. | Capacity admission disables PUT early ACK; MPU waits for the full tail. | +| Startup metadata migration | Sealed OfflineRecovery mode. | Explicit TailDrained. | + +Publication and OfflineRecovery are separate capabilities. These local owner +rules do not establish durable distributed generation authority. + +## Selection and evidence + +After the normal `ci` test run, inspect its actual selection: + +```bash +cargo nextest list --profile ci --all --exclude e2e_test --message-format json > core-test-listing.json +python3 scripts/check_test_wiring.py --check-core core-test-listing.json +``` + +`python3 scripts/check_test_wiring.py --self-test` exercises missing, ignored, +filtered and malformed listing cases, and missing or altered required fixtures. +Those cases must fail; an optional legacy fixture test that returns early is +not evidence for a required fixture. + +The `test-and-lint` artifact contains the native log, listing, JUnit and +execution receipt. A native test that reopens a disk in the same process does +not establish process-crash or power-loss durability. + +The closed-receiver child uses the existing rollback matrix and is reaped on +normal exit. Its harness has no independent total deadline; a hang is not +covered by the passing matrix. + +## Process and network faults + +Use the existing `e2e-full`, `e2e-nightly` and `e2e-distributed` lanes for real +server-process restart, fresh-drive and network-fault scenarios. Their +membership remains pinned by `.config/e2e-*-selection.txt`; the lane checks +selection before execution. See [distributed-e2e.md](distributed-e2e.md) for +the named scenarios, platform requirements and failure artifacts. + +The protocol membership includes WebDAV quota reporting from #8197: 19 tests +on Linux and 17 on macOS, where two SFTP compliance cases are Linux-only. +The updated pins retain every previously selected test. + +Record the source commit, selected test names, actual fault reached, process +logs, metadata observations and JUnit for each run. A capability early return, +setup failure, missing report or abstract protocol model cannot establish +that the fault ran. Native ownership tests do not prove the durable ordered +authority described in +[unified-object-generation.md](../architecture/unified-object-generation.md). diff --git a/docs/testing/ecstore-validation-suite-design.md b/docs/testing/ecstore-validation-suite-design.md index ba8536db3..59f99e5d6 100644 --- a/docs/testing/ecstore-validation-suite-design.md +++ b/docs/testing/ecstore-validation-suite-design.md @@ -70,6 +70,8 @@ The object tests reuse `rename_fanout_barrier::arm(object, disk_slot, phase)` an Select these checks with `cargo nextest list -p rustfs-ecstore --features test-util -E 'test(tail_drained_put) | test(early_ack_tail_drain) | test(no_lock_put_waits_for_rename_tail) | test(object_transaction_fencing_persists_epoch_on_multipart_commit) | test(transition_and_restore_reclaim) | test(decommission_durable_ilm_receipt_pagination) | test(dispatch_completion_cas)'`, then run the same expression under the default and CI profiles without retries. Remaining crash, reopen, rollback, and lock-loss schedules use the existing domain tests; this completion fixture is not a replacement for those checks. +`seeded_put_completion_pressure_drains_every_real_tail` uses seed `0x2248_0009` for four finite rounds: Quorum and TailDrained, each with a normal or cancelled caller. Every round parks the target and two non-inline pressure PUTs on a real four-disk set, observes their three-disk metadata quorum, and releases their rename phases in a seeded order. It checks the target's namespace owner, quorum versus tail ACK, cancellation handoff, exact bodies, immediate same-key CAS, zero remaining fan-out tasks, physical mutation-lease release, and staging drain. Select this one test with `cargo nextest list --locked -p rustfs-ecstore --lib -E 'test(=set_disk::ops::object::put_object_tmp_cleanup_tests::seeded_put_completion_pressure_drains_every_real_tail)'`; run the same expression with `cargo nextest run --profile default --retries 0`, then `--profile ci --retries 0`. Both profiles must select and pass one test; the test performs all four rounds without environment overrides or sleeps. + ### Coverage gate `full` and `destructive` run `cargo llvm-cov -p rustfs-ecstore --lib` and fail when line coverage of the gate scope is below `--unit-coverage-min`. The default minimum and the 100% target for EC read, write, decode, heal, metadata-quorum, and rollback paths are the `UNIT_COVERAGE_*` constants at the top of the runner. `cargo-llvm-cov` must be installed unless `--skip-coverage` is passed explicitly. The default scope `ec-critical` is: diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 38f2828fa..09e668fa7 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -7358,32 +7358,36 @@ where { let store = current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; - with_config_object_write_lock(store, SITE_REPLICATION_JOIN_ADMISSION_LOCK_PATH.to_string(), move || async move { - let fresh = load_site_replication_state().await?; - let fresh_local_peer = local_peer_at_endpoint(local_endpoint.clone(), &fresh); - if join_request_is_superseded(&fresh, join_req.updated_at) { - let peer = fresh - .peers - .get(&fresh_local_peer.deployment_id) - .cloned() - .unwrap_or(fresh_local_peer); - return Ok(PeerJoinOutcome::Superseded(peer)); - } - - apply_iam(join_req.clone()).await?; - - let incoming_updated_at = join_req.updated_at; - update_site_replication_state_when_changed(move |state| { - let local_peer = local_peer_at_endpoint(local_endpoint, state); - if join_request_is_superseded(state, incoming_updated_at) { - let peer = state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer); - return Ok(StateCommit::Unchanged(PeerJoinOutcome::Superseded(peer))); + with_config_object_write_lock( + store, + SITE_REPLICATION_JOIN_ADMISSION_LOCK_PATH.to_string(), + move |_write_guard| async move { + let fresh = load_site_replication_state().await?; + let fresh_local_peer = local_peer_at_endpoint(local_endpoint.clone(), &fresh); + if join_request_is_superseded(&fresh, join_req.updated_at) { + let peer = fresh + .peers + .get(&fresh_local_peer.deployment_id) + .cloned() + .unwrap_or(fresh_local_peer); + return Ok(PeerJoinOutcome::Superseded(peer)); } - apply_peer_join(state, &local_peer, join_req, defer_sync_state_enable); - Ok(StateCommit::Changed(PeerJoinOutcome::Applied(Box::new(state.clone()), local_peer))) - }) - .await - }) + + apply_iam(join_req.clone()).await?; + + let incoming_updated_at = join_req.updated_at; + update_site_replication_state_when_changed(move |state| { + let local_peer = local_peer_at_endpoint(local_endpoint, state); + if join_request_is_superseded(state, incoming_updated_at) { + let peer = state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer); + return Ok(StateCommit::Unchanged(PeerJoinOutcome::Superseded(peer))); + } + apply_peer_join(state, &local_peer, join_req, defer_sync_state_enable); + Ok(StateCommit::Changed(PeerJoinOutcome::Applied(Box::new(state.clone()), local_peer))) + }) + .await + }, + ) .await .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("lock site replication join admission failed: {e}")))? } diff --git a/rustfs/src/admin/handlers/table_catalog/mod.rs b/rustfs/src/admin/handlers/table_catalog/mod.rs index 144319a1c..37c06d295 100644 --- a/rustfs/src/admin/handlers/table_catalog/mod.rs +++ b/rustfs/src/admin/handlers/table_catalog/mod.rs @@ -1726,8 +1726,18 @@ where object: &str, data: Vec, precondition: crate::table_catalog::TableCatalogPutPrecondition, + write_guards: Vec, ) -> crate::table_catalog::TableCatalogStoreResult<()> { - self.put_object(bucket, object, data, precondition).await + self.authorize(bucket, object, S3Action::PutObjectAction).await?; + self.ensure_observation_allowed(bucket, object)?; + let observation = TableCommitObservedObject { + identity: TableCommitObjectIdentity::ContentSha256(hex_sha256(&data, str::to_string)), + max_size: None, + }; + self.backend + .put_object_unlocked(bucket, object, data, precondition, write_guards) + .await?; + self.record_observation(bucket, object, observation) } async fn delete_object(&self, bucket: &str, object: &str) -> crate::table_catalog::TableCatalogStoreResult<()> { diff --git a/rustfs/src/admin/handlers/table_catalog/tests.rs b/rustfs/src/admin/handlers/table_catalog/tests.rs index 00509c58f..2cab5a7b2 100644 --- a/rustfs/src/admin/handlers/table_catalog/tests.rs +++ b/rustfs/src/admin/handlers/table_catalog/tests.rs @@ -87,6 +87,80 @@ async fn call_load_table_handler(request: S3Request) -> S3Result(()) }) @@ -1154,9 +1154,9 @@ mod tests { let transaction_order = transaction_order.clone(); tokio::spawn(async move { let transaction_store = store.clone(); - let transaction = with_admin_server_config_write_lock(store, move || async move { + let transaction = with_admin_server_config_write_lock(store, move |write_guard| async move { let _ = oidc_entered_tx.send(()); - let mut config = read_admin_config_without_migrate_no_lock(transaction_store.clone()).await?; + let mut config = read_admin_config_without_migrate_no_lock(transaction_store.clone(), &write_guard).await?; transaction_order.lock().expect("transaction order lock").push("oidc-read"); let mut provider = KVS::new(); @@ -1170,7 +1170,7 @@ mod tests { .entry(IDENTITY_OPENID_SUB_SYS.to_string()) .or_default() .insert("concurrent-oidc".to_string(), provider); - save_admin_server_config_no_lock(transaction_store, &config).await?; + save_admin_server_config_no_lock(transaction_store, &config, &write_guard).await?; transaction_order.lock().expect("transaction order lock").push("oidc-save"); Ok::<(), StorageError>(()) }); @@ -1449,9 +1449,9 @@ mod tests { let (writer_entered_tx, mut writer_entered_rx) = tokio::sync::oneshot::channel(); let writer = tokio::spawn(async move { let transaction_store = writer_store.clone(); - let transaction = with_admin_server_config_write_lock(writer_store, move || async move { + let transaction = with_admin_server_config_write_lock(writer_store, move |write_guard| async move { writer_entered_tx.send(()).expect("signal writer entry"); - save_admin_server_config_no_lock(transaction_store, &latest).await + save_admin_server_config_no_lock(transaction_store, &latest, &write_guard).await }); tokio::pin!(transaction); let mut writer_polled_tx = Some(writer_polled_tx); diff --git a/rustfs/src/admin/service/site_replication.rs b/rustfs/src/admin/service/site_replication.rs index 3f47b584e..f8504b134 100644 --- a/rustfs/src/admin/service/site_replication.rs +++ b/rustfs/src/admin/service/site_replication.rs @@ -118,13 +118,13 @@ pub async fn reload_site_replication_runtime_state_for_context(context: Option<& // fan-out cannot overwrite a concurrent state writer. IO must be the // no-lock variants — the boundary already holds the object lock. let lock_store = store.clone(); - with_site_replication_state_lock_on(lock_store, move || async move { + with_site_replication_state_lock_on(lock_store, move |write_guard| async move { match read_config_no_lock(store.clone(), SITE_REPLICATION_STATE_PATH).await { Ok(data) => { if let Some(normalized) = normalize_site_replication_state_json(&data) .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e))? { - save_config_no_lock(store, SITE_REPLICATION_STATE_PATH, normalized) + save_config_no_lock(store, SITE_REPLICATION_STATE_PATH, normalized, &write_guard) .await .map_err(|e| { S3Error::with_message( diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index beade7fc2..3f56bcb2d 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -826,8 +826,11 @@ pub(crate) async fn read_existing_admin_server_config_no_lock(api: Arc) } #[cfg(test)] -pub(crate) async fn read_admin_config_without_migrate_no_lock(api: Arc) -> Result { - ecstore_config::com::read_config_without_migrate_no_lock(api).await +pub(crate) async fn read_admin_config_without_migrate_no_lock( + api: Arc, + guard: &crate::storage::storage_api::ecstore_object::WriteCommitGuard, +) -> Result { + ecstore_config::com::read_config_without_migrate_no_lock(api, guard).await } pub(crate) type AdminServerConfigSnapshot = ecstore_config::com::ServerConfigSnapshot; @@ -850,14 +853,15 @@ pub(crate) async fn save_admin_server_config(api: Arc, cfg: &rustfs_con pub(crate) async fn save_admin_server_config_no_lock( api: Arc, cfg: &rustfs_config::server_config::Config, + guard: &crate::storage::storage_api::ecstore_object::WriteCommitGuard, ) -> Result<()> { - ecstore_config::com::save_server_config_no_lock(api, cfg).await + ecstore_config::com::save_server_config_no_lock(api, cfg, guard).await } #[cfg(test)] pub(crate) async fn with_admin_server_config_write_lock(api: Arc, operation: F) -> Result where - F: FnOnce() -> Fut + Send + 'static, + F: FnOnce(crate::storage::storage_api::ecstore_object::WriteCommitGuard) -> Fut + Send + 'static, Fut: std::future::Future + Send + 'static, T: Send + 'static, { @@ -1033,6 +1037,7 @@ pub(crate) mod metrics { pub(crate) mod object { pub(crate) use crate::storage::storage_api::StorageObjectOptions; + pub(crate) use crate::storage::storage_api::ecstore_object::WriteCommitGuard; } pub(crate) mod rebalance { diff --git a/rustfs/src/app/object/copy.rs b/rustfs/src/app/object/copy.rs index 4ce0774ff..e674375cd 100644 --- a/rustfs/src/app/object/copy.rs +++ b/rustfs/src/app/object/copy.rs @@ -96,13 +96,17 @@ fn copy_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err } } -async fn acquire_self_copy_namespace_lock(store: &S, bucket: &str, object: &str) -> S3Result +async fn acquire_self_copy_namespace_lock( + store: &S, + bucket: &str, + object: &str, +) -> S3Result where S: NamespaceLocking + ?Sized, { let object = encode_dir_object(object); let lock = store.new_ns_lock(bucket, &object).await.map_err(ApiError::from)?; - lock.get_write_lock(get_lock_acquire_timeout()) + crate::app::storage_api::object_usecase::WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout()) .await .map_err(|err| ApiError::from(copy_namespace_lock_error(bucket, &object, "write", err)).into()) } @@ -420,7 +424,8 @@ impl DefaultObjectUsecase { None }; if let Some(guard) = _self_copy_lock_guard.as_ref() { - dst_opts.add_namespace_lock_guard(guard); + dst_opts.add_write_commit_guard(guard); + dst_opts.write_completion = crate::app::storage_api::object_usecase::WriteCompletion::TailDrained; } dst_opts.expected_current_version_id = expected_current_version_id.clone(); @@ -1098,7 +1103,7 @@ mod tests { let payload = b"object whose key equals its bucket".to_vec(); let mut reader = PutObjReader::from_vec(payload.clone()); let setup_opts = ObjectOptions { - no_lock: true, + write_completion: crate::app::storage_api::object_usecase::WriteCompletion::TailDrained, ..Default::default() }; store diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 23e63777b..4e15fc1e6 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -1181,6 +1181,7 @@ pub(crate) mod bucket_usecase { pub(crate) mod object_usecase { pub(crate) use super::storage_contracts::BUCKET_LIFECYCLE_LOCK_OBJECT; + pub(crate) use crate::storage::storage_api::ecstore_object::{WriteCommitGuard, WriteCompletion}; pub(crate) mod object_cache { #[cfg(test)] diff --git a/rustfs/src/protocols/client.rs b/rustfs/src/protocols/client.rs index ef0246f2e..abbcbb65f 100644 --- a/rustfs/src/protocols/client.rs +++ b/rustfs/src/protocols/client.rs @@ -15,19 +15,17 @@ use crate::runtime_sources::current_action_credentials; #[cfg(feature = "webdav")] use crate::runtime_sources::current_object_store_handle; -#[cfg(feature = "webdav")] use crate::shared_types::RemoteAddr; #[cfg(feature = "webdav")] use crate::storage_api::protocols::client::capacity; -use crate::storage_api::protocols::client::{FS, ReqInfo, RequestContext}; +use crate::storage_api::protocols::client::{FS, ReqInfo, RequestContext, authorize_request}; use http::{HeaderMap, Method}; use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode}; use rustfs_credentials; -#[cfg(feature = "webdav")] use rustfs_protocols::common::SessionContext; #[cfg(feature = "webdav")] use rustfs_protocols::common::client::s3::{BucketCapacity, SessionCapacityView}; -#[cfg(feature = "webdav")] +use rustfs_protocols::common::gateway::{AuthorizationError, S3Action, is_operation_supported}; use rustfs_trusted_proxies::ClientInfo; use rustfs_utils::MaskedAccessKey; use s3s::dto::*; @@ -211,6 +209,22 @@ fn build_object_uri(bucket: &str, key: &str, query: &[(&str, Option<&str>)]) -> parse_protocol_uri(uri, format!("bucket={bucket} key={key}")) } +fn build_list_objects_uri(input: &ListObjectsV2Input) -> S3Result { + let max_keys = input.max_keys.map(|value| value.to_string()); + let query: Vec<_> = [ + ("list-type", Some("2")), + ("prefix", input.prefix.as_deref()), + ("delimiter", input.delimiter.as_deref()), + ("max-keys", max_keys.as_deref()), + ("continuation-token", input.continuation_token.as_deref()), + ("start-after", input.start_after.as_deref()), + ] + .into_iter() + .filter(|(_, value)| value.is_some()) + .collect(); + build_bucket_uri(&input.bucket, &query) +} + /// Request parameters for creating S3 requests #[derive(Debug)] struct RequestParams<'a> { @@ -232,6 +246,52 @@ impl ProtocolStorageClient { Self { fs } } + async fn authorize_protocol_request( + &self, + session: &SessionContext, + action: &S3Action, + bucket: &str, + object: Option<&str>, + listing: Option<&ListObjectsV2Input>, + ) -> S3Result<()> { + let context = self.fs.server_ctx().app_context(); + let (credentials, is_owner) = + crate::auth::check_key_valid_with_context("", session.access_key(), context.as_deref()).await?; + let uri = if let Some(input) = listing { + build_list_objects_uri(input)? + } else if let Some(key) = object { + build_object_uri(bucket, key, &[])? + } else { + build_bucket_uri(bucket, &[])? + }; + let mut request = Self::create_request( + (), + Method::GET, + uri, + RequestParams { + bucket: Some(bucket.to_owned()), + object: object.map(str::to_owned), + credentials: &credentials, + }, + )?; + request.extensions.insert(self.fs.server_ctx().clone()); + if let Some(info) = request.extensions.get_mut::() { + info.is_owner = is_owner; + } + let remote_addr = std::net::SocketAddr::new(session.source_ip, 0); + request.extensions.insert(Some(RemoteAddr(remote_addr))); + let mut client_info = ClientInfo::direct(remote_addr); + client_info.forwarded_proto = Some(if session.secure_transport { "https" } else { "http" }.to_owned()); + request.extensions.insert(client_info); + request.headers = session.request_headers.clone(); + let policy_action = if matches!(action, S3Action::HeadBucket) { + S3Action::ListBucket.into() + } else { + action.clone().into() + }; + authorize_request(&mut request, policy_action).await + } + /// Create a proper S3Request with ReqInfo extension for authorization fn create_request(input: T, method: Method, uri: http::Uri, params: RequestParams<'_>) -> S3Result> { let mut extensions = http::Extensions::default(); @@ -278,6 +338,41 @@ impl ProtocolStorageClient { impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageClient { type Error = s3s::S3Error; + async fn authorize_operation( + &self, + session: &SessionContext, + action: &S3Action, + bucket: &str, + object: Option<&str>, + ) -> Result<(), AuthorizationError> { + // ListBucket requires the complete listing input to evaluate request conditions. + if matches!(action, S3Action::ListBucket) || !is_operation_supported(session.protocol, action) { + return Err(AuthorizationError::AccessDenied); + } + self.authorize_protocol_request(session, action, bucket, object, None) + .await + .map_err(|error| match error.code() { + s3s::S3ErrorCode::AccessDenied | s3s::S3ErrorCode::InvalidAccessKeyId => AuthorizationError::AccessDenied, + _ => AuthorizationError::IamUnavailable, + }) + } + + async fn authorize_list_objects( + &self, + session: &SessionContext, + input: &ListObjectsV2Input, + ) -> Result<(), AuthorizationError> { + if !is_operation_supported(session.protocol, &S3Action::ListBucket) { + return Err(AuthorizationError::AccessDenied); + } + self.authorize_protocol_request(session, &S3Action::ListBucket, &input.bucket, None, Some(input)) + .await + .map_err(|error| match error.code() { + s3s::S3ErrorCode::AccessDenied | s3s::S3ErrorCode::InvalidAccessKeyId => AuthorizationError::AccessDenied, + _ => AuthorizationError::IamUnavailable, + }) + } + async fn get_object( &self, bucket: &str, @@ -483,7 +578,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli trace_protocol_request("list_objects_v2", Some(&input.bucket), None); let bucket = input.bucket.clone(); - let uri = build_bucket_uri(&bucket, &[("list-type", Some("2"))])?; + let uri = build_list_objects_uri(&input)?; let req = Self::create_request( input, Method::GET, diff --git a/rustfs/src/protocols/mod.rs b/rustfs/src/protocols/mod.rs index c072afd3b..22cf43bf8 100644 --- a/rustfs/src/protocols/mod.rs +++ b/rustfs/src/protocols/mod.rs @@ -15,3 +15,6 @@ pub mod client; pub use client::ProtocolStorageClient; + +#[cfg(test)] +mod policy_test; diff --git a/rustfs/src/protocols/policy_test.rs b/rustfs/src/protocols/policy_test.rs new file mode 100644 index 000000000..5ebeb1643 --- /dev/null +++ b/rustfs/src/protocols/policy_test.rs @@ -0,0 +1,475 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Regression coverage for policy enforcement at the protocol/backend boundary. + +use super::client::ProtocolStorageClient; +use crate::runtime_sources::{AppContext, IamInterface, KmsInterface, ServerContextSlot}; +use crate::storage_api::protocols::client::{BucketOperations, FS, ListObjectsV2Input, MakeBucketOptions}; +use rustfs_credentials::Credentials; +use rustfs_iam::{ + store::{ + Store, UserType, + object::{IAM_CONFIG_PREFIX, ObjectStore}, + }, + sys::IamSys, +}; +use rustfs_kms::KmsServiceManager; +use rustfs_protocols::common::{ + client::s3::StorageBackend, + gateway::{AuthorizationError, S3Action}, + session::{Protocol, ProtocolPrincipal, SessionContext}, +}; +use serde_json::json; +use std::sync::Arc; + +struct TestIam(Arc>); +impl IamInterface for TestIam { + fn handle(&self) -> Arc> { + self.0.clone() + } + fn is_ready(&self) -> bool { + true + } +} +struct TestKms; +impl KmsInterface for TestKms { + fn handle(&self) -> Arc { + Arc::new(KmsServiceManager::new()) + } +} + +fn decode_policy(value: serde_json::Value) -> Result { + serde_json::from_str(&value.to_string()) +} + +#[test] +#[serial_test::serial] +fn protocol_policy_denials_and_request_conditions() { + crate::app::gating_test_env::run_large_stack_test("protocol-policy-authorization", || async { + let (_temp, _paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await; + ObjectStore::new(store.clone()) + .save_iam_config(json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX)) + .await + .expect("seed IAM format"); + let iam = rustfs_iam::init_iam_sys_for_context(store.clone()) + .await + .expect("initialize IAM"); + let root = Credentials { + access_key: "protocol-root".into(), + secret_key: "protocol-root-secret".into(), + status: "on".into(), + ..Default::default() + }; + let context = Arc::new(AppContext::new(store.clone(), Arc::new(TestIam(iam.clone())), Arc::new(TestKms))); + assert!(context.publish_action_credentials(root.clone())); + let slot = ServerContextSlot::new(); + assert!(slot.install(context)); + let backend = ProtocolStorageClient::new(FS::with_server_ctx(slot)); + let bucket = "protocol-policy-test"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + iam.create_user( + "protocol-user", + &rustfs_madmin::AddOrUpdateUserReq { + secret_key: "protocol-user-secret".into(), + policy: None, + status: rustfs_madmin::AccountStatus::Enabled, + }, + ) + .await + .expect("create protocol user"); + let identity = iam + .check_key("protocol-user") + .await + .expect("load user") + .0 + .expect("user exists"); + let allow = json!({"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:ListBucket"], + "Resource":[format!("arn:aws:s3:::{bucket}"),format!("arn:aws:s3:::{bucket}/*")]}); + iam.set_policy( + "protocol-policy", + decode_policy(json!({"Version":"2012-10-17","Statement":[allow.clone()]})).expect("parse allow"), + ) + .await + .expect("save allow"); + iam.policy_db_set("protocol-user", UserType::Reg, false, "protocol-policy") + .await + .expect("attach policy"); + use crate::storage_api::protocols::client::{ObjectIO, StorageObjectOptions, StoragePutObjReader}; + let mut reader = StoragePutObjReader::from_vec(b"protected content".to_vec()); + store + .put_object(bucket, "secret.txt", &mut reader, &StorageObjectOptions::default()) + .await + .expect("seed protected object"); + let mut sessions: Vec<_> = [Protocol::WebDav, Protocol::Ftps, Protocol::Sftp] + .into_iter() + .map(|protocol| { + SessionContext::new( + ProtocolPrincipal::new(Arc::new(identity.clone())), + protocol, + "127.0.0.1".parse().expect("loopback"), + ) + }) + .collect(); + for session in &sessions { + backend + .authorize_operation(session, &S3Action::GetObject, bucket, Some("secret.txt")) + .await + .expect("IAM allow works"); + } + let bucket_policy = json!({"Version":"2012-10-17","Statement":[{ + "Effect":"Deny","Principal":{"AWS":"*"},"Action":["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:ListBucket"], + "Resource":[format!("arn:aws:s3:::{bucket}"),format!("arn:aws:s3:::{bucket}/*")], + "Condition":{"StringLike":{"aws:username":"*"}} + }]}); + store + .update_bucket_metadata_config(bucket, "policy.json", serde_json::to_vec(&bucket_policy).expect("encode policy")) + .await + .expect("save bucket deny"); + for session in &sessions { + let list_input = ListObjectsV2Input::builder() + .bucket(bucket.to_owned()) + .prefix(Some("secret/".into())) + .build() + .expect("list input"); + assert!( + matches!( + backend.authorize_list_objects(session, &list_input).await, + Err(AuthorizationError::AccessDenied) + ), + "bucket deny constrains listing" + ); + for action in [ + S3Action::GetObject, + S3Action::HeadObject, + S3Action::PutObject, + S3Action::DeleteObject, + S3Action::HeadBucket, + ] { + let object = if matches!(action, S3Action::HeadBucket) { + None + } else { + Some("secret.txt") + }; + assert!( + matches!( + backend.authorize_operation(session, &action, bucket, object).await, + Err(AuthorizationError::AccessDenied) + ), + "bucket deny must constrain {action:?} over {:?}", + session.protocol + ); + } + } + let root_session = SessionContext::new( + ProtocolPrincipal::new(Arc::new(rustfs_policy::auth::UserIdentity { + credentials: root, + ..Default::default() + })), + Protocol::Sftp, + "127.0.0.1".parse().expect("loopback"), + ); + assert!( + matches!( + backend + .authorize_operation(&root_session, &S3Action::GetObject, bucket, Some("secret.txt")) + .await, + Err(AuthorizationError::AccessDenied) + ), + "bucket deny constrains root object access" + ); + store + .update_bucket_metadata_config( + bucket, + "policy.json", + serde_json::to_vec(&json!({"Version":"2012-10-17","Statement":[]})).expect("encode empty policy"), + ) + .await + .expect("clear bucket policy"); + let list_input = ListObjectsV2Input::builder() + .bucket(bucket.to_owned()) + .prefix(Some("private /+&=照片/".into())) + .delimiter(Some("/".into())) + .max_keys(Some(2)) + .build() + .expect("conditioned listing input"); + // Request condition values must come from the same DTO as execution, + // including percent-encoded prefixes, explicit empty strings and zero. + for (key, operator, value, input) in [ + ("s3:prefix", "StringEquals", json!("private /+&=照片/"), list_input.clone()), + ( + "s3:prefix", + "StringEquals", + json!(""), + ListObjectsV2Input { + prefix: Some(String::new()), + ..list_input.clone() + }, + ), + ("s3:delimiter", "StringEquals", json!("/"), list_input.clone()), + ("s3:max-keys", "NumericLessThanEquals", json!("2"), list_input.clone()), + ( + "s3:max-keys", + "NumericEquals", + json!("0"), + ListObjectsV2Input { + max_keys: Some(0), + ..list_input.clone() + }, + ), + ] { + let mut non_matching = input.clone(); + let mut absent = input.clone(); + match key { + "s3:prefix" => { + non_matching.prefix = Some("other/".into()); + absent.prefix = None; + } + "s3:delimiter" => { + non_matching.delimiter = Some("|".into()); + absent.delimiter = None; + } + "s3:max-keys" => { + non_matching.max_keys = Some(3); + absent.max_keys = None; + } + _ => unreachable!("known list condition"), + } + for source in ["iam-allow", "iam-deny", "bucket-deny"] { + let conditional = json!({"Effect":if source == "iam-allow" { "Allow" } else { "Deny" }, + "Action":"s3:ListBucket","Resource":format!("arn:aws:s3:::{bucket}"), + "Condition":{(operator):{(key):value.clone()}}}); + let iam_statements = match source { + "iam-allow" => json!([conditional.clone()]), + "iam-deny" => json!([allow.clone(), conditional.clone()]), + _ => json!([allow.clone()]), + }; + iam.set_policy( + "protocol-policy", + decode_policy(json!({"Version":"2012-10-17","Statement":iam_statements})).expect("parse list policy"), + ) + .await + .expect("save list policy"); + let bucket_statements = if source == "bucket-deny" { + let mut statement = conditional; + statement["Principal"] = json!({"AWS":"*"}); + json!([statement]) + } else { + json!([]) + }; + store + .update_bucket_metadata_config( + bucket, + "policy.json", + serde_json::to_vec(&json!({"Version":"2012-10-17","Statement":bucket_statements})) + .expect("encode list bucket policy"), + ) + .await + .expect("save list bucket policy"); + for session in &sessions { + for (request, matches_condition) in [(&input, true), (&non_matching, false), (&absent, false)] { + let result = backend.authorize_list_objects(session, request).await; + let should_allow = if source == "iam-allow" { + matches_condition + } else { + !matches_condition + }; + if should_allow { + result.unwrap_or_else(|error| panic!("{source} {key} should allow {:?}: {error}", session.protocol)); + } else { + assert!( + matches!(result, Err(AuthorizationError::AccessDenied)), + "{source} {key} must deny {:?}", + session.protocol + ); + } + } + } + } + } + store + .update_bucket_metadata_config( + bucket, + "policy.json", + serde_json::to_vec(&json!({"Version":"2012-10-17","Statement":[]})).expect("encode empty list policy"), + ) + .await + .expect("clear list bucket policy"); + for (condition, should_deny) in [ + (json!({"IpAddress":{"aws:SourceIp":"127.0.0.0/8"}}), true), + (json!({"IpAddress":{"aws:SourceIp":"192.0.2.0/24"}}), false), + (json!({"NotIpAddress":{"aws:SourceIp":"127.0.0.0/8"}}), false), + (json!({"NotIpAddress":{"aws:SourceIp":"192.0.2.0/24"}}), true), + ] { + iam.set_policy( + "protocol-policy", + decode_policy(json!({"Version":"2012-10-17","Statement":[allow.clone(),{ + "Effect":"Deny","Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{bucket}/*"),"Condition":condition + }]})) + .expect("parse condition policy"), + ) + .await + .expect("save condition policy"); + for session in &mut sessions { + session + .request_headers + .insert("x-forwarded-for", "192.0.2.1".parse().expect("forwarded IP")); + let result = backend + .authorize_operation(session, &S3Action::GetObject, bucket, Some("secret.txt")) + .await; + if should_deny { + assert!(matches!(result, Err(AuthorizationError::AccessDenied)), "matching IAM deny must apply"); + } else { + result.expect("non-matching deny must permit the request"); + } + } + } + for (header, condition_key) in [("user-agent", "aws:UserAgent"), ("referer", "aws:Referer")] { + iam.set_policy( + "protocol-policy", + decode_policy(json!({"Version":"2012-10-17","Statement":[allow.clone(),{ + "Effect":"Deny","Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{bucket}/*"), + "Condition":{"StringEquals":{(condition_key):"blocked-client"}} + }]})) + .expect("parse header condition"), + ) + .await + .expect("save header condition"); + let session = &mut sessions[0]; + session.request_headers.insert( + http::header::HeaderName::from_static(header), + "blocked-client".parse().expect("header value"), + ); + assert!( + matches!( + backend + .authorize_operation(session, &S3Action::GetObject, bucket, Some("secret.txt")) + .await, + Err(AuthorizationError::AccessDenied) + ), + "WebDAV header condition must be enforced" + ); + session.request_headers.remove(header); + backend + .authorize_operation(session, &S3Action::GetObject, bucket, Some("secret.txt")) + .await + .expect("absent header follows condition semantics"); + } + let conditional_allow = json!({"Version":"2012-10-17","Statement":[{ + "Effect":"Allow","Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{bucket}/*"), + "Condition":{"IpAddress":{"aws:SourceIp":"127.0.0.0/8"}} + }]}); + iam.set_policy("protocol-policy", decode_policy(conditional_allow).expect("parse conditional allow")) + .await + .expect("save conditional allow"); + for session in &sessions { + backend + .authorize_operation(session, &S3Action::GetObject, bucket, Some("secret.txt")) + .await + .expect("matching conditional allow"); + assert!( + matches!( + backend + .authorize_operation(session, &S3Action::PutObject, bucket, Some("secret.txt")) + .await, + Err(AuthorizationError::AccessDenied) + ), + "read-only user cannot write" + ); + } + #[cfg(feature = "ftps")] + { + // Drive libunftp's real authentication handoff: a lost peer address would + // make this conditional Allow fail even though a manually built session works. + let reservation = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("reserve FTP address"); + let address = reservation.local_addr().expect("FTP address"); + drop(reservation); + let server = rustfs_protocols::FtpsServer::new( + rustfs_protocols::FtpsConfig { + bind_addr: address, + tls_enabled: false, + ftps_required: false, + ..Default::default() + }, + backend.clone(), + ) + .await + .expect("create FTP server"); + let (shutdown, receiver) = tokio::sync::broadcast::channel(1); + let server_task = tokio::spawn(async move { server.start(receiver).await }); + use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; + let stream = tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + match tokio::net::TcpStream::connect(address).await { + Ok(stream) => break stream, + Err(_) => tokio::time::sleep(std::time::Duration::from_millis(20)).await, + } + } + }) + .await + .expect("FTP listener ready"); + let (read, mut write) = stream.into_split(); + let mut read = tokio::io::BufReader::new(read); + async fn reply(read: &mut tokio::io::BufReader) -> String { + let mut line = String::new(); + tokio::time::timeout(std::time::Duration::from_secs(10), read.read_line(&mut line)) + .await + .expect("FTP response deadline") + .expect("read FTP response"); + line + } + assert!(reply(&mut read).await.starts_with("220")); + for (command, status) in [ + ("USER protocol-user\r\n", "331"), + ("PASS protocol-user-secret\r\n", "230"), + ("TYPE I\r\n", "200"), + ] { + write.write_all(command.as_bytes()).await.expect("write FTP command"); + let response = reply(&mut read).await; + assert!(response.starts_with(status), "unexpected FTP response: {response}"); + } + write + .write_all(format!("SIZE /{bucket}/secret.txt\r\n").as_bytes()) + .await + .expect("send SIZE"); + let response = reply(&mut read).await; + assert!(response.starts_with("213"), "real FTP peer must satisfy conditional Allow: {response}"); + iam.set_policy( + "protocol-policy", + decode_policy(json!({"Version":"2012-10-17","Statement":[allow,{ + "Effect":"Deny","Action":"s3:GetObject","Resource":format!("arn:aws:s3:::{bucket}/*"), + "Condition":{"IpAddress":{"aws:SourceIp":"127.0.0.0/8"}} + }]})) + .expect("parse FTP deny"), + ) + .await + .expect("save FTP deny"); + write + .write_all(format!("SIZE /{bucket}/secret.txt\r\n").as_bytes()) + .await + .expect("send denied SIZE"); + let response = reply(&mut read).await; + assert!(response.starts_with("550"), "real FTP peer must match conditional Deny: {response}"); + write.write_all(b"QUIT\r\n").await.expect("send QUIT"); + assert!(reply(&mut read).await.starts_with("221")); + shutdown.send(()).expect("stop FTP server"); + server_task.await.expect("FTP server task").expect("FTP server shutdown"); + } + }); +} diff --git a/rustfs/src/server/module_switch.rs b/rustfs/src/server/module_switch.rs index fdca6fe6e..4143f842e 100644 --- a/rustfs/src/server/module_switch.rs +++ b/rustfs/src/server/module_switch.rs @@ -275,11 +275,11 @@ where // waiting for the local guard. serialize_module_switch_rmw(|| async move { let save_store = store.clone(); - with_config_object_write_lock(store, MODULE_SWITCH_CONFIG_PATH.to_string(), move || async move { + with_config_object_write_lock(store, MODULE_SWITCH_CONFIG_PATH.to_string(), move |write_guard| async move { save_persisted_module_switches_inner( config, move |data| async move { - save_config_no_lock(save_store, MODULE_SWITCH_CONFIG_PATH, data) + save_config_no_lock(save_store, MODULE_SWITCH_CONFIG_PATH, data, &write_guard) .await .map_err(|e| format!("failed to save module switch config: {e}")) }, @@ -631,14 +631,14 @@ mod tests { async move { writer_started_tx.send(()).expect("signal remote writer start"); let save_store = store.clone(); - with_config_object_write_lock(store, MODULE_SWITCH_CONFIG_PATH.to_string(), move || async move { + with_config_object_write_lock(store, MODULE_SWITCH_CONFIG_PATH.to_string(), move |write_guard| async move { writer_entered.store(true, AtomicOrdering::SeqCst); let data = serde_json::to_vec(&PersistedModuleSwitches { notify_enabled: false, audit_enabled: false, }) .expect("serialize disabled module state"); - save_config_no_lock(save_store, MODULE_SWITCH_CONFIG_PATH, data) + save_config_no_lock(save_store, MODULE_SWITCH_CONFIG_PATH, data, &write_guard) .await .map_err(|err| err.to_string())?; publish_order.lock().expect("module publish order lock").push("new-false"); diff --git a/rustfs/src/site_replication/hooks.rs b/rustfs/src/site_replication/hooks.rs index a705985e0..79a03f5b0 100644 --- a/rustfs/src/site_replication/hooks.rs +++ b/rustfs/src/site_replication/hooks.rs @@ -46,7 +46,7 @@ where store, SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(), move || async move { - with_config_object_write_lock(mutation_store, mutation_path, operation) + with_config_object_write_lock(mutation_store, mutation_path, move |_write_guard| operation()) .await .map_err(|err| S3Error::from(ApiError::from(err))) }, @@ -68,9 +68,13 @@ where Fut: std::future::Future> + Send + 'static, T: Send + 'static, { - with_config_object_write_lock(store, SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(), operation) - .await - .map_err(|err| S3Error::from(ApiError::from(err)))? + with_config_object_write_lock( + store, + SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(), + move |_write_guard| operation(), + ) + .await + .map_err(|err| S3Error::from(ApiError::from(err)))? } #[derive(Debug, Default)] @@ -978,9 +982,11 @@ pub(crate) async fn commit_site_replication_delete_bucket(intent: &SiteReplicati reservations: intent.reservations.clone(), displaced: Vec::new(), }; - match with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { - broadcast_site_replication_delete_bucket(&delivery_intent).await - }) + match with_config_object_write_lock( + store, + SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), + move |_write_guard| async move { broadcast_site_replication_delete_bucket(&delivery_intent).await }, + ) .await { Ok(result) => result, diff --git a/rustfs/src/site_replication/repair.rs b/rustfs/src/site_replication/repair.rs index ec341d70f..ca9523263 100644 --- a/rustfs/src/site_replication/repair.rs +++ b/rustfs/src/site_replication/repair.rs @@ -169,10 +169,11 @@ pub(crate) async fn load_site_replication_repair_state_from_store(store: Arc, state: &SiteReplicationRepairState, + guard: &crate::storage_api::site_replication::WriteCommitGuard, ) -> S3Result<()> { let data = serde_json::to_vec(state) .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize repair state failed: {e}")))?; - save_config_no_lock(store, SITE_REPLICATION_REPAIR_STATE_PATH, data) + save_config_no_lock(store, SITE_REPLICATION_REPAIR_STATE_PATH, data, guard) .await .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("save repair state failed: {e}"))) } @@ -197,10 +198,10 @@ where current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; let read_store = store.clone(); let save_store = store.clone(); - with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_STATE_PATH.to_string(), move || async move { + with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_STATE_PATH.to_string(), move |write_guard| async move { let mut state = load_site_replication_repair_state_from_store(read_store).await?; let result = update(&mut state)?; - save_site_replication_repair_state_to_store(save_store, &state).await?; + save_site_replication_repair_state_to_store(save_store, &state, &write_guard).await?; Ok(result) }) .await @@ -709,9 +710,11 @@ pub(crate) async fn execute_site_replication_repair( ) -> S3Result> { let store = current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; - with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { - execute_site_replication_repair_locked(request).await - }) + with_config_object_write_lock( + store, + SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), + move |_write_guard| async move { execute_site_replication_repair_locked(request).await }, + ) .await .map_err(|_| { S3Error::with_message(S3ErrorCode::ClientTokenConflict, "another site replication repair is active".to_string()) diff --git a/rustfs/src/site_replication/retry.rs b/rustfs/src/site_replication/retry.rs index 383cb15f7..e1a3b9599 100644 --- a/rustfs/src/site_replication/retry.rs +++ b/rustfs/src/site_replication/retry.rs @@ -1700,23 +1700,27 @@ async fn drain_site_replication_retry_queue_lightweight_inner() -> S3Result<()> // timestamp, and the locked reload below decides what may actually send. promote_reachable_deferred_retry_events(&runtime, &actionable, deferred).await?; - with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { - let Some(runtime) = runtime_site_replication_targets().await? else { - return Ok(()); - }; - if runtime.state.pending_endpoint_refresh.is_some() - || runtime.state.pending_remove.is_some() - || runtime.state.pending_rotation.is_some() - { - return Ok(()); - } - let now = OffsetDateTime::now_utc(); - let (actionable, _) = lightweight_retry_drain_partition(&runtime.state, now); - if actionable.is_empty() { - return Ok(()); - } - drain_site_replication_retry_queue_lightweight_locked(Arc::new(runtime), actionable, now).await - }) + with_config_object_write_lock( + store, + SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), + move |_write_guard| async move { + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + if runtime.state.pending_endpoint_refresh.is_some() + || runtime.state.pending_remove.is_some() + || runtime.state.pending_rotation.is_some() + { + return Ok(()); + } + let now = OffsetDateTime::now_utc(); + let (actionable, _) = lightweight_retry_drain_partition(&runtime.state, now); + if actionable.is_empty() { + return Ok(()); + } + drain_site_replication_retry_queue_lightweight_locked(Arc::new(runtime), actionable, now).await + }, + ) .await .map_err(ApiError::from)? } @@ -1762,26 +1766,30 @@ pub(crate) async fn drain_site_replication_retry_queue_inner() -> S3Result<()> { // re-runs the dry-run. The lock elects one server to replay the queue; // after acquiring it, reload state so a settled event or deleted bucket // cannot be replayed from this admission snapshot. - with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move { - // Runtime and queue snapshots captured before this distributed lock - // are only admission hints. Another node may have settled the event, - // or a local bucket may have been deleted, while this node waited. - let Some(runtime) = runtime_site_replication_targets().await? else { - return Ok(()); - }; - if runtime.state.pending_endpoint_refresh.is_some() - || runtime.state.pending_remove.is_some() - || runtime.state.pending_rotation.is_some() - { - return Ok(()); - } - let horizon = heavyweight_retry_drain_horizon(OffsetDateTime::now_utc()); - let actionable = actionable_site_replication_retry_events(&runtime.state, horizon); - if actionable.is_empty() { - return Ok(()); - } - drain_site_replication_retry_queue_locked(runtime, actionable).await - }) + with_config_object_write_lock( + store, + SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), + move |_write_guard| async move { + // Runtime and queue snapshots captured before this distributed lock + // are only admission hints. Another node may have settled the event, + // or a local bucket may have been deleted, while this node waited. + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(()); + }; + if runtime.state.pending_endpoint_refresh.is_some() + || runtime.state.pending_remove.is_some() + || runtime.state.pending_rotation.is_some() + { + return Ok(()); + } + let horizon = heavyweight_retry_drain_horizon(OffsetDateTime::now_utc()); + let actionable = actionable_site_replication_retry_events(&runtime.state, horizon); + if actionable.is_empty() { + return Ok(()); + } + drain_site_replication_retry_queue_locked(runtime, actionable).await + }, + ) .await .map_err(ApiError::from)? } diff --git a/rustfs/src/site_replication/state.rs b/rustfs/src/site_replication/state.rs index 731dd5496..beb7e94e8 100644 --- a/rustfs/src/site_replication/state.rs +++ b/rustfs/src/site_replication/state.rs @@ -382,7 +382,11 @@ pub(crate) async fn load_site_replication_state_no_lock(store: Arc) -> /// Persist-or-clear under an already-held state object lock. Normalizes the /// peer map exactly once (the historical persist path normalized twice with /// two full clones — P2-22). -pub(crate) async fn persist_site_replication_state_no_lock(store: Arc, mut state: SiteReplicationState) -> S3Result<()> { +pub(crate) async fn persist_site_replication_state_no_lock( + store: Arc, + mut state: SiteReplicationState, + guard: &crate::storage_api::site_replication::WriteCommitGuard, +) -> S3Result<()> { state.peers = normalize_peer_map_by_identity(state.peers); if state.peers.len() <= 1 && state.pending_rotation.is_none() && state.pending_remove.is_none() { match delete_config_no_lock(store, SITE_REPLICATION_STATE_PATH).await { @@ -392,7 +396,7 @@ pub(crate) async fn persist_site_replication_state_no_lock(store: Arc, } else { let data = serde_json::to_vec(&state) .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize state failed: {e}")))?; - save_config_no_lock(store, SITE_REPLICATION_STATE_PATH, data) + save_config_no_lock(store, SITE_REPLICATION_STATE_PATH, data, guard) .await .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("save state failed: {e}"))) } @@ -437,13 +441,13 @@ where F: FnOnce(SiteReplicationState) -> Fut + Send + 'static, Fut: std::future::Future)>> + Send + 'static, { - with_site_replication_state_lock(move || async move { + with_site_replication_state_lock(move |write_guard| async move { let store = current_object_store_handle() .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; let state = load_site_replication_state_no_lock(store.clone()).await?; let (result, changed) = transaction(state).await?; if let Some(state) = changed { - persist_site_replication_state_no_lock(store, state).await?; + persist_site_replication_state_no_lock(store, state, &write_guard).await?; } Ok(result) }) @@ -457,13 +461,13 @@ where T: Send + 'static, F: FnOnce(&mut SiteReplicationState) -> S3Result> + Send + 'static, { - with_site_replication_state_lock(move || async move { + with_site_replication_state_lock(move |write_guard| async move { let store = current_object_store_handle() .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?; let mut state = load_site_replication_state_no_lock(store.clone()).await?; match update(&mut state)? { StateCommit::Changed(result) => { - persist_site_replication_state_no_lock(store, state).await?; + persist_site_replication_state_no_lock(store, state, &write_guard).await?; Ok(result) } StateCommit::Unchanged(result) => Ok(result), diff --git a/rustfs/src/site_replication/state_lock.rs b/rustfs/src/site_replication/state_lock.rs index 9eb228882..5ca14153c 100644 --- a/rustfs/src/site_replication/state_lock.rs +++ b/rustfs/src/site_replication/state_lock.rs @@ -54,7 +54,7 @@ pub(crate) const SITE_REPLICATION_STATE_PATH: &str = "config/site-replication/st pub(crate) async fn with_site_replication_state_lock(operation: F) -> S3Result where T: Send + 'static, - F: FnOnce() -> Fut + Send + 'static, + F: FnOnce(crate::storage_api::site_replication::WriteCommitGuard) -> Fut + Send + 'static, Fut: std::future::Future> + Send + 'static, { let store = current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?; @@ -91,7 +91,7 @@ where pub(crate) async fn with_site_replication_state_lock_on(store: Arc, operation: F) -> S3Result where T: Send + 'static, - F: FnOnce() -> Fut + Send + 'static, + F: FnOnce(crate::storage_api::site_replication::WriteCommitGuard) -> Fut + Send + 'static, Fut: std::future::Future> + Send + 'static, { with_config_object_write_lock(store, SITE_REPLICATION_STATE_PATH.to_string(), operation) diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 4217c43b5..298a13946 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -9307,7 +9307,7 @@ mod tests { "object", &mut reader, &ObjectOptions { - no_lock: true, + write_completion: crate::storage::storage_api::ecstore_object::WriteCompletion::TailDrained, ..Default::default() }, ), diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 01588dbae..43b25f1d8 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -577,9 +577,9 @@ pub(crate) mod ecstore_object { pub(crate) use rustfs_ecstore::api::object::{ EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, ObjectEncryptionResolver, ObjectMutationHook, PrepareSelectObjectSnapshotError, ReadEncryptionMaterial, - ReadEncryptionMode, ReadEncryptionRequest, SelectObjectSnapshot, WriteCompletion, get_object_body_cache_plaintext_len, - lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook, - unregister_get_object_body_cache_hook, unregister_object_mutation_hook, + ReadEncryptionMode, ReadEncryptionRequest, SelectObjectSnapshot, WriteCommitGuard, WriteCompletion, + get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, + register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook, }; } @@ -1300,8 +1300,13 @@ pub(crate) fn replication_queue_current_count() -> Option { get_global_replication_stats().and_then(|stats| stats.queue_current_count()) } -pub(crate) async fn save_config_no_lock(api: Arc, file: &str, data: Vec) -> Result<()> { - ecstore_config::com::save_config_no_lock(api, file, data).await +pub(crate) async fn save_config_no_lock( + api: Arc, + file: &str, + data: Vec, + guard: &ecstore_object::WriteCommitGuard, +) -> Result<()> { + ecstore_config::com::save_config_no_lock(api, file, data, guard).await } pub(crate) async fn delete_config_no_lock(api: Arc, file: &str) -> Result<()> { @@ -1310,7 +1315,7 @@ pub(crate) async fn delete_config_no_lock(api: Arc, file: &str) -> Resu pub(crate) async fn with_config_object_write_lock(api: Arc, object: String, operation: F) -> Result where - F: FnOnce() -> Fut + Send + 'static, + F: FnOnce(ecstore_object::WriteCommitGuard) -> Fut + Send + 'static, Fut: std::future::Future + Send + 'static, T: Send + 'static, { diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index ea42490df..69cdfd774 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -128,9 +128,17 @@ pub(crate) mod kms { pub(crate) mod protocols { pub(crate) mod client { - pub(crate) use crate::storage::storage_api::access_consumer::ReqInfo; + pub(crate) use crate::storage::storage_api::access_consumer::{ReqInfo, authorize_request}; + #[cfg(test)] + pub(crate) use crate::storage::storage_api::contract::bucket::{BucketOperations, MakeBucketOptions}; + #[cfg(test)] + pub(crate) use crate::storage::storage_api::contract::object::ObjectIO; + #[cfg(test)] + pub(crate) use crate::storage::storage_api::{StorageObjectOptions, StoragePutObjReader}; pub(crate) type FS = crate::storage::storage_api::FS; pub(crate) use crate::storage::storage_api::request_context_consumer::RequestContext; + #[cfg(test)] + pub(crate) use s3s::dto::ListObjectsV2Input; /// WebDAV quota reporting surface (`session_capacity_view`): bucket /// quota config, cached usage, and the console's erasure-aware usable @@ -289,6 +297,7 @@ pub(crate) mod site_replication { pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata::{ BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BucketMetadata, }; + pub(crate) use crate::storage::storage_api::ecstore_object::WriteCommitGuard; #[cfg(test)] pub(crate) use crate::storage::storage_api::ecstore_bucket::replication::merge_incoming_replication_config; @@ -443,6 +452,7 @@ pub(crate) mod workload { } pub(crate) mod table { + pub(crate) use crate::storage::storage_api::ecstore_object::{WriteCommitGuard, WriteCompletion}; pub(crate) mod contract { pub(crate) mod http { pub(crate) use super::super::super::storage_contracts::HTTPPreconditions; diff --git a/rustfs/src/table_catalog/store/mod.rs b/rustfs/src/table_catalog/store/mod.rs index a8c9c3eef..8f8bba826 100644 --- a/rustfs/src/table_catalog/store/mod.rs +++ b/rustfs/src/table_catalog/store/mod.rs @@ -14,6 +14,7 @@ use super::identifier::is_valid_table_metadata_file_name; use super::*; +use crate::storage_api::table::{WriteCommitGuard, WriteCompletion}; mod migration; mod object; @@ -573,6 +574,7 @@ pub(crate) struct TableCatalogObjectMetadata { pub(crate) struct TableCatalogLockGuard { _guard: Box, lock_lost: Option>, + write_commit_guard: Option, } impl TableCatalogLockGuard { @@ -584,6 +586,7 @@ impl TableCatalogLockGuard { Self { _guard: Box::new(guard), lock_lost: None, + write_commit_guard: None, } } @@ -592,11 +595,25 @@ impl TableCatalogLockGuard { Self { _guard: Box::new(guard), lock_lost, + write_commit_guard: None, } } + fn write_namespace(guard: WriteCommitGuard) -> Self { + Self { + _guard: Box::new(()), + lock_lost: None, + write_commit_guard: Some(guard), + } + } + + fn write_commit_guards(&self) -> Vec { + self.write_commit_guard.iter().cloned().collect() + } + pub(crate) fn is_lock_lost(&self) -> bool { self.lock_lost.as_ref().is_some_and(|signal| signal.is_lost()) + || self.write_commit_guard.as_ref().is_some_and(WriteCommitGuard::is_lock_lost) } } @@ -722,6 +739,7 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static { object: &str, data: Vec, precondition: TableCatalogPutPrecondition, + _write_guards: Vec, ) -> TableCatalogStoreResult<()> { self.put_object(bucket, object, data, precondition).await } @@ -1946,7 +1964,7 @@ where data: Vec, precondition: TableCatalogPutPrecondition, ) -> TableCatalogStoreResult<()> { - self.put_object_with_options(bucket, object, data, precondition, false).await + self.put_object_with_options(bucket, object, data, precondition, None).await } async fn put_object_unlocked( @@ -1955,8 +1973,10 @@ where object: &str, data: Vec, precondition: TableCatalogPutPrecondition, + write_guards: Vec, ) -> TableCatalogStoreResult<()> { - self.put_object_with_options(bucket, object, data, precondition, true).await + self.put_object_with_options(bucket, object, data, precondition, Some(write_guards)) + .await } async fn delete_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> { @@ -2043,11 +2063,10 @@ where .new_ns_lock(bucket, object) .await .map_err(|err| storage_error_to_catalog("create catalog table lock", err))?; - let guard = lock - .get_write_lock(get_lock_acquire_timeout()) + let guard = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout()) .await .map_err(|err| catalog_lock_acquisition_error("acquire catalog table lock", err))?; - Ok(TableCatalogLockGuard::namespace(guard)) + Ok(TableCatalogLockGuard::write_namespace(guard)) } async fn acquire_read_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult { @@ -2120,14 +2139,22 @@ where object: &str, data: Vec, precondition: TableCatalogPutPrecondition, - no_lock: bool, + write_guards: Option>, ) -> TableCatalogStoreResult<()> { let mut reader = PutObjReader::from_vec(data); - let opts = ObjectOptions { + let mut opts = ObjectOptions { http_preconditions: http_preconditions_for_catalog_put(precondition), - no_lock, ..Default::default() }; + if let Some(write_guards) = write_guards { + if write_guards.is_empty() { + return Err(TableCatalogStoreError::Internal("catalog write has no namespace owner".to_string())); + } + for guard in &write_guards { + opts.add_write_commit_guard(guard); + } + opts.write_completion = WriteCompletion::TailDrained; + } self.store .put_object(bucket, object, &mut reader, &opts) .await diff --git a/rustfs/src/table_catalog/store/object.rs b/rustfs/src/table_catalog/store/object.rs index 347d096af..daf11ee60 100644 --- a/rustfs/src/table_catalog/store/object.rs +++ b/rustfs/src/table_catalog/store/object.rs @@ -747,13 +747,16 @@ where object: &str, entry: &T, precondition: TableCatalogPutPrecondition, + write_guards: Vec, ) -> TableCatalogStoreResult<()> where T: Serialize, { let data = serde_json::to_vec(entry) .map_err(|err| TableCatalogStoreError::Internal(format!("failed to serialize catalog entry {object}: {err}")))?; - self.backend.put_object_unlocked(bucket, object, data, precondition).await + self.backend + .put_object_unlocked(bucket, object, data, precondition, write_guards) + .await } async fn write_exact_entry_unlocked( @@ -762,11 +765,14 @@ where object: &str, entry: &T, precondition: TableCatalogPutPrecondition, + write_guards: Vec, ) -> TableCatalogStoreResult where T: DeserializeOwned + PartialEq + Serialize, { - let write_result = self.write_entry_unlocked(bucket, object, entry, precondition).await; + let write_result = self + .write_entry_unlocked(bucket, object, entry, precondition, write_guards) + .await; let current = self.read_entry_unlocked::(bucket, object).await?; match current { Some((current, Some(etag))) if current == *entry => Ok(etag), @@ -878,14 +884,21 @@ where intent: &mut TableRenameIntent, etag: String, state: TableRenameIntentState, + write_guards: Vec, ) -> TableCatalogStoreResult { if intent.state >= state { return Ok(etag); } intent.state = state; intent.updated_at = OffsetDateTime::now_utc().to_string(); - self.write_exact_entry_unlocked(self.catalog_bucket(), object, intent, TableCatalogPutPrecondition::IfMatch(etag)) - .await + self.write_exact_entry_unlocked( + self.catalog_bucket(), + object, + intent, + TableCatalogPutPrecondition::IfMatch(etag), + write_guards, + ) + .await } async fn recover_active_table_rename( @@ -1027,6 +1040,10 @@ where &source_object, &source_fence, TableCatalogPutPrecondition::IfMatch(current_etag), + _catalog_guards + .iter() + .flat_map(TableCatalogLockGuard::write_commit_guards) + .collect(), ) .await?; } @@ -1039,7 +1056,13 @@ where } } intent_etag = self - .advance_table_rename_intent_unlocked(&intent_object, &mut intent, intent_etag, TableRenameIntentState::SourceFenced) + .advance_table_rename_intent_unlocked( + &intent_object, + &mut intent, + intent_etag, + TableRenameIntentState::SourceFenced, + _intent_guard.write_commit_guards(), + ) .await?; let mut destination_fence = intent.destination.clone(); @@ -1055,6 +1078,10 @@ where &destination_object, &destination_fence, TableCatalogPutPrecondition::IfAbsent, + _catalog_guards + .iter() + .flat_map(TableCatalogLockGuard::write_commit_guards) + .collect(), ) .await?; } @@ -1067,6 +1094,10 @@ where &destination_object, &destination_fence, TableCatalogPutPrecondition::IfMatch(current_etag), + _catalog_guards + .iter() + .flat_map(TableCatalogLockGuard::write_commit_guards) + .collect(), ) .await?; } @@ -1084,6 +1115,7 @@ where &mut intent, intent_etag, TableRenameIntentState::DestinationWritten, + _intent_guard.write_commit_guards(), ) .await?; @@ -1098,6 +1130,10 @@ where &source_object, &source_tombstone, TableCatalogPutPrecondition::IfMatch(current_etag), + _catalog_guards + .iter() + .flat_map(TableCatalogLockGuard::write_commit_guards) + .collect(), ) .await?; } @@ -1115,6 +1151,7 @@ where &mut intent, intent_etag, TableRenameIntentState::SourceTombstoned, + _intent_guard.write_commit_guards(), ) .await?; @@ -1139,6 +1176,10 @@ where &index_object, &destination_index, TableCatalogPutPrecondition::IfMatch(intent.warehouse_index_etag.clone()), + _catalog_guards + .iter() + .flat_map(TableCatalogLockGuard::write_commit_guards) + .collect(), ) .await?; } @@ -1148,6 +1189,7 @@ where &mut intent, intent_etag, TableRenameIntentState::IndexPublished, + _intent_guard.write_commit_guards(), ) .await?; match self @@ -1161,6 +1203,10 @@ where &destination_object, &intent.destination, TableCatalogPutPrecondition::IfMatch(current_etag), + _catalog_guards + .iter() + .flat_map(TableCatalogLockGuard::write_commit_guards) + .collect(), ) .await?; } @@ -1178,10 +1224,17 @@ where &mut intent, intent_etag, TableRenameIntentState::DestinationPublished, + _intent_guard.write_commit_guards(), ) .await?; - self.advance_table_rename_intent_unlocked(&intent_object, &mut intent, intent_etag, TableRenameIntentState::Completed) - .await?; + self.advance_table_rename_intent_unlocked( + &intent_object, + &mut intent, + intent_etag, + TableRenameIntentState::Completed, + _intent_guard.write_commit_guards(), + ) + .await?; let Some((mut bucket_entry, bucket_etag)) = self.read_table_bucket_with_etag_unlocked(table_bucket).await? else { return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); @@ -1196,12 +1249,17 @@ where &bucket_object, &bucket_entry, TableCatalogPutPrecondition::IfMatch(bucket_etag), + _bucket_guard.write_commit_guards(), ) .await?; Ok(()) } - async fn write_warehouse_index_state_unlocked(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { + async fn write_warehouse_index_state_unlocked( + &self, + table_bucket: &str, + write_guards: Vec, + ) -> TableCatalogStoreResult<()> { let state = TableWarehouseIndexStateEntry { version: TABLE_WAREHOUSE_INDEX_STATE_VERSION, table_bucket: table_bucket.to_string(), @@ -1212,6 +1270,7 @@ where &self.paths.warehouse_index_state_path(table_bucket), &state, TableCatalogPutPrecondition::Any, + write_guards, ) .await } @@ -1324,6 +1383,7 @@ where object, replacement, TableCatalogPutPrecondition::IfMatch(current_etag), + _guard.write_commit_guards(), ) .await?; Ok(true) @@ -1606,7 +1666,13 @@ where .await? else { return self - .write_entry_unlocked(self.catalog_bucket(), &object, &tombstone, TableCatalogPutPrecondition::IfAbsent) + .write_entry_unlocked( + self.catalog_bucket(), + &object, + &tombstone, + TableCatalogPutPrecondition::IfAbsent, + _guard.write_commit_guards(), + ) .await; }; validate_table_warehouse_index_entry_object(&self.paths, &object, ¤t)?; @@ -1626,6 +1692,7 @@ where &object, &tombstone, TableCatalogPutPrecondition::IfMatch(current_etag), + _guard.write_commit_guards(), ) .await } @@ -1927,7 +1994,8 @@ where ) .await?; } - self.write_warehouse_index_state_unlocked(table_bucket).await + self.write_warehouse_index_state_unlocked(table_bucket, _guard.write_commit_guards()) + .await } async fn require_table_bucket(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { @@ -2178,7 +2246,13 @@ where )); } let result = self - .write_entry_unlocked(self.catalog_bucket(), &table_path, &entry, precondition) + .write_entry_unlocked( + self.catalog_bucket(), + &table_path, + &entry, + precondition, + _table_guard.write_commit_guards(), + ) .await; if result.is_err() { self.rollback_table_warehouse_index_reservation(&entry, reservation, "table entry write failed") @@ -2244,7 +2318,7 @@ where "view creation publication fence was lost before catalog update".to_string(), )); } - self.write_entry_unlocked(self.catalog_bucket(), &view_path, &entry, precondition) + self.write_entry_unlocked(self.catalog_bucket(), &view_path, &entry, precondition, _view_guard.write_commit_guards()) .await } @@ -4799,8 +4873,14 @@ where } entry.updated_at = Some(next_table_catalog_update_time(current.updated_at.as_deref())); } - self.write_entry_unlocked(self.catalog_bucket(), &object, &entry, TableCatalogPutPrecondition::Any) - .await + self.write_entry_unlocked( + self.catalog_bucket(), + &object, + &entry, + TableCatalogPutPrecondition::Any, + _guard.write_commit_guards(), + ) + .await } async fn create_namespace(&self, entry: NamespaceEntry) -> TableCatalogStoreResult<()> { @@ -4849,8 +4929,14 @@ where TableCatalogPutPrecondition::IfAbsent } }; - self.write_entry_unlocked(self.catalog_bucket(), &object, &entry, precondition) - .await + self.write_entry_unlocked( + self.catalog_bucket(), + &object, + &entry, + precondition, + _namespace_guard.write_commit_guards(), + ) + .await } async fn list_namespaces(&self, table_bucket: &str) -> TableCatalogStoreResult> { @@ -4954,8 +5040,14 @@ where if before == next { return Ok(result); } - self.write_entry_unlocked(self.catalog_bucket(), &namespace_path, &next, precondition) - .await?; + self.write_entry_unlocked( + self.catalog_bucket(), + &namespace_path, + &next, + precondition, + _namespace_guard.write_commit_guards(), + ) + .await?; Ok(result) } @@ -5313,6 +5405,7 @@ where &index_object, &source_index, TableCatalogPutPrecondition::IfAbsent, + _index_guard.write_commit_guards(), ) .await? } @@ -5338,6 +5431,7 @@ where &intent_object, &intent, TableCatalogPutPrecondition::IfAbsent, + _intent_guard.write_commit_guards(), ) .await?; bucket_entry.active_rename_id = Some(rename_id); @@ -5347,6 +5441,7 @@ where &bucket_object, &bucket_entry, TableCatalogPutPrecondition::IfMatch(bucket_etag), + _bucket_guard.write_commit_guards(), ) .await?; } @@ -5796,6 +5891,7 @@ where &table_path, &next, TableCatalogPutPrecondition::IfMatch(current_etag), + _guard.write_commit_guards(), ) .await; record_table_commit_cas_result(&request.operation, cas_started, &cas_result); @@ -6091,6 +6187,7 @@ where &view_path, &next, TableCatalogPutPrecondition::IfMatch(current_etag), + _guard.write_commit_guards(), ) .await; if let Err(err) = write_result { diff --git a/scripts/test_functional_chain.py b/scripts/test_functional_chain.py index 172f45a17..f36ff671b 100644 --- a/scripts/test_functional_chain.py +++ b/scripts/test_functional_chain.py @@ -439,7 +439,7 @@ class WorkflowTimeoutTests(unittest.TestCase): line in (" id: test", " id: pool_test") for line in step)] self.assertEqual(len(primary), 1) if suite == "pool-expand": - self.assertIn(" timeout-minutes: ${{ inputs.pool_timeout_minutes || 240 }}", primary[0]) + self.assertIn(" timeout-minutes: ${{ fromJSON(inputs.pool_timeout_minutes || '240') }}", primary[0]) for event in ("workflow_call", "workflow_dispatch"): event_block = yaml_block(source.splitlines(), event, 2) timeout_input = yaml_block(event_block, "pool_timeout_minutes", 6)